diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
index 776b1ab944f69..bb3c75f10aaea 100644
--- a/.buildkite/pipelines/intake.yml
+++ b/.buildkite/pipelines/intake.yml
@@ -62,7 +62,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
index e9c743885d78d..12729a9b6ebda 100644
--- a/.buildkite/pipelines/periodic-packaging.yml
+++ b/.buildkite/pipelines/periodic-packaging.yml
@@ -322,8 +322,8 @@ steps:
         env:
           BWC_VERSION: 7.16.3
 
-      - label: "{{matrix.image}} / 7.17.23 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.23
+      - label: "{{matrix.image}} / 7.17.24 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.24
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -337,7 +337,7 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 7.17.23
+          BWC_VERSION: 7.17.24
 
       - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.1
@@ -577,8 +577,8 @@ steps:
         env:
           BWC_VERSION: 8.13.4
 
-      - label: "{{matrix.image}} / 8.14.4 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.4
+      - label: "{{matrix.image}} / 8.14.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -592,10 +592,10 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.14.4
+          BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0
+      - label: "{{matrix.image}} / 8.15.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -609,7 +609,7 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.15.0
+          BWC_VERSION: 8.15.1
 
       - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0
diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
index f908b946bb523..740fec13d1790 100644
--- a/.buildkite/pipelines/periodic.yml
+++ b/.buildkite/pipelines/periodic.yml
@@ -342,8 +342,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 7.17.23 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.23#bwcTest
+      - label: 7.17.24 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.24#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -353,7 +353,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 7.17.23
+          BWC_VERSION: 7.17.24
         retry:
           automatic:
             - exit_status: "-1"
@@ -642,8 +642,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.14.4 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.4#bwcTest
+      - label: 8.14.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -653,7 +653,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.14.4
+          BWC_VERSION: 8.14.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -662,8 +662,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.0#bwcTest
+      - label: 8.15.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -673,7 +673,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.15.0
+          BWC_VERSION: 8.15.1
         retry:
           automatic:
             - exit_status: "-1"
@@ -771,7 +771,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -821,7 +821,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
diff --git a/.buildkite/scripts/gradle-build-cache-validation.sh b/.buildkite/scripts/gradle-build-cache-validation.sh
new file mode 100755
index 0000000000000..75dc9b264b8bc
--- /dev/null
+++ b/.buildkite/scripts/gradle-build-cache-validation.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+set -euo pipefail
+
+VALIDATION_SCRIPTS_VERSION=2.5.1
+GRADLE_ENTERPRISE_ACCESS_KEY=$(vault kv get -field=value secret/ci/elastic-elasticsearch/gradle-enterprise-api-key)
+export GRADLE_ENTERPRISE_ACCESS_KEY
+
+curl -s -L -O https://github.com/gradle/gradle-enterprise-build-validation-scripts/releases/download/v$VALIDATION_SCRIPTS_VERSION/gradle-enterprise-gradle-build-validation-$VALIDATION_SCRIPTS_VERSION.zip && unzip -q -o gradle-enterprise-gradle-build-validation-$VALIDATION_SCRIPTS_VERSION.zip
+
+# Create a temporary file
+tmpOutputFile=$(mktemp)
+trap "rm $tmpOutputFile" EXIT
+
+set +e
+gradle-enterprise-gradle-build-validation/03-validate-local-build-caching-different-locations.sh -r https://github.com/elastic/elasticsearch.git -b $BUILDKITE_BRANCH --gradle-enterprise-server https://gradle-enterprise.elastic.co -t precommit --fail-if-not-fully-cacheable | tee $tmpOutputFile
+# Capture the return value
+retval=$?
+set -e
+
+# Now read the content from the temporary file into a variable
+perfOutput=$(cat $tmpOutputFile | sed -n '/Performance Characteristics/,/See https:\/\/gradle.com\/bvs\/main\/Gradle.md#performance-characteristics for details./p' | sed '$d' | sed 's/\x1b\[[0-9;]*m//g')
+investigationOutput=$(cat $tmpOutputFile | sed -n '/Investigation Quick Links/,$p' | sed 's/\x1b\[[0-9;]*m//g')
+
+# Initialize HTML output variable
+summaryHtml="<h4>Build Cache Performance Characteristics</h4>"
+summaryHtml+="<ul>"
+
+# Process each line of the string
+while IFS=: read -r label value; do
+    if [[ -n "$label" && -n "$value" ]]; then
+        # Trim whitespace from label and value
+        trimmed_label=$(echo "$label" | xargs)
+        trimmed_value=$(echo "$value" | xargs)
+
+        # Append to HTML output variable
+        summaryHtml+="<li><strong>$trimmed_label:</strong> $trimmed_value</li>"
+    fi
+done <<< "$perfOutput"
+
+summaryHtml+="</ul>"
+
+# generate html for links
+summaryHtml+="<h4>Investigation Links</h4>"
+summaryHtml+="<ul>"
+
+# Process each line of the string
+while IFS= read -r line; do
+    if [[ "$line" =~ http.* ]]; then
+        # Extract URL and description using awk
+        url=$(echo "$line" | awk '{print $NF}')
+        description=$(echo "$line" | sed -e "s/:.*//")
+
+        # Append to HTML output variable
+        summaryHtml+="    <li><a href=\"$url\">$description</a></li>"
+    fi
+done <<< "$investigationOutput"
+
+# End of the HTML content
+summaryHtml+="</ul>"
+
+cat << EOF | buildkite-agent annotate --context "ctx-validation-summary" --style "info"
+$summaryHtml
+EOF
+
+# Check if the command was successful
+if [ $retval -eq 0 ]; then
+    echo "Experiment completed successfully"
+elif [ $retval -eq 1 ]; then
+    echo "An invalid input was provided while attempting to run the experiment"
+elif [ $retval -eq 2 ]; then
+    echo "One of the builds that is part of the experiment failed"
+elif [ $retval -eq 3 ]; then
+    echo "The build was not fully cacheable for the given task graph"
+elif [ $retval -eq 3 ]; then
+    echo "An unclassified, fatal error happened while running the experiment"
+fi
+
+exit $retval
+
diff --git a/.buildkite/scripts/gradle-cache-validation.sh b/.buildkite/scripts/gradle-cache-validation.sh
deleted file mode 100755
index fbb957bc3b26b..0000000000000
--- a/.buildkite/scripts/gradle-cache-validation.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-VALIDATION_SCRIPTS_VERSION=2.5.1
-GRADLE_ENTERPRISE_ACCESS_KEY=$(vault kv get -field=value secret/ci/elastic-elasticsearch/gradle-enterprise-api-key)
-export GRADLE_ENTERPRISE_ACCESS_KEY
-
-curl -s -L -O https://github.com/gradle/gradle-enterprise-build-validation-scripts/releases/download/v$VALIDATION_SCRIPTS_VERSION/gradle-enterprise-gradle-build-validation-$VALIDATION_SCRIPTS_VERSION.zip && unzip -q -o gradle-enterprise-gradle-build-validation-$VALIDATION_SCRIPTS_VERSION.zip
-
-# Create a temporary file
-tmpOutputFile=$(mktemp)
-trap "rm $tmpOutputFile" EXIT
-
-gradle-enterprise-gradle-build-validation/03-validate-local-build-caching-different-locations.sh -r https://github.com/elastic/elasticsearch.git -b $BUILDKITE_BRANCH --gradle-enterprise-server https://gradle-enterprise.elastic.co -t precommit --fail-if-not-fully-cacheable | tee $tmpOutputFile
-
-# Capture the return value
-retval=$?
-
-# Now read the content from the temporary file into a variable
-perfOutput=$(cat $tmpOutputFile | sed -n '/Performance Characteristics/,/See https:\/\/gradle.com\/bvs\/main\/Gradle.md#performance-characteristics for details./p' | sed '$d' | sed 's/\x1b\[[0-9;]*m//g')
-investigationOutput=$(cat $tmpOutputFile | sed -n '/Investigation Quick Links/,$p' | sed 's/\x1b\[[0-9;]*m//g')
-
-# Initialize HTML output variable
-summaryHtml="<h4>Performance Characteristics</h4>"
-summaryHtml+="<ul>"
-
-# Process each line of the string
-while IFS=: read -r label value; do
-    if [[ -n "$label" && -n "$value" ]]; then
-        # Trim whitespace from label and value
-        trimmed_label=$(echo "$label" | xargs)
-        trimmed_value=$(echo "$value" | xargs)
-
-        # Append to HTML output variable
-        summaryHtml+="<li><strong>$trimmed_label:</strong> $trimmed_value</li>"
-    fi
-done <<< "$perfOutput"
-
-summaryHtml+="</ul>"
-
-# generate html for links
-summaryHtml+="<h4>Investigation Links</h4>"
-summaryHtml+="<ul>"
-
-# Process each line of the string
-while IFS= read -r line; do
-    if [[ "$line" =~ http.* ]]; then
-        # Extract URL and description using awk
-        url=$(echo "$line" | awk '{print $NF}')
-        description=$(echo "$line" | sed -e "s/:.*//")
-
-        # Append to HTML output variable
-        summaryHtml+="    <li><a href=\"$url\">$description</a></li>"
-    fi
-done <<< "$investigationOutput"
-
-# End of the HTML content
-summaryHtml+="</ul>"
-
-cat << EOF | buildkite-agent annotate --context "ctx-validation-summary" --style "info"
-$summaryHtml
-EOF
-
-# Check if the command was successful
-if [ $retval -eq 0 ]; then
-    echo "Experiment completed successfully"
-elif [ $retval -eq 1 ]; then
-    echo "An invalid input was provided while attempting to run the experiment"
-elif [ $retval -eq 2 ]; then
-    echo "One of the builds that is part of the experiment failed"
-elif [ $retval -eq 3 ]; then
-    echo "The build was not fully cacheable for the given task graph"
-elif [ $retval -eq 3 ]; then
-    echo "An unclassified, fatal error happened while running the experiment"
-fi
-
-exit $retval
-
diff --git a/.buildkite/scripts/gradle-configuration-cache-validation.sh b/.buildkite/scripts/gradle-configuration-cache-validation.sh
new file mode 100755
index 0000000000000..8249155c5ffc5
--- /dev/null
+++ b/.buildkite/scripts/gradle-configuration-cache-validation.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# TODO/ FIXIT without a full resolved gradle home, we see issues configuration cache reuse
+./gradlew --max-workers=8 --parallel --scan --no-daemon precommit
+
+./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
+
+# Create a temporary file
+tmpOutputFile=$(mktemp)
+trap "rm $tmpOutputFile" EXIT
+
+echo "2nd run"
+# TODO run-gradle.sh script causes issues because of init script handling
+./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
+
+# Check if the command was successful
+if grep -q "Configuration cache entry reused." $tmpOutputFile; then
+    echo "Gradle configuration cache reused"
+    exit 0
+else
+    echo "Failed to reuse Gradle configuration cache."
+    exit 1
+fi
+
+
diff --git a/.buildkite/scripts/lucene-snapshot/update-branch.sh b/.buildkite/scripts/lucene-snapshot/update-branch.sh
index d02123f3236e7..6a2d1e3df05f7 100755
--- a/.buildkite/scripts/lucene-snapshot/update-branch.sh
+++ b/.buildkite/scripts/lucene-snapshot/update-branch.sh
@@ -2,17 +2,17 @@
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then
-  echo "Error: This script should only be run on the lucene_snapshot branch"
+if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot"* ]]; then
+  echo "Error: This script should only be run on lucene_snapshot branches"
   exit 1
 fi
 
-echo --- Updating lucene_snapshot branch with main
+echo --- Updating "$BUILDKITE_BRANCH" branch with main
 
 git config --global user.name elasticsearchmachine
 git config --global user.email 'infra-root+elasticsearchmachine@elastic.co'
 
-git checkout lucene_snapshot
+git checkout "$BUILDKITE_BRANCH"
 git fetch origin main
 git merge --no-edit origin/main
-git push origin lucene_snapshot
+git push origin "$BUILDKITE_BRANCH"
diff --git a/.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh b/.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh
index 75f42a32cb590..7bec83d055139 100755
--- a/.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh
+++ b/.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh
@@ -2,8 +2,8 @@
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then
-  echo "Error: This script should only be run on the lucene_snapshot branch"
+if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot"* ]]; then
+  echo "Error: This script should only be run on the lucene_snapshot branches"
   exit 1
 fi
 
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index 776be80e0d291..e43b3333dd755 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -16,7 +16,7 @@ BWC_VERSION:
   - "7.14.2"
   - "7.15.2"
   - "7.16.3"
-  - "7.17.23"
+  - "7.17.24"
   - "8.0.1"
   - "8.1.3"
   - "8.2.3"
@@ -31,6 +31,6 @@ BWC_VERSION:
   - "8.11.4"
   - "8.12.2"
   - "8.13.4"
-  - "8.14.4"
-  - "8.15.0"
+  - "8.14.3"
+  - "8.15.1"
   - "8.16.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
index f5f7f7a7d4ecb..2eea118e57e2a 100644
--- a/.ci/snapshotBwcVersions
+++ b/.ci/snapshotBwcVersions
@@ -1,5 +1,4 @@
 BWC_VERSION:
-  - "7.17.23"
-  - "8.14.4"
-  - "8.15.0"
+  - "7.17.24"
+  - "8.15.1"
   - "8.16.0"
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 0f7e3073ed022..5b98444c044d2 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -27,8 +27,12 @@ libs/logstash-bridge @elastic/logstash
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @elastic/kibana-security
 
 # APM Data index templates, etc.
-x-pack/plugin/apm-data/src/main/resources @elastic/apm-server
-x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/apm-server
+x-pack/plugin/apm-data/src/main/resources @elastic/obs-ds-intake-services
+x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/obs-ds-intake-services
+
+# OTel
+x-pack/plugin/otel-data/src/main/resources @elastic/obs-ds-intake-services
+x-pack/plugin/otel-data/src/yamlRestTest/resources @elastic/obs-ds-intake-services
 
 # Delivery
 gradle @elastic/es-delivery
diff --git a/README.asciidoc b/README.asciidoc
index dc27735d3c015..c1945e56b025b 100644
--- a/README.asciidoc
+++ b/README.asciidoc
@@ -1,6 +1,6 @@
 = Elasticsearch
 
-Elasticsearch is a distributed search and analytics engine optimized for speed and relevance on production-scale workloads. Elasticsearch is the foundation of Elastic's open Stack platform. Search in near real-time over massive datasets, perform vector searches, integrate with generative AI applications, and much more.
+Elasticsearch is a distributed search and analytics engine, scalable data store and vector database optimized for speed and relevance on production-scale workloads. Elasticsearch is the foundation of Elastic's open Stack platform. Search in near real-time over massive datasets, perform vector searches, integrate with generative AI applications, and much more.
 
 Use cases enabled by Elasticsearch include:
 
@@ -33,76 +33,144 @@ https://www.elastic.co/downloads/elasticsearch[elastic.co/downloads/elasticsearc
 === Run Elasticsearch locally
 
 //// 
-IMPORTANT: This content is replicated in the Elasticsearch guide. 
-If you make changes, you must also update setup/set-up-local-dev-deployment.asciidoc.
+IMPORTANT: This content is replicated in the Elasticsearch guide. See `run-elasticsearch-locally.asciidoc`.
+Both will soon be replaced by a quickstart script.
 ////
 
-To try out Elasticsearch on your own machine, we recommend using Docker
-and running both Elasticsearch and Kibana.
-Docker images are available from the https://www.docker.elastic.co[Elastic Docker registry].
+[WARNING]
+==== 
+DO NOT USE THESE INSTRUCTIONS FOR PRODUCTION DEPLOYMENTS.
 
-NOTE: Starting in Elasticsearch 8.0, security is enabled by default. 
-The first time you start Elasticsearch, TLS encryption is configured automatically, 
-a password is generated for the `elastic` user, 
-and a Kibana enrollment token is created so you can connect Kibana to your secured cluster.
+This setup is intended for local development and testing only.
+====
 
-For other installation options, see the
-https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html[Elasticsearch installation documentation].
+The following commands help you very quickly spin up a single-node Elasticsearch cluster, together with Kibana in Docker.
+Use this setup for local development or testing.
 
-**Start Elasticsearch**
+==== Prerequisites
 
-. Install and start https://www.docker.com/products/docker-desktop[Docker
-Desktop]. Go to **Preferences > Resources > Advanced** and set Memory to at least 4GB.
+If you don't have Docker installed, https://www.docker.com/products/docker-desktop[download and install Docker Desktop] for your operating system.
 
-. Start an Elasticsearch container:
-+
+==== Set environment variables
+
+Configure the following environment variables.
+
+[source,sh]
+----
+export ELASTIC_PASSWORD="<ES_PASSWORD>"  # password for "elastic" username
+export KIBANA_PASSWORD="<KIB_PASSWORD>"   # Used internally by Kibana, must be at least 6 characters long
+----
+
+==== Create a Docker network
+
+To run both Elasticsearch and Kibana, you'll need to create a Docker network:
+
+[source,sh]
 ----
-docker network create elastic
-docker pull docker.elastic.co/elasticsearch/elasticsearch:{version} <1>
-docker run --name elasticsearch --net elastic -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -t docker.elastic.co/elasticsearch/elasticsearch:{version}
+docker network create elastic-net
 ----
-<1> Replace {version} with the version of Elasticsearch you want to run.
-+
-When you start Elasticsearch for the first time, the generated `elastic` user password and
-Kibana enrollment token are output to the terminal.
-+
-NOTE: You might need to scroll back a bit in the terminal to view the password 
-and enrollment token.
 
-. Copy the generated password and enrollment token and save them in a secure 
-location. These values are shown only when you start Elasticsearch for the first time.
-You'll use these to enroll Kibana with your Elasticsearch cluster and log in.
+==== Run Elasticsearch
+
+Start the Elasticsearch container with the following command:
 
-**Start Kibana**
+[source,sh]
+----
+docker run -p 127.0.0.1:9200:9200 -d --name elasticsearch --network elastic-net \
+  -e ELASTIC_PASSWORD=$ELASTIC_PASSWORD \
+  -e "discovery.type=single-node" \
+  -e "xpack.security.http.ssl.enabled=false" \
+  -e "xpack.license.self_generated.type=trial" \
+  docker.elastic.co/elasticsearch/elasticsearch:{version}
+----
 
-Kibana enables you to easily send requests to Elasticsearch and analyze, visualize, and manage data interactively.
+==== Run Kibana (optional)
 
-. In a new terminal session, start Kibana and connect it to your Elasticsearch container:
-+
+To run Kibana, you must first set the `kibana_system` password in the Elasticsearch container.
+
+[source,sh]
 ----
-docker pull docker.elastic.co/kibana/kibana:{version} <1>
-docker run --name kibana --net elastic -p 5601:5601 docker.elastic.co/kibana/kibana:{version}
+# configure the Kibana password in the ES container
+curl -u elastic:$ELASTIC_PASSWORD \
+  -X POST \
+  http://localhost:9200/_security/user/kibana_system/_password \
+  -d '{"password":"'"$KIBANA_PASSWORD"'"}' \
+  -H 'Content-Type: application/json'
 ----
-<1> Replace {version} with the version of Kibana you want to run.
-+
-When you start Kibana, a unique URL is output to your terminal.
+// NOTCONSOLE
 
-. To access Kibana, open the generated URL in your browser.
+Start the Kibana container with the following command:
 
-  .. Paste the enrollment token that you copied when starting
-  Elasticsearch and click the button to connect your Kibana instance with Elasticsearch.
+[source,sh]
+----
+docker run -p 127.0.0.1:5601:5601 -d --name kibana --network elastic-net \
+  -e ELASTICSEARCH_URL=http://elasticsearch:9200 \
+  -e ELASTICSEARCH_HOSTS=http://elasticsearch:9200 \
+  -e ELASTICSEARCH_USERNAME=kibana_system \
+  -e ELASTICSEARCH_PASSWORD=$KIBANA_PASSWORD \
+  -e "xpack.security.enabled=false" \
+  -e "xpack.license.self_generated.type=trial" \
+  docker.elastic.co/kibana/kibana:{version}
+----
 
-  .. Log in to Kibana as the `elastic` user with the password that was generated
-  when you started Elasticsearch.
+.Trial license
+[%collapsible]
+====
+The service is started with a trial license. The trial license enables all features of Elasticsearch for a trial period of 30 days. After the trial period expires, the license is downgraded to a basic license, which is free forever. If you prefer to skip the trial and use the basic license, set the value of the `xpack.license.self_generated.type` variable to basic instead. For a detailed feature comparison between the different licenses, refer to our https://www.elastic.co/subscriptions[subscriptions page].
+====
 
-**Send requests to Elasticsearch**
+==== Send requests to Elasticsearch
 
 You send data and other requests to Elasticsearch through REST APIs. 
 You can interact with Elasticsearch using any client that sends HTTP requests, 
 such as the https://www.elastic.co/guide/en/elasticsearch/client/index.html[Elasticsearch
 language clients] and https://curl.se[curl]. 
+
+===== Using curl 
+
+Here's an example curl command to create a new Elasticsearch index, using basic auth:
+
+[source,sh]
+----
+curl -u elastic:$ELASTIC_PASSWORD \
+  -X PUT \
+  http://localhost:9200/my-new-index \
+  -H 'Content-Type: application/json'
+----
+// NOTCONSOLE
+
+===== Using a language client
+
+To connect to your local dev Elasticsearch cluster with a language client, you can use basic authentication with the `elastic` username and the password you set in the environment variable.
+
+You'll use the following connection details:
+
+* **Elasticsearch endpoint**: `http://localhost:9200`
+* **Username**: `elastic`
+* **Password**: `$ELASTIC_PASSWORD` (Value you set in the environment variable)
+
+For example, to connect with the Python `elasticsearch` client:
+
+[source,python]
+----
+import os
+from elasticsearch import Elasticsearch
+
+username = 'elastic'
+password = os.getenv('ELASTIC_PASSWORD') # Value you set in the environment variable
+
+client = Elasticsearch(
+    "http://localhost:9200",
+    basic_auth=(username, password)
+)
+
+print(client.info())
+----
+
+===== Using the Dev Tools Console
+
 Kibana's developer console provides an easy way to experiment and test requests. 
-To access the console, go to **Management > Dev Tools**.
+To access the console, open Kibana, then go to **Management** > **Dev Tools**.
 
 **Add data**
 
diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
index 49e81a67e85f9..b16621aaaa471 100644
--- a/benchmarks/build.gradle
+++ b/benchmarks/build.gradle
@@ -1,4 +1,5 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.TestUtil
 
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
@@ -29,6 +30,7 @@ tasks.named("javadoc").configure { enabled = false }
 configurations {
   expression
   painless
+  nativeLib
 }
 
 dependencies {
@@ -37,6 +39,7 @@ dependencies {
     // us to invoke the JMH uberjar as usual.
     exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'
   }
+  api(project(':libs:elasticsearch-h3'))
   api(project(':modules:aggregations'))
   api(project(':x-pack:plugin:esql-core'))
   api(project(':x-pack:plugin:esql'))
@@ -44,6 +47,7 @@ dependencies {
   implementation project(path: ':libs:elasticsearch-simdvec')
   expression(project(path: ':modules:lang-expression', configuration: 'zip'))
   painless(project(path: ':modules:lang-painless', configuration: 'zip'))
+  nativeLib(project(':libs:elasticsearch-native'))
   api "org.openjdk.jmh:jmh-core:$versions.jmh"
   annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
   // Dependencies of JMH
@@ -75,17 +79,8 @@ tasks.register("copyPainless", Copy) {
 tasks.named("run").configure {
   executable = "${BuildParams.runtimeJavaHome}/bin/java"
   args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index"
-  dependsOn "copyExpression", "copyPainless"
-  systemProperty 'java.library.path', file("../libs/native/libraries/build/platform/${platformName()}-${os.arch}")
-}
-
-String platformName() {
-  String name = System.getProperty("os.name");
-  if (name.startsWith("Mac")) {
-    return "darwin";
-  } else {
-    return name.toLowerCase(Locale.ROOT);
-  }
+  dependsOn "copyExpression", "copyPainless", configurations.nativeLib
+  systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString())
 }
 
 spotless {
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java
index 49603043e7bcc..59fdfff3025a1 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java
@@ -20,13 +20,10 @@
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BooleanBigArrayBlock;
 import org.elasticsearch.compute.data.BooleanBigArrayVector;
-import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BooleanVector;
-import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.DoubleBigArrayBlock;
 import org.elasticsearch.compute.data.DoubleBigArrayVector;
-import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.DoubleVector;
 import org.elasticsearch.compute.data.IntBigArrayBlock;
 import org.elasticsearch.compute.data.IntBigArrayVector;
@@ -34,39 +31,13 @@
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.LongBigArrayBlock;
 import org.elasticsearch.compute.data.LongBigArrayVector;
-import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.LongVector;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Level;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OperationsPerInvocation;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-import org.openjdk.jmh.annotations.Warmup;
 
 import java.util.ArrayList;
 import java.util.BitSet;
-import java.util.Collections;
-import java.util.List;
 import java.util.Random;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.IntStream;
-
-@Warmup(iterations = 5)
-@Measurement(iterations = 7)
-@BenchmarkMode(Mode.AverageTime)
-@OutputTimeUnit(TimeUnit.NANOSECONDS)
-@State(Scope.Thread)
-@Fork(1)
-public class BlockBenchmark {
 
+public class BlockBenchmark {
     /**
      * All data type/block kind combinations to be loaded before the benchmark.
      * It is important to be exhaustive here so that all implementers of {@link IntBlock#getInt(int)} are actually loaded when we benchmark
@@ -114,35 +85,12 @@ public class BlockBenchmark {
     private static final int MAX_MV_ELEMENTS = 100;
     private static final int MAX_BYTES_REF_LENGTH = 255;
 
-    private static final Random random = new Random();
-
-    private static final BlockFactory blockFactory = BlockFactory.getInstance(
-        new NoopCircuitBreaker("noop"),
-        BigArrays.NON_RECYCLING_INSTANCE
-    );
-
-    static {
-        // Smoke test all the expected values and force loading subclasses more like prod
-        int totalPositions = 10;
-        long[] actualCheckSums = new long[NUM_BLOCKS_PER_ITERATION];
-
-        for (String paramString : RELEVANT_TYPE_BLOCK_COMBINATIONS) {
-            String[] params = paramString.split("/");
-            String dataType = params[0];
-            String blockKind = params[1];
-
-            BenchmarkBlocks data = buildBlocks(dataType, blockKind, totalPositions);
-            int[][] traversalOrders = createTraversalOrders(data.blocks, false);
-            run(dataType, data, traversalOrders, actualCheckSums);
-            assertCheckSums(data, actualCheckSums);
-        }
-    }
+    static final Random random = new Random();
 
-    private record BenchmarkBlocks(Block[] blocks, long[] checkSums) {};
+    static final BlockFactory blockFactory = BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE);
 
-    private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, int totalPositions) {
+    static Block[] buildBlocks(String dataType, String blockKind, int totalPositions) {
         Block[] blocks = new Block[NUM_BLOCKS_PER_ITERATION];
-        long[] checkSums = new long[NUM_BLOCKS_PER_ITERATION];
 
         switch (dataType) {
             case "boolean" -> {
@@ -237,11 +185,6 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
                         }
                     }
                 }
-
-                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
-                    BooleanBlock block = (BooleanBlock) blocks[blockIndex];
-                    checkSums[blockIndex] = computeBooleanCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray());
-                }
             }
             case "BytesRef" -> {
                 for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
@@ -294,11 +237,6 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
                         }
                     }
                 }
-
-                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
-                    BytesRefBlock block = (BytesRefBlock) blocks[blockIndex];
-                    checkSums[blockIndex] = computeBytesRefCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray());
-                }
             }
             case "double" -> {
                 for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
@@ -386,11 +324,6 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
                         }
                     }
                 }
-
-                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
-                    DoubleBlock block = (DoubleBlock) blocks[blockIndex];
-                    checkSums[blockIndex] = computeDoubleCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray());
-                }
             }
             case "int" -> {
                 for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
@@ -478,11 +411,6 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
                         }
                     }
                 }
-
-                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
-                    IntBlock block = (IntBlock) blocks[blockIndex];
-                    checkSums[blockIndex] = computeIntCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray());
-                }
             }
             case "long" -> {
                 for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
@@ -570,36 +498,12 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in
                         }
                     }
                 }
-
-                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
-                    LongBlock block = (LongBlock) blocks[blockIndex];
-                    checkSums[blockIndex] = computeLongCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray());
-                }
             }
             default -> {
                 throw new IllegalStateException("illegal data type [" + dataType + "]");
             }
         }
-
-        return new BenchmarkBlocks(blocks, checkSums);
-    }
-
-    private static int[][] createTraversalOrders(Block[] blocks, boolean randomized) {
-        int[][] orders = new int[blocks.length][];
-
-        for (int i = 0; i < blocks.length; i++) {
-            IntStream positionsStream = IntStream.range(0, blocks[i].getPositionCount());
-
-            if (randomized) {
-                List<Integer> positions = new java.util.ArrayList<>(positionsStream.boxed().toList());
-                Collections.shuffle(positions, random);
-                orders[i] = positions.stream().mapToInt(x -> x).toArray();
-            } else {
-                orders[i] = positionsStream.toArray();
-            }
-        }
-
-        return orders;
+        return blocks;
     }
 
     private static int[] randomFirstValueIndexes(int totalPositions) {
@@ -631,220 +535,4 @@ private static BitSet randomNulls(int positionCount) {
 
         return nulls;
     }
-
-    private static void run(String dataType, BenchmarkBlocks data, int[][] traversalOrders, long[] resultCheckSums) {
-        switch (dataType) {
-            case "boolean" -> {
-                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
-                    BooleanBlock block = (BooleanBlock) data.blocks[blockIndex];
-
-                    resultCheckSums[blockIndex] = computeBooleanCheckSum(block, traversalOrders[blockIndex]);
-                }
-            }
-            case "BytesRef" -> {
-                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
-                    BytesRefBlock block = (BytesRefBlock) data.blocks[blockIndex];
-
-                    resultCheckSums[blockIndex] = computeBytesRefCheckSum(block, traversalOrders[blockIndex]);
-                }
-            }
-            case "double" -> {
-                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
-                    DoubleBlock block = (DoubleBlock) data.blocks[blockIndex];
-
-                    resultCheckSums[blockIndex] = computeDoubleCheckSum(block, traversalOrders[blockIndex]);
-                }
-            }
-            case "int" -> {
-                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
-                    IntBlock block = (IntBlock) data.blocks[blockIndex];
-
-                    resultCheckSums[blockIndex] = computeIntCheckSum(block, traversalOrders[blockIndex]);
-                }
-            }
-            case "long" -> {
-                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
-                    LongBlock block = (LongBlock) data.blocks[blockIndex];
-
-                    resultCheckSums[blockIndex] = computeLongCheckSum(block, traversalOrders[blockIndex]);
-                }
-            }
-            default -> {
-                throw new IllegalStateException();
-            }
-        }
-    }
-
-    private static void assertCheckSums(BenchmarkBlocks data, long[] actualCheckSums) {
-        for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
-            if (actualCheckSums[blockIndex] != data.checkSums[blockIndex]) {
-                throw new AssertionError("checksums do not match for block [" + blockIndex + "]");
-            }
-        }
-    }
-
-    private static long computeBooleanCheckSum(BooleanBlock block, int[] traversalOrder) {
-        long sum = 0;
-
-        for (int position : traversalOrder) {
-            if (block.isNull(position)) {
-                continue;
-            }
-            int start = block.getFirstValueIndex(position);
-            int end = start + block.getValueCount(position);
-            for (int i = start; i < end; i++) {
-                sum += block.getBoolean(i) ? 1 : 0;
-            }
-        }
-
-        return sum;
-    }
-
-    private static long computeBytesRefCheckSum(BytesRefBlock block, int[] traversalOrder) {
-        long sum = 0;
-        BytesRef currentValue = new BytesRef();
-
-        for (int position : traversalOrder) {
-            if (block.isNull(position)) {
-                continue;
-            }
-            int start = block.getFirstValueIndex(position);
-            int end = start + block.getValueCount(position);
-            for (int i = start; i < end; i++) {
-                block.getBytesRef(i, currentValue);
-                sum += currentValue.length > 0 ? currentValue.bytes[0] : 0;
-            }
-        }
-
-        return sum;
-    }
-
-    private static long computeDoubleCheckSum(DoubleBlock block, int[] traversalOrder) {
-        long sum = 0;
-
-        for (int position : traversalOrder) {
-            if (block.isNull(position)) {
-                continue;
-            }
-            int start = block.getFirstValueIndex(position);
-            int end = start + block.getValueCount(position);
-            for (int i = start; i < end; i++) {
-                // Use an operation that is not affected by rounding errors. Otherwise, the result may depend on the traversalOrder.
-                sum += (long) block.getDouble(i);
-            }
-        }
-
-        return sum;
-    }
-
-    private static long computeIntCheckSum(IntBlock block, int[] traversalOrder) {
-        int sum = 0;
-
-        for (int position : traversalOrder) {
-            if (block.isNull(position)) {
-                continue;
-            }
-            int start = block.getFirstValueIndex(position);
-            int end = start + block.getValueCount(position);
-            for (int i = start; i < end; i++) {
-                sum += block.getInt(i);
-            }
-        }
-
-        return sum;
-    }
-
-    private static long computeLongCheckSum(LongBlock block, int[] traversalOrder) {
-        long sum = 0;
-
-        for (int position : traversalOrder) {
-            if (block.isNull(position)) {
-                continue;
-            }
-            int start = block.getFirstValueIndex(position);
-            int end = start + block.getValueCount(position);
-            for (int i = start; i < end; i++) {
-                sum += block.getLong(i);
-            }
-        }
-
-        return sum;
-    }
-
-    private static boolean isRandom(String accessType) {
-        return accessType.equalsIgnoreCase("random");
-    }
-
-    /**
-     * Must be a subset of {@link BlockBenchmark#RELEVANT_TYPE_BLOCK_COMBINATIONS}
-     */
-    @Param(
-        {
-            "boolean/array",
-            "boolean/array-multivalue-null",
-            "boolean/big-array",
-            "boolean/big-array-multivalue-null",
-            "boolean/vector",
-            "boolean/vector-big-array",
-            "boolean/vector-const",
-            "BytesRef/array",
-            "BytesRef/array-multivalue-null",
-            "BytesRef/vector",
-            "BytesRef/vector-const",
-            "double/array",
-            "double/array-multivalue-null",
-            "double/big-array",
-            "double/big-array-multivalue-null",
-            "double/vector",
-            "double/vector-big-array",
-            "double/vector-const",
-            "int/array",
-            "int/array-multivalue-null",
-            "int/big-array",
-            "int/big-array-multivalue-null",
-            "int/vector",
-            "int/vector-big-array",
-            "int/vector-const",
-            "long/array",
-            "long/array-multivalue-null",
-            "long/big-array",
-            "long/big-array-multivalue-null",
-            "long/vector",
-            "long/vector-big-array",
-            "long/vector-const" }
-    )
-    public String dataTypeAndBlockKind;
-
-    @Param({ "sequential", "random" })
-    public String accessType;
-
-    private BenchmarkBlocks data;
-
-    private int[][] traversalOrders;
-
-    private final long[] actualCheckSums = new long[NUM_BLOCKS_PER_ITERATION];
-
-    @Setup
-    public void setup() {
-        String[] params = dataTypeAndBlockKind.split("/");
-        String dataType = params[0];
-        String blockKind = params[1];
-
-        data = buildBlocks(dataType, blockKind, BLOCK_TOTAL_POSITIONS);
-        traversalOrders = createTraversalOrders(data.blocks, isRandom(accessType));
-    }
-
-    @Benchmark
-    @OperationsPerInvocation(NUM_BLOCKS_PER_ITERATION * BLOCK_TOTAL_POSITIONS)
-    public void run() {
-        String[] params = dataTypeAndBlockKind.split("/");
-        String dataType = params[0];
-
-        run(dataType, data, traversalOrders, actualCheckSums);
-    }
-
-    @TearDown(Level.Iteration)
-    public void assertCheckSums() {
-        assertCheckSums(data, actualCheckSums);
-    }
 }
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java
new file mode 100644
index 0000000000000..23048ad188a37
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java
@@ -0,0 +1,295 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.benchmark.compute.operator;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OperationsPerInvocation;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 5)
+@Measurement(iterations = 7)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Thread)
+@Fork(1)
+public class BlockKeepMaskBenchmark extends BlockBenchmark {
+    static {
+        // Smoke test all the expected values and force loading subclasses more like prod
+        int totalPositions = 10;
+        for (String paramString : RELEVANT_TYPE_BLOCK_COMBINATIONS) {
+            String[] params = paramString.split("/");
+            String dataType = params[0];
+            String blockKind = params[1];
+            BooleanVector mask = buildMask(totalPositions);
+
+            BenchmarkBlocks data = buildBenchmarkBlocks(dataType, blockKind, mask, totalPositions);
+            Block[] results = new Block[NUM_BLOCKS_PER_ITERATION];
+            run(data, mask, results);
+            assertCheckSums(dataType, blockKind, data, results, totalPositions);
+        }
+    }
+
+    record BenchmarkBlocks(Block[] blocks, long[] checkSums) {};
+
+    static BenchmarkBlocks buildBenchmarkBlocks(String dataType, String blockKind, BooleanVector mask, int totalPositions) {
+        Block[] blocks = BlockBenchmark.buildBlocks(dataType, blockKind, totalPositions);
+        return new BenchmarkBlocks(blocks, checksumsFor(dataType, blocks, mask));
+    }
+
+    static long[] checksumsFor(String dataType, Block[] blocks, BooleanVector mask) {
+        long[] checkSums = new long[NUM_BLOCKS_PER_ITERATION];
+        switch (dataType) {
+            case "boolean" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    BooleanBlock block = (BooleanBlock) blocks[blockIndex];
+                    checkSums[blockIndex] = computeBooleanCheckSum(block, mask);
+                }
+            }
+            case "BytesRef" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    BytesRefBlock block = (BytesRefBlock) blocks[blockIndex];
+                    checkSums[blockIndex] = computeBytesRefCheckSum(block, mask);
+                }
+            }
+            case "double" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    DoubleBlock block = (DoubleBlock) blocks[blockIndex];
+                    checkSums[blockIndex] = computeDoubleCheckSum(block, mask);
+                }
+            }
+            case "int" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    IntBlock block = (IntBlock) blocks[blockIndex];
+                    checkSums[blockIndex] = computeIntCheckSum(block, mask);
+                }
+            }
+            case "long" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    LongBlock block = (LongBlock) blocks[blockIndex];
+                    checkSums[blockIndex] = computeLongCheckSum(block, mask);
+                }
+            }
+            // TODO float
+            default -> throw new IllegalStateException("illegal data type [" + dataType + "]");
+        }
+        return checkSums;
+    }
+
+    static BooleanVector buildMask(int totalPositions) {
+        try (BooleanVector.FixedBuilder builder = blockFactory.newBooleanVectorFixedBuilder(totalPositions)) {
+            for (int p = 0; p < totalPositions; p++) {
+                builder.appendBoolean(p % 2 == 0);
+            }
+            return builder.build();
+        }
+    }
+
+    private static void run(BenchmarkBlocks data, BooleanVector mask, Block[] results) {
+        for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+            results[blockIndex] = data.blocks[blockIndex].keepMask(mask);
+        }
+    }
+
+    private static void assertCheckSums(String dataType, String blockKind, BenchmarkBlocks data, Block[] results, int positionCount) {
+        long[] checkSums = checksumsFor(dataType, results, blockFactory.newConstantBooleanVector(true, positionCount));
+        for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+            if (checkSums[blockIndex] != data.checkSums[blockIndex]) {
+                throw new AssertionError(
+                    "checksums do not match for block ["
+                        + blockIndex
+                        + "]["
+                        + dataType
+                        + "]["
+                        + blockKind
+                        + "]: "
+                        + checkSums[blockIndex]
+                        + " vs "
+                        + data.checkSums[blockIndex]
+                );
+            }
+        }
+    }
+
+    private static long computeBooleanCheckSum(BooleanBlock block, BooleanVector mask) {
+        long sum = 0;
+
+        for (int p = 0; p < block.getPositionCount(); p++) {
+            if (block.isNull(p) || mask.getBoolean(p) == false) {
+                continue;
+            }
+            int start = block.getFirstValueIndex(p);
+            int end = start + block.getValueCount(p);
+            for (int i = start; i < end; i++) {
+                sum += block.getBoolean(i) ? 1 : 0;
+            }
+        }
+
+        return sum;
+    }
+
+    private static long computeBytesRefCheckSum(BytesRefBlock block, BooleanVector mask) {
+        long sum = 0;
+        BytesRef scratch = new BytesRef();
+
+        for (int p = 0; p < block.getPositionCount(); p++) {
+            if (block.isNull(p) || mask.getBoolean(p) == false) {
+                continue;
+            }
+            int start = block.getFirstValueIndex(p);
+            int end = start + block.getValueCount(p);
+            for (int i = start; i < end; i++) {
+                BytesRef v = block.getBytesRef(i, scratch);
+                sum += v.length > 0 ? v.bytes[v.offset] : 0;
+            }
+        }
+
+        return sum;
+    }
+
+    private static long computeDoubleCheckSum(DoubleBlock block, BooleanVector mask) {
+        long sum = 0;
+
+        for (int p = 0; p < block.getPositionCount(); p++) {
+            if (block.isNull(p) || mask.getBoolean(p) == false) {
+                continue;
+            }
+            int start = block.getFirstValueIndex(p);
+            int end = start + block.getValueCount(p);
+            for (int i = start; i < end; i++) {
+                sum += (long) block.getDouble(i);
+            }
+        }
+
+        return sum;
+    }
+
+    private static long computeIntCheckSum(IntBlock block, BooleanVector mask) {
+        int sum = 0;
+
+        for (int p = 0; p < block.getPositionCount(); p++) {
+            if (block.isNull(p) || mask.getBoolean(p) == false) {
+                continue;
+            }
+            int start = block.getFirstValueIndex(p);
+            int end = start + block.getValueCount(p);
+            for (int i = start; i < end; i++) {
+                sum += block.getInt(i);
+            }
+        }
+
+        return sum;
+    }
+
+    private static long computeLongCheckSum(LongBlock block, BooleanVector mask) {
+        long sum = 0;
+
+        for (int p = 0; p < block.getPositionCount(); p++) {
+            if (block.isNull(p) || mask.getBoolean(p) == false) {
+                continue;
+            }
+            int start = block.getFirstValueIndex(p);
+            int end = start + block.getValueCount(p);
+            for (int i = start; i < end; i++) {
+                sum += block.getLong(i);
+            }
+        }
+
+        return sum;
+    }
+
+    /**
+     * Must be a subset of {@link BlockBenchmark#RELEVANT_TYPE_BLOCK_COMBINATIONS}
+     */
+    @Param(
+        {
+            "boolean/array",
+            "boolean/array-multivalue-null",
+            "boolean/big-array",
+            "boolean/big-array-multivalue-null",
+            "boolean/vector",
+            "boolean/vector-big-array",
+            "boolean/vector-const",
+            "BytesRef/array",
+            "BytesRef/array-multivalue-null",
+            "BytesRef/vector",
+            "BytesRef/vector-const",
+            "double/array",
+            "double/array-multivalue-null",
+            "double/big-array",
+            "double/big-array-multivalue-null",
+            "double/vector",
+            "double/vector-big-array",
+            "double/vector-const",
+            "int/array",
+            "int/array-multivalue-null",
+            "int/big-array",
+            "int/big-array-multivalue-null",
+            "int/vector",
+            "int/vector-big-array",
+            "int/vector-const",
+            "long/array",
+            "long/array-multivalue-null",
+            "long/big-array",
+            "long/big-array-multivalue-null",
+            "long/vector",
+            "long/vector-big-array",
+            "long/vector-const" }
+    )
+    public String dataTypeAndBlockKind;
+
+    private BenchmarkBlocks data;
+
+    private final BooleanVector mask = buildMask(BLOCK_TOTAL_POSITIONS);
+
+    private final Block[] results = new Block[NUM_BLOCKS_PER_ITERATION];
+
+    @Setup
+    public void setup() {
+        String[] params = dataTypeAndBlockKind.split("/");
+        String dataType = params[0];
+        String blockKind = params[1];
+
+        data = buildBenchmarkBlocks(dataType, blockKind, mask, BLOCK_TOTAL_POSITIONS);
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_BLOCKS_PER_ITERATION * BLOCK_TOTAL_POSITIONS)
+    public void run() {
+        run(data, mask, results);
+    }
+
+    @TearDown(Level.Iteration)
+    public void assertCheckSums() {
+        String[] params = dataTypeAndBlockKind.split("/");
+        String dataType = params[0];
+        String blockKind = params[1];
+        assertCheckSums(dataType, blockKind, data, results, BLOCK_TOTAL_POSITIONS);
+    }
+}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java
new file mode 100644
index 0000000000000..327dcfcff3a28
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java
@@ -0,0 +1,319 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.benchmark.compute.operator;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.util.*;
+import org.elasticsearch.compute.data.*;
+import org.openjdk.jmh.annotations.*;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
+
+@Warmup(iterations = 5)
+@Measurement(iterations = 7)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Thread)
+@Fork(1)
+public class BlockReadBenchmark extends BlockBenchmark {
+    static {
+        // Smoke test all the expected values and force loading subclasses more like prod
+        int totalPositions = 10;
+        long[] actualCheckSums = new long[NUM_BLOCKS_PER_ITERATION];
+
+        for (String paramString : RELEVANT_TYPE_BLOCK_COMBINATIONS) {
+            String[] params = paramString.split("/");
+            String dataType = params[0];
+            String blockKind = params[1];
+
+            BenchmarkBlocks data = buildBenchmarkBlocks(dataType, blockKind, totalPositions);
+            int[][] traversalOrders = createTraversalOrders(data.blocks(), false);
+            run(dataType, data, traversalOrders, actualCheckSums);
+            assertCheckSums(data, actualCheckSums);
+        }
+    }
+
+    private static int[][] createTraversalOrders(Block[] blocks, boolean randomized) {
+        int[][] orders = new int[blocks.length][];
+
+        for (int i = 0; i < blocks.length; i++) {
+            IntStream positionsStream = IntStream.range(0, blocks[i].getPositionCount());
+
+            if (randomized) {
+                List<Integer> positions = new ArrayList<>(positionsStream.boxed().toList());
+                Collections.shuffle(positions, random);
+                orders[i] = positions.stream().mapToInt(x -> x).toArray();
+            } else {
+                orders[i] = positionsStream.toArray();
+            }
+        }
+
+        return orders;
+    }
+
+    record BenchmarkBlocks(Block[] blocks, long[] checkSums) {};
+
+    static BenchmarkBlocks buildBenchmarkBlocks(String dataType, String blockKind, int totalPositions) {
+        Block[] blocks = BlockBenchmark.buildBlocks(dataType, blockKind, totalPositions);
+        long[] checkSums = new long[NUM_BLOCKS_PER_ITERATION];
+        switch (dataType) {
+            case "boolean" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    BooleanBlock block = (BooleanBlock) blocks[blockIndex];
+                    checkSums[blockIndex] = computeBooleanCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray());
+                }
+            }
+            case "BytesRef" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    BytesRefBlock block = (BytesRefBlock) blocks[blockIndex];
+                    checkSums[blockIndex] = computeBytesRefCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray());
+                }
+            }
+            case "double" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    DoubleBlock block = (DoubleBlock) blocks[blockIndex];
+                    checkSums[blockIndex] = computeDoubleCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray());
+                }
+            }
+            case "int" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    IntBlock block = (IntBlock) blocks[blockIndex];
+                    checkSums[blockIndex] = computeIntCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray());
+                }
+            }
+            case "long" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    LongBlock block = (LongBlock) blocks[blockIndex];
+                    checkSums[blockIndex] = computeLongCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray());
+                }
+            }
+            // TODO float
+            default -> throw new IllegalStateException("illegal data type [" + dataType + "]");
+        }
+        return new BenchmarkBlocks(blocks, checkSums);
+    }
+
+    private static void run(String dataType, BenchmarkBlocks data, int[][] traversalOrders, long[] resultCheckSums) {
+        switch (dataType) {
+            case "boolean" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    BooleanBlock block = (BooleanBlock) data.blocks[blockIndex];
+
+                    resultCheckSums[blockIndex] = computeBooleanCheckSum(block, traversalOrders[blockIndex]);
+                }
+            }
+            case "BytesRef" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    BytesRefBlock block = (BytesRefBlock) data.blocks[blockIndex];
+
+                    resultCheckSums[blockIndex] = computeBytesRefCheckSum(block, traversalOrders[blockIndex]);
+                }
+            }
+            case "double" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    DoubleBlock block = (DoubleBlock) data.blocks[blockIndex];
+
+                    resultCheckSums[blockIndex] = computeDoubleCheckSum(block, traversalOrders[blockIndex]);
+                }
+            }
+            case "int" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    IntBlock block = (IntBlock) data.blocks[blockIndex];
+
+                    resultCheckSums[blockIndex] = computeIntCheckSum(block, traversalOrders[blockIndex]);
+                }
+            }
+            case "long" -> {
+                for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+                    LongBlock block = (LongBlock) data.blocks[blockIndex];
+
+                    resultCheckSums[blockIndex] = computeLongCheckSum(block, traversalOrders[blockIndex]);
+                }
+            }
+            default -> {
+                throw new IllegalStateException();
+            }
+        }
+    }
+
+    private static void assertCheckSums(BenchmarkBlocks data, long[] actualCheckSums) {
+        for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) {
+            if (actualCheckSums[blockIndex] != data.checkSums[blockIndex]) {
+                throw new AssertionError("checksums do not match for block [" + blockIndex + "]");
+            }
+        }
+    }
+
+    private static long computeBooleanCheckSum(BooleanBlock block, int[] traversalOrder) {
+        long sum = 0;
+
+        for (int position : traversalOrder) {
+            if (block.isNull(position)) {
+                continue;
+            }
+            int start = block.getFirstValueIndex(position);
+            int end = start + block.getValueCount(position);
+            for (int i = start; i < end; i++) {
+                sum += block.getBoolean(i) ? 1 : 0;
+            }
+        }
+
+        return sum;
+    }
+
+    private static long computeBytesRefCheckSum(BytesRefBlock block, int[] traversalOrder) {
+        long sum = 0;
+        BytesRef scratch = new BytesRef();
+
+        for (int position : traversalOrder) {
+            if (block.isNull(position)) {
+                continue;
+            }
+            int start = block.getFirstValueIndex(position);
+            int end = start + block.getValueCount(position);
+            for (int i = start; i < end; i++) {
+                BytesRef v = block.getBytesRef(i, scratch);
+                sum += v.length > 0 ? v.bytes[v.offset] : 0;
+            }
+        }
+
+        return sum;
+    }
+
+    private static long computeDoubleCheckSum(DoubleBlock block, int[] traversalOrder) {
+        long sum = 0;
+
+        for (int position : traversalOrder) {
+            if (block.isNull(position)) {
+                continue;
+            }
+            int start = block.getFirstValueIndex(position);
+            int end = start + block.getValueCount(position);
+            for (int i = start; i < end; i++) {
+                // Use an operation that is not affected by rounding errors. Otherwise, the result may depend on the traversalOrder.
+                sum += (long) block.getDouble(i);
+            }
+        }
+
+        return sum;
+    }
+
+    private static long computeIntCheckSum(IntBlock block, int[] traversalOrder) {
+        int sum = 0;
+
+        for (int position : traversalOrder) {
+            if (block.isNull(position)) {
+                continue;
+            }
+            int start = block.getFirstValueIndex(position);
+            int end = start + block.getValueCount(position);
+            for (int i = start; i < end; i++) {
+                sum += block.getInt(i);
+            }
+        }
+
+        return sum;
+    }
+
+    private static long computeLongCheckSum(LongBlock block, int[] traversalOrder) {
+        long sum = 0;
+
+        for (int position : traversalOrder) {
+            if (block.isNull(position)) {
+                continue;
+            }
+            int start = block.getFirstValueIndex(position);
+            int end = start + block.getValueCount(position);
+            for (int i = start; i < end; i++) {
+                sum += block.getLong(i);
+            }
+        }
+
+        return sum;
+    }
+
+    private static boolean isRandom(String accessType) {
+        return accessType.equalsIgnoreCase("random");
+    }
+
+    /**
+     * Must be a subset of {@link BlockBenchmark#RELEVANT_TYPE_BLOCK_COMBINATIONS}
+     */
+    @Param(
+        {
+            "boolean/array",
+            "boolean/array-multivalue-null",
+            "boolean/big-array",
+            "boolean/big-array-multivalue-null",
+            "boolean/vector",
+            "boolean/vector-big-array",
+            "boolean/vector-const",
+            "BytesRef/array",
+            "BytesRef/array-multivalue-null",
+            "BytesRef/vector",
+            "BytesRef/vector-const",
+            "double/array",
+            "double/array-multivalue-null",
+            "double/big-array",
+            "double/big-array-multivalue-null",
+            "double/vector",
+            "double/vector-big-array",
+            "double/vector-const",
+            "int/array",
+            "int/array-multivalue-null",
+            "int/big-array",
+            "int/big-array-multivalue-null",
+            "int/vector",
+            "int/vector-big-array",
+            "int/vector-const",
+            "long/array",
+            "long/array-multivalue-null",
+            "long/big-array",
+            "long/big-array-multivalue-null",
+            "long/vector",
+            "long/vector-big-array",
+            "long/vector-const" }
+    )
+    public String dataTypeAndBlockKind;
+
+    @Param({ "sequential", "random" })
+    public String accessType;
+
+    private BenchmarkBlocks data;
+
+    private int[][] traversalOrders;
+
+    private final long[] actualCheckSums = new long[NUM_BLOCKS_PER_ITERATION];
+
+    @Setup
+    public void setup() {
+        String[] params = dataTypeAndBlockKind.split("/");
+        String dataType = params[0];
+        String blockKind = params[1];
+
+        data = buildBenchmarkBlocks(dataType, blockKind, BLOCK_TOTAL_POSITIONS);
+        traversalOrders = createTraversalOrders(data.blocks(), isRandom(accessType));
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(NUM_BLOCKS_PER_ITERATION * BLOCK_TOTAL_POSITIONS)
+    public void run() {
+        String[] params = dataTypeAndBlockKind.split("/");
+        String dataType = params[0];
+
+        run(dataType, data, traversalOrders, actualCheckSums);
+    }
+
+    @TearDown(Level.Iteration)
+    public void assertCheckSums() {
+        assertCheckSums(data, actualCheckSums);
+    }
+}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3Benchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3Benchmark.java
new file mode 100644
index 0000000000000..2441acab7d405
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3Benchmark.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.benchmark.h3;
+
+import org.elasticsearch.h3.H3;
+import org.openjdk.jmh.Main;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.concurrent.TimeUnit;
+
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(1)
+public class H3Benchmark {
+
+    @Benchmark
+    public void pointToH3(H3State state, Blackhole bh) {
+        for (int i = 0; i < state.points.length; i++) {
+            for (int res = 0; res <= 15; res++) {
+                bh.consume(H3.geoToH3(state.points[i][0], state.points[i][1], res));
+            }
+        }
+    }
+
+    @Benchmark
+    public void h3Boundary(H3State state, Blackhole bh) {
+        for (int i = 0; i < state.h3.length; i++) {
+            bh.consume(H3.h3ToGeoBoundary(state.h3[i]));
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        Main.main(args);
+    }
+}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3State.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3State.java
new file mode 100644
index 0000000000000..5707e692a0750
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3State.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.benchmark.h3;
+
+import org.elasticsearch.h3.H3;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+
+import java.io.IOException;
+import java.util.Random;
+
+@State(Scope.Benchmark)
+public class H3State {
+
+    double[][] points = new double[1000][2];
+    long[] h3 = new long[1000];
+
+    @Setup(Level.Trial)
+    public void setupTrial() throws IOException {
+        Random random = new Random(1234);
+        for (int i = 0; i < points.length; i++) {
+            points[i][0] = random.nextDouble() * 180 - 90;   // lat
+            points[i][1] = random.nextDouble() * 360 - 180;  // lon
+            int res = random.nextInt(16);            // resolution
+            h3[i] = H3.geoToH3(points[i][0], points[i][1], res);
+        }
+    }
+}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java
index 8c5de05a01648..d7a72615f4b93 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -167,7 +168,7 @@ public void setUp() throws Exception {
             .build();
 
         Settings settings = Settings.builder().put("node.name", ShardsAvailabilityHealthIndicatorBenchmark.class.getSimpleName()).build();
-        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
 
         ClusterService clusterService = new ClusterService(
             Settings.EMPTY,
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java
index 5a27abe8be2a4..fe221ec980dc3 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java
@@ -186,6 +186,11 @@ public void setDocument(int docid) {
                 public boolean needs_score() {
                     return false;
                 }
+
+                @Override
+                public boolean needs_termStats() {
+                    return false;
+                }
             };
         };
     }
diff --git a/branches.json b/branches.json
index b852cd1fa5dbd..1d860501cbc87 100644
--- a/branches.json
+++ b/branches.json
@@ -7,9 +7,6 @@
     {
       "branch": "8.15"
     },
-    {
-      "branch": "8.14"
-    },
     {
       "branch": "7.17"
     }
diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
index efe2ff3449216..9036682bf0f0c 100644
--- a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
+++ b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip
+distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy
index 172bf9115d152..a89a26d2800d4 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy
@@ -15,17 +15,23 @@ abstract class AbstractGradleInternalPluginFuncTest extends AbstractJavaGradleFu
     abstract <T extends Plugin> Class<T> getPluginClassUnderTest();
 
     def setup() {
+        settingsFile.text = """
+        plugins {
+            id 'elasticsearch.java-toolchain'
+        }
+        """ + settingsFile.text
+
         buildFile << """
         import ${getPluginClassUnderTest().getName()}
-        
+
         plugins {
           // bring in build-tools-internal onto the classpath
           id 'elasticsearch.global-build-info'
         }
         // internally used plugins do not have a plugin id as they are
-        // not intended to be used directly from build scripts 
+        // not intended to be used directly from build scripts
         plugins.apply(${getPluginClassUnderTest().getSimpleName()})
-        
+
         """
     }
 }
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy
index d74dce6924e32..c63eca8680179 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy
@@ -19,22 +19,22 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest {
         given:
         someLibProject()
         subProject("some-depending-lib") {
-            buildFile << """               
+            buildFile << """
                 plugins {
                     id 'elasticsearch.java-doc'
                     id 'java'
                 }
                 group = 'org.acme.depending'
-                
+
                 dependencies {
                     implementation project(':some-lib')
                 }
             """
             classFile('org.acme.depending.SomeDepending') << """
                 package org.acme.depending;
-                
+
                 import org.acme.Something;
-                
+
                 public class SomeDepending {
                     public Something createSomething() {
                         return new Something();
@@ -66,16 +66,17 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest {
 
     def "sources of shadowed dependencies are added to projects javadoc"() {
         given:
+        settingsFile.text = ""
         someLibProject() << """version = 1.0"""
         subProject("some-depending-lib") {
-            buildFile << """               
+            buildFile << """
                 plugins {
                     id 'elasticsearch.java-doc'
                     id 'com.github.johnrengelman.shadow' version '7.1.2'
                     id 'java'
                 }
                 group = 'org.acme.depending'
-                
+
                 dependencies {
                     implementation project(':some-lib')
                     shadow project(':some-shadowed-lib')
@@ -83,9 +84,9 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest {
             """
             classFile('org.acme.depending.SomeDepending') << """
                 package org.acme.depending;
-                
+
                 import org.acme.Something;
-                
+
                 public class SomeDepending {
                     public Something createSomething() {
                         return new Something();
@@ -94,9 +95,9 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest {
             """
             classFile('org.acme.depending.SomeShadowedDepending') << """
                 package org.acme.depending;
-                
+
                 import org.acme.shadowed.Shadowed;
-                
+
                 public class SomeShadowedDepending {
                     public Shadowed createShadowed() {
                         return new Shadowed();
@@ -114,7 +115,7 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest {
             """
             classFile('org.acme.shadowed.Shadowed') << """
                 package org.acme.shadowed;
-                
+
                 public class Shadowed {
                 }
             """
@@ -145,22 +146,22 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest {
             tasks.named("javadoc").configure { enabled = false }
         """
         subProject("some-depending-lib") {
-            buildFile << """               
+            buildFile << """
                 plugins {
                     id 'elasticsearch.java-doc'
                     id 'java'
                 }
                 group = 'org.acme.depending'
-                
+
                 dependencies {
                     implementation project(':some-lib')
                 }
             """
             classFile('org.acme.depending.SomeDepending') << """
                 package org.acme.depending;
-                
+
                 import org.acme.Something;
-                
+
                 public class SomeDepending {
                     public Something createSomething() {
                         return new Something();
@@ -264,7 +265,7 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest {
 
             classFile('org.acme.Something') << """
                 package org.acme;
-                
+
                 public class Something {
                 }
             """
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy
index b365624b5749a..c6a4572cb8a86 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy
@@ -18,6 +18,7 @@ import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
 import org.elasticsearch.gradle.fixtures.AbstractGradleInternalPluginFuncTest
 import org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersPrecommitPlugin
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin
+import org.gradle.testkit.runner.GradleRunner
 import org.gradle.testkit.runner.TaskOutcome
 
 
@@ -211,6 +212,10 @@ class ThirdPartyAuditTaskFuncTest extends AbstractGradleInternalPluginFuncTest {
         loggingDynamicType.toJar(targetFile(dir("${baseGroupFolderPath}/broken-log4j/0.0.1/"), "broken-log4j-0.0.1.jar"))
     }
 
+    GradleRunner gradleRunner(Object... arguments) {
+        return super.gradleRunner(arguments).withEnvironment([RUNTIME_JAVA_HOME: System.getProperty("java.home")])
+    }
+
     static File targetFile(File dir, String fileName) {
         new File(dir, fileName)
     }
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy
index 94fa329af1715..5e96fa524268a 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy
@@ -12,6 +12,7 @@ import spock.lang.IgnoreIf
 
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
+import org.gradle.testkit.runner.GradleRunner
 import org.gradle.testkit.runner.TaskOutcome
 
 @IgnoreIf({ os.isWindows() })
@@ -205,4 +206,8 @@ echo "Running elasticsearch \$0"
             }
         """
     }
+
+    GradleRunner gradleRunner(Object... arguments) {
+        return super.gradleRunner(arguments).withEnvironment([RUNTIME_JAVA_HOME: System.getProperty("java.home")])
+    }
 }
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
index 6cb22dad9bc79..285c3a61b08c2 100644
--- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
+++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -168,8 +168,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
               '-ea',
               '-Djava.security.manager=allow',
               '-Djava.locale.providers=SPI,COMPAT',
-              '-Djava.library.path=' + testLibraryPath,
-              '-Djna.library.path=' + testLibraryPath,
+              '-Des.nativelibs.path=' + testLibraryPath,
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',
               '--add-opens=java.base/java.nio.channels=ALL-UNNAMED',
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle b/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle
index f85ceed18604b..3400be77a588d 100644
--- a/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle
+++ b/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle
@@ -25,7 +25,6 @@ configure(allprojects) {
         JvmVendorSpec.ORACLE :
         JvmVendorSpec.matching(VersionProperties.bundledJdkVendor)
     }
-
   project.tasks.withType(Test).configureEach { Test test ->
         if (BuildParams.getIsRuntimeJavaHomeSet()) {
             test.executable = "${BuildParams.runtimeJavaHome}/bin/java" +
@@ -47,12 +46,4 @@ configure(allprojects) {
             }
         }
     }
-    project.plugins.withType(ThirdPartyAuditPrecommitPlugin) {
-      project.getTasks().withType(ThirdPartyAuditTask.class).configureEach {
-        if (BuildParams.getIsRuntimeJavaHomeSet() == false) {
-          javaHome.set(launcher.map { it.metadata.installationPath.asFile.path })
-          targetCompatibility.set(providers.provider(() -> JavaVersion.toVersion(launcher.map { it.metadata.javaRuntimeVersion }.get())))
-        }
-      }
-    }
 }
diff --git a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy
index e454d2ee38fff..658e2623cbbd7 100644
--- a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy
+++ b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy
@@ -13,6 +13,7 @@ import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.internal.test.AntFixture
 import org.gradle.api.file.FileSystemOperations
 import org.gradle.api.file.ProjectLayout
+import org.gradle.api.provider.ProviderFactory
 import org.gradle.api.tasks.Internal
 import org.gradle.process.ExecOperations
 
@@ -24,14 +25,17 @@ abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
     AntFixture fixture
 
     @Inject
-    AntFixtureStop(ProjectLayout projectLayout, ExecOperations execOperations, FileSystemOperations fileSystemOperations) {
-       super(projectLayout, execOperations, fileSystemOperations)
+    AntFixtureStop(ProjectLayout projectLayout,
+                   ExecOperations execOperations,
+                   FileSystemOperations fileSystemOperations,
+                   ProviderFactory providerFactory) {
+        super(projectLayout, execOperations, fileSystemOperations, providerFactory)
     }
 
     void setFixture(AntFixture fixture) {
         assert this.fixture == null
         this.fixture = fixture;
-        final Object pid = "${ -> this.fixture.pid }"
+        final Object pid = "${-> this.fixture.pid}"
         onlyIf("pidFile exists") { fixture.pidFile.exists() }
         doFirst {
             logger.info("Shutting down ${fixture.name} with pid ${pid}")
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
index b513fd7b93631..b27f480df4e63 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
@@ -61,7 +61,7 @@ public void apply(Project target) {
             : System.getenv("BUILDKITE_BUILD_NUMBER");
         String performanceTest = System.getenv("BUILD_PERFORMANCE_TEST");
         if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false) {
-            File targetFile = target.file("build/" + buildNumber + ".tar.bz2");
+            File targetFile = calculateTargetFile(target, buildNumber);
             File projectDir = target.getProjectDir();
             File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/");
             DevelocityConfiguration extension = target.getExtensions().getByType(DevelocityConfiguration.class);
@@ -86,9 +86,19 @@ public void apply(Project target) {
         }
     }
 
+    private File calculateTargetFile(Project target, String buildNumber) {
+        File uploadFile = target.file("build/" + buildNumber + ".tar.bz2");
+        int artifactIndex = 1;
+        while (uploadFile.exists()) {
+            uploadFile = target.file("build/" + buildNumber + "-" + artifactIndex++ + ".tar.bz2");
+        }
+        return uploadFile;
+    }
+
     private List<File> resolveProjectLogs(File projectDir) {
         var projectDirFiles = getFileOperations().fileTree(projectDir);
         projectDirFiles.include("**/*.hprof");
+        projectDirFiles.include("**/build/reports/configuration-cache/**");
         projectDirFiles.include("**/build/test-results/**/*.xml");
         projectDirFiles.include("**/build/testclusters/**");
         projectDirFiles.include("**/build/testrun/*/temp/**");
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java
index f95d9d72a473f..a3b1dd9731591 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java
@@ -189,9 +189,7 @@ private static void configureNativeLibraryPath(Project project) {
             var libraryPath = (Supplier<String>) () -> TestUtil.getTestLibraryPath(nativeConfigFiles.getAsPath());
 
             test.dependsOn(nativeConfigFiles);
-            // we may use JNA or the JDK's foreign function api to load libraries, so we set both sysprops
-            systemProperties.systemProperty("java.library.path", libraryPath);
-            systemProperties.systemProperty("jna.library.path", libraryPath);
+            systemProperties.systemProperty("es.nativelibs.path", libraryPath);
         });
     }
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java
index e9e75a711a8ff..03b8f19d10b13 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java
@@ -43,17 +43,16 @@ void impl(String implName, Project implProject) {
         });
 
         String manifestTaskName = "generate" + capitalName + "ProviderManifest";
-        Provider<Directory> generatedResourcesDir = project.getLayout().getBuildDirectory().dir("generated-resources");
+        Provider<Directory> generatedResourcesRoot = project.getLayout().getBuildDirectory().dir("generated-resources");
         var generateProviderManifest = project.getTasks().register(manifestTaskName, GenerateProviderManifest.class);
         generateProviderManifest.configure(t -> {
-            t.getManifestFile().set(generatedResourcesDir.map(d -> d.file("LISTING.TXT")));
+            t.getManifestFile().set(generatedResourcesRoot.map(d -> d.dir(manifestTaskName).file("LISTING.TXT")));
             t.getProviderImplClasspath().from(implConfig);
         });
-
         String implTaskName = "generate" + capitalName + "ProviderImpl";
         var generateProviderImpl = project.getTasks().register(implTaskName, Sync.class);
         generateProviderImpl.configure(t -> {
-            t.into(generatedResourcesDir);
+            t.into(generatedResourcesRoot.map(d -> d.dir(implTaskName)));
             t.into("IMPL-JARS/" + implName, spec -> {
                 spec.from(implConfig);
                 spec.from(generateProviderManifest);
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java
index b090f05c14c83..964784643936b 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java
@@ -26,15 +26,15 @@
 import static java.util.Objects.requireNonNull;
 
 public class BuildParams {
-    private static File runtimeJavaHome;
+    private static Provider<File> runtimeJavaHome;
     private static Boolean isRuntimeJavaHomeSet;
     private static List<JavaHome> javaVersions;
     private static JavaVersion minimumCompilerVersion;
     private static JavaVersion minimumRuntimeVersion;
     private static JavaVersion gradleJavaVersion;
-    private static JavaVersion runtimeJavaVersion;
+    private static Provider<JavaVersion> runtimeJavaVersion;
     private static Provider<? extends Action<JavaToolchainSpec>> javaToolChainSpec;
-    private static String runtimeJavaDetails;
+    private static Provider<String> runtimeJavaDetails;
     private static Boolean inFipsJvm;
     private static String gitRevision;
     private static String gitOrigin;
@@ -58,7 +58,7 @@ public static void init(Consumer<MutableBuildParams> initializer) {
     }
 
     public static File getRuntimeJavaHome() {
-        return value(runtimeJavaHome);
+        return value(runtimeJavaHome).get();
     }
 
     public static Boolean getIsRuntimeJavaHomeSet() {
@@ -82,11 +82,11 @@ public static JavaVersion getGradleJavaVersion() {
     }
 
     public static JavaVersion getRuntimeJavaVersion() {
-        return value(runtimeJavaVersion);
+        return value(runtimeJavaVersion.get());
     }
 
     public static String getRuntimeJavaDetails() {
-        return value(runtimeJavaDetails);
+        return value(runtimeJavaDetails.get());
     }
 
     public static Boolean isInFipsJvm() {
@@ -126,7 +126,7 @@ public static Boolean isCi() {
     }
 
     public static Boolean isGraalVmRuntime() {
-        return value(runtimeJavaDetails.toLowerCase().contains("graalvm"));
+        return value(runtimeJavaDetails.get().toLowerCase().contains("graalvm"));
     }
 
     public static Integer getDefaultParallel() {
@@ -182,16 +182,18 @@ public void reset() {
             });
         }
 
-        public void setRuntimeJavaHome(File runtimeJavaHome) {
-            try {
-                BuildParams.runtimeJavaHome = requireNonNull(runtimeJavaHome).getCanonicalFile();
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
+        public void setRuntimeJavaHome(Provider<File> runtimeJavaHome) {
+            BuildParams.runtimeJavaHome = runtimeJavaHome.map(javaHome -> {
+                try {
+                    return javaHome.getCanonicalFile();
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            });
         }
 
-        public void setIsRuntimeJavaHomeSet(boolean isRutimeJavaHomeSet) {
-            BuildParams.isRuntimeJavaHomeSet = isRutimeJavaHomeSet;
+        public void setIsRuntimeJavaHomeSet(boolean isRuntimeJavaHomeSet) {
+            BuildParams.isRuntimeJavaHomeSet = isRuntimeJavaHomeSet;
         }
 
         public void setJavaVersions(List<JavaHome> javaVersions) {
@@ -210,11 +212,11 @@ public void setGradleJavaVersion(JavaVersion gradleJavaVersion) {
             BuildParams.gradleJavaVersion = requireNonNull(gradleJavaVersion);
         }
 
-        public void setRuntimeJavaVersion(JavaVersion runtimeJavaVersion) {
+        public void setRuntimeJavaVersion(Provider<JavaVersion> runtimeJavaVersion) {
             BuildParams.runtimeJavaVersion = requireNonNull(runtimeJavaVersion);
         }
 
-        public void setRuntimeJavaDetails(String runtimeJavaDetails) {
+        public void setRuntimeJavaDetails(Provider<String> runtimeJavaDetails) {
             BuildParams.runtimeJavaDetails = runtimeJavaDetails;
         }
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
index e61bbefc9a973..b287815854098 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.gradle.internal.info;
 
 import org.apache.commons.io.IOUtils;
+import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.BwcVersions;
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
 import org.elasticsearch.gradle.internal.conventions.info.ParallelDetector;
@@ -31,7 +32,6 @@
 import org.gradle.internal.jvm.inspection.JvmMetadataDetector;
 import org.gradle.internal.jvm.inspection.JvmVendor;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
-import org.gradle.jvm.toolchain.JavaLauncher;
 import org.gradle.jvm.toolchain.JavaToolchainService;
 import org.gradle.jvm.toolchain.JavaToolchainSpec;
 import org.gradle.jvm.toolchain.JvmVendorSpec;
@@ -48,10 +48,8 @@
 import java.nio.file.Files;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
-import java.util.Optional;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
@@ -98,9 +96,11 @@ public void apply(Project project) {
         JavaVersion minimumCompilerVersion = JavaVersion.toVersion(getResourceContents("/minimumCompilerVersion"));
         JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion"));
 
-        Optional<File> selectedRuntimeJavaHome = findRuntimeJavaHome();
-        File actualRuntimeJavaHome = selectedRuntimeJavaHome.orElse(Jvm.current().getJavaHome());
-        boolean isRuntimeJavaHomeSet = selectedRuntimeJavaHome.isPresent();
+        Provider<File> explicitRuntimeJavaHome = findRuntimeJavaHome();
+        boolean isExplicitRuntimeJavaHomeSet = explicitRuntimeJavaHome.isPresent();
+        Provider<File> actualRuntimeJavaHome = isExplicitRuntimeJavaHomeSet
+            ? explicitRuntimeJavaHome
+            : resolveJavaHomeFromToolChainService(VersionProperties.getBundledJdkMajorVersion());
 
         GitInfo gitInfo = GitInfo.gitInfo(project.getRootDir());
 
@@ -108,16 +108,22 @@ public void apply(Project project) {
             params.reset();
             params.setRuntimeJavaHome(actualRuntimeJavaHome);
             params.setJavaToolChainSpec(resolveToolchainSpecFromEnv());
+            Provider<JvmInstallationMetadata> runtimeJdkMetaData = actualRuntimeJavaHome.map(
+                runtimeJavaHome -> metadataDetector.getMetadata(getJavaInstallation(runtimeJavaHome))
+            );
             params.setRuntimeJavaVersion(
-                determineJavaVersion(
-                    "runtime java.home",
-                    actualRuntimeJavaHome,
-                    isRuntimeJavaHomeSet ? minimumRuntimeVersion : Jvm.current().getJavaVersion()
+                actualRuntimeJavaHome.map(
+                    javaHome -> determineJavaVersion(
+                        "runtime java.home",
+                        javaHome,
+                        isExplicitRuntimeJavaHomeSet
+                            ? minimumRuntimeVersion
+                            : JavaVersion.toVersion(VersionProperties.getBundledJdkMajorVersion())
+                    )
                 )
             );
-            params.setIsRuntimeJavaHomeSet(isRuntimeJavaHomeSet);
-            JvmInstallationMetadata runtimeJdkMetaData = metadataDetector.getMetadata(getJavaInstallation(actualRuntimeJavaHome));
-            params.setRuntimeJavaDetails(formatJavaVendorDetails(runtimeJdkMetaData));
+            params.setIsRuntimeJavaHomeSet(isExplicitRuntimeJavaHomeSet);
+            params.setRuntimeJavaDetails(runtimeJdkMetaData.map(m -> formatJavaVendorDetails(m)));
             params.setJavaVersions(getAvailableJavaVersions());
             params.setMinimumCompilerVersion(minimumCompilerVersion);
             params.setMinimumRuntimeVersion(minimumRuntimeVersion);
@@ -300,62 +306,30 @@ private static void assertMinimumCompilerVersion(JavaVersion minimumCompilerVers
         }
     }
 
-    private Optional<File> findRuntimeJavaHome() {
+    private Provider<File> findRuntimeJavaHome() {
         String runtimeJavaProperty = System.getProperty("runtime.java");
 
         if (runtimeJavaProperty != null) {
-            return Optional.of(resolveJavaHomeFromToolChainService(runtimeJavaProperty));
+            return resolveJavaHomeFromToolChainService(runtimeJavaProperty);
         }
-        String env = System.getenv("RUNTIME_JAVA_HOME");
-        if (env != null) {
-            return Optional.of(new File(env));
+        if (System.getenv("RUNTIME_JAVA_HOME") != null) {
+            return providers.provider(() -> new File(System.getenv("RUNTIME_JAVA_HOME")));
         }
         // fall back to tool chain if set.
-        env = System.getenv("JAVA_TOOLCHAIN_HOME");
-        return env == null ? Optional.empty() : Optional.of(new File(env));
-    }
-
-    @NotNull
-    private String resolveJavaHomeFromEnvVariable(String javaHomeEnvVar) {
-        Provider<String> javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv");
-        // Provide a useful error if we're looking for a Java home version that we haven't told Gradle about yet
-        Arrays.stream(javaHomeNames.get().split(","))
-            .filter(s -> s.equals(javaHomeEnvVar))
-            .findFirst()
-            .orElseThrow(
-                () -> new GradleException(
-                    "Environment variable '"
-                        + javaHomeEnvVar
-                        + "' is not registered with Gradle installation supplier. Ensure 'org.gradle.java.installations.fromEnv' is "
-                        + "updated in gradle.properties file."
-                )
-            );
-        String versionedJavaHome = System.getenv(javaHomeEnvVar);
-        if (versionedJavaHome == null) {
-            final String exceptionMessage = String.format(
-                Locale.ROOT,
-                "$%s must be set to build Elasticsearch. "
-                    + "Note that if the variable was just set you "
-                    + "might have to run `./gradlew --stop` for "
-                    + "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details.",
-                javaHomeEnvVar
-            );
-            throw new GradleException(exceptionMessage);
-        }
-        return versionedJavaHome;
+        String env = System.getenv("JAVA_TOOLCHAIN_HOME");
+        return providers.provider(() -> {
+            if (env == null) {
+                return null;
+            }
+            return new File(env);
+        });
     }
 
     @NotNull
-    private File resolveJavaHomeFromToolChainService(String version) {
+    private Provider<File> resolveJavaHomeFromToolChainService(String version) {
         Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
-        Provider<JavaLauncher> javaLauncherProvider = toolChainService.launcherFor(javaToolchainSpec -> {
-            javaToolchainSpec.getLanguageVersion().value(value);
-        });
-        return javaLauncherProvider.get().getMetadata().getInstallationPath().getAsFile();
-    }
-
-    private static String getJavaHomeEnvVarName(String version) {
-        return "JAVA" + version + "_HOME";
+        return toolChainService.launcherFor(javaToolchainSpec -> javaToolchainSpec.getLanguageVersion().value(value))
+            .map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile());
     }
 
     public static String getResourceContents(String resourcePath) {
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java
index c52ea9aaeb6f5..533066168c604 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java
@@ -102,7 +102,7 @@ public class LicenseAnalyzer {
             AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
             LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
             OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-            SOFTWARE\\.
+            SOFTWARE\\.?
             """).replaceAll("\\s+", "\\\\s*"), Pattern.DOTALL)),
         new LicenseMatcher(
             "MIT-0",
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
index 1fc030be42480..9e40d96438e48 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
@@ -66,10 +66,8 @@ public TaskProvider<? extends Task> createTask(Project project) {
                     )
                 );
             t.dependsOn(resourcesTask);
-            if (BuildParams.getIsRuntimeJavaHomeSet()) {
-                t.getJavaHome().set(project.provider(BuildParams::getRuntimeJavaHome).map(File::getPath));
-            }
             t.getTargetCompatibility().set(project.provider(BuildParams::getRuntimeJavaVersion));
+            t.getJavaHome().set(project.provider(BuildParams::getRuntimeJavaHome).map(File::getPath));
             t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile());
             t.getJdkJarHellClasspath().from(jdkJarHellConfig);
             t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnly));
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java
index 08abb02ea831e..ec79fe20492e1 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java
@@ -52,7 +52,7 @@ public void apply(Project project) {
 
         project.getTasks().register("extractCurrentVersions", ExtractCurrentVersionsTask.class);
         project.getTasks().register("tagVersions", TagVersionsTask.class);
-        project.getTasks().register("setCompatibleVersions", SetCompatibleVersionsTask.class);
+        project.getTasks().register("setCompatibleVersions", SetCompatibleVersionsTask.class, t -> t.setThisVersion(version));
 
         final FileTree yamlFiles = projectDirectory.dir("docs/changelog")
             .getAsFileTree()
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTask.java
index 15e0a0cc345d5..17761e5183b31 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTask.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTask.java
@@ -14,6 +14,7 @@
 import com.github.javaparser.ast.expr.NameExpr;
 import com.github.javaparser.printer.lexicalpreservation.LexicalPreservingPrinter;
 
+import org.elasticsearch.gradle.Version;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.options.Option;
 import org.gradle.initialization.layout.BuildLayout;
@@ -28,6 +29,8 @@
 
 public class SetCompatibleVersionsTask extends AbstractVersionsTask {
 
+    private Version thisVersion;
+    private Version releaseVersion;
     private Map<String, Integer> versionIds = Map.of();
 
     @Inject
@@ -35,21 +38,35 @@ public SetCompatibleVersionsTask(BuildLayout layout) {
         super(layout);
     }
 
+    public void setThisVersion(Version version) {
+        thisVersion = version;
+    }
+
     @Option(option = "version-id", description = "Version id used for the release. Of the form <VersionType>:<id>.")
     public void versionIds(List<String> version) {
         this.versionIds = splitVersionIds(version);
     }
 
+    @Option(option = "release", description = "The version being released")
+    public void releaseVersion(String version) {
+        releaseVersion = Version.fromString(version);
+    }
+
     @TaskAction
     public void executeTask() throws IOException {
         if (versionIds.isEmpty()) {
             throw new IllegalArgumentException("No version ids specified");
         }
+
+        if (releaseVersion.getMajor() < thisVersion.getMajor()) {
+            // don't need to update CCS version - this is for a different major
+            return;
+        }
+
         Integer transportVersion = versionIds.get(TRANSPORT_VERSION_TYPE);
         if (transportVersion == null) {
             throw new IllegalArgumentException("TransportVersion id not specified");
         }
-
         Path versionJava = rootDir.resolve(TRANSPORT_VERSIONS_FILE_PATH);
         CompilationUnit file = LexicalPreservingPrinter.setup(StaticJavaParser.parse(versionJava));
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java
index 96fde95d0dd17..965f3964c9a38 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.gradle.Architecture;
 import org.elasticsearch.gradle.ElasticsearchDistribution;
 
-import java.io.File;
 import java.util.Locale;
 
 public class TestUtil {
@@ -19,8 +18,7 @@ public class TestUtil {
     public static String getTestLibraryPath(String nativeLibsDir) {
         String arch = Architecture.current().toString().toLowerCase(Locale.ROOT);
         String platform = String.format(Locale.ROOT, "%s-%s", ElasticsearchDistribution.CURRENT_PLATFORM, arch);
-        String existingLibraryPath = System.getProperty("java.library.path");
 
-        return String.format(Locale.ROOT, "%s/%s%c%s", nativeLibsDir, platform, File.pathSeparatorChar, existingLibraryPath);
+        return String.format(Locale.ROOT, "%s/%s", nativeLibsDir, platform);
     }
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SourceDirectoryCommandLineArgumentProvider.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SourceDirectoryCommandLineArgumentProvider.java
new file mode 100644
index 0000000000000..30141f021935b
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SourceDirectoryCommandLineArgumentProvider.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.util;
+
+import org.gradle.api.file.Directory;
+import org.gradle.api.tasks.InputDirectory;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
+import org.gradle.process.CommandLineArgumentProvider;
+
+import java.util.Arrays;
+
+public class SourceDirectoryCommandLineArgumentProvider implements CommandLineArgumentProvider {
+
+    private final Directory sourceDirectory;
+
+    public SourceDirectoryCommandLineArgumentProvider(Directory sourceDirectory) {
+        this.sourceDirectory = sourceDirectory;
+    }
+
+    public Iterable<String> asArguments() {
+        return Arrays.asList("-s", sourceDirectory.getAsFile().getAbsolutePath());
+    }
+
+    @InputDirectory
+    @PathSensitive(PathSensitivity.RELATIVE)
+    public Directory getSourceDirectory() {
+        return sourceDirectory;
+    }
+}
diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion
index f7b1c8ff61774..8d04a0f38fab0 100644
--- a/build-tools-internal/src/main/resources/minimumGradleVersion
+++ b/build-tools-internal/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-8.9
\ No newline at end of file
+8.10
\ No newline at end of file
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java
index 6087482db278d..3a425d11ccf17 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java
@@ -17,6 +17,8 @@
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.provider.MapProperty;
 import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Optional;
@@ -92,17 +94,45 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat
     private String output;
 
     @Inject
-    public LoggedExec(ProjectLayout projectLayout, ExecOperations execOperations, FileSystemOperations fileSystemOperations) {
+    public LoggedExec(
+        ProjectLayout projectLayout,
+        ExecOperations execOperations,
+        FileSystemOperations fileSystemOperations,
+        ProviderFactory providerFactory
+    ) {
         this.projectLayout = projectLayout;
         this.execOperations = execOperations;
         this.fileSystemOperations = fileSystemOperations;
         getWorkingDir().convention(projectLayout.getProjectDirectory().getAsFile());
         // For now mimic default behaviour of Gradle Exec task here
-        getEnvironment().putAll(System.getenv());
+        setupDefaultEnvironment(providerFactory);
         getCaptureOutput().convention(false);
         getSpoolOutput().convention(false);
     }
 
+    /**
+     * We explicitly configure the environment variables that are passed to the executed process.
+     * This is required to make sure that the build cache and Gradle configuration cache is correctly configured
+     * can be reused across different build invocations.
+     * */
+    private void setupDefaultEnvironment(ProviderFactory providerFactory) {
+        getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("BUILDKITE"));
+        getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("GRADLE_BUILD_CACHE"));
+        getEnvironment().putAll(providerFactory.environmentVariablesPrefixedBy("VAULT"));
+        Provider<String> javaToolchainHome = providerFactory.environmentVariable("JAVA_TOOLCHAIN_HOME");
+        if (javaToolchainHome.isPresent()) {
+            getEnvironment().put("JAVA_TOOLCHAIN_HOME", javaToolchainHome);
+        }
+        Provider<String> javaRuntimeHome = providerFactory.environmentVariable("RUNTIME_JAVA_HOME");
+        if (javaRuntimeHome.isPresent()) {
+            getEnvironment().put("RUNTIME_JAVA_HOME", javaRuntimeHome);
+        }
+        Provider<String> path = providerFactory.environmentVariable("PATH");
+        if (path.isPresent()) {
+            getEnvironment().put("PATH", path);
+        }
+    }
+
     @TaskAction
     public void run() {
         boolean spoolOutput = getSpoolOutput().get();
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java
index a1da860abe26a..9593a281686e7 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java
@@ -22,6 +22,9 @@ public void apply(Project project) {
             test.systemProperty("tests.gradle", true);
             test.systemProperty("tests.task", test.getPath());
 
+            // Flag is required for later Java versions since our tests use a custom security manager
+            test.jvmArgs("-Djava.security.manager=allow");
+
             SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider();
             // don't track these as inputs since they contain absolute paths and break cache relocatability
             nonInputProperties.systemProperty("gradle.dist.lib", gradle.getGradleHomeDir().getAbsolutePath() + "/lib");
diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
index 41f6f445f58ec..5ea970c533474 100644
--- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
+++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
@@ -13,6 +13,7 @@ import org.elasticsearch.gradle.internal.test.BuildConfigurationAwareGradleRunne
 import org.elasticsearch.gradle.internal.test.InternalAwareGradleRunner
 import org.elasticsearch.gradle.internal.test.NormalizeOutputGradleRunner
 import org.elasticsearch.gradle.internal.test.TestResultExtension
+import org.gradle.internal.component.external.model.ComponentVariant
 import org.gradle.testkit.runner.BuildResult
 import org.gradle.testkit.runner.GradleRunner
 import org.junit.Rule
@@ -22,6 +23,7 @@ import spock.lang.TempDir
 
 import java.lang.management.ManagementFactory
 import java.nio.file.Files
+import java.io.File
 import java.nio.file.Path
 import java.util.jar.JarEntry
 import java.util.jar.JarOutputStream
diff --git a/catalog-info.yaml b/catalog-info.yaml
index dfeeae51c1b3a..e57841c9de268 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -125,7 +125,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
         SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
-      branch_configuration: lucene_snapshot
+      branch_configuration: lucene_snapshot lucene_snapshot_10
       default_branch: lucene_snapshot
       teams:
         elasticsearch-team: {}
@@ -142,6 +142,10 @@ spec:
           branch: lucene_snapshot
           cronline: "0 2 * * * America/New_York"
           message: "Builds a new lucene snapshot 1x per day"
+        Periodically on lucene_snapshot_10:
+          branch: lucene_snapshot_10
+          cronline: "0 2 * * * America/New_York"
+          message: "Builds a new lucene snapshot 1x per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -169,7 +173,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
         SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
-      branch_configuration: lucene_snapshot
+      branch_configuration: lucene_snapshot lucene_snapshot_10
       default_branch: lucene_snapshot
       teams:
         elasticsearch-team: {}
@@ -186,6 +190,10 @@ spec:
           branch: lucene_snapshot
           cronline: "0 6 * * * America/New_York"
           message: "Merges main into lucene_snapshot branch 1x per day"
+        Periodically on lucene_snapshot_10:
+          branch: lucene_snapshot_10
+          cronline: "0 6 * * * America/New_York"
+          message: "Merges main into lucene_snapshot_10 branch 1x per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -213,7 +221,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
         SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
-      branch_configuration: lucene_snapshot
+      branch_configuration: lucene_snapshot lucene_snapshot_10
       default_branch: lucene_snapshot
       teams:
         elasticsearch-team: {}
@@ -230,6 +238,10 @@ spec:
           branch: lucene_snapshot
           cronline: "0 9,12,15,18 * * * America/New_York"
           message: "Runs tests against lucene_snapshot branch several times per day"
+        Periodically on lucene_snapshot_10:
+          branch: lucene_snapshot_10
+          cronline: "0 9,12,15,18 * * * America/New_York"
+          message: "Runs tests against lucene_snapshot_10 branch several times per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java
index 9896fd6c84599..01ec6c118bf24 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugin.noop.NoopPlugin;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java
index 790b6bfd6deca..871cdb860a9a9 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugin.noop.NoopPlugin;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.InternalAggregations;
diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile
index 32f35b05015b9..2a2a77a6df820 100644
--- a/distribution/docker/src/docker/Dockerfile
+++ b/distribution/docker/src/docker/Dockerfile
@@ -22,7 +22,7 @@
 <% if (docker_base == 'iron_bank') { %>
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9
-ARG BASE_TAG=9.3
+ARG BASE_TAG=9.4
 <% } %>
 
 ################################################################################
diff --git a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml
index 38ce16a413af2..f4364c5008c09 100644
--- a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml
+++ b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml
@@ -14,7 +14,7 @@ tags:
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.3"
+  BASE_TAG: "9.4"
 
 # Docker image labels
 labels:
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
index 7b904d4cb5a89..bea7fbb7f63e8 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
@@ -32,6 +32,7 @@
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * The main CLI for running Elasticsearch.
@@ -44,6 +45,8 @@ class ServerCli extends EnvironmentAwareCommand {
     private final OptionSpecBuilder quietOption;
     private final OptionSpec<String> enrollmentTokenOption;
 
+    // flag for indicating shutdown has begun. we use an AtomicBoolean to double as a synchronization object
+    private final AtomicBoolean shuttingDown = new AtomicBoolean(false);
     private volatile ServerProcess server;
 
     // visible for testing
@@ -98,7 +101,14 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
             syncPlugins(terminal, env, processInfo);
 
             ServerArgs args = createArgs(options, env, secrets, processInfo);
-            this.server = startServer(terminal, processInfo, args);
+            synchronized (shuttingDown) {
+                // if we are shutting down there is no reason to start the server
+                if (shuttingDown.get()) {
+                    terminal.println("CLI is shutting down, skipping starting server process");
+                    return;
+                }
+                this.server = startServer(terminal, processInfo, args);
+            }
         }
 
         if (options.has(daemonizeOption)) {
@@ -233,8 +243,11 @@ private ServerArgs createArgs(OptionSet options, Environment env, SecureSettings
 
     @Override
     public void close() throws IOException {
-        if (server != null) {
-            server.stop();
+        synchronized (shuttingDown) {
+            shuttingDown.set(true);
+            if (server != null) {
+                server.stop();
+            }
         }
     }
 
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
index 2a89f18209d11..94e2d538c0ad0 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
@@ -10,11 +10,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.core.SuppressForbidden;
 
-import java.io.File;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -25,7 +21,6 @@ final class SystemJvmOptions {
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
-        String libraryPath = findLibraryPath(sysprops);
 
         return Stream.concat(
             Stream.of(
@@ -73,8 +68,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 maybeOverrideDockerCgroup(distroType),
                 maybeSetActiveProcessorCount(nodeSettings),
                 setReplayFile(distroType, isHotspot),
-                "-Djava.library.path=" + libraryPath,
-                "-Djna.library.path=" + libraryPath,
                 // Pass through distribution type
                 "-Des.distribution.type=" + distroType
             ),
@@ -144,38 +137,4 @@ private static Stream<String> maybeWorkaroundG1Bug() {
         }
         return Stream.of();
     }
-
-    private static String findLibraryPath(Map<String, String> sysprops) {
-        // working dir is ES installation, so we use relative path here
-        Path platformDir = Paths.get("lib", "platform");
-        String existingPath = sysprops.get("java.library.path");
-        assert existingPath != null;
-
-        String osname = sysprops.get("os.name");
-        String os;
-        if (osname.startsWith("Windows")) {
-            os = "windows";
-        } else if (osname.startsWith("Linux")) {
-            os = "linux";
-        } else if (osname.startsWith("Mac OS")) {
-            os = "darwin";
-        } else {
-            os = "unsupported_os[" + osname + "]";
-        }
-        String archname = sysprops.get("os.arch");
-        String arch;
-        if (archname.equals("amd64") || archname.equals("x86_64")) {
-            arch = "x64";
-        } else if (archname.equals("aarch64")) {
-            arch = archname;
-        } else {
-            arch = "unsupported_arch[" + archname + "]";
-        }
-        return platformDir.resolve(os + "-" + arch).toAbsolutePath() + getPathSeparator() + existingPath;
-    }
-
-    @SuppressForbidden(reason = "no way to get path separator with nio")
-    private static String getPathSeparator() {
-        return File.pathSeparator;
-    }
 }
diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java
index 87b7894a9135a..fc889f036a795 100644
--- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java
+++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java
@@ -17,7 +17,6 @@
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.StringReader;
-import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
@@ -30,12 +29,10 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
@@ -44,14 +41,7 @@
 @WithoutSecurityManager
 public class JvmOptionsParserTests extends ESTestCase {
 
-    private static final Map<String, String> TEST_SYSPROPS = Map.of(
-        "os.name",
-        "Linux",
-        "os.arch",
-        "aarch64",
-        "java.library.path",
-        "/usr/lib"
-    );
+    private static final Map<String, String> TEST_SYSPROPS = Map.of("os.name", "Linux", "os.arch", "aarch64");
 
     public void testSubstitution() {
         final List<String> jvmOptions = JvmOptionsParser.substitutePlaceholders(
@@ -390,40 +380,4 @@ public void testCommandLineDistributionType() {
         final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops);
         assertThat(jvmOptions, hasItem("-Des.distribution.type=testdistro"));
     }
-
-    public void testLibraryPath() {
-        assertLibraryPath("Mac OS", "aarch64", "darwin-aarch64");
-        assertLibraryPath("Mac OS", "amd64", "darwin-x64");
-        assertLibraryPath("Mac OS", "x86_64", "darwin-x64");
-        assertLibraryPath("Linux", "aarch64", "linux-aarch64");
-        assertLibraryPath("Linux", "amd64", "linux-x64");
-        assertLibraryPath("Linux", "x86_64", "linux-x64");
-        assertLibraryPath("Windows", "amd64", "windows-x64");
-        assertLibraryPath("Windows", "x86_64", "windows-x64");
-        assertLibraryPath("Unknown", "aarch64", "unsupported_os[Unknown]-aarch64");
-        assertLibraryPath("Mac OS", "Unknown", "darwin-unsupported_arch[Unknown]");
-    }
-
-    private void assertLibraryPath(String os, String arch, String expected) {
-        String existingPath = "/usr/lib";
-        var sysprops = Map.of("os.name", os, "os.arch", arch, "java.library.path", existingPath);
-        final List<String> jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops);
-        Map<String, String> options = new HashMap<>();
-        for (var jvmOption : jvmOptions) {
-            if (jvmOption.startsWith("-D")) {
-                String[] parts = jvmOption.substring(2).split("=");
-                assert parts.length == 2;
-                options.put(parts[0], parts[1]);
-            }
-        }
-        String separator = FileSystems.getDefault().getSeparator();
-        assertThat(
-            options,
-            hasEntry(equalTo("java.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath)))
-        );
-        assertThat(
-            options,
-            hasEntry(equalTo("jna.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath)))
-        );
-    }
 }
diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java
index 38a64a778fc27..e603790051c0c 100644
--- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java
+++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java
@@ -36,6 +36,8 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
@@ -50,6 +52,7 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.matchesRegex;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class ServerCliTests extends CommandTestCase {
 
@@ -383,6 +386,52 @@ public void testSecureSettingsLoaderWithNullPassword() throws Exception {
         assertEquals("", loader.password);
     }
 
+    public void testProcessCreationRace() throws Exception {
+        for (int i = 0; i < 10; ++i) {
+            CyclicBarrier raceStart = new CyclicBarrier(2);
+            TestServerCli cli = new TestServerCli() {
+                @Override
+                void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception {
+                    super.syncPlugins(terminal, env, processInfo);
+                    raceStart.await();
+                }
+
+                @Override
+                public void close() throws IOException {
+                    try {
+                        raceStart.await();
+                    } catch (InterruptedException ie) {
+                        Thread.currentThread().interrupt();
+                        throw new AssertionError(ie);
+                    } catch (BrokenBarrierException e) {
+                        throw new AssertionError(e);
+                    }
+                    super.close();
+                }
+            };
+            Thread closeThread = new Thread(() -> {
+                try {
+                    cli.close();
+                } catch (IOException e) {
+                    throw new AssertionError(e);
+                }
+            });
+            closeThread.start();
+            cli.main(new String[] {}, terminal, new ProcessInfo(sysprops, envVars, esHomeDir));
+            closeThread.join();
+
+            if (cli.getServer() == null) {
+                // close won the race, so server should never have been started
+                assertThat(cli.startServerCalled, is(false));
+            } else {
+                // creation won the race, so check we correctly waited on it and stopped
+                assertThat(cli.getServer(), sameInstance(mockServer));
+                assertThat(mockServer.waitForCalled, is(true));
+                assertThat(mockServer.stopCalled, is(true));
+            }
+        }
+    }
+
     private MockSecureSettingsLoader loadWithMockSecureSettingsLoader() throws Exception {
         var loader = new MockSecureSettingsLoader();
         this.mockSecureSettingsLoader = loader;
@@ -465,9 +514,9 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
     }
 
     private class MockServerProcess extends ServerProcess {
-        boolean detachCalled = false;
-        boolean waitForCalled = false;
-        boolean stopCalled = false;
+        volatile boolean detachCalled = false;
+        volatile boolean waitForCalled = false;
+        volatile boolean stopCalled = false;
 
         MockServerProcess() {
             super(null, null);
@@ -505,6 +554,8 @@ void reset() {
     }
 
     private class TestServerCli extends ServerCli {
+        boolean startServerCalled = false;
+
         @Override
         protected Command loadTool(String toolname, String libs) {
             if (toolname.equals("auto-configure-node")) {
@@ -551,20 +602,21 @@ protected SecureSettingsLoader secureSettingsLoader(Environment env) {
 
             return new KeystoreSecureSettingsLoader();
         }
+
+        @Override
+        protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws Exception {
+            startServerCalled = true;
+            if (argsValidator != null) {
+                argsValidator.accept(args);
+            }
+            mockServer.reset();
+            return mockServer;
+        }
     }
 
     @Override
     protected Command newCommand() {
-        return new TestServerCli() {
-            @Override
-            protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) {
-                if (argsValidator != null) {
-                    argsValidator.accept(args);
-                }
-                mockServer.reset();
-                return mockServer;
-            }
-        };
+        return new TestServerCli();
     }
 
     static class MockSecureSettingsLoader implements SecureSettingsLoader {
diff --git a/docs/changelog/101373.yaml b/docs/changelog/101373.yaml
deleted file mode 100644
index 53b5680301c79..0000000000000
--- a/docs/changelog/101373.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 101373
-summary: Adding aggregations support for the `_ignored` field
-area: Search
-type: feature
-issues:
- - 59946
diff --git a/docs/changelog/103374.yaml b/docs/changelog/103374.yaml
deleted file mode 100644
index fcdee9185eb92..0000000000000
--- a/docs/changelog/103374.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-pr: 103374
-summary: Cut over stored fields to ZSTD for compression
-area: Search
-type: enhancement
-issues: []
-highlight:
-  title: Stored fields are now compressed with ZStandard instead of LZ4/DEFLATE
-  body: |-
-    Stored fields are now compressed by splitting documents into blocks, which
-    are then compressed independently with ZStandard. `index.codec: default`
-    (default) uses blocks of at most 14kB or 128 documents compressed with level
-    0, while `index.codec: best_compression` uses blocks of at most 240kB or
-    2048 documents compressed at level 3. On most datasets that we tested
-    against, this yielded storage improvements in the order of 10%, slightly
-    faster indexing and similar retrieval latencies.
-  notable: true
diff --git a/docs/changelog/105792.yaml b/docs/changelog/105792.yaml
deleted file mode 100644
index b9190e60cc96d..0000000000000
--- a/docs/changelog/105792.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-pr: 105792
-summary: "Change `skip_unavailable` remote cluster setting default value to true"
-area: Search
-type: breaking
-issues: []
-breaking:
-  title: "Change `skip_unavailable` remote cluster setting default value to true"
-  area: Cluster and node setting
-  details: The default value of the `skip_unavailable` setting is now set to true.
-    All existing and future remote clusters that do not define this setting will use the new default.
-    This setting only affects cross-cluster searches using the _search or _async_search API.
-  impact: Unavailable remote clusters in a cross-cluster search will no longer cause the search to fail unless
-    skip_unavailable is configured to be `false` in elasticsearch.yml or via the `_cluster/settings` API. 
-    Unavailable clusters with `skip_unavailable`=`true` (either explicitly or by using the new default) are marked 
-    as SKIPPED in the search response metadata section and do not fail the entire search. If users want to ensure that a
-    search returns a failure when a particular remote cluster is not available, `skip_unavailable` must be now be 
-    set explicitly.    
-  notable: true
diff --git a/docs/changelog/105829.yaml b/docs/changelog/105829.yaml
deleted file mode 100644
index d9f8439e4b887..0000000000000
--- a/docs/changelog/105829.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105829
-summary: Log shard movements
-area: Allocation
-type: enhancement
-issues: []
diff --git a/docs/changelog/106252.yaml b/docs/changelog/106252.yaml
deleted file mode 100644
index 5e3f084632b9d..0000000000000
--- a/docs/changelog/106252.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106252
-summary: Add min/max range of the `event.ingested` field to cluster state for searchable
-  snapshots
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/106486.yaml b/docs/changelog/106486.yaml
deleted file mode 100644
index b33df50780e02..0000000000000
--- a/docs/changelog/106486.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-pr: 106486
-summary: Create custom parser for ISO-8601 datetimes
-area: Infra/Core
-type: enhancement
-issues:
-  - 102063
-highlight:
-  title: New custom parser for ISO-8601 datetimes
-  body: |-
-    This introduces a new custom parser for ISO-8601 datetimes, for the `iso8601`, `strict_date_optional_time`, and
-    `strict_date_optional_time_nanos` built-in date formats. This provides a performance improvement over the
-    default Java date-time parsing. Whilst it maintains much of the same behaviour,
-    the new parser does not accept nonsensical date-time strings that have multiple fractional seconds fields
-    or multiple timezone specifiers. If the new parser fails to parse a string, it will then use the previous parser
-    to parse it. If a large proportion of the input data consists of these invalid strings, this may cause
-    a small performance degradation. If you wish to force the use of the old parsers regardless,
-    set the JVM property `es.datetime.java_time_parsers=true` on all ES nodes.
diff --git a/docs/changelog/106553.yaml b/docs/changelog/106553.yaml
deleted file mode 100644
index 0ec5b1bb02da8..0000000000000
--- a/docs/changelog/106553.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106553
-summary: Add support for hiragana_uppercase & katakana_uppercase token filters in kuromoji analysis plugin
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/106591.yaml b/docs/changelog/106591.yaml
deleted file mode 100644
index 6a7814cb9cede..0000000000000
--- a/docs/changelog/106591.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106591
-summary: Make dense vector field type updatable
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/106820.yaml b/docs/changelog/106820.yaml
deleted file mode 100644
index d854e3984c13d..0000000000000
--- a/docs/changelog/106820.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106820
-summary: Add a capabilities API to check node and cluster capabilities
-area: Infra/REST API
-type: feature
-issues: []
diff --git a/docs/changelog/107081.yaml b/docs/changelog/107081.yaml
deleted file mode 100644
index 2acd2f919b476..0000000000000
--- a/docs/changelog/107081.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107081
-summary: Implement synthetic source support for range fields
-area: Mapping
-type: feature
-issues: []
diff --git a/docs/changelog/107088.yaml b/docs/changelog/107088.yaml
deleted file mode 100644
index 01a926f185eea..0000000000000
--- a/docs/changelog/107088.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107088
-summary: Introduce role description field
-area: Authorization
-type: enhancement
-issues: []
diff --git a/docs/changelog/107191.yaml b/docs/changelog/107191.yaml
deleted file mode 100644
index 5ef6297c0f3f1..0000000000000
--- a/docs/changelog/107191.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-pr: 107191
-summary: Stricter failure handling in multi-repo get-snapshots request handling
-area: Snapshot/Restore
-type: bug
-issues: []
-highlight:
-  title: Stricter failure handling in multi-repo get-snapshots request handling
-  body: |
-    If a multi-repo get-snapshots request encounters a failure in one of the
-    targeted repositories then earlier versions of Elasticsearch would proceed
-    as if the faulty repository did not exist, except for a per-repository
-    failure report in a separate section of the response body. This makes it
-    impossible to paginate the results properly in the presence of failures. In
-    versions 8.15.0 and later this API's failure handling behaviour has been
-    made stricter, reporting an overall failure if any targeted repository's
-    contents cannot be listed.
-  notable: true
diff --git a/docs/changelog/107216.yaml b/docs/changelog/107216.yaml
deleted file mode 100644
index 7144eedf9bea4..0000000000000
--- a/docs/changelog/107216.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107216
-summary: Add per-field KNN vector format to Index Segments API
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/107240.yaml b/docs/changelog/107240.yaml
deleted file mode 100644
index baf4c222a9a27..0000000000000
--- a/docs/changelog/107240.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107240
-summary: Include doc size info in ingest stats
-area: Ingest Node
-type: enhancement
-issues:
- - 106386
diff --git a/docs/changelog/107244.yaml b/docs/changelog/107244.yaml
deleted file mode 100644
index f805796674f93..0000000000000
--- a/docs/changelog/107244.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107244
-summary: Support effective watermark thresholds in node stats API
-area: Allocation
-type: enhancement
-issues: [106676]
diff --git a/docs/changelog/107279.yaml b/docs/changelog/107279.yaml
deleted file mode 100644
index a2940ecc9ba2d..0000000000000
--- a/docs/changelog/107279.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107279
-summary: Introduce _transform/_node_stats API
-area: Transform
-type: feature
-issues: []
diff --git a/docs/changelog/107409.yaml b/docs/changelog/107409.yaml
deleted file mode 100644
index 6f2350239772f..0000000000000
--- a/docs/changelog/107409.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107409
-summary: "ESQL: Introduce a casting operator, `::`"
-area: ES|QL
-type: feature
-issues: []
diff --git a/docs/changelog/107410.yaml b/docs/changelog/107410.yaml
deleted file mode 100644
index 5026e88cfa762..0000000000000
--- a/docs/changelog/107410.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107410
-summary: Cluster-state based Security role mapper
-area: Authorization
-type: enhancement
-issues: []
diff --git a/docs/changelog/107415.yaml b/docs/changelog/107415.yaml
deleted file mode 100644
index 8877d0426c60d..0000000000000
--- a/docs/changelog/107415.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107415
-summary: Fix `DecayFunctions'` `toString`
-area: Search
-type: bug
-issues:
- - 100870
diff --git a/docs/changelog/107426.yaml b/docs/changelog/107426.yaml
deleted file mode 100644
index 2feed3df56108..0000000000000
--- a/docs/changelog/107426.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107426
-summary: Support wait indefinitely for search tasks to complete on node shutdown
-area: Infra/Node Lifecycle
-type: bug
-issues: []
diff --git a/docs/changelog/107435.yaml b/docs/changelog/107435.yaml
deleted file mode 100644
index ae5d2215419c4..0000000000000
--- a/docs/changelog/107435.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107435
-summary: '`NoSuchRemoteClusterException` should not be thrown when a remote is configured'
-area: Network
-type: bug
-issues:
- - 107381
diff --git a/docs/changelog/107493.yaml b/docs/changelog/107493.yaml
deleted file mode 100644
index dfd45e1493c95..0000000000000
--- a/docs/changelog/107493.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107493
-summary: Remote cluster - API key security model - cluster privileges
-area: Security
-type: enhancement
-issues: []
diff --git a/docs/changelog/107545.yaml b/docs/changelog/107545.yaml
deleted file mode 100644
index ad457cc5a533f..0000000000000
--- a/docs/changelog/107545.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107545
-summary: "ESQL: Union Types Support"
-area: ES|QL
-type: enhancement
-issues:
- - 100603
diff --git a/docs/changelog/107549.yaml b/docs/changelog/107549.yaml
deleted file mode 100644
index 36250cf65b4d9..0000000000000
--- a/docs/changelog/107549.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107549
-summary: Add synthetic source support for binary fields
-area: Mapping
-type: feature
-issues: []
diff --git a/docs/changelog/107567.yaml b/docs/changelog/107567.yaml
deleted file mode 100644
index 558b5b570b1fb..0000000000000
--- a/docs/changelog/107567.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107567
-summary: Add ignored field values to synthetic source
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/107579.yaml b/docs/changelog/107579.yaml
deleted file mode 100644
index fdee59424b8de..0000000000000
--- a/docs/changelog/107579.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107579
-summary: Adding `hits_time_in_millis` and `misses_time_in_millis` to enrich cache
-  stats
-area: Ingest Node
-type: enhancement
-issues: []
diff --git a/docs/changelog/107593.yaml b/docs/changelog/107593.yaml
deleted file mode 100644
index 2e3d2cbc80119..0000000000000
--- a/docs/changelog/107593.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107593
-summary: Add auto-sharding APM metrics
-area: Infra/Metrics
-type: enhancement
-issues: []
diff --git a/docs/changelog/107640.yaml b/docs/changelog/107640.yaml
deleted file mode 100644
index 9871943481f20..0000000000000
--- a/docs/changelog/107640.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107640
-summary: "Unified Highlighter to support matched_fields "
-area: Highlighting
-type: enhancement
-issues:
-  - 5172
diff --git a/docs/changelog/107645.yaml b/docs/changelog/107645.yaml
deleted file mode 100644
index 93fc0f2a89b3a..0000000000000
--- a/docs/changelog/107645.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-pr: 107645
-summary: Add `_name` support for top level `knn` clauses
-area: Search
-type: enhancement
-issues:
-  - 106254
-  - 107448
diff --git a/docs/changelog/107647.yaml b/docs/changelog/107647.yaml
deleted file mode 100644
index 97d98a7c91079..0000000000000
--- a/docs/changelog/107647.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107647
-summary: Adding human readable times to geoip stats
-area: Ingest Node
-type: enhancement
-issues: []
diff --git a/docs/changelog/107663.yaml b/docs/changelog/107663.yaml
deleted file mode 100644
index a7c3dc185425a..0000000000000
--- a/docs/changelog/107663.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107663
-summary: Optimize `GeoBounds` and `GeoCentroid` aggregations for single value fields
-area: Geo
-type: enhancement
-issues: []
diff --git a/docs/changelog/107675.yaml b/docs/changelog/107675.yaml
deleted file mode 100644
index b1d51cd3f8538..0000000000000
--- a/docs/changelog/107675.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-pr: 107675
-summary: Interpret `?timeout=-1` as infinite ack timeout
-area: Cluster Coordination
-type: breaking
-issues: []
-breaking:
-  title: Interpret `?timeout=-1` as infinite ack timeout
-  area: REST API
-  details: |
-    Today {es} accepts the parameter `?timeout=-1` in many APIs, but interprets
-    this to mean the same as `?timeout=0`. From 8.15 onwards `?timeout=-1` will
-    mean to wait indefinitely, aligning the behaviour of this parameter with
-    other similar parameters such as `?master_timeout`.
-  impact: |
-    Use `?timeout=0` to force relevant operations to time out immediately
-    instead of `?timeout=-1`
-  notable: false
diff --git a/docs/changelog/107676.yaml b/docs/changelog/107676.yaml
deleted file mode 100644
index b14bc29e66efd..0000000000000
--- a/docs/changelog/107676.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107676
-summary: Add model download progress to the download task status
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/107706.yaml b/docs/changelog/107706.yaml
deleted file mode 100644
index 76b7f662bf0e0..0000000000000
--- a/docs/changelog/107706.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107706
-summary: Add rate limiting support for the Inference API
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/107735.yaml b/docs/changelog/107735.yaml
deleted file mode 100644
index 372cb59ba8b1f..0000000000000
--- a/docs/changelog/107735.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107735
-summary: Implement synthetic source support for annotated text field
-area: Mapping
-type: feature
-issues: []
diff --git a/docs/changelog/107739.yaml b/docs/changelog/107739.yaml
deleted file mode 100644
index c55a0e332b4f6..0000000000000
--- a/docs/changelog/107739.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107739
-summary: Binary field enables doc values by default for index mode with synthe…
-area: Mapping
-type: enhancement
-issues:
- - 107554
diff --git a/docs/changelog/107764.yaml b/docs/changelog/107764.yaml
deleted file mode 100644
index 3f83efc789014..0000000000000
--- a/docs/changelog/107764.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107764
-summary: Increase size of big arrays only when there is an actual value in the aggregators
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/107779.yaml b/docs/changelog/107779.yaml
deleted file mode 100644
index a41c19a2329e0..0000000000000
--- a/docs/changelog/107779.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107779
-summary: Allow rescorer with field collapsing
-area: Search
-type: enhancement
-issues:
-  - 27243
\ No newline at end of file
diff --git a/docs/changelog/107792.yaml b/docs/changelog/107792.yaml
deleted file mode 100644
index bd9730d49d5d6..0000000000000
--- a/docs/changelog/107792.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107792
-summary: Halt Indexer on Stop/Abort API
-area: Transform
-type: bug
-issues: []
diff --git a/docs/changelog/107813.yaml b/docs/changelog/107813.yaml
deleted file mode 100644
index 1cbb518a8be5b..0000000000000
--- a/docs/changelog/107813.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107813
-summary: Increase size of big arrays only when there is an actual value in the aggregators
-  (Analytics module)
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/107827.yaml b/docs/changelog/107827.yaml
deleted file mode 100644
index 7cf217567b745..0000000000000
--- a/docs/changelog/107827.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107827
-summary: Add permission to secure access to certain config files
-area: Security
-type: bug
-issues: []
diff --git a/docs/changelog/107832.yaml b/docs/changelog/107832.yaml
deleted file mode 100644
index 491c491736005..0000000000000
--- a/docs/changelog/107832.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107832
-summary: Optimise few metric aggregations for single value fields
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/107862.yaml b/docs/changelog/107862.yaml
deleted file mode 100644
index 77f7a8c9fb02a..0000000000000
--- a/docs/changelog/107862.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107862
-summary: Fix serialization of put-shutdown request
-area: Infra/Node Lifecycle
-type: bug
-issues:
- - 107857
diff --git a/docs/changelog/107876.yaml b/docs/changelog/107876.yaml
deleted file mode 100644
index 21624cacf7e1d..0000000000000
--- a/docs/changelog/107876.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107876
-summary: "ESQL: Add aggregates node level reduction"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/107877.yaml b/docs/changelog/107877.yaml
deleted file mode 100644
index cf458b3aa3a42..0000000000000
--- a/docs/changelog/107877.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107877
-summary: Support metrics counter types in ESQL
-area: "ES|QL"
-type: enhancement
-issues: []
diff --git a/docs/changelog/107886.yaml b/docs/changelog/107886.yaml
deleted file mode 100644
index a328bc2a2a208..0000000000000
--- a/docs/changelog/107886.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107886
-summary: Cluster state role mapper file settings service
-area: Authorization
-type: enhancement
-issues: []
diff --git a/docs/changelog/107892.yaml b/docs/changelog/107892.yaml
deleted file mode 100644
index 5fd5404c48d02..0000000000000
--- a/docs/changelog/107892.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107892
-summary: Optimise cardinality aggregations for single value fields
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/107893.yaml b/docs/changelog/107893.yaml
deleted file mode 100644
index 61f0f4d76e679..0000000000000
--- a/docs/changelog/107893.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107893
-summary: Optimise histogram aggregations for single value fields
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/107897.yaml b/docs/changelog/107897.yaml
deleted file mode 100644
index e4a2a5270475d..0000000000000
--- a/docs/changelog/107897.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107897
-summary: Optimise composite aggregations for single value fields
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/107917.yaml b/docs/changelog/107917.yaml
deleted file mode 100644
index 18125bf46f4b7..0000000000000
--- a/docs/changelog/107917.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107917
-summary: Exit gracefully when deleted
-area: Transform
-type: bug
-issues:
- - 107266
diff --git a/docs/changelog/107922.yaml b/docs/changelog/107922.yaml
deleted file mode 100644
index e28d0f6262af4..0000000000000
--- a/docs/changelog/107922.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107922
-summary: Feature/annotated text store defaults
-area: Mapping
-type: enhancement
-issues:
- - 107734
diff --git a/docs/changelog/107930.yaml b/docs/changelog/107930.yaml
deleted file mode 100644
index 90af5c55b8604..0000000000000
--- a/docs/changelog/107930.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107930
-summary: Optimise terms aggregations for single value fields
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/107937.yaml b/docs/changelog/107937.yaml
deleted file mode 100644
index 5938c8e8b6602..0000000000000
--- a/docs/changelog/107937.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107937
-summary: Optimise multiterms aggregation for single value fields
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/107947.yaml b/docs/changelog/107947.yaml
deleted file mode 100644
index 637ac3c005779..0000000000000
--- a/docs/changelog/107947.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107947
-summary: "ESQL: Fix equals `hashCode` for functions"
-area: ES|QL
-type: bug
-issues:
- - 104393
diff --git a/docs/changelog/107967.yaml b/docs/changelog/107967.yaml
deleted file mode 100644
index 159370e44f236..0000000000000
--- a/docs/changelog/107967.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107967
-summary: Sort time series indices by time range in `GetDataStreams` API
-area: TSDB
-type: bug
-issues:
- - 102088
diff --git a/docs/changelog/107972.yaml b/docs/changelog/107972.yaml
deleted file mode 100644
index 3ec83d6a56954..0000000000000
--- a/docs/changelog/107972.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107972
-summary: Require question to be non-null in `QuestionAnsweringConfig`
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/107977.yaml b/docs/changelog/107977.yaml
deleted file mode 100644
index fdbbb57d7e48f..0000000000000
--- a/docs/changelog/107977.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107977
-summary: Fix off by one error when handling null values in range fields
-area: Mapping
-type: bug
-issues:
- - 107282
diff --git a/docs/changelog/107978.yaml b/docs/changelog/107978.yaml
deleted file mode 100644
index 50115df9ee092..0000000000000
--- a/docs/changelog/107978.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107978
-summary: Drop shards close timeout when stopping node.
-area: Engine
-type: enhancement
-issues:
- - 107938
diff --git a/docs/changelog/107987.yaml b/docs/changelog/107987.yaml
deleted file mode 100644
index e8afebde0b190..0000000000000
--- a/docs/changelog/107987.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107987
-summary: "ESQL: Implement LOOKUP, an \"inline\" enrich"
-area: ES|QL
-type: enhancement
-issues:
- - 107306
diff --git a/docs/changelog/107990.yaml b/docs/changelog/107990.yaml
deleted file mode 100644
index 80cb96aca4426..0000000000000
--- a/docs/changelog/107990.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107990
-summary: Optimise `time_series` aggregation for single value fields
-area: TSDB
-type: enhancement
-issues: []
diff --git a/docs/changelog/108016.yaml b/docs/changelog/108016.yaml
deleted file mode 100644
index 0aa3f86a6f859..0000000000000
--- a/docs/changelog/108016.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108016
-summary: Optimise `BinaryRangeAggregator` for single value fields
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/108019.yaml b/docs/changelog/108019.yaml
deleted file mode 100644
index 69e8e9fd371f8..0000000000000
--- a/docs/changelog/108019.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108019
-summary: Ignore additional cpu.stat fields
-area: Infra/Core
-type: bug
-issues:
-  - 107983
diff --git a/docs/changelog/108051.yaml b/docs/changelog/108051.yaml
deleted file mode 100644
index a47e1192c6090..0000000000000
--- a/docs/changelog/108051.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108051
-summary: Track synthetic source for disabled objects
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/108065.yaml b/docs/changelog/108065.yaml
deleted file mode 100644
index 2ec93bf6e6295..0000000000000
--- a/docs/changelog/108065.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108065
-summary: '`DenseVectorFieldMapper` fixed typo'
-area: Mapping
-type: bug
-issues: []
diff --git a/docs/changelog/108070.yaml b/docs/changelog/108070.yaml
deleted file mode 100644
index cde191aa50804..0000000000000
--- a/docs/changelog/108070.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108070
-summary: Redirect `VersionConflict` to reset code
-area: Transform
-type: bug
-issues: []
diff --git a/docs/changelog/108088.yaml b/docs/changelog/108088.yaml
deleted file mode 100644
index 95c58f6dc19f1..0000000000000
--- a/docs/changelog/108088.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108088
-summary: Add a SIMD (AVX2) optimised vector distance function for int7 on x64
-area: "Search"
-type: enhancement
-issues: []
diff --git a/docs/changelog/108089.yaml b/docs/changelog/108089.yaml
deleted file mode 100644
index 02fb6349185a6..0000000000000
--- a/docs/changelog/108089.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108089
-summary: "ES|QL: limit query depth to 500 levels"
-area: ES|QL
-type: bug
-issues:
- - 107752
diff --git a/docs/changelog/108106.yaml b/docs/changelog/108106.yaml
deleted file mode 100644
index e9dd438e620c4..0000000000000
--- a/docs/changelog/108106.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108106
-summary: Simulate should succeed if `ignore_missing_pipeline`
-area: Ingest Node
-type: bug
-issues:
- - 107314
diff --git a/docs/changelog/108118.yaml b/docs/changelog/108118.yaml
deleted file mode 100644
index b9b0f1c1406e0..0000000000000
--- a/docs/changelog/108118.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108118
-summary: Optimize for single value in ordinals grouping
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/108122.yaml b/docs/changelog/108122.yaml
deleted file mode 100644
index 981ab39b9dad8..0000000000000
--- a/docs/changelog/108122.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108122
-summary: Correct query profiling for conjunctions
-area: Search
-type: bug
-issues:
- - 108116
diff --git a/docs/changelog/108130.yaml b/docs/changelog/108130.yaml
deleted file mode 100644
index 5b431bdb0cc1b..0000000000000
--- a/docs/changelog/108130.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108130
-summary: Optimise frequent item sets aggregation for single value fields
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/108131.yaml b/docs/changelog/108131.yaml
deleted file mode 100644
index 7a4286c1e44a0..0000000000000
--- a/docs/changelog/108131.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108131
-summary: "Inference Processor: skip inference when all fields are missing"
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/108144.yaml b/docs/changelog/108144.yaml
deleted file mode 100644
index 6ff5b1d600d0e..0000000000000
--- a/docs/changelog/108144.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108144
-summary: Bump Tika dependency to 2.9.2
-area: Ingest Node
-type: upgrade
-issues: []
diff --git a/docs/changelog/108145.yaml b/docs/changelog/108145.yaml
deleted file mode 100644
index b8c9428c1e3a8..0000000000000
--- a/docs/changelog/108145.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108145
-summary: Async close of `IndexShard`
-area: Engine
-type: bug
-issues: []
diff --git a/docs/changelog/108146.yaml b/docs/changelog/108146.yaml
deleted file mode 100644
index 2a4f917134090..0000000000000
--- a/docs/changelog/108146.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108146
-summary: Allow deletion of the ELSER inference service when reference in ingest
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/108155.yaml b/docs/changelog/108155.yaml
deleted file mode 100644
index 57db86b4005b9..0000000000000
--- a/docs/changelog/108155.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108155
-summary: Upgrade to Netty 4.1.109
-area: Network
-type: upgrade
-issues: []
diff --git a/docs/changelog/108161.yaml b/docs/changelog/108161.yaml
deleted file mode 100644
index 73fa41e2089d3..0000000000000
--- a/docs/changelog/108161.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108161
-summary: Refactor TextEmbeddingResults to use primitives rather than objects
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/108165.yaml b/docs/changelog/108165.yaml
deleted file mode 100644
index b88b0f5e217dd..0000000000000
--- a/docs/changelog/108165.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108165
-summary: Add `BlockHash` for 3 `BytesRefs`
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/108171.yaml b/docs/changelog/108171.yaml
deleted file mode 100644
index 1ec17bb3e411d..0000000000000
--- a/docs/changelog/108171.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108171
-summary: "add Elastic-internal stable bridge api for use by Logstash"
-area: Infra/Core
-type: enhancement
-issues: []
diff --git a/docs/changelog/108222.yaml b/docs/changelog/108222.yaml
deleted file mode 100644
index 701b853441e32..0000000000000
--- a/docs/changelog/108222.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108222
-summary: Add generic fallback implementation for synthetic source
-area: Mapping
-type: feature
-issues: []
diff --git a/docs/changelog/108223.yaml b/docs/changelog/108223.yaml
deleted file mode 100644
index ba8756a8f9c68..0000000000000
--- a/docs/changelog/108223.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108223
-summary: Upgrade bouncy castle (non-fips) to 1.78.1
-area: Security
-type: upgrade
-issues: []
diff --git a/docs/changelog/108227.yaml b/docs/changelog/108227.yaml
deleted file mode 100644
index 79f69bc4aaff6..0000000000000
--- a/docs/changelog/108227.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108227
-summary: "Apm-data: improve indexing resilience"
-area: Data streams
-type: enhancement
-issues: []
diff --git a/docs/changelog/108254.yaml b/docs/changelog/108254.yaml
deleted file mode 100644
index 3bf08e8b8f5fc..0000000000000
--- a/docs/changelog/108254.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108254
-summary: Add `sparse_vector` query
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/108266.yaml b/docs/changelog/108266.yaml
deleted file mode 100644
index 5a189cfcdc258..0000000000000
--- a/docs/changelog/108266.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108266
-summary: Log details of non-green indicators in `HealthPeriodicLogger`
-area: Health
-type: enhancement
-issues: []
diff --git a/docs/changelog/108300.yaml b/docs/changelog/108300.yaml
deleted file mode 100644
index c4d6e468113a4..0000000000000
--- a/docs/changelog/108300.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108300
-summary: "ESQL: Add more time span units"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/108306.yaml b/docs/changelog/108306.yaml
deleted file mode 100644
index 7a104ce880f43..0000000000000
--- a/docs/changelog/108306.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108306
-summary: Enable inter-segment concurrency for low cardinality numeric terms aggs
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/108333.yaml b/docs/changelog/108333.yaml
deleted file mode 100644
index c3152500ce1b2..0000000000000
--- a/docs/changelog/108333.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108333
-summary: Allow `read_slm` to call GET /_slm/status
-area: ILM+SLM
-type: bug
-issues: []
diff --git a/docs/changelog/108340.yaml b/docs/changelog/108340.yaml
deleted file mode 100644
index fb2ea72c0a0f5..0000000000000
--- a/docs/changelog/108340.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108340
-summary: "Apm-data: increase version for templates"
-area: Data streams
-type: enhancement
-issues: []
diff --git a/docs/changelog/108349.yaml b/docs/changelog/108349.yaml
deleted file mode 100644
index 6d9ea3d658dca..0000000000000
--- a/docs/changelog/108349.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108349
-summary: "Ecs@mappings: reduce scope for `ecs_geo_point`"
-area: Data streams
-type: bug
-issues:
- - 108338
diff --git a/docs/changelog/108379.yaml b/docs/changelog/108379.yaml
deleted file mode 100644
index 312856a5db33d..0000000000000
--- a/docs/changelog/108379.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108379
-summary: Create a new `NodeRequest` for every `NodesDataTiersUsageTransport` use
-area: Indices APIs
-type: bug
-issues: []
diff --git a/docs/changelog/108394.yaml b/docs/changelog/108394.yaml
deleted file mode 100644
index 58f48fa548c6e..0000000000000
--- a/docs/changelog/108394.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108394
-summary: Handle `IndexNotFoundException`
-area: Transform
-type: bug
-issues:
- - 107263
diff --git a/docs/changelog/108395.yaml b/docs/changelog/108395.yaml
deleted file mode 100644
index c33cf169a99fa..0000000000000
--- a/docs/changelog/108395.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108395
-summary: "ESQL: change from quoting from backtick to quote"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/108396.yaml b/docs/changelog/108396.yaml
deleted file mode 100644
index 63937646b755c..0000000000000
--- a/docs/changelog/108396.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108396
-summary: "Apm-data: improve default pipeline performance"
-area: Data streams
-type: enhancement
-issues:
- - 108290
diff --git a/docs/changelog/108409.yaml b/docs/changelog/108409.yaml
deleted file mode 100644
index 6cff86cf93930..0000000000000
--- a/docs/changelog/108409.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108409
-summary: Support multiple associated groups for TopN
-area: Application
-type: enhancement
-issues:
- - 108018
diff --git a/docs/changelog/108410.yaml b/docs/changelog/108410.yaml
deleted file mode 100644
index 5fd831231a3be..0000000000000
--- a/docs/changelog/108410.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108410
-summary: GeoIP tasks should wait longer for master
-area: Ingest Node
-type: bug
-issues: []
diff --git a/docs/changelog/108417.yaml b/docs/changelog/108417.yaml
deleted file mode 100644
index bb650922f1be5..0000000000000
--- a/docs/changelog/108417.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108417
-summary: Track source for arrays of objects
-area: Mapping
-type: enhancement
-issues:
- - 90708
diff --git a/docs/changelog/108421.yaml b/docs/changelog/108421.yaml
deleted file mode 100644
index 1f077a4a2cb7c..0000000000000
--- a/docs/changelog/108421.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108421
-summary: "[ES|QL] Support Named and Positional Parameters in `EsqlQueryRequest`"
-area: ES|QL
-type: enhancement
-issues:
- - 107029
diff --git a/docs/changelog/108429.yaml b/docs/changelog/108429.yaml
deleted file mode 100644
index 562454a0de256..0000000000000
--- a/docs/changelog/108429.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108429
-summary: Fix `ClassCastException` in Significant Terms
-area: Aggregations
-type: bug
-issues:
- - 108427
diff --git a/docs/changelog/108444.yaml b/docs/changelog/108444.yaml
deleted file mode 100644
index c946ab24f939a..0000000000000
--- a/docs/changelog/108444.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108444
-summary: "Apm-data: ignore malformed fields, and too many dynamic fields"
-area: Data streams
-type: enhancement
-issues: []
diff --git a/docs/changelog/108452.yaml b/docs/changelog/108452.yaml
deleted file mode 100644
index fdf531602c806..0000000000000
--- a/docs/changelog/108452.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108452
-summary: Add the rerank task to the Elasticsearch internal inference service
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/108455.yaml b/docs/changelog/108455.yaml
deleted file mode 100644
index 8397af7b07cf1..0000000000000
--- a/docs/changelog/108455.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108455
-summary: "[ES|QL] Convert string to datetime when the other size of an arithmetic\
-  \ operator is `date_period` or `time_duration`"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/108459.yaml b/docs/changelog/108459.yaml
deleted file mode 100644
index 5e05797f284be..0000000000000
--- a/docs/changelog/108459.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108459
-summary: Do not use global ordinals strategy if the leaf reader context cannot be
-  obtained
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/108472.yaml b/docs/changelog/108472.yaml
deleted file mode 100644
index 82481e4edec3a..0000000000000
--- a/docs/changelog/108472.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108472
-summary: Add support for Azure AI Studio embeddings and completions to the inference service.
-area: Machine Learning
-type: feature
-issues: []
diff --git a/docs/changelog/108517.yaml b/docs/changelog/108517.yaml
deleted file mode 100644
index 359c8302fdf6c..0000000000000
--- a/docs/changelog/108517.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108517
-summary: Forward `indexServiceSafe` exception to listener
-area: Transform
-type: bug
-issues:
- - 108418
diff --git a/docs/changelog/108521.yaml b/docs/changelog/108521.yaml
deleted file mode 100644
index adc7c11a4decd..0000000000000
--- a/docs/changelog/108521.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108521
-summary: Adding override for lintian false positive on `libvec.so`
-area: "Packaging"
-type: bug
-issues:
- - 108514
diff --git a/docs/changelog/108522.yaml b/docs/changelog/108522.yaml
deleted file mode 100644
index 5bc064d7995e9..0000000000000
--- a/docs/changelog/108522.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108522
-summary: Ensure we return non-negative scores when scoring scalar dot-products
-area: Vector Search
-type: bug
-issues: []
diff --git a/docs/changelog/108537.yaml b/docs/changelog/108537.yaml
deleted file mode 100644
index 1c0228a71d449..0000000000000
--- a/docs/changelog/108537.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108537
-summary: Limit the value in prefix query
-area: Search
-type: enhancement
-issues:
- - 108486
diff --git a/docs/changelog/108538.yaml b/docs/changelog/108538.yaml
deleted file mode 100644
index 10ae49f0c1670..0000000000000
--- a/docs/changelog/108538.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108538
-summary: Adding RankFeature search phase implementation
-area: Search
-type: feature
-issues: []
diff --git a/docs/changelog/108574.yaml b/docs/changelog/108574.yaml
deleted file mode 100644
index b3c957721e01e..0000000000000
--- a/docs/changelog/108574.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108574
-summary: "[ESQL] CBRT function"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/108602.yaml b/docs/changelog/108602.yaml
deleted file mode 100644
index d544c89980123..0000000000000
--- a/docs/changelog/108602.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108602
-summary: "[Inference API] Extract optional long instead of integer in `RateLimitSettings#of`"
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/108606.yaml b/docs/changelog/108606.yaml
deleted file mode 100644
index 04780bff58800..0000000000000
--- a/docs/changelog/108606.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-pr: 108606
-summary: "Extend ISO8601 datetime parser to specify forbidden fields, allowing it to be used\
-  \ on more formats"
-area: Infra/Core
-type: enhancement
-issues: []
-highlight:
-  title: New custom parser for more ISO-8601 date formats
-  body: |-
-    Following on from #106486, this extends the custom ISO-8601 datetime parser to cover the `strict_year`,
-    `strict_year_month`, `strict_date_time`, `strict_date_time_no_millis`, `strict_date_hour_minute_second`,
-    `strict_date_hour_minute_second_millis`, and `strict_date_hour_minute_second_fraction` date formats.
-    As before, the parser will use the existing java.time parser if there are parsing issues, and the
-    `es.datetime.java_time_parsers=true` JVM property will force the use of the old parsers regardless.
diff --git a/docs/changelog/108607.yaml b/docs/changelog/108607.yaml
deleted file mode 100644
index 9ad4cf91e67b9..0000000000000
--- a/docs/changelog/108607.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108607
-summary: Specify parse index when error occurs on multiple datetime parses
-area: Infra/Core
-type: bug
-issues: []
diff --git a/docs/changelog/108612.yaml b/docs/changelog/108612.yaml
deleted file mode 100644
index 7a3dfa2b7ba44..0000000000000
--- a/docs/changelog/108612.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108612
-summary: "[Connector API] Change `UpdateConnectorFiltering` API to have better defaults"
-area: Application
-type: enhancement
-issues: []
diff --git a/docs/changelog/108624.yaml b/docs/changelog/108624.yaml
deleted file mode 100644
index 0da1fd2902c03..0000000000000
--- a/docs/changelog/108624.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-pr: 108624
-summary: Disallow new rollup jobs in clusters with no rollup usage
-area: Rollup
-type: breaking
-issues:
- - 108381
-breaking:
-  title: Disallow new rollup jobs in clusters with no rollup usage
-  area: Rollup
-  details: The put rollup API will fail with an error when a rollup job is created in a cluster with no rollup usage
-  impact: Clusters with no rollup usage (either no rollup job or index) can not create new rollup jobs
-  notable: true
diff --git a/docs/changelog/108639.yaml b/docs/changelog/108639.yaml
deleted file mode 100644
index e4964cbeb0285..0000000000000
--- a/docs/changelog/108639.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108639
-summary: Add support for the 'Domain' database to the geoip processor
-area: Ingest Node
-type: enhancement
-issues: []
diff --git a/docs/changelog/108643.yaml b/docs/changelog/108643.yaml
deleted file mode 100644
index f71a943673326..0000000000000
--- a/docs/changelog/108643.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108643
-summary: Use `scheduleUnlessShuttingDown` in `LeaderChecker`
-area: Cluster Coordination
-type: bug
-issues:
- - 108642
diff --git a/docs/changelog/108651.yaml b/docs/changelog/108651.yaml
deleted file mode 100644
index 227c464909d50..0000000000000
--- a/docs/changelog/108651.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108651
-summary: Add support for the 'ISP' database to the geoip processor
-area: Ingest Node
-type: enhancement
-issues: []
diff --git a/docs/changelog/108672.yaml b/docs/changelog/108672.yaml
deleted file mode 100644
index e1261fcf6f232..0000000000000
--- a/docs/changelog/108672.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108672
-summary: Add bounds checking to parsing ISO8601 timezone offset values
-area: Infra/Core
-type: bug
-issues: []
diff --git a/docs/changelog/108679.yaml b/docs/changelog/108679.yaml
deleted file mode 100644
index 62cd82a52c5bb..0000000000000
--- a/docs/changelog/108679.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108679
-summary: Suppress deprecation warnings from ingest pipelines when deleting trained model
-area: Machine Learning
-type: bug
-issues:
- - 105004
diff --git a/docs/changelog/108682.yaml b/docs/changelog/108682.yaml
deleted file mode 100644
index bd566acab8306..0000000000000
--- a/docs/changelog/108682.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108682
-summary: Adding support for explain in rrf
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/108683.yaml b/docs/changelog/108683.yaml
deleted file mode 100644
index b9e7df5fefc18..0000000000000
--- a/docs/changelog/108683.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-pr: 108683
-summary: Add support for the 'Connection Type' database to the geoip processor
-area: Ingest Node
-type: enhancement
-issues: []
-highlight:
-  title: "Preview: Support for the 'Connection Type, 'Domain', and 'ISP' databases in the geoip processor"
-  body: |-
-    As a Technical Preview, the {ref}/geoip-processor.html[`geoip`] processor can now use the commercial
-    https://dev.maxmind.com/geoip/docs/databases/connection-type[GeoIP2 'Connection Type'],
-    https://dev.maxmind.com/geoip/docs/databases/domain[GeoIP2 'Domain'],
-    and
-    https://dev.maxmind.com/geoip/docs/databases/isp[GeoIP2 'ISP']
-    databases from MaxMind.
diff --git a/docs/changelog/108684.yaml b/docs/changelog/108684.yaml
deleted file mode 100644
index 91684d2998be6..0000000000000
--- a/docs/changelog/108684.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108684
-summary: Check if `CsvTests` required capabilities exist
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/108687.yaml b/docs/changelog/108687.yaml
deleted file mode 100644
index 771516d551567..0000000000000
--- a/docs/changelog/108687.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108687
-summary: Adding `user_type` support for the enterprise database for the geoip processor
-area: Ingest Node
-type: enhancement
-issues: []
diff --git a/docs/changelog/108693.yaml b/docs/changelog/108693.yaml
deleted file mode 100644
index ee701e0f57736..0000000000000
--- a/docs/changelog/108693.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108693
-summary: Test pipeline run after reroute
-area: Ingest Node
-type: enhancement
-issues: []
diff --git a/docs/changelog/108705.yaml b/docs/changelog/108705.yaml
deleted file mode 100644
index fd08734831018..0000000000000
--- a/docs/changelog/108705.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108705
-summary: Associate restore snapshot task to parent mount task
-area: Distributed
-type: bug
-issues:
- - 105830
diff --git a/docs/changelog/108713.yaml b/docs/changelog/108713.yaml
deleted file mode 100644
index d6b1ddabd6c1e..0000000000000
--- a/docs/changelog/108713.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108713
-summary: Rewrite away type converting functions that do not convert types
-area: ES|QL
-type: enhancement
-issues:
- - 107716
diff --git a/docs/changelog/108726.yaml b/docs/changelog/108726.yaml
deleted file mode 100644
index 2e800a45e6975..0000000000000
--- a/docs/changelog/108726.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108726
-summary: Allow RA metrics to be reported upon parsing completed or accumulated
-area: Infra/Metrics
-type: enhancement
-issues: []
diff --git a/docs/changelog/108733.yaml b/docs/changelog/108733.yaml
deleted file mode 100644
index 76a969219ea4c..0000000000000
--- a/docs/changelog/108733.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108733
-summary: Query Roles API
-area: Security
-type: feature
-issues: []
diff --git a/docs/changelog/108746.yaml b/docs/changelog/108746.yaml
deleted file mode 100644
index 93ed917f3b56e..0000000000000
--- a/docs/changelog/108746.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108746
-summary: Support synthetic source for `aggregate_metric_double` when ignore_malf…
-area: Mapping
-type: feature
-issues: []
diff --git a/docs/changelog/108759.yaml b/docs/changelog/108759.yaml
deleted file mode 100644
index dfc2b30fe6c57..0000000000000
--- a/docs/changelog/108759.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108759
-summary: Expose `?master_timeout` in autoscaling APIs
-area: Autoscaling
-type: bug
-issues: []
diff --git a/docs/changelog/108761.yaml b/docs/changelog/108761.yaml
deleted file mode 100644
index 92aa67ebe0bfe..0000000000000
--- a/docs/changelog/108761.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108761
-summary: Add some missing timeout params to REST API specs
-area: Infra/REST API
-type: bug
-issues: []
diff --git a/docs/changelog/108764.yaml b/docs/changelog/108764.yaml
deleted file mode 100644
index 94de27eb52c9b..0000000000000
--- a/docs/changelog/108764.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108764
-summary: ST_DISTANCE Function
-area: ES|QL
-type: enhancement
-issues:
-  - 108212
diff --git a/docs/changelog/108780.yaml b/docs/changelog/108780.yaml
deleted file mode 100644
index 40e66326e6b9b..0000000000000
--- a/docs/changelog/108780.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108780
-summary: Add `continent_code` support to the geoip processor
-area: Ingest Node
-type: enhancement
-issues:
- - 85820
diff --git a/docs/changelog/108786.yaml b/docs/changelog/108786.yaml
deleted file mode 100644
index 1c07a3ceac900..0000000000000
--- a/docs/changelog/108786.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108786
-summary: Make ingest byte stat names more descriptive
-area: Ingest Node
-type: enhancement
-issues: []
diff --git a/docs/changelog/108793.yaml b/docs/changelog/108793.yaml
deleted file mode 100644
index 87668c8ee009b..0000000000000
--- a/docs/changelog/108793.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108793
-summary: Add `SparseVectorStats`
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/108796.yaml b/docs/changelog/108796.yaml
deleted file mode 100644
index 808247cf347d9..0000000000000
--- a/docs/changelog/108796.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108796
-summary: Return ingest byte stats even when 0-valued
-area: Ingest Node
-type: enhancement
-issues: []
diff --git a/docs/changelog/108814.yaml b/docs/changelog/108814.yaml
deleted file mode 100644
index 94298838c372e..0000000000000
--- a/docs/changelog/108814.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108814
-summary: Deserialize publish requests on generic thread-pool
-area: Cluster Coordination
-type: bug
-issues:
- - 106352
diff --git a/docs/changelog/108818.yaml b/docs/changelog/108818.yaml
deleted file mode 100644
index ed60fb5f64abd..0000000000000
--- a/docs/changelog/108818.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108818
-summary: Store source for nested objects
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/108820.yaml b/docs/changelog/108820.yaml
deleted file mode 100644
index 55045ffce3dfa..0000000000000
--- a/docs/changelog/108820.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108820
-summary: Allow `LuceneSourceOperator` to early terminate
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/108822.yaml b/docs/changelog/108822.yaml
deleted file mode 100644
index 8cec4da5dbc7f..0000000000000
--- a/docs/changelog/108822.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108822
-summary: Update ASM to 9.7 for plugin scanner
-area: Infra/Plugins
-type: upgrade
-issues:
- - 108776
diff --git a/docs/changelog/108831.yaml b/docs/changelog/108831.yaml
deleted file mode 100644
index 496bc0108f9d2..0000000000000
--- a/docs/changelog/108831.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108831
-summary: Rename rule query and add support for multiple rulesets
-area: Application
-type: enhancement
-issues: [ ]
diff --git a/docs/changelog/108849.yaml b/docs/changelog/108849.yaml
deleted file mode 100644
index 7c503efe9187b..0000000000000
--- a/docs/changelog/108849.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108849
-summary: "[Osquery] Extend `kibana_system` role with an access to new `osquery_manager`\
-  \ index"
-area: Authorization
-type: enhancement
-issues: []
diff --git a/docs/changelog/108856.yaml b/docs/changelog/108856.yaml
deleted file mode 100644
index 9b8f42248a442..0000000000000
--- a/docs/changelog/108856.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108856
-summary: Return noop instance `DocSizeObserver` for updates with scripts
-area: Infra/Metrics
-type: enhancement
-issues: []
diff --git a/docs/changelog/108860.yaml b/docs/changelog/108860.yaml
deleted file mode 100644
index 93aa8ce7c08ff..0000000000000
--- a/docs/changelog/108860.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108860
-summary: "Apm-data: enable plugin by default"
-area: Data streams
-type: enhancement
-issues: []
diff --git a/docs/changelog/108862.yaml b/docs/changelog/108862.yaml
deleted file mode 100644
index ddba15f11e8f5..0000000000000
--- a/docs/changelog/108862.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108862
-summary: "Apm-data: set codec: best_compression for logs-apm.* data streams"
-area: Data streams
-type: enhancement
-issues: []
diff --git a/docs/changelog/108868.yaml b/docs/changelog/108868.yaml
deleted file mode 100644
index d0643f056cce8..0000000000000
--- a/docs/changelog/108868.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108868
-summary: GA the update trained model action
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/108870.yaml b/docs/changelog/108870.yaml
deleted file mode 100644
index 435eea9845f16..0000000000000
--- a/docs/changelog/108870.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108870
-summary: Adding score from `RankDoc` to `SearchHit`
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/108871.yaml b/docs/changelog/108871.yaml
deleted file mode 100644
index 46bf8ca9d8404..0000000000000
--- a/docs/changelog/108871.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108871
-summary: "Reapply \"ESQL: Expose \"_ignored\" metadata field\""
-area: ES|QL
-type: feature
-issues: []
diff --git a/docs/changelog/108878.yaml b/docs/changelog/108878.yaml
deleted file mode 100644
index 1a8127869a647..0000000000000
--- a/docs/changelog/108878.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108878
-summary: Support arrays in fallback synthetic source implementation
-area: Mapping
-type: feature
-issues: []
diff --git a/docs/changelog/108881.yaml b/docs/changelog/108881.yaml
deleted file mode 100644
index b6de1129cfa03..0000000000000
--- a/docs/changelog/108881.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108881
-summary: Add synthetic source support for `geo_shape` via fallback implementation
-area: Mapping
-type: feature
-issues: []
diff --git a/docs/changelog/108885.yaml b/docs/changelog/108885.yaml
deleted file mode 100644
index c66843e082e29..0000000000000
--- a/docs/changelog/108885.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108885
-summary: "Apm-data: increase priority above Fleet templates"
-area: Data streams
-type: enhancement
-issues: []
diff --git a/docs/changelog/108886.yaml b/docs/changelog/108886.yaml
deleted file mode 100644
index 18df59e577713..0000000000000
--- a/docs/changelog/108886.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108886
-summary: Expose `?master_timeout` on get-shutdown API
-area: Infra/Node Lifecycle
-type: bug
-issues: []
diff --git a/docs/changelog/108891.yaml b/docs/changelog/108891.yaml
deleted file mode 100644
index 8282b616b34a9..0000000000000
--- a/docs/changelog/108891.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108891
-summary: Fix NPE during destination index creation
-area: Transform
-type: bug
-issues:
- - 108890
diff --git a/docs/changelog/108895.yaml b/docs/changelog/108895.yaml
deleted file mode 100644
index 15293896b20c5..0000000000000
--- a/docs/changelog/108895.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108895
-summary: Add permission to secure access to certain config files specified by settings
-area: "Security"
-type: bug
-issues: []
diff --git a/docs/changelog/108896.yaml b/docs/changelog/108896.yaml
deleted file mode 100644
index c52f074b65605..0000000000000
--- a/docs/changelog/108896.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108896
-summary: Introduce `logs` index mode as Tech Preview
-area: Logs
-type: feature
-issues:
- - 108896
diff --git a/docs/changelog/108911.yaml b/docs/changelog/108911.yaml
deleted file mode 100644
index 8832e01f7426e..0000000000000
--- a/docs/changelog/108911.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108911
-summary: Store source for fields in objects with `dynamic` override
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/108942.yaml b/docs/changelog/108942.yaml
deleted file mode 100644
index c58b06a92cee8..0000000000000
--- a/docs/changelog/108942.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108942
-summary: Fix NPE in trained model assignment updater
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/108947.yaml b/docs/changelog/108947.yaml
deleted file mode 100644
index 8aa4293242985..0000000000000
--- a/docs/changelog/108947.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108947
-summary: Provide the `DocumentSizeReporter` with index mode
-area: Infra/Metrics
-type: enhancement
-issues: []
diff --git a/docs/changelog/108999.yaml b/docs/changelog/108999.yaml
deleted file mode 100644
index 089d765b4e2d0..0000000000000
--- a/docs/changelog/108999.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108999
-summary: Use default translog durability on AD results index
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/109007.yaml b/docs/changelog/109007.yaml
deleted file mode 100644
index c828db64220fb..0000000000000
--- a/docs/changelog/109007.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109007
-summary: Multivalue Sparse Vector Support
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/109017.yaml b/docs/changelog/109017.yaml
new file mode 100644
index 0000000000000..80bcdd6fc0e25
--- /dev/null
+++ b/docs/changelog/109017.yaml
@@ -0,0 +1,6 @@
+pr: 109017
+summary: "ESQL: Add `MV_PSERIES_WEIGHTED_SUM` for score calculations used by security\
+  \ solution"
+area: ES|QL
+type: "feature"
+issues: [ ]
diff --git a/docs/changelog/109025.yaml b/docs/changelog/109025.yaml
deleted file mode 100644
index 38d19cab13d30..0000000000000
--- a/docs/changelog/109025.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109025
-summary: Introduce a setting controlling the activation of the `logs` index mode in logs@settings
-area: Logs
-type: feature
-issues:
- - 108762
diff --git a/docs/changelog/109042.yaml b/docs/changelog/109042.yaml
deleted file mode 100644
index 5aa80db991c0d..0000000000000
--- a/docs/changelog/109042.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109042
-summary: Add Create or update query rule API call
-area: Application
-type: enhancement
-issues: [ ]
diff --git a/docs/changelog/109043.yaml b/docs/changelog/109043.yaml
deleted file mode 100644
index bdfe3addea8e9..0000000000000
--- a/docs/changelog/109043.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109043
-summary: "Apm-data: set concrete values for `metricset.interval`"
-area: Data streams
-type: bug
-issues: []
diff --git a/docs/changelog/109044.yaml b/docs/changelog/109044.yaml
deleted file mode 100644
index 9e50c377606a0..0000000000000
--- a/docs/changelog/109044.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109044
-summary: Enable fallback synthetic source for `token_count`
-area: Mapping
-type: feature
-issues: []
diff --git a/docs/changelog/109047.yaml b/docs/changelog/109047.yaml
deleted file mode 100644
index 85a8808353a08..0000000000000
--- a/docs/changelog/109047.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109047
-summary: Prevent concurrent jobs during cleanup
-area: Transform
-type: bug
-issues: []
diff --git a/docs/changelog/109070.yaml b/docs/changelog/109070.yaml
deleted file mode 100644
index 8dbc0ec1c6cf2..0000000000000
--- a/docs/changelog/109070.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109070
-summary: "ESQL: Add `ip_prefix` function"
-area: ES|QL
-type: feature
-issues:
- - 99064
diff --git a/docs/changelog/109071.yaml b/docs/changelog/109071.yaml
deleted file mode 100644
index 275a5433cc1d8..0000000000000
--- a/docs/changelog/109071.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109071
-summary: Better handling of multiple rescorers clauses with LTR
-area: "Search"
-type: bug
-issues: []
diff --git a/docs/changelog/109078.yaml b/docs/changelog/109078.yaml
deleted file mode 100644
index f602ee9b131bc..0000000000000
--- a/docs/changelog/109078.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109078
-summary: Expose API Key cache metrics
-area: Authentication
-type: enhancement
-issues: []
diff --git a/docs/changelog/109084.yaml b/docs/changelog/109084.yaml
deleted file mode 100644
index 67ff5610c5a66..0000000000000
--- a/docs/changelog/109084.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109084
-summary: Add AVX-512 optimised vector distance functions for int7 on x64
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/109104.yaml b/docs/changelog/109104.yaml
deleted file mode 100644
index 985cf14bc5952..0000000000000
--- a/docs/changelog/109104.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109104
-summary: Offload request to generic threadpool
-area: Machine Learning
-type: bug
-issues:
- - 109100
diff --git a/docs/changelog/109123.yaml b/docs/changelog/109123.yaml
deleted file mode 100644
index dfd7e52b33e7f..0000000000000
--- a/docs/changelog/109123.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109123
-summary: "[Inference API] Check for related pipelines on delete inference endpoint"
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/109126.yaml b/docs/changelog/109126.yaml
deleted file mode 100644
index 248eacc76b65c..0000000000000
--- a/docs/changelog/109126.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109126
-summary: Correctly handle duplicate model ids for the `_cat` trained models api and usage statistics
-area: Machine Learning
-type: bug
-issues: [ ]
diff --git a/docs/changelog/109167.yaml b/docs/changelog/109167.yaml
deleted file mode 100644
index e366b2302263c..0000000000000
--- a/docs/changelog/109167.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109167
-summary: Fixes cluster state-based role mappings not recovered from disk
-area: Authorization
-type: bug
-issues: []
diff --git a/docs/changelog/109174.yaml b/docs/changelog/109174.yaml
deleted file mode 100644
index 5cd57ebd34ac6..0000000000000
--- a/docs/changelog/109174.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109174
-summary: "ESQL: Change \"substring\" function to not return null on empty string"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/109185.yaml b/docs/changelog/109185.yaml
deleted file mode 100644
index 4da72c4b20ffb..0000000000000
--- a/docs/changelog/109185.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109185
-summary: Handle unmatching remote cluster wildcards properly for `IndicesRequest.SingleIndexNoWildcards`
-  requests
-area: Authorization
-type: bug
-issues: []
diff --git a/docs/changelog/109194.yaml b/docs/changelog/109194.yaml
deleted file mode 100644
index bf50139547f62..0000000000000
--- a/docs/changelog/109194.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109194
-summary: "[Inference API] Add Mistral Embeddings Support to Inference API"
-area: Machine Learning
-type: enhancement
-issues: [ ]
diff --git a/docs/changelog/109196.yaml b/docs/changelog/109196.yaml
deleted file mode 100644
index 7f5ca3efbc8d4..0000000000000
--- a/docs/changelog/109196.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109196
-summary: Handle nullable `DocsStats` and `StoresStats`
-area: Distributed
-type: bug
-issues: []
diff --git a/docs/changelog/109204.yaml b/docs/changelog/109204.yaml
deleted file mode 100644
index b5b22ef1a06f9..0000000000000
--- a/docs/changelog/109204.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109204
-summary: Detect long-running tasks on network threads
-area: Network
-type: enhancement
-issues: []
diff --git a/docs/changelog/109205.yaml b/docs/changelog/109205.yaml
deleted file mode 100644
index 10f13a6549fbc..0000000000000
--- a/docs/changelog/109205.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109205
-summary: "ESQL: Fix `IpPrefix` function not handling correctly `ByteRefs`"
-area: ES|QL
-type: bug
-issues:
- - 109198
diff --git a/docs/changelog/109219.yaml b/docs/changelog/109219.yaml
deleted file mode 100644
index abf4f49235166..0000000000000
--- a/docs/changelog/109219.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-pr: 109219
-summary: Update Lucene version to 9.11
-area: Search
-type: feature
-issues: []
-highlight:
-  title: "Update Elasticsearch to Lucene 9.11"
-  body: |-
-    Elasticsearch is now updated using the latest Lucene version 9.11.
-    Here are the full release notes:
-    But, here are some particular highlights:
-    - Usage of MADVISE for better memory management: https://github.com/apache/lucene/pull/13196
-    - Use RWLock to access LRUQueryCache to reduce contention: https://github.com/apache/lucene/pull/13306
-    - Speedup multi-segment HNSW graph search for nested kNN queries: https://github.com/apache/lucene/pull/13121
-    - Add a MemorySegment Vector scorer - for scoring without copying on-heap vectors: https://github.com/apache/lucene/pull/13339
diff --git a/docs/changelog/109220.yaml b/docs/changelog/109220.yaml
deleted file mode 100644
index b8efa8f784d7a..0000000000000
--- a/docs/changelog/109220.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109220
-summary: "ESQL: add REPEAT string function"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/109233.yaml b/docs/changelog/109233.yaml
deleted file mode 100644
index 36010273c80db..0000000000000
--- a/docs/changelog/109233.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109233
-summary: Fix trappy timeouts in security settings APIs
-area: Security
-type: bug
-issues: []
diff --git a/docs/changelog/109236.yaml b/docs/changelog/109236.yaml
deleted file mode 100644
index e2eb917ea0343..0000000000000
--- a/docs/changelog/109236.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109236
-summary: Use proper executor for failing requests when connection closes
-area: Network
-type: bug
-issues:
- - 109225
diff --git a/docs/changelog/109240.yaml b/docs/changelog/109240.yaml
deleted file mode 100644
index a9fad3abdc47f..0000000000000
--- a/docs/changelog/109240.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109240
-summary: Fix trappy timeout in allocation explain API
-area: Allocation
-type: bug
-issues: []
diff --git a/docs/changelog/109241.yaml b/docs/changelog/109241.yaml
deleted file mode 100644
index b7343b9df1841..0000000000000
--- a/docs/changelog/109241.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109241
-summary: Fix misc trappy allocation API timeouts
-area: Allocation
-type: bug
-issues: []
diff --git a/docs/changelog/109256.yaml b/docs/changelog/109256.yaml
deleted file mode 100644
index 30c15ed77f9b9..0000000000000
--- a/docs/changelog/109256.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-pr: 109256
-summary: "[ESQL] Migrate `SimplifyComparisonArithmetics` optimization"
-area: ES|QL
-type: bug
-issues:
- - 108388
- - 108743
diff --git a/docs/changelog/109312.yaml b/docs/changelog/109312.yaml
deleted file mode 100644
index 594d3f90e8fd1..0000000000000
--- a/docs/changelog/109312.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109312
-summary: Enable fallback synthetic source for `point` and `shape`
-area: Mapping
-type: feature
-issues: []
diff --git a/docs/changelog/109317.yaml b/docs/changelog/109317.yaml
deleted file mode 100644
index 1d8595d99c2a6..0000000000000
--- a/docs/changelog/109317.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-pr: 109317
-summary: Add new int4 quantization to dense_vector
-area: Search
-type: feature
-issues: []
-highlight:
-  title: Add new int4 quantization to dense_vector
-  body: |-
-    New int4 (half-byte) scalar quantization support via two knew index types: `int4_hnsw` and `int4_flat`.
-    This gives an 8x reduction from `float32` with some accuracy loss. In addition to less memory required, this
-    improves query and merge speed significantly when compared to raw vectors.
-  notable: true
-
diff --git a/docs/changelog/109332.yaml b/docs/changelog/109332.yaml
deleted file mode 100644
index 3d03523fd518b..0000000000000
--- a/docs/changelog/109332.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109332
-summary: "ES|QL: vectorize eval"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/109358.yaml b/docs/changelog/109358.yaml
deleted file mode 100644
index af47b4129d874..0000000000000
--- a/docs/changelog/109358.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109358
-summary: Use the multi node routing action for internal inference services
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/109359.yaml b/docs/changelog/109359.yaml
deleted file mode 100644
index 37202eb5a28ec..0000000000000
--- a/docs/changelog/109359.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109359
-summary: Adding hamming distance function to painless for `dense_vector` fields
-area: Vector Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/109370.yaml b/docs/changelog/109370.yaml
deleted file mode 100644
index 32b190d1a1c94..0000000000000
--- a/docs/changelog/109370.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109370
-summary: Enable fallback synthetic source by default
-area: Mapping
-type: feature
-issues:
- - 106460
diff --git a/docs/changelog/109384.yaml b/docs/changelog/109384.yaml
deleted file mode 100644
index 303da23d57d8e..0000000000000
--- a/docs/changelog/109384.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109384
-summary: Fix serialising inference delete response
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/109386.yaml b/docs/changelog/109386.yaml
deleted file mode 100644
index 984ee96dde063..0000000000000
--- a/docs/changelog/109386.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109386
-summary: "ESQL: `top_list` aggregation"
-area: ES|QL
-type: feature
-issues:
- - 109213
diff --git a/docs/changelog/109395.yaml b/docs/changelog/109395.yaml
deleted file mode 100644
index e5336695afa48..0000000000000
--- a/docs/changelog/109395.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109395
-summary: Correct positioning for unique token filter
-area: Analysis
-type: bug
-issues: []
diff --git a/docs/changelog/109410.yaml b/docs/changelog/109410.yaml
deleted file mode 100644
index e8c4dcdab42c6..0000000000000
--- a/docs/changelog/109410.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109410
-summary: Support synthetic source for date fields when `ignore_malformed` is used
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/109414.yaml b/docs/changelog/109414.yaml
new file mode 100644
index 0000000000000..81b7541bde35b
--- /dev/null
+++ b/docs/changelog/109414.yaml
@@ -0,0 +1,6 @@
+pr: 109414
+summary: Don't fail retention lease sync actions due to capacity constraints
+area: CRUD
+type: bug
+issues:
+ - 105926
diff --git a/docs/changelog/109444.yaml b/docs/changelog/109444.yaml
deleted file mode 100644
index 8c56fe2dd9f02..0000000000000
--- a/docs/changelog/109444.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109444
-summary: "Aggs: Scripted metric allow list"
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/109449.yaml b/docs/changelog/109449.yaml
deleted file mode 100644
index 90cb908227f1b..0000000000000
--- a/docs/changelog/109449.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109449
-summary: Reset max page size to settings value
-area: Transform
-type: bug
-issues:
- - 109308
diff --git a/docs/changelog/109462.yaml b/docs/changelog/109462.yaml
deleted file mode 100644
index a05f4a04e80ae..0000000000000
--- a/docs/changelog/109462.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109462
-summary: Add `wait_for_completion` parameter to delete snapshot request
-area: Distributed
-type: enhancement
-issues:
- - 101300
diff --git a/docs/changelog/109470.yaml b/docs/changelog/109470.yaml
deleted file mode 100644
index 837c1664b775a..0000000000000
--- a/docs/changelog/109470.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109470
-summary: Enabling profiling for `RankBuilders` and adding tests for RRF
-area: Ranking
-type: enhancement
-issues: []
diff --git a/docs/changelog/109480.yaml b/docs/changelog/109480.yaml
deleted file mode 100644
index 3a6f48e9bd840..0000000000000
--- a/docs/changelog/109480.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109480
-summary: "[Connector API] Add claim sync job endpoint"
-area: Application
-type: feature
-issues: []
diff --git a/docs/changelog/109481.yaml b/docs/changelog/109481.yaml
deleted file mode 100644
index e8251788a90bd..0000000000000
--- a/docs/changelog/109481.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109481
-summary: Fork freeing search/scroll contexts to GENERIC pool
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/109487.yaml b/docs/changelog/109487.yaml
deleted file mode 100644
index c69c77203f12d..0000000000000
--- a/docs/changelog/109487.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109487
-summary: Start Trained Model Deployment API request query params now override body params
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/109501.yaml b/docs/changelog/109501.yaml
deleted file mode 100644
index 6e81f98816cbf..0000000000000
--- a/docs/changelog/109501.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-pr: 109501
-summary: Reflect latest changes in synthetic source documentation
-area: Mapping
-type: enhancement
-issues: []
-highlight:
-  title: Synthetic `_source` improvements
-  body: |-
-    There are multiple improvements to synthetic `_source` functionality:
-
-    * Synthetic `_source` is now supported for all field types including `nested` and `object`. `object` fields are supported with `enabled` set to `false`.
-
-    * Synthetic `_source` can be enabled together with `ignore_malformed` and `ignore_above` parameters for all field types that support them.
-  notable: false
diff --git a/docs/changelog/109506.yaml b/docs/changelog/109506.yaml
deleted file mode 100644
index 3a7570ed0b93a..0000000000000
--- a/docs/changelog/109506.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109506
-summary: Support synthetic source for `scaled_float` and `unsigned_long` when `ignore_malformed`
-  is used
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/109534.yaml b/docs/changelog/109534.yaml
deleted file mode 100644
index c6eb520bb70a8..0000000000000
--- a/docs/changelog/109534.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109534
-summary: Propagate accurate deployment timeout
-area: Machine Learning
-type: bug
-issues:
- - 109407
diff --git a/docs/changelog/109540.yaml b/docs/changelog/109540.yaml
deleted file mode 100644
index 722c60a30fb97..0000000000000
--- a/docs/changelog/109540.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109540
-summary: Add metrics@custom component template to metrics-*-* index template
-area: Data streams
-type: enhancement
-issues:
- - 109475
diff --git a/docs/changelog/109551.yaml b/docs/changelog/109551.yaml
deleted file mode 100644
index f4949669091d9..0000000000000
--- a/docs/changelog/109551.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109551
-summary: Avoid `InferenceRunner` deadlock
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/109554.yaml b/docs/changelog/109554.yaml
deleted file mode 100644
index 4e78a8f3044c7..0000000000000
--- a/docs/changelog/109554.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109554
-summary: "[Query Rules] Add API calls to get or delete individual query rules within\
-  \ a ruleset"
-area: Relevance
-type: enhancement
-issues: []
diff --git a/docs/changelog/109563.yaml b/docs/changelog/109563.yaml
deleted file mode 100644
index 9099064b6b040..0000000000000
--- a/docs/changelog/109563.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109563
-summary: Add allocation explain output for THROTTLING shards
-area: Infra/Core
-type: enhancement
-issues: []
diff --git a/docs/changelog/109583.yaml b/docs/changelog/109583.yaml
new file mode 100644
index 0000000000000..84757e307b4fb
--- /dev/null
+++ b/docs/changelog/109583.yaml
@@ -0,0 +1,29 @@
+pr: 109583
+summary: "ESQL: INLINESTATS"
+area: ES|QL
+type: feature
+issues:
+ - 107589
+highlight:
+  title: "ESQL: INLINESTATS"
+  body: |-
+    This adds the `INLINESTATS` command to ESQL which performs a STATS and
+    then enriches the results into the output stream. So, this query:
+
+    [source,esql]
+    ----
+    FROM test
+    | INLINESTATS m=MAX(a * b) BY b
+    | WHERE m == a * b
+    | SORT a DESC, b DESC
+    | LIMIT 3
+    ----
+
+    Produces output like:
+
+    |  a  |  b  |   m   |
+    | --- | --- | ----- |
+    |  99 | 999 | 98901 |
+    |  99 | 998 | 98802 |
+    |  99 | 997 | 98703 |
+  notable: true
diff --git a/docs/changelog/109597.yaml b/docs/changelog/109597.yaml
deleted file mode 100644
index 9b99df85da6a3..0000000000000
--- a/docs/changelog/109597.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109597
-summary: Opt `scripted_metric` out of parallelization
-area: Aggregations
-type: feature
-issues: []
diff --git a/docs/changelog/109603.yaml b/docs/changelog/109603.yaml
deleted file mode 100644
index 2d6e8b94aa8d0..0000000000000
--- a/docs/changelog/109603.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109603
-summary: Update translog `writeLocation` for `flushListener` after commit
-area: Engine
-type: enhancement
-issues: []
diff --git a/docs/changelog/109606.yaml b/docs/changelog/109606.yaml
deleted file mode 100644
index 6c9089c4c4fde..0000000000000
--- a/docs/changelog/109606.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109606
-summary: Avoid NPE if `users_roles` file does not exist
-area: Authentication
-type: bug
-issues: []
diff --git a/docs/changelog/109613.yaml b/docs/changelog/109613.yaml
deleted file mode 100644
index 21d152ac1d6de..0000000000000
--- a/docs/changelog/109613.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109613
-summary: Consider `error_trace` supported by all endpoints
-area: Infra/REST API
-type: bug
-issues:
- - 109612
diff --git a/docs/changelog/109618.yaml b/docs/changelog/109618.yaml
deleted file mode 100644
index f28bb15a53d96..0000000000000
--- a/docs/changelog/109618.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109618
-summary: Fail cluster state API if blocked
-area: Cluster Coordination
-type: bug
-issues:
- - 107503
diff --git a/docs/changelog/109634.yaml b/docs/changelog/109634.yaml
deleted file mode 100644
index 4c6358578b6de..0000000000000
--- a/docs/changelog/109634.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109634
-summary: "[Query Rules] Require Enterprise License for Query Rules"
-area: Relevance
-type: enhancement
-issues: []
diff --git a/docs/changelog/109651.yaml b/docs/changelog/109651.yaml
deleted file mode 100644
index 982e6a5b536cc..0000000000000
--- a/docs/changelog/109651.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109651
-summary: Support synthetic source for `geo_point` when `ignore_malformed` is used
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/109653.yaml b/docs/changelog/109653.yaml
deleted file mode 100644
index 665163ec2a91b..0000000000000
--- a/docs/changelog/109653.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109653
-summary: Handle the "JSON memory allocator bytes" field
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/109657.yaml b/docs/changelog/109657.yaml
deleted file mode 100644
index 35b315b7568c9..0000000000000
--- a/docs/changelog/109657.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109657
-summary: Track `RequestedRangeNotSatisfiedException` separately in S3 Metrics
-area: Snapshot/Restore
-type: enhancement
-issues: []
diff --git a/docs/changelog/109672.yaml b/docs/changelog/109672.yaml
deleted file mode 100644
index bb6532ab7accf..0000000000000
--- a/docs/changelog/109672.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109672
-summary: Log repo UUID at generation/registration time
-area: Snapshot/Restore
-type: enhancement
-issues: []
diff --git a/docs/changelog/109717.yaml b/docs/changelog/109717.yaml
deleted file mode 100644
index 326657ea4ce21..0000000000000
--- a/docs/changelog/109717.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109717
-summary: Bump jackson version in modules:repository-azure
-area: Snapshot/Restore
-type: upgrade
-issues: []
diff --git a/docs/changelog/109720.yaml b/docs/changelog/109720.yaml
deleted file mode 100644
index b029726c84427..0000000000000
--- a/docs/changelog/109720.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109720
-summary: "DocsStats: Add human readable bytesize"
-area: Stats
-type: enhancement
-issues: []
diff --git a/docs/changelog/109746.yaml b/docs/changelog/109746.yaml
deleted file mode 100644
index 5360f545333ac..0000000000000
--- a/docs/changelog/109746.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109746
-summary: ES|QL Add primitive float support to the Compute Engine
-area: ES|QL
-type: enhancement
-issues:
- - 109178
diff --git a/docs/changelog/109779.yaml b/docs/changelog/109779.yaml
deleted file mode 100644
index 4ccd8d475ec8d..0000000000000
--- a/docs/changelog/109779.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109779
-summary: Include component templates in retention validaiton
-area: Data streams
-type: bug
-issues: []
diff --git a/docs/changelog/109781.yaml b/docs/changelog/109781.yaml
deleted file mode 100644
index df74645b53d84..0000000000000
--- a/docs/changelog/109781.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109781
-summary: ES|QL Add primitive float variants of all aggregators to the compute engine
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/109794.yaml b/docs/changelog/109794.yaml
deleted file mode 100644
index d244c69a903ba..0000000000000
--- a/docs/changelog/109794.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109794
-summary: Provide document size reporter with `MapperService`
-area: Infra/Metrics
-type: bug
-issues: []
diff --git a/docs/changelog/109807.yaml b/docs/changelog/109807.yaml
deleted file mode 100644
index 5cf8a2c896c4e..0000000000000
--- a/docs/changelog/109807.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109807
-summary: "ESQL: Fix LOOKUP attribute shadowing"
-area: ES|QL
-type: bug
-issues:
- - 109392
diff --git a/docs/changelog/109813.yaml b/docs/changelog/109813.yaml
deleted file mode 100644
index edcef17e87606..0000000000000
--- a/docs/changelog/109813.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109813
-summary: Add text similarity reranker retriever
-area: Ranking
-type: feature
-issues: []
diff --git a/docs/changelog/109848.yaml b/docs/changelog/109848.yaml
deleted file mode 100644
index 858bbe84ef3a4..0000000000000
--- a/docs/changelog/109848.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109848
-summary: Denser in-memory representation of `ShardBlobsToDelete`
-area: Snapshot/Restore
-type: enhancement
-issues: []
diff --git a/docs/changelog/109873.yaml b/docs/changelog/109873.yaml
deleted file mode 100644
index c77197cc22d0a..0000000000000
--- a/docs/changelog/109873.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109873
-summary: "ESQL: add Arrow dataframes output format"
-area: ES|QL
-type: feature
-issues: []
diff --git a/docs/changelog/109876.yaml b/docs/changelog/109876.yaml
deleted file mode 100644
index 4a65b4e17c4a3..0000000000000
--- a/docs/changelog/109876.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109876
-summary: Always pick the user `maxPageSize` value
-area: Transform
-type: bug
-issues:
- - 109844
diff --git a/docs/changelog/109880.yaml b/docs/changelog/109880.yaml
deleted file mode 100644
index 71c7209824a8a..0000000000000
--- a/docs/changelog/109880.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-pr: 109880
-summary: Deprecate `text_expansion` and `weighted_tokens` queries
-area: Machine Learning
-type: deprecation
-issues: [ ]
-deprecation:
-  title: Deprecate `text_expansion` and `weighted_tokens` queries
-  area: REST API
-  details: The `text_expansion` and `weighted_tokens` queries have been replaced by `sparse_vector`.
-  impact: Please update your existing `text_expansion` and `weighted_tokens` queries to use `sparse_vector.`
diff --git a/docs/changelog/109882.yaml b/docs/changelog/109882.yaml
deleted file mode 100644
index 0f0fed01c5a7a..0000000000000
--- a/docs/changelog/109882.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109882
-summary: Support synthetic source together with `ignore_malformed` in histogram fields
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/109893.yaml b/docs/changelog/109893.yaml
deleted file mode 100644
index df6d6e51236c8..0000000000000
--- a/docs/changelog/109893.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109893
-summary: Add Anthropic messages integration to Inference API
-area: Machine Learning
-type: enhancement
-issues: [ ]
diff --git a/docs/changelog/109908.yaml b/docs/changelog/109908.yaml
deleted file mode 100644
index cdf2acf17096c..0000000000000
--- a/docs/changelog/109908.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109908
-summary: "Update checkpoints after post-replication actions, even on failure"
-area: CRUD
-type: bug
-issues: []
diff --git a/docs/changelog/109931.yaml b/docs/changelog/109931.yaml
deleted file mode 100644
index 3575cfd49176f..0000000000000
--- a/docs/changelog/109931.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109931
-summary: Apply FLS to the contents of `IgnoredSourceFieldMapper`
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/109957.yaml b/docs/changelog/109957.yaml
deleted file mode 100644
index 6bbcd8175501c..0000000000000
--- a/docs/changelog/109957.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109957
-summary: Add request metric to `RestController` to track success/failure (by status
-  code)
-area: Infra/Metrics
-type: enhancement
-issues: []
diff --git a/docs/changelog/109963.yaml b/docs/changelog/109963.yaml
deleted file mode 100644
index 1745d549582d4..0000000000000
--- a/docs/changelog/109963.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109963
-summary: Propagate mapper builder context flags across nested mapper builder context
-  creation
-area: Mapping
-type: bug
-issues: []
diff --git a/docs/changelog/109967.yaml b/docs/changelog/109967.yaml
deleted file mode 100644
index cfc6b6462954b..0000000000000
--- a/docs/changelog/109967.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109967
-summary: Default the HF service to cosine similarity
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/109981.yaml b/docs/changelog/109981.yaml
deleted file mode 100644
index cf9388f79e29c..0000000000000
--- a/docs/changelog/109981.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109981
-summary: Limit number of synonym rules that can be created
-area: Mapping
-type: bug
-issues: [108785]
diff --git a/docs/changelog/109989.yaml b/docs/changelog/109989.yaml
deleted file mode 100644
index f1f5972b60eb3..0000000000000
--- a/docs/changelog/109989.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109989
-summary: "ESQL: Fix Join references"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/109993.yaml b/docs/changelog/109993.yaml
deleted file mode 100644
index 40d161b6b5c24..0000000000000
--- a/docs/changelog/109993.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109993
-summary: "[ES|QL] `weighted_avg`"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/110004.yaml b/docs/changelog/110004.yaml
deleted file mode 100644
index f680016527a9c..0000000000000
--- a/docs/changelog/110004.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-pr: 110004
-summary: Mark Query Rules as GA
-area: Relevance
-type: feature
-issues: []
-highlight:
-  title: Mark Query Rules as GA
-  body: |-
-    This PR marks query rules as Generally Available. All APIs are no longer
-    in tech preview.
-  notable: true
diff --git a/docs/changelog/110016.yaml b/docs/changelog/110016.yaml
deleted file mode 100644
index 28ad55aa796c8..0000000000000
--- a/docs/changelog/110016.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110016
-summary: Opt in keyword field into fallback synthetic source if needed
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/110019.yaml b/docs/changelog/110019.yaml
deleted file mode 100644
index 632e79008d351..0000000000000
--- a/docs/changelog/110019.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110019
-summary: Improve mechanism for extracting the result of a `PlainActionFuture`
-area: Distributed
-type: enhancement
-issues:
- - 108125
diff --git a/docs/changelog/110046.yaml b/docs/changelog/110046.yaml
deleted file mode 100644
index 6ebe440e7aced..0000000000000
--- a/docs/changelog/110046.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110046
-summary: "ESQL: make named params objects truly per request"
-area: ES|QL
-type: bug
-issues:
- - 110028
diff --git a/docs/changelog/110059.yaml b/docs/changelog/110059.yaml
deleted file mode 100644
index ba160c091cdc2..0000000000000
--- a/docs/changelog/110059.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-pr: 110059
-summary: Adds new `bit` `element_type` for `dense_vectors`
-area: Vector Search
-type: feature
-issues: []
-highlight:
-  title: Adds new `bit` `element_type` for `dense_vectors`
-  body: |-
-    This adds `bit` vector support by adding `element_type: bit` for
-    vectors. This new element type works for indexed and non-indexed
-    vectors. Additionally, it works with `hnsw` and `flat` index types. No
-    quantization based codec works with this element type, this is
-    consistent with `byte` vectors.
-
-    `bit` vectors accept up to `32768` dimensions in size and expect vectors
-    that are being indexed to be encoded either as a hexidecimal string or a
-    `byte[]` array where each element of the `byte` array represents `8`
-    bits of the vector.
-
-    `bit` vectors support script usage and regular query usage. When
-    indexed, all comparisons done are `xor` and `popcount` summations (aka,
-    hamming distance), and the scores are transformed and normalized given
-    the vector dimensions.
-
-    For scripts, `l1norm` is the same as `hamming` distance and `l2norm` is
-    `sqrt(l1norm)`. `dotProduct` and `cosineSimilarity` are not supported. 
-
-    Note, the dimensions expected by this element_type are always to be
-    divisible by `8`, and the `byte[]` vectors provided for index must be
-    have size `dim/8` size, where each byte element represents `8` bits of
-    the vectors.
-  notable: true
diff --git a/docs/changelog/110061.yaml b/docs/changelog/110061.yaml
deleted file mode 100644
index 1880a2a197722..0000000000000
--- a/docs/changelog/110061.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110061
-summary: Avoiding running watch jobs in TickerScheduleTriggerEngine if it is paused
-area: Watcher
-type: bug
-issues:
- - 105933
diff --git a/docs/changelog/110066.yaml b/docs/changelog/110066.yaml
deleted file mode 100644
index 920c6304b63ae..0000000000000
--- a/docs/changelog/110066.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110066
-summary: Support flattened fields and multi-fields as dimensions in downsampling
-area: Downsampling
-type: bug
-issues:
- - 99297
diff --git a/docs/changelog/110096.yaml b/docs/changelog/110096.yaml
deleted file mode 100644
index 3d6616c289266..0000000000000
--- a/docs/changelog/110096.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110096
-summary: Fix `ClassCastException` with MV_EXPAND on missing field
-area: ES|QL
-type: bug
-issues:
- - 109974
diff --git a/docs/changelog/110102.yaml b/docs/changelog/110102.yaml
deleted file mode 100644
index d1b9b53e2dfc5..0000000000000
--- a/docs/changelog/110102.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110102
-summary: Optimize ST_DISTANCE filtering with Lucene circle intersection query
-area: ES|QL
-type: enhancement
-issues:
- - 109972
diff --git a/docs/changelog/110112.yaml b/docs/changelog/110112.yaml
deleted file mode 100644
index eca5fd9af15ce..0000000000000
--- a/docs/changelog/110112.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110112
-summary: Increase response size limit for batched requests
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/110146.yaml b/docs/changelog/110146.yaml
deleted file mode 100644
index 61ba35cec319b..0000000000000
--- a/docs/changelog/110146.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110146
-summary: Fix trailing slash in `ml.get_categories` specification
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/110160.yaml b/docs/changelog/110160.yaml
deleted file mode 100644
index 0c38c23c69067..0000000000000
--- a/docs/changelog/110160.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110160
-summary: Opt in number fields into fallback synthetic source when doc values a…
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/110176.yaml b/docs/changelog/110176.yaml
deleted file mode 100644
index ae1d7d10d6dc4..0000000000000
--- a/docs/changelog/110176.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110176
-summary: Fix trailing slash in two rollup specifications
-area: Rollup
-type: bug
-issues: []
diff --git a/docs/changelog/110177.yaml b/docs/changelog/110177.yaml
deleted file mode 100644
index 0ac5328d88df4..0000000000000
--- a/docs/changelog/110177.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110177
-summary: Fix trailing slash in `security.put_privileges` specification
-area: Authorization
-type: bug
-issues: []
diff --git a/docs/changelog/110179.yaml b/docs/changelog/110179.yaml
deleted file mode 100644
index b99a390c8586f..0000000000000
--- a/docs/changelog/110179.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110179
-summary: Make repository analysis API available to non-operators
-area: Snapshot/Restore
-type: enhancement
-issues:
- - 100318
diff --git a/docs/changelog/110186.yaml b/docs/changelog/110186.yaml
deleted file mode 100644
index 23eaab118e2ab..0000000000000
--- a/docs/changelog/110186.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110186
-summary: Don't sample calls to `ReduceContext#consumeBucketsAndMaybeBreak` ins `InternalDateHistogram`
-  and `InternalHistogram` during reduction
-area: Aggregations
-type: bug
-issues: []
diff --git a/docs/changelog/110201.yaml b/docs/changelog/110201.yaml
deleted file mode 100644
index a880638881948..0000000000000
--- a/docs/changelog/110201.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110201
-summary: "ES|QL: Fix DISSECT that overwrites input"
-area: ES|QL
-type: bug
-issues:
- - 110184
diff --git a/docs/changelog/110214.yaml b/docs/changelog/110214.yaml
deleted file mode 100644
index 20f61cac64454..0000000000000
--- a/docs/changelog/110214.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110214
-summary: Handle `ignore_above` in synthetic source for flattened fields
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/110216.yaml b/docs/changelog/110216.yaml
new file mode 100644
index 0000000000000..00ab20b230e2c
--- /dev/null
+++ b/docs/changelog/110216.yaml
@@ -0,0 +1,5 @@
+pr: 110216
+summary: Register SLM run before snapshotting to save stats
+area: ILM+SLM
+type: enhancement
+issues: []
diff --git a/docs/changelog/110233.yaml b/docs/changelog/110233.yaml
deleted file mode 100644
index d9ce4057090a4..0000000000000
--- a/docs/changelog/110233.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110233
-summary: Support k parameter for knn query
-area: Vector Search
-type: enhancement
-issues:
- - 108473
diff --git a/docs/changelog/110234.yaml b/docs/changelog/110234.yaml
deleted file mode 100644
index 0656ba5fb6636..0000000000000
--- a/docs/changelog/110234.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110234
-summary: Upgrade to Lucene-9.11.1
-area: Search
-type: upgrade
-issues: []
diff --git a/docs/changelog/110236.yaml b/docs/changelog/110236.yaml
deleted file mode 100644
index e2dbff7fbf768..0000000000000
--- a/docs/changelog/110236.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-pr: 110236
-summary: '`ParseHeapRatioOrDeprecatedByteSizeValue` for `indices.breaker.total.limit`'
-area: Infra/Settings
-type: deprecation
-issues: []
-deprecation:
-  title: 'Deprecate absolute size values for `indices.breaker.total.limit` setting'
-  area: Cluster and node setting
-  details: Previously, the value of `indices.breaker.total.limit` could be specified as
-    an absolute size in bytes. This setting controls the overal amount of
-    memory the server is allowed to use before taking remedial actions. Setting
-    this to a specific number of bytes led to strange behaviour when the node
-    maximum heap size changed because the circut breaker limit would remain
-    unchanged. This would either leave the value too low, causing part of the
-    heap to remain unused; or it would leave the value too high, causing the
-    circuit breaker to be ineffective at preventing OOM errors.  The only
-    reasonable behaviour for this setting is that it scales with the size of
-    the heap, and so absolute byte limits are now deprecated.
-  impact: Users must change their configuration to specify a percentage instead of
-    an absolute number of bytes for `indices.breaker.total.limit`, or else
-    accept the default, which is already specified as a percentage.
diff --git a/docs/changelog/110248.yaml b/docs/changelog/110248.yaml
deleted file mode 100644
index 85739528b69c6..0000000000000
--- a/docs/changelog/110248.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110248
-summary: "[Inference API] Add Amazon Bedrock Support to Inference API"
-area: Machine Learning
-type: enhancement
-issues: [ ]
diff --git a/docs/changelog/110251.yaml b/docs/changelog/110251.yaml
deleted file mode 100644
index a3b0c3128be35..0000000000000
--- a/docs/changelog/110251.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-pr: 110251
-summary: Support index sorting with nested fields
-area: Logs
-type: enhancement
-issues:
- - 107349
-highlight:
-  title: Index sorting on indexes with nested fields
-  body: |-
-    Index sorting is now supported for indexes with mappings containing nested objects.
-    The index sort spec (as specified by `index.sort.field`) can't contain any nested
-    fields, still.
-  notable: false
diff --git a/docs/changelog/110334.yaml b/docs/changelog/110334.yaml
deleted file mode 100644
index f83ac04ded773..0000000000000
--- a/docs/changelog/110334.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110334
-summary: Sentence Chunker
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/110337.yaml b/docs/changelog/110337.yaml
deleted file mode 100644
index bf21a95c9157f..0000000000000
--- a/docs/changelog/110337.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110337
-summary: Support `ignore_above` on keyword dimensions
-area: TSDB
-type: enhancement
-issues: []
diff --git a/docs/changelog/110338.yaml b/docs/changelog/110338.yaml
deleted file mode 100644
index 2334a1cbc9283..0000000000000
--- a/docs/changelog/110338.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110338
-summary: Add `semantic_text` field type and `semantic` query
-area: Mapping
-type: feature
-issues: []
diff --git a/docs/changelog/110347.yaml b/docs/changelog/110347.yaml
deleted file mode 100644
index 8727128230935..0000000000000
--- a/docs/changelog/110347.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110347
-summary: "ESQL: Renamed `TopList` to Top"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/110361.yaml b/docs/changelog/110361.yaml
deleted file mode 100644
index 8558c88e06049..0000000000000
--- a/docs/changelog/110361.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-pr: 110361
-summary: Don't detect `PlainActionFuture` deadlock on concurrent complete
-area: Distributed
-type: bug
-issues:
- - 110181
- - 110360
diff --git a/docs/changelog/110369.yaml b/docs/changelog/110369.yaml
deleted file mode 100644
index 770294605b444..0000000000000
--- a/docs/changelog/110369.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110369
-summary: Run terms concurrently when cardinality is only lower than shard size
-area: Aggregations
-type: bug
-issues:
- - 105505
diff --git a/docs/changelog/110383.yaml b/docs/changelog/110383.yaml
deleted file mode 100644
index 5e9bddd4bfcd2..0000000000000
--- a/docs/changelog/110383.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110383
-summary: Add bulk delete roles API
-area: Security
-type: enhancement
-issues: []
diff --git a/docs/changelog/110391.yaml b/docs/changelog/110391.yaml
deleted file mode 100644
index 1e00eda970398..0000000000000
--- a/docs/changelog/110391.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110391
-summary: Fix ST_DISTANCE Lucene push-down for complex predicates
-area: ES|QL
-type: bug
-issues:
- - 110349
diff --git a/docs/changelog/110395.yaml b/docs/changelog/110395.yaml
deleted file mode 100644
index 690be55abb5b2..0000000000000
--- a/docs/changelog/110395.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
-pr: 110395
-summary: Mark the Redact processor as Generally Available
-area: Ingest Node
-type: feature
-issues: []
-highlight:
-  title: The Redact processor is Generally Available
-  body: The Redact processor uses the Grok rules engine to obscure text in the input document matching the given Grok patterns. The Redact processor was initially released as Technical Preview in `8.7.0`, and is now released as Generally Available.
-  notable: true
diff --git a/docs/changelog/110431.yaml b/docs/changelog/110431.yaml
deleted file mode 100644
index 0dd93ef718ef9..0000000000000
--- a/docs/changelog/110431.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110431
-summary: "[Inference API] Fix serialization for inference delete endpoint response"
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/110476.yaml b/docs/changelog/110476.yaml
deleted file mode 100644
index bc12b3711a366..0000000000000
--- a/docs/changelog/110476.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-pr: 110476
-summary: Fix bug in union-types with type-casting in grouping key of STATS
-area: ES|QL
-type: bug
-issues:
- - 109922
- - 110477
diff --git a/docs/changelog/110488.yaml b/docs/changelog/110488.yaml
deleted file mode 100644
index fbb439f20fc96..0000000000000
--- a/docs/changelog/110488.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110488
-summary: "ESQL: Validate unique plan attribute names"
-area: ES|QL
-type: bug
-issues:
- - 110541
diff --git a/docs/changelog/110524.yaml b/docs/changelog/110524.yaml
new file mode 100644
index 0000000000000..6274c99b09998
--- /dev/null
+++ b/docs/changelog/110524.yaml
@@ -0,0 +1,5 @@
+pr: 110524
+summary: Introduce mode `subobjects=auto` for objects
+area: Mapping
+type: enhancement
+issues: []
diff --git a/docs/changelog/110540.yaml b/docs/changelog/110540.yaml
deleted file mode 100644
index 5e4994da80704..0000000000000
--- a/docs/changelog/110540.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-pr: 110540
-summary: Deprecate using slm privileges to access ilm
-area: ILM+SLM
-type: deprecation
-issues: []
-deprecation:
-  title: Deprecate using slm privileges to access ilm
-  area: REST API
-  details: The `read_slm` privilege can get the ILM status, and
-    the `manage_slm` privilege can start and stop ILM. Access to these 
-    APIs should be granted using the `read_ilm` and `manage_ilm` privileges
-    instead. Access to ILM APIs will be removed from SLM privileges in
-    a future major release, and is now deprecated.
-  impact: Users that need access to the ILM status API should now 
-    use the `read_ilm` privilege. Users that need to start and stop ILM, 
-    should use the `manage_ilm` privilege.
diff --git a/docs/changelog/110586.yaml b/docs/changelog/110586.yaml
deleted file mode 100644
index cc2bcb85a2dac..0000000000000
--- a/docs/changelog/110586.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110586
-summary: "ESQL: Fix Max doubles bug with negatives and add tests for Max and Min"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/110630.yaml b/docs/changelog/110630.yaml
new file mode 100644
index 0000000000000..9bf78e1209753
--- /dev/null
+++ b/docs/changelog/110630.yaml
@@ -0,0 +1,5 @@
+pr: 110630
+summary: Telemetry for inference adaptive allocations
+area: Machine Learning
+type: feature
+issues: []
diff --git a/docs/changelog/110633.yaml b/docs/changelog/110633.yaml
new file mode 100644
index 0000000000000..d4d1dc68cdbcc
--- /dev/null
+++ b/docs/changelog/110633.yaml
@@ -0,0 +1,5 @@
+pr: 110633
+summary: Add manage roles privilege
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/110651.yaml b/docs/changelog/110651.yaml
deleted file mode 100644
index c25c63ee0284a..0000000000000
--- a/docs/changelog/110651.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110651
-summary: "Remove `default_field: message` from metrics index templates"
-area: Data streams
-type: enhancement
-issues: []
diff --git a/docs/changelog/110665.yaml b/docs/changelog/110665.yaml
deleted file mode 100644
index fa6db3190fe60..0000000000000
--- a/docs/changelog/110665.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110665
-summary: "[ESQL] Fix parsing of large magnitude negative numbers"
-area: ES|QL
-type: bug
-issues:
- - 104323
diff --git a/docs/changelog/110666.yaml b/docs/changelog/110666.yaml
deleted file mode 100644
index d96f8e2024c81..0000000000000
--- a/docs/changelog/110666.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110666
-summary: Removing the use of Stream::peek from `GeoIpDownloader::cleanDatabases`
-area: Ingest Node
-type: bug
-issues: []
diff --git a/docs/changelog/110707.yaml b/docs/changelog/110707.yaml
deleted file mode 100644
index e13688c73c743..0000000000000
--- a/docs/changelog/110707.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110707
-summary: Fix issue with returning incomplete fragment for plain highlighter
-area: Highlighting
-type: bug
-issues: []
diff --git a/docs/changelog/110710.yaml b/docs/changelog/110710.yaml
deleted file mode 100644
index bf3349ee25cdd..0000000000000
--- a/docs/changelog/110710.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 110710
-summary: Add a cluster listener to fix missing node features after upgrading from a version prior to 8.13
-area: Infra/Core
-type: bug
-issues:
- - 109254
diff --git a/docs/changelog/110734.yaml b/docs/changelog/110734.yaml
new file mode 100644
index 0000000000000..d6dce144b89cd
--- /dev/null
+++ b/docs/changelog/110734.yaml
@@ -0,0 +1,5 @@
+pr: 110734
+summary: Fix bug in ML serverless autoscaling which prevented trained model updates from triggering a scale up
+area: Machine Learning
+type: bug
+issues: [ ]
diff --git a/docs/changelog/110793.yaml b/docs/changelog/110793.yaml
deleted file mode 100644
index 8f1f3ba9afeb7..0000000000000
--- a/docs/changelog/110793.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-pr: 110793
-summary: Fix for union-types for multiple columns with the same name
-area: ES|QL
-type: bug
-issues:
- - 110490
- - 109916
diff --git a/docs/changelog/110796.yaml b/docs/changelog/110796.yaml
new file mode 100644
index 0000000000000..a54a9a08bbd27
--- /dev/null
+++ b/docs/changelog/110796.yaml
@@ -0,0 +1,5 @@
+pr: 110796
+summary: Remove needless forking to GENERIC in `TransportMultiSearchAction`
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/110816.yaml b/docs/changelog/110816.yaml
new file mode 100644
index 0000000000000..bf707376ec9ea
--- /dev/null
+++ b/docs/changelog/110816.yaml
@@ -0,0 +1,6 @@
+pr: 110816
+summary: GET _cluster/settings with include_defaults returns the expected fallback value if defined in elasticsearch.yml
+area: Infra/Settings
+type: bug
+issues:
+ - 110815
diff --git a/docs/changelog/110824.yaml b/docs/changelog/110824.yaml
deleted file mode 100644
index 4fe97d6692865..0000000000000
--- a/docs/changelog/110824.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110824
-summary: "[ESQL] Count_distinct(_source) should return a 400"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/110844.yaml b/docs/changelog/110844.yaml
deleted file mode 100644
index ea879f13f3e67..0000000000000
--- a/docs/changelog/110844.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110844
-summary: Directly download commercial ip geolocation databases from providers
-area: Ingest Node
-type: feature
-issues: []
diff --git a/docs/changelog/110847.yaml b/docs/changelog/110847.yaml
new file mode 100644
index 0000000000000..214adc97ac7cb
--- /dev/null
+++ b/docs/changelog/110847.yaml
@@ -0,0 +1,5 @@
+pr: 110847
+summary: SLM Interval based scheduling
+area: ILM+SLM
+type: feature
+issues: []
diff --git a/docs/changelog/110901.yaml b/docs/changelog/110901.yaml
new file mode 100644
index 0000000000000..599cb7ce9ec98
--- /dev/null
+++ b/docs/changelog/110901.yaml
@@ -0,0 +1,15 @@
+pr: 110901
+summary: Set lenient to true by default when using updateable synonyms
+area: Analysis
+type: breaking
+issues: []
+breaking:
+  title: Set lenient to true by default when using updateable synonyms
+  area: Analysis
+  details: |
+    When a `synonym` or `synonym_graph` token filter is configured with `updateable: true`, the default `lenient`
+    value will now be `true`.
+  impact: |
+    `synonym` or `synonym_graph` token filters configured with `updateable: true` will ignore invalid synonyms by
+    default. This prevents shard initialization errors on invalid synonyms.
+  notable: true
diff --git a/docs/changelog/110906.yaml b/docs/changelog/110906.yaml
deleted file mode 100644
index 6123b1108fd17..0000000000000
--- a/docs/changelog/110906.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110906
-summary: "Add comma before charset parameter in WWW-Authenticate response header"
-area: Authentication
-type: bug
-issues: []
diff --git a/docs/changelog/110922.yaml b/docs/changelog/110922.yaml
deleted file mode 100644
index 6a85ce57de103..0000000000000
--- a/docs/changelog/110922.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110922
-summary: Speed up collecting zero document string terms
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/110927.yaml b/docs/changelog/110927.yaml
deleted file mode 100644
index 3602ce3e811fa..0000000000000
--- a/docs/changelog/110927.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 110927
-summary: Fix leak in collapsing search results
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/110971.yaml b/docs/changelog/110971.yaml
new file mode 100644
index 0000000000000..3579f77dc0d1d
--- /dev/null
+++ b/docs/changelog/110971.yaml
@@ -0,0 +1,5 @@
+pr: 110971
+summary: "Search in ES|QL: Add MATCH operator"
+area: ES|QL
+type: feature
+issues: []
diff --git a/docs/changelog/110974.yaml b/docs/changelog/110974.yaml
new file mode 100644
index 0000000000000..c9e8c9b78675e
--- /dev/null
+++ b/docs/changelog/110974.yaml
@@ -0,0 +1,5 @@
+pr: 110974
+summary: Add custom rule parameters to force time shift
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/110986.yaml b/docs/changelog/110986.yaml
new file mode 100644
index 0000000000000..4e320b19c9578
--- /dev/null
+++ b/docs/changelog/110986.yaml
@@ -0,0 +1,6 @@
+pr: 110986
+summary: Fix unnecessary mustache template evaluation
+area: Ingest Node
+type: enhancement
+issues:
+  - 110191
diff --git a/docs/changelog/110993.yaml b/docs/changelog/110993.yaml
new file mode 100644
index 0000000000000..9eb653a09e3a4
--- /dev/null
+++ b/docs/changelog/110993.yaml
@@ -0,0 +1,5 @@
+pr: 110993
+summary:  Add link to Max Shards Per Node exception message
+area: Distributed
+type: enhancement
+issues: []
diff --git a/docs/changelog/111071.yaml b/docs/changelog/111071.yaml
new file mode 100644
index 0000000000000..5e8ab53db3d03
--- /dev/null
+++ b/docs/changelog/111071.yaml
@@ -0,0 +1,5 @@
+pr: 111071
+summary: Use native scalar scorer for int8_flat index
+area: Vector Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/111091.yaml b/docs/changelog/111091.yaml
new file mode 100644
index 0000000000000..8444681a14a48
--- /dev/null
+++ b/docs/changelog/111091.yaml
@@ -0,0 +1,5 @@
+pr: 111091
+summary: "X-pack/plugin/otel: introduce x-pack-otel plugin"
+area: Data streams
+type: feature
+issues: []
diff --git a/docs/changelog/111105.yaml b/docs/changelog/111105.yaml
new file mode 100644
index 0000000000000..ed32bd1ef7fc3
--- /dev/null
+++ b/docs/changelog/111105.yaml
@@ -0,0 +1,5 @@
+pr: 111105
+summary: "ESQL: TOP aggregation IP support"
+area: ES|QL
+type: feature
+issues: []
diff --git a/docs/changelog/111118.yaml b/docs/changelog/111118.yaml
new file mode 100644
index 0000000000000..c9fe6cb443688
--- /dev/null
+++ b/docs/changelog/111118.yaml
@@ -0,0 +1,5 @@
+pr: 111118
+summary: "[ES|QL] Simplify patterns for subfields"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/111123.yaml b/docs/changelog/111123.yaml
new file mode 100644
index 0000000000000..605b8607f4082
--- /dev/null
+++ b/docs/changelog/111123.yaml
@@ -0,0 +1,5 @@
+pr: 111123
+summary: Add Lucene segment-level fields stats
+area: Mapping
+type: enhancement
+issues: []
diff --git a/docs/changelog/111154.yaml b/docs/changelog/111154.yaml
new file mode 100644
index 0000000000000..3297f5005a811
--- /dev/null
+++ b/docs/changelog/111154.yaml
@@ -0,0 +1,5 @@
+pr: 111154
+summary: EIS integration
+area: Inference
+type: feature
+issues: []
diff --git a/docs/changelog/111181.yaml b/docs/changelog/111181.yaml
new file mode 100644
index 0000000000000..7f9f5937b7652
--- /dev/null
+++ b/docs/changelog/111181.yaml
@@ -0,0 +1,5 @@
+pr: 111181
+summary: "[Inference API] Add Alibaba Cloud AI Search Model support to Inference API"
+area: Machine Learning
+type: enhancement
+issues: [ ]
diff --git a/docs/changelog/111193.yaml b/docs/changelog/111193.yaml
new file mode 100644
index 0000000000000..9e56facb60d3a
--- /dev/null
+++ b/docs/changelog/111193.yaml
@@ -0,0 +1,6 @@
+pr: 111193
+summary: Fix cases of collections with one point
+area: Geo
+type: bug
+issues:
+ - 110982
diff --git a/docs/changelog/111212.yaml b/docs/changelog/111212.yaml
new file mode 100644
index 0000000000000..67d1513b3ff6f
--- /dev/null
+++ b/docs/changelog/111212.yaml
@@ -0,0 +1,6 @@
+pr: 111212
+summary: Fix score count validation in reranker response
+area: Ranking
+type: bug
+issues:
+ - 111202
diff --git a/docs/changelog/111215.yaml b/docs/changelog/111215.yaml
new file mode 100644
index 0000000000000..dc044c2283fc4
--- /dev/null
+++ b/docs/changelog/111215.yaml
@@ -0,0 +1,6 @@
+pr: 111215
+summary: Make `SnapshotLifecycleStats` immutable so `SnapshotLifecycleMetadata.EMPTY`
+  isn't changed as side-effect
+area: ILM+SLM
+type: bug
+issues: []
diff --git a/docs/changelog/111225.yaml b/docs/changelog/111225.yaml
new file mode 100644
index 0000000000000..bcd344847cfd2
--- /dev/null
+++ b/docs/changelog/111225.yaml
@@ -0,0 +1,5 @@
+pr: 111225
+summary: Upgrade Azure SDK
+area: Snapshot/Restore
+type: upgrade
+issues: []
diff --git a/docs/changelog/111238.yaml b/docs/changelog/111238.yaml
new file mode 100644
index 0000000000000..b918b754ff595
--- /dev/null
+++ b/docs/changelog/111238.yaml
@@ -0,0 +1,6 @@
+pr: 111238
+summary: Fix validation of TEXT fields with case insensitive comparison
+area: EQL
+type: bug
+issues:
+ - 111235
diff --git a/docs/changelog/111245.yaml b/docs/changelog/111245.yaml
new file mode 100644
index 0000000000000..384373d52cb20
--- /dev/null
+++ b/docs/changelog/111245.yaml
@@ -0,0 +1,6 @@
+pr: 111245
+summary: Truncating watcher history if it is too large
+area: Watcher
+type: bug
+issues:
+ - 94745
diff --git a/docs/changelog/111274.yaml b/docs/changelog/111274.yaml
new file mode 100644
index 0000000000000..e26bcc03ce118
--- /dev/null
+++ b/docs/changelog/111274.yaml
@@ -0,0 +1,5 @@
+pr: 111274
+summary: Include account name in Azure settings exceptions
+area: Snapshot/Restore
+type: enhancement
+issues: []
diff --git a/docs/changelog/111284.yaml b/docs/changelog/111284.yaml
new file mode 100644
index 0000000000000..f87649a134af6
--- /dev/null
+++ b/docs/changelog/111284.yaml
@@ -0,0 +1,6 @@
+pr: 111284
+summary: Update `semantic_text` field to support indexing numeric and boolean data
+  types
+area: Mapping
+type: enhancement
+issues: []
diff --git a/docs/changelog/111285.yaml b/docs/changelog/111285.yaml
new file mode 100644
index 0000000000000..e4856482b4d6e
--- /dev/null
+++ b/docs/changelog/111285.yaml
@@ -0,0 +1,5 @@
+pr: 111285
+summary: "[Bugfix] Add `accessDeclaredMembers` permission to allow search application templates to parse floats"
+area: Relevance
+type: bug
+issues: []
diff --git a/docs/changelog/111311.yaml b/docs/changelog/111311.yaml
new file mode 100644
index 0000000000000..5786e11e885e2
--- /dev/null
+++ b/docs/changelog/111311.yaml
@@ -0,0 +1,6 @@
+pr: 111311
+summary: Adding support for data streams with a match-all template
+area: Data streams
+type: bug
+issues:
+ - 111204
diff --git a/docs/changelog/111315.yaml b/docs/changelog/111315.yaml
new file mode 100644
index 0000000000000..0e2e56898b51c
--- /dev/null
+++ b/docs/changelog/111315.yaml
@@ -0,0 +1,5 @@
+pr: 111315
+summary: Add link to flood-stage watermark exception message
+area: Allocation
+type: enhancement
+issues: []
diff --git a/docs/changelog/111316.yaml b/docs/changelog/111316.yaml
new file mode 100644
index 0000000000000..0d915cd1ec3ea
--- /dev/null
+++ b/docs/changelog/111316.yaml
@@ -0,0 +1,5 @@
+pr: 111316
+summary: "[Service Account] Add `AutoOps` account"
+area: Security
+type: enhancement
+issues: []
diff --git a/docs/changelog/111344.yaml b/docs/changelog/111344.yaml
new file mode 100644
index 0000000000000..3d5988054749d
--- /dev/null
+++ b/docs/changelog/111344.yaml
@@ -0,0 +1,5 @@
+pr: 111344
+summary: Add support for Azure Managed Identity
+area: Snapshot/Restore
+type: enhancement
+issues: []
diff --git a/docs/changelog/111367.yaml b/docs/changelog/111367.yaml
new file mode 100644
index 0000000000000..89e6c1d3b4da4
--- /dev/null
+++ b/docs/changelog/111367.yaml
@@ -0,0 +1,5 @@
+pr: 111367
+summary: "ESQL: Add Values aggregation tests, fix `ConstantBytesRefBlock` memory handling"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/111412.yaml b/docs/changelog/111412.yaml
new file mode 100644
index 0000000000000..297fa77cd2664
--- /dev/null
+++ b/docs/changelog/111412.yaml
@@ -0,0 +1,6 @@
+pr: 111412
+summary: Make enrich cache based on memory usage
+area: Ingest Node
+type: enhancement
+issues:
+ - 106081
diff --git a/docs/changelog/111420.yaml b/docs/changelog/111420.yaml
new file mode 100644
index 0000000000000..4e2640ac5762a
--- /dev/null
+++ b/docs/changelog/111420.yaml
@@ -0,0 +1,5 @@
+pr: 111420
+summary: "[Query rules] Add `exclude` query rule type"
+area: Relevance
+type: feature
+issues: []
diff --git a/docs/changelog/111437.yaml b/docs/changelog/111437.yaml
new file mode 100644
index 0000000000000..a50312ffdd1aa
--- /dev/null
+++ b/docs/changelog/111437.yaml
@@ -0,0 +1,5 @@
+pr: 111437
+summary: "[ES|QL] Create `Range` in `PushFiltersToSource` for qualified pushable filters on the same field"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/111445.yaml b/docs/changelog/111445.yaml
new file mode 100644
index 0000000000000..9ba8e4371bd0c
--- /dev/null
+++ b/docs/changelog/111445.yaml
@@ -0,0 +1,5 @@
+pr: 111445
+summary: Support booleans in routing path
+area: TSDB
+type: enhancement
+issues: []
diff --git a/docs/changelog/111457.yaml b/docs/changelog/111457.yaml
new file mode 100644
index 0000000000000..f4ad4ee53eb0a
--- /dev/null
+++ b/docs/changelog/111457.yaml
@@ -0,0 +1,6 @@
+pr: 111457
+summary: Add support for boolean dimensions
+area: TSDB
+type: enhancement
+issues:
+ - 111338
diff --git a/docs/changelog/111475.yaml b/docs/changelog/111475.yaml
new file mode 100644
index 0000000000000..264c975444868
--- /dev/null
+++ b/docs/changelog/111475.yaml
@@ -0,0 +1,6 @@
+pr: 111475
+summary: "ESQL: Fix for overzealous validation in case of invalid mapped fields"
+area: ES|QL
+type: bug
+issues:
+ - 111452
diff --git a/docs/changelog/111490.yaml b/docs/changelog/111490.yaml
new file mode 100644
index 0000000000000..b67c16189cc62
--- /dev/null
+++ b/docs/changelog/111490.yaml
@@ -0,0 +1,5 @@
+pr: 111490
+summary: Temporarily return both `modelId` and `inferenceId` for GET /_inference until we migrate clients to only `inferenceId`
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/111501.yaml b/docs/changelog/111501.yaml
new file mode 100644
index 0000000000000..a424142376e52
--- /dev/null
+++ b/docs/changelog/111501.yaml
@@ -0,0 +1,6 @@
+pr: 111501
+summary: "[ES|QL] Combine Disjunctive CIDRMatch"
+area: ES|QL
+type: enhancement
+issues:
+ - 105143
diff --git a/docs/changelog/111516.yaml b/docs/changelog/111516.yaml
new file mode 100644
index 0000000000000..96e8bd843f750
--- /dev/null
+++ b/docs/changelog/111516.yaml
@@ -0,0 +1,5 @@
+pr: 111516
+summary: Adding support for `allow_partial_search_results` in PIT
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/111519.yaml b/docs/changelog/111519.yaml
new file mode 100644
index 0000000000000..8cc62fb8ed903
--- /dev/null
+++ b/docs/changelog/111519.yaml
@@ -0,0 +1,5 @@
+pr: 111519
+summary: "ESQL: Don't mutate the `BoolQueryBuilder` in plan"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/111523.yaml b/docs/changelog/111523.yaml
new file mode 100644
index 0000000000000..202d16c5a426d
--- /dev/null
+++ b/docs/changelog/111523.yaml
@@ -0,0 +1,5 @@
+pr: 111523
+summary: Search coordinator uses `event.ingested` in cluster state to do rewrites
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/111535.yaml b/docs/changelog/111535.yaml
new file mode 100644
index 0000000000000..4beebbf28d4e1
--- /dev/null
+++ b/docs/changelog/111535.yaml
@@ -0,0 +1,5 @@
+pr: 111535
+summary: Fix remote cluster credential secure settings reload
+area: Authorization
+type: bug
+issues: []
diff --git a/docs/changelog/111544.yaml b/docs/changelog/111544.yaml
new file mode 100644
index 0000000000000..d4c46f485e664
--- /dev/null
+++ b/docs/changelog/111544.yaml
@@ -0,0 +1,5 @@
+pr: 111544
+summary: "ESQL: Strings support for MAX and MIN aggregations"
+area: ES|QL
+type: feature
+issues: []
diff --git a/docs/changelog/111552.yaml b/docs/changelog/111552.yaml
new file mode 100644
index 0000000000000..d9991788d4fa9
--- /dev/null
+++ b/docs/changelog/111552.yaml
@@ -0,0 +1,5 @@
+pr: 111552
+summary: Siem ea 9521 improve test
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/111576.yaml b/docs/changelog/111576.yaml
new file mode 100644
index 0000000000000..6d3c331f4bbd5
--- /dev/null
+++ b/docs/changelog/111576.yaml
@@ -0,0 +1,6 @@
+pr: 111576
+summary: Execute shard snapshot tasks in shard-id order
+area: Snapshot/Restore
+type: enhancement
+issues:
+ - 108739
diff --git a/docs/changelog/111600.yaml b/docs/changelog/111600.yaml
new file mode 100644
index 0000000000000..0c1e01e1c2e23
--- /dev/null
+++ b/docs/changelog/111600.yaml
@@ -0,0 +1,5 @@
+pr: 111600
+summary: Make ecs@mappings work with OTel attributes
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/111624.yaml b/docs/changelog/111624.yaml
new file mode 100644
index 0000000000000..7b04b244ef7a7
--- /dev/null
+++ b/docs/changelog/111624.yaml
@@ -0,0 +1,6 @@
+pr: 111624
+summary: Extend logging for dropped warning headers
+area: Infra/Core
+type: enhancement
+issues:
+ - 90527
diff --git a/docs/changelog/111644.yaml b/docs/changelog/111644.yaml
new file mode 100644
index 0000000000000..3705d697c95e3
--- /dev/null
+++ b/docs/changelog/111644.yaml
@@ -0,0 +1,6 @@
+pr: 111644
+summary: Force using the last centroid during merging
+area: Aggregations
+type: bug
+issues:
+ - 111065
diff --git a/docs/changelog/111655.yaml b/docs/changelog/111655.yaml
new file mode 100644
index 0000000000000..077714d15a712
--- /dev/null
+++ b/docs/changelog/111655.yaml
@@ -0,0 +1,5 @@
+pr: 111655
+summary: Migrate Inference to `ChunkedToXContent`
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/111673.yaml b/docs/changelog/111673.yaml
new file mode 100644
index 0000000000000..ebc211633fcab
--- /dev/null
+++ b/docs/changelog/111673.yaml
@@ -0,0 +1,5 @@
+pr: 111673
+summary: Properly handle filters on `TextSimilarityRank` retriever
+area: Ranking
+type: bug
+issues: []
diff --git a/docs/changelog/111683.yaml b/docs/changelog/111683.yaml
new file mode 100644
index 0000000000000..cbb2e5ad71ddc
--- /dev/null
+++ b/docs/changelog/111683.yaml
@@ -0,0 +1,6 @@
+pr: 111683
+summary: Only emit product origin in deprecation log if present
+area: Infra/Logging
+type: bug
+issues:
+ - 81757
diff --git a/docs/changelog/111689.yaml b/docs/changelog/111689.yaml
new file mode 100644
index 0000000000000..ccb3d4d4f87c5
--- /dev/null
+++ b/docs/changelog/111689.yaml
@@ -0,0 +1,6 @@
+pr: 111689
+summary: Add nanos support to `ZonedDateTime` serialization
+area: Infra/Core
+type: enhancement
+issues:
+ - 68292
diff --git a/docs/changelog/111690.yaml b/docs/changelog/111690.yaml
new file mode 100644
index 0000000000000..36e715744ad88
--- /dev/null
+++ b/docs/changelog/111690.yaml
@@ -0,0 +1,5 @@
+pr: 111690
+summary: "ESQL: Support INLINESTATS grouped on expressions"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/111729.yaml b/docs/changelog/111729.yaml
new file mode 100644
index 0000000000000..c75c14a997da9
--- /dev/null
+++ b/docs/changelog/111729.yaml
@@ -0,0 +1,6 @@
+pr: 111729
+summary: Speed up dense/sparse vector stats
+area: Vector Search
+type: bug
+issues:
+ - 111715
diff --git a/docs/changelog/111740.yaml b/docs/changelog/111740.yaml
new file mode 100644
index 0000000000000..48b7ee200e45e
--- /dev/null
+++ b/docs/changelog/111740.yaml
@@ -0,0 +1,6 @@
+pr: 111740
+summary: Fix Start Trial API output acknowledgement header for features
+area: License
+type: bug
+issues:
+ - 111739
diff --git a/docs/changelog/111749.yaml b/docs/changelog/111749.yaml
new file mode 100644
index 0000000000000..77e0c65005dd6
--- /dev/null
+++ b/docs/changelog/111749.yaml
@@ -0,0 +1,6 @@
+pr: 111749
+summary: "ESQL: Added `mv_percentile` function"
+area: ES|QL
+type: feature
+issues:
+ - 111591
diff --git a/docs/changelog/111756.yaml b/docs/changelog/111756.yaml
new file mode 100644
index 0000000000000..e58345dbe696a
--- /dev/null
+++ b/docs/changelog/111756.yaml
@@ -0,0 +1,6 @@
+pr: 111756
+summary: Fix `NullPointerException` when doing knn search on empty index without dims
+area: Vector Search
+type: bug
+issues:
+ - 111733
diff --git a/docs/changelog/111758.yaml b/docs/changelog/111758.yaml
new file mode 100644
index 0000000000000..c95cdf48bc8a7
--- /dev/null
+++ b/docs/changelog/111758.yaml
@@ -0,0 +1,6 @@
+pr: 111758
+summary: Revert "Avoid bucket copies in Aggs"
+area: Aggregations
+type: bug
+issues:
+ - 111679
diff --git a/docs/changelog/111779.yaml b/docs/changelog/111779.yaml
new file mode 100644
index 0000000000000..52c635490e1e4
--- /dev/null
+++ b/docs/changelog/111779.yaml
@@ -0,0 +1,7 @@
+pr: 111779
+summary: "ESQL: Fix serialization during `can_match`"
+area: ES|QL
+type: bug
+issues:
+ - 111701
+ - 111726
diff --git a/docs/changelog/111797.yaml b/docs/changelog/111797.yaml
new file mode 100644
index 0000000000000..00b793a19d9c3
--- /dev/null
+++ b/docs/changelog/111797.yaml
@@ -0,0 +1,6 @@
+pr: 111797
+summary: "ESQL: fix for missing indices error message"
+area: ES|QL
+type: bug
+issues:
+ - 111712
diff --git a/docs/changelog/111807.yaml b/docs/changelog/111807.yaml
new file mode 100644
index 0000000000000..97c5e58461c34
--- /dev/null
+++ b/docs/changelog/111807.yaml
@@ -0,0 +1,5 @@
+pr: 111807
+summary: Explain Function Score Query
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/111809.yaml b/docs/changelog/111809.yaml
new file mode 100644
index 0000000000000..5a2f220e3a697
--- /dev/null
+++ b/docs/changelog/111809.yaml
@@ -0,0 +1,5 @@
+pr: 111809
+summary: Add Field caps support for Semantic Text
+area: Mapping
+type: enhancement
+issues: []
diff --git a/docs/changelog/111818.yaml b/docs/changelog/111818.yaml
new file mode 100644
index 0000000000000..c3a632861aae6
--- /dev/null
+++ b/docs/changelog/111818.yaml
@@ -0,0 +1,5 @@
+pr: 111818
+summary: Add tier preference to security index settings allowlist
+area: Security
+type: enhancement
+issues: []
diff --git a/docs/changelog/111840.yaml b/docs/changelog/111840.yaml
new file mode 100644
index 0000000000000..c40a9e2aef621
--- /dev/null
+++ b/docs/changelog/111840.yaml
@@ -0,0 +1,5 @@
+pr: 111840
+summary: "ESQL: Add async ID and `is_running` headers to ESQL async query"
+area: ES|QL
+type: feature
+issues: []
diff --git a/docs/changelog/111843.yaml b/docs/changelog/111843.yaml
new file mode 100644
index 0000000000000..c8b20036520f3
--- /dev/null
+++ b/docs/changelog/111843.yaml
@@ -0,0 +1,5 @@
+pr: 111843
+summary: Add maximum nested depth check to WKT parser
+area: Geo
+type: bug
+issues: []
diff --git a/docs/changelog/111855.yaml b/docs/changelog/111855.yaml
new file mode 100644
index 0000000000000..3f15e9c20135a
--- /dev/null
+++ b/docs/changelog/111855.yaml
@@ -0,0 +1,5 @@
+pr: 111855
+summary: "ESQL: Profile more timing information"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/111863.yaml b/docs/changelog/111863.yaml
new file mode 100644
index 0000000000000..1724cd83f984b
--- /dev/null
+++ b/docs/changelog/111863.yaml
@@ -0,0 +1,6 @@
+pr: 111863
+summary: Fixing incorrect bulk request took time
+area: Ingest Node
+type: bug
+issues:
+ - 111854
diff --git a/docs/changelog/111866.yaml b/docs/changelog/111866.yaml
new file mode 100644
index 0000000000000..34bf56da4dc9e
--- /dev/null
+++ b/docs/changelog/111866.yaml
@@ -0,0 +1,6 @@
+pr: 111866
+summary: Fix windows memory locking
+area: Infra/Core
+type: bug
+issues:
+ - 111847
diff --git a/docs/changelog/111874.yaml b/docs/changelog/111874.yaml
new file mode 100644
index 0000000000000..26ec90aa6cd4c
--- /dev/null
+++ b/docs/changelog/111874.yaml
@@ -0,0 +1,8 @@
+pr: 111874
+summary: "ESQL: BUCKET: allow numerical spans as whole numbers"
+area: ES|QL
+type: enhancement
+issues:
+ - 104646
+ - 109340
+ - 105375
diff --git a/docs/changelog/111879.yaml b/docs/changelog/111879.yaml
new file mode 100644
index 0000000000000..b8c2111e1d286
--- /dev/null
+++ b/docs/changelog/111879.yaml
@@ -0,0 +1,6 @@
+pr: 111879
+summary: "ESQL: Have BUCKET generate friendlier intervals"
+area: ES|QL
+type: enhancement
+issues:
+ - 110916
diff --git a/docs/changelog/111915.yaml b/docs/changelog/111915.yaml
new file mode 100644
index 0000000000000..f64c45b82d10c
--- /dev/null
+++ b/docs/changelog/111915.yaml
@@ -0,0 +1,6 @@
+pr: 111915
+summary: Fix DLS & FLS sometimes being enforced when it is disabled
+area: Authorization
+type: bug
+issues:
+ - 94709
diff --git a/docs/changelog/111917.yaml b/docs/changelog/111917.yaml
new file mode 100644
index 0000000000000..0dc760d76a698
--- /dev/null
+++ b/docs/changelog/111917.yaml
@@ -0,0 +1,7 @@
+pr: 111917
+summary: "[ES|QL] Cast mixed numeric types to a common numeric type for Coalesce and\
+  \ In at Analyzer"
+area: ES|QL
+type: enhancement
+issues:
+ - 111486
diff --git a/docs/changelog/111932.yaml b/docs/changelog/111932.yaml
new file mode 100644
index 0000000000000..ce840ecebcff0
--- /dev/null
+++ b/docs/changelog/111932.yaml
@@ -0,0 +1,6 @@
+pr: 111932
+summary: Fix union-types where one index is missing the field
+area: ES|QL
+type: bug
+issues:
+ - 111912
diff --git a/docs/changelog/111937.yaml b/docs/changelog/111937.yaml
new file mode 100644
index 0000000000000..7d856e29d54c5
--- /dev/null
+++ b/docs/changelog/111937.yaml
@@ -0,0 +1,6 @@
+pr: 111937
+summary: Handle `BigInteger` in xcontent copy
+area: Infra/Core
+type: bug
+issues: 
+  - 111812
diff --git a/docs/changelog/111943.yaml b/docs/changelog/111943.yaml
new file mode 100644
index 0000000000000..6b9f03ccee31c
--- /dev/null
+++ b/docs/changelog/111943.yaml
@@ -0,0 +1,6 @@
+pr: 111943
+summary: Fix synthetic source for empty nested objects
+area: Mapping
+type: bug
+issues:
+ - 111811
diff --git a/docs/changelog/111947.yaml b/docs/changelog/111947.yaml
new file mode 100644
index 0000000000000..0aff0b9c7b8be
--- /dev/null
+++ b/docs/changelog/111947.yaml
@@ -0,0 +1,5 @@
+pr: 111947
+summary: Improve performance of grok pattern cycle detection
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/111948.yaml b/docs/changelog/111948.yaml
new file mode 100644
index 0000000000000..a3a592abaf1ca
--- /dev/null
+++ b/docs/changelog/111948.yaml
@@ -0,0 +1,5 @@
+pr: 111948
+summary: Upgrade xcontent to Jackson 2.17.0
+area: Infra/Core
+type: upgrade
+issues: []
diff --git a/docs/changelog/111950.yaml b/docs/changelog/111950.yaml
new file mode 100644
index 0000000000000..3f23c17d8e652
--- /dev/null
+++ b/docs/changelog/111950.yaml
@@ -0,0 +1,6 @@
+pr: 111950
+summary: "[ES|QL] Name parameter with leading underscore"
+area: ES|QL
+type: enhancement
+issues:
+ - 111821
diff --git a/docs/changelog/111955.yaml b/docs/changelog/111955.yaml
new file mode 100644
index 0000000000000..ebc518203b7cc
--- /dev/null
+++ b/docs/changelog/111955.yaml
@@ -0,0 +1,7 @@
+pr: 111955
+summary: Clean up dangling S3 multipart uploads
+area: Snapshot/Restore
+type: enhancement
+issues:
+ - 101169
+ - 44971
diff --git a/docs/changelog/111966.yaml b/docs/changelog/111966.yaml
new file mode 100644
index 0000000000000..facf0a61c4d8a
--- /dev/null
+++ b/docs/changelog/111966.yaml
@@ -0,0 +1,5 @@
+pr: 111966
+summary: No error when `store_array_source` is used without synthetic source
+area: Mapping
+type: bug
+issues: []
diff --git a/docs/changelog/111968.yaml b/docs/changelog/111968.yaml
new file mode 100644
index 0000000000000..9d758c76369e9
--- /dev/null
+++ b/docs/changelog/111968.yaml
@@ -0,0 +1,6 @@
+pr: 111968
+summary: "ESQL: don't lose the original casting error message"
+area: ES|QL
+type: bug
+issues:
+ - 111967
diff --git a/docs/changelog/111969.yaml b/docs/changelog/111969.yaml
new file mode 100644
index 0000000000000..2d276850c4988
--- /dev/null
+++ b/docs/changelog/111969.yaml
@@ -0,0 +1,5 @@
+pr: 111969
+summary: "[Profiling] add `container.id` field to event index template"
+area: Application
+type: enhancement
+issues: []
diff --git a/docs/changelog/111972.yaml b/docs/changelog/111972.yaml
new file mode 100644
index 0000000000000..58477c68f0e7c
--- /dev/null
+++ b/docs/changelog/111972.yaml
@@ -0,0 +1,15 @@
+pr: 111972
+summary: Introduce global retention in data stream lifecycle.
+area: Data streams
+type: feature
+issues: []
+highlight:
+  title: Add global retention in data stream lifecycle
+  body: "Data stream lifecycle now supports configuring retention on a cluster level,\
+    \ namely global retention. Global retention \nallows us to configure two different\
+    \ retentions:\n\n- `data_streams.lifecycle.retention.default` is applied to all\
+    \ data streams managed by the data stream lifecycle that do not have retention\n\
+    defined on the data stream level.\n- `data_streams.lifecycle.retention.max` is\
+    \ applied to all data streams managed by the data stream lifecycle and it allows\
+    \ any data stream \ndata to be deleted after the `max_retention` has passed."
+  notable: true
diff --git a/docs/changelog/111983.yaml b/docs/changelog/111983.yaml
new file mode 100644
index 0000000000000..d5043d0b44155
--- /dev/null
+++ b/docs/changelog/111983.yaml
@@ -0,0 +1,6 @@
+pr: 111983
+summary: Avoid losing error message in failure collector
+area: ES|QL
+type: bug
+issues:
+ - 111894
diff --git a/docs/changelog/111994.yaml b/docs/changelog/111994.yaml
new file mode 100644
index 0000000000000..ee62651c43987
--- /dev/null
+++ b/docs/changelog/111994.yaml
@@ -0,0 +1,6 @@
+pr: 111994
+summary: Merge multiple ignored source entires for the same field
+area: Logs
+type: bug
+issues:
+ - 111694
diff --git a/docs/changelog/112005.yaml b/docs/changelog/112005.yaml
new file mode 100644
index 0000000000000..2d84381e632b3
--- /dev/null
+++ b/docs/changelog/112005.yaml
@@ -0,0 +1,6 @@
+pr: 112005
+summary: Check for valid `parentDoc` before retrieving its previous
+area: Mapping
+type: bug
+issues:
+ - 111990
diff --git a/docs/changelog/112019.yaml b/docs/changelog/112019.yaml
new file mode 100644
index 0000000000000..7afb207864ed7
--- /dev/null
+++ b/docs/changelog/112019.yaml
@@ -0,0 +1,5 @@
+pr: 112019
+summary: Display effective retention in the relevant data stream APIs
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/112038.yaml b/docs/changelog/112038.yaml
new file mode 100644
index 0000000000000..6cbfb373b7420
--- /dev/null
+++ b/docs/changelog/112038.yaml
@@ -0,0 +1,6 @@
+pr: 112038
+summary: Semantic reranking should fail whenever inference ID does not exist
+area: Relevance
+type: bug
+issues:
+ - 111934
diff --git a/docs/changelog/112046.yaml b/docs/changelog/112046.yaml
new file mode 100644
index 0000000000000..f3cda1ed7a7d2
--- /dev/null
+++ b/docs/changelog/112046.yaml
@@ -0,0 +1,5 @@
+pr: 112046
+summary: Fix calculation of parent offset for ignored source in some cases
+area: Mapping
+type: bug
+issues: []
diff --git a/docs/changelog/112058.yaml b/docs/changelog/112058.yaml
new file mode 100644
index 0000000000000..e974b3413582e
--- /dev/null
+++ b/docs/changelog/112058.yaml
@@ -0,0 +1,5 @@
+pr: 112058
+summary: Fix RRF validation for `rank_constant` < 1
+area: Ranking
+type: bug
+issues: []
diff --git a/docs/changelog/112066.yaml b/docs/changelog/112066.yaml
new file mode 100644
index 0000000000000..5dd846766bc8e
--- /dev/null
+++ b/docs/changelog/112066.yaml
@@ -0,0 +1,6 @@
+pr: 112066
+summary: Do not treat replica as unassigned if primary recently created and unassigned
+  time is below a threshold
+area: Health
+type: enhancement
+issues: []
diff --git a/docs/changelog/112090.yaml b/docs/changelog/112090.yaml
new file mode 100644
index 0000000000000..6d6e4d0851523
--- /dev/null
+++ b/docs/changelog/112090.yaml
@@ -0,0 +1,6 @@
+pr: 112090
+summary: Always check `crsType` when folding spatial functions
+area: Geo
+type: bug
+issues:
+ - 112089
diff --git a/docs/changelog/112100.yaml b/docs/changelog/112100.yaml
new file mode 100644
index 0000000000000..9135edecb4d77
--- /dev/null
+++ b/docs/changelog/112100.yaml
@@ -0,0 +1,5 @@
+pr: 112100
+summary: Exclude internal data streams from global retention
+area: Data streams
+type: bug
+issues: []
diff --git a/docs/changelog/112123.yaml b/docs/changelog/112123.yaml
new file mode 100644
index 0000000000000..0c0d7ac44cd17
--- /dev/null
+++ b/docs/changelog/112123.yaml
@@ -0,0 +1,5 @@
+pr: 112123
+summary: SLM interval schedule followup - add back `getFieldName` style getters
+area: ILM+SLM
+type: enhancement
+issues: []
diff --git a/docs/changelog/112126.yaml b/docs/changelog/112126.yaml
new file mode 100644
index 0000000000000..f6a7aeb893a5e
--- /dev/null
+++ b/docs/changelog/112126.yaml
@@ -0,0 +1,5 @@
+pr: 112126
+summary: Add support for spatial relationships in point field mapper
+area: Geo
+type: enhancement
+issues: []
diff --git a/docs/changelog/112133.yaml b/docs/changelog/112133.yaml
new file mode 100644
index 0000000000000..11109402b7373
--- /dev/null
+++ b/docs/changelog/112133.yaml
@@ -0,0 +1,5 @@
+pr: 112133
+summary: Add telemetry for repository usage
+area: Snapshot/Restore
+type: enhancement
+issues: []
diff --git a/docs/changelog/112139.yaml b/docs/changelog/112139.yaml
new file mode 100644
index 0000000000000..d6d992ec1dcf2
--- /dev/null
+++ b/docs/changelog/112139.yaml
@@ -0,0 +1,6 @@
+pr: 112139
+summary: Fix NPE when executing doc value queries over shape geometries with empty
+  segments
+area: Geo
+type: bug
+issues: []
diff --git a/docs/changelog/112151.yaml b/docs/changelog/112151.yaml
new file mode 100644
index 0000000000000..f5cbfd8da07c2
--- /dev/null
+++ b/docs/changelog/112151.yaml
@@ -0,0 +1,5 @@
+pr: 112151
+summary: Store original source for keywords using a normalizer
+area: Logs
+type: enhancement
+issues: []
diff --git a/docs/changelog/112173.yaml b/docs/changelog/112173.yaml
new file mode 100644
index 0000000000000..9a43b0d1bf1fa
--- /dev/null
+++ b/docs/changelog/112173.yaml
@@ -0,0 +1,7 @@
+pr: 112173
+summary: Prevent synthetic field loaders accessing stored fields from using stale
+  data
+area: Mapping
+type: bug
+issues:
+ - 112156
diff --git a/docs/changelog/112178.yaml b/docs/changelog/112178.yaml
new file mode 100644
index 0000000000000..f1011291542b8
--- /dev/null
+++ b/docs/changelog/112178.yaml
@@ -0,0 +1,6 @@
+pr: 112178
+summary: Avoid wrapping rejection exception in exchange
+area: ES|QL
+type: bug
+issues:
+ - 112106
diff --git a/docs/changelog/112199.yaml b/docs/changelog/112199.yaml
new file mode 100644
index 0000000000000..eb22f215f9828
--- /dev/null
+++ b/docs/changelog/112199.yaml
@@ -0,0 +1,5 @@
+pr: 112199
+summary: Support docvalues only query in shape field
+area: Geo
+type: enhancement
+issues: []
diff --git a/docs/changelog/112200.yaml b/docs/changelog/112200.yaml
new file mode 100644
index 0000000000000..0c2c3d71e3ddf
--- /dev/null
+++ b/docs/changelog/112200.yaml
@@ -0,0 +1,6 @@
+pr: 112200
+summary: "ES|QL: better validation of GROK patterns"
+area: ES|QL
+type: bug
+issues:
+ - 112111
diff --git a/docs/changelog/112214.yaml b/docs/changelog/112214.yaml
new file mode 100644
index 0000000000000..430f95a72bb3f
--- /dev/null
+++ b/docs/changelog/112214.yaml
@@ -0,0 +1,5 @@
+pr: 112214
+summary: '`ByteArrayStreamInput:` Return -1 when there are no more bytes to read'
+area: Infra/Core
+type: bug
+issues: []
diff --git a/docs/changelog/112217.yaml b/docs/changelog/112217.yaml
new file mode 100644
index 0000000000000..bb367d6128001
--- /dev/null
+++ b/docs/changelog/112217.yaml
@@ -0,0 +1,5 @@
+pr: 112217
+summary: Fix template alias parsing livelock
+area: Indices APIs
+type: bug
+issues: []
diff --git a/docs/changelog/112218.yaml b/docs/changelog/112218.yaml
new file mode 100644
index 0000000000000..c426dd7ade4ed
--- /dev/null
+++ b/docs/changelog/112218.yaml
@@ -0,0 +1,9 @@
+pr: 112218
+summary: "ESQL: Fix a bug in `MV_PERCENTILE`"
+area: ES|QL
+type: bug
+issues:
+ - 112193
+ - 112180
+ - 112187
+ - 112188
diff --git a/docs/changelog/112226.yaml b/docs/changelog/112226.yaml
new file mode 100644
index 0000000000000..ac36c0c0fe4e2
--- /dev/null
+++ b/docs/changelog/112226.yaml
@@ -0,0 +1,6 @@
+pr: 112226
+summary: "Fix \"unexpected field [remote_cluster]\" for CCS (RCS 1.0) when using API\
+  \ key that references `remote_cluster`"
+area: Security
+type: bug
+issues: []
diff --git a/docs/changelog/112230.yaml b/docs/changelog/112230.yaml
new file mode 100644
index 0000000000000..ef12dc3f78267
--- /dev/null
+++ b/docs/changelog/112230.yaml
@@ -0,0 +1,5 @@
+pr: 112230
+summary: Fix connection timeout for `OpenIdConnectAuthenticator` get Userinfo
+area: Security
+type: bug
+issues: []
diff --git a/docs/changelog/112242.yaml b/docs/changelog/112242.yaml
new file mode 100644
index 0000000000000..7292a00166de2
--- /dev/null
+++ b/docs/changelog/112242.yaml
@@ -0,0 +1,5 @@
+pr: 112242
+summary: Fix toReleaseVersion() when called on the current version id
+area: Infra/Core
+type: bug
+issues: [111900]
diff --git a/docs/changelog/112260.yaml b/docs/changelog/112260.yaml
new file mode 100644
index 0000000000000..3f5642188a367
--- /dev/null
+++ b/docs/changelog/112260.yaml
@@ -0,0 +1,6 @@
+pr: 112260
+summary: Fix DLS over Runtime Fields
+area: "Authorization"
+type: bug
+issues:
+ - 111637
diff --git a/docs/changelog/112270.yaml b/docs/changelog/112270.yaml
new file mode 100644
index 0000000000000..1e6b9c7fc9290
--- /dev/null
+++ b/docs/changelog/112270.yaml
@@ -0,0 +1,5 @@
+pr: 112270
+summary: Support sparse embedding models in the elasticsearch inference service
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/112273.yaml b/docs/changelog/112273.yaml
new file mode 100644
index 0000000000000..3182a1884a145
--- /dev/null
+++ b/docs/changelog/112273.yaml
@@ -0,0 +1,5 @@
+pr: 111181
+summary: "[Inference API] Add Docs for AlibabaCloud AI Search Support for the Inference API"
+area: Machine Learning
+type: enhancement
+issues: [ ]
diff --git a/docs/changelog/112277.yaml b/docs/changelog/112277.yaml
new file mode 100644
index 0000000000000..eac474555999a
--- /dev/null
+++ b/docs/changelog/112277.yaml
@@ -0,0 +1,5 @@
+pr: 112277
+summary: Upgrade `repository-azure` dependencies
+area: Snapshot/Restore
+type: upgrade
+issues: []
diff --git a/docs/changelog/112320.yaml b/docs/changelog/112320.yaml
new file mode 100644
index 0000000000000..d35a08dfa4e91
--- /dev/null
+++ b/docs/changelog/112320.yaml
@@ -0,0 +1,5 @@
+pr: 112320
+summary: Upgrade xcontent to Jackson 2.17.2
+area: Infra/Core
+type: upgrade
+issues: []
diff --git a/docs/changelog/112341.yaml b/docs/changelog/112341.yaml
new file mode 100644
index 0000000000000..8f44b53ad9998
--- /dev/null
+++ b/docs/changelog/112341.yaml
@@ -0,0 +1,5 @@
+pr: 112341
+summary: Fix DLS using runtime fields and synthetic source
+area: Authorization
+type: bug
+issues: []
diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
index d7d837b2f8364..16879450c65d8 100644
--- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
@@ -6,6 +6,8 @@
 
 A metric aggregation that executes using scripts to provide a metric output.
 
+WARNING: `scripted_metric` is not available in {serverless-full}.
+
 WARNING: Using scripts can result in slower search speeds. See
 <<scripts-and-search-speed>>.
 
@@ -127,7 +129,7 @@ init_script::       Executed prior to any collection of documents. Allows the ag
 +
 In the above example, the `init_script` creates an array `transactions` in the `state` object.
 
-map_script::        Executed once per document collected. This is a required script. 
+map_script::        Executed once per document collected. This is a required script.
 +
 In the above example, the `map_script` checks the value of the type field. If the value is 'sale' the value of the amount field
 is added to the transactions array. If the value of the type field is not 'sale' the negated value of the amount field is added
@@ -282,4 +284,4 @@ params::           Optional. An object whose contents will be passed as variable
 
 If a parent bucket of the scripted metric aggregation does not collect any documents an empty aggregation response will be returned from the
 shard with a `null` value. In this case the `reduce_script`'s `states` variable will contain `null` as a response from that shard.
-`reduce_script`'s should therefore expect and deal with `null` responses from shards.  
+`reduce_script`'s should therefore expect and deal with `null` responses from shards.
diff --git a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc
index e37118019a55c..f0fa4f30fd83f 100644
--- a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc
@@ -87,8 +87,8 @@ changes to synonym files. Only to be used for search analyzers.
 * `expand` (defaults to `true`).
 Expands definitions for equivalent synonym rules.
 See <<synonym-graph-tokenizer-expand-equivalent-synonyms,expand equivalent synonyms>>.
-* `lenient` (defaults to `false`).
-If `true` ignores errors while parsing the synonym configuration.
+* `lenient` (defaults to the value of the `updateable` setting).
+If `true` ignores errors while parsing the synonym rules.
 It is important to note that only those synonym rules which cannot get parsed are ignored.
 See <<synonym-graph-tokenizer-stop-token-filter,synonyms and stop token filters>> for an example of `lenient` behaviour for invalid synonym rules.
 
@@ -181,11 +181,11 @@ This can can cause errors on the synonym rule.
 
 [WARNING]
 ====
-Invalid synonym rules can cause errors when applying analyzer changes.
+If `lenient` is set to `false`, invalid synonym rules can cause errors when applying analyzer changes.
 For reloadable analyzers, this prevents reloading and applying changes.
 You must correct errors in the synonym rules and reload the analyzer.
 
-An index with invalid synonym rules cannot be reopened, making it inoperable when:
+When `lenient` is set to `false`, an index with invalid synonym rules cannot be reopened, making it inoperable when:
 
 * A node containing the index starts
 * The index is opened from a closed state
diff --git a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc
index 1658f016db60b..b0020a1120fc0 100644
--- a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc
@@ -75,8 +75,8 @@ changes to synonym files. Only to be used for search analyzers.
 * `expand` (defaults to `true`).
 Expands definitions for equivalent synonym rules.
 See <<synonym-tokenizer-expand-equivalent-synonyms,expand equivalent synonyms>>.
-* `lenient` (defaults to `false`).
-If `true` ignores errors while parsing the synonym configuration.
+* `lenient` (defaults to the value of the `updateable` setting).
+If `true` ignores errors while parsing the synonym rules.
 It is important to note that only those synonym rules which cannot get parsed are ignored.
 See <<synonym-tokenizer-stop-token-filter,synonyms and stop token filters>> for an example of `lenient` behaviour for invalid synonym rules.
 
@@ -169,11 +169,11 @@ This can can cause errors on the synonym rule.
 
 [WARNING]
 ====
-Invalid synonym rules can cause errors when applying analyzer changes.
+If `lenient` is set to `false`, invalid synonym rules can cause errors when applying analyzer changes.
 For reloadable analyzers, this prevents reloading and applying changes.
 You must correct errors in the synonym rules and reload the analyzer.
 
-An index with invalid synonym rules cannot be reopened, making it inoperable when:
+When `lenient` is set to `false`, an index with invalid synonym rules cannot be reopened, making it inoperable when:
 
 * A node containing the index starts
 * The index is opened from a closed state
diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc
index 25881b707d724..f8d925945401e 100644
--- a/docs/reference/api-conventions.asciidoc
+++ b/docs/reference/api-conventions.asciidoc
@@ -334,6 +334,7 @@ All REST API parameters (both request parameters and JSON body) support
 providing boolean "false" as the value `false` and boolean "true" as the
 value `true`. All other values will raise an error.
 
+[[api-conventions-number-values]]
 [discrete]
 === Number Values
 
diff --git a/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc b/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc
index 090eda5ef5436..e4da2c45ee978 100644
--- a/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc
+++ b/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc
@@ -4,7 +4,7 @@
 
 NOTE: {cloud-only}
 
-You can use the following APIs to perform autoscaling operations.
+You can use the following APIs to perform {cloud}/ec-autoscaling.html[autoscaling operations].
 
 [discrete]
 [[autoscaling-api-top-level]]
diff --git a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc
index 608b7bd7cb903..190428485a003 100644
--- a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc
+++ b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc
@@ -7,7 +7,7 @@
 
 NOTE: {cloud-only}
 
-Delete autoscaling policy.
+Delete {cloud}/ec-autoscaling.html[autoscaling] policy.
 
 [[autoscaling-delete-autoscaling-policy-request]]
 ==== {api-request-title}
diff --git a/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc
index 05724b9c48b6e..d635d8c8f7bd0 100644
--- a/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc
+++ b/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc
@@ -7,7 +7,7 @@
 
 NOTE: {cloud-only}
 
-Get autoscaling capacity.
+Get {cloud}/ec-autoscaling.html[autoscaling] capacity.
 
 [[autoscaling-get-autoscaling-capacity-request]]
 ==== {api-request-title}
diff --git a/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc
index ad00d69d1aeb2..973eedcb361c9 100644
--- a/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc
+++ b/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc
@@ -7,7 +7,7 @@
 
 NOTE: {cloud-only}
 
-Get autoscaling policy.
+Get {cloud}/ec-autoscaling.html[autoscaling] policy.
 
 [[autoscaling-get-autoscaling-policy-request]]
 ==== {api-request-title}
diff --git a/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc
index ff79def51ebb9..e564f83411eb4 100644
--- a/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc
+++ b/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc
@@ -7,7 +7,7 @@
 
 NOTE: {cloud-only}
 
-Creates or updates an autoscaling policy.
+Creates or updates an {cloud}/ec-autoscaling.html[autoscaling] policy.
 
 [[autoscaling-put-autoscaling-policy-request]]
 ==== {api-request-title}
diff --git a/docs/reference/autoscaling/deciders/fixed-decider.asciidoc b/docs/reference/autoscaling/deciders/fixed-decider.asciidoc
index c46d1dffe2cc8..5a8b009d9f063 100644
--- a/docs/reference/autoscaling/deciders/fixed-decider.asciidoc
+++ b/docs/reference/autoscaling/deciders/fixed-decider.asciidoc
@@ -6,7 +6,7 @@ experimental[]
 [WARNING]
 The fixed decider is intended for testing only. Do not use this decider in production.
 
-The `fixed` decider responds with a fixed required capacity. It is not enabled
+The {cloud}/ec-autoscaling.html[autoscaling] `fixed` decider responds with a fixed required capacity. It is not enabled
 by default but can be enabled for any policy by explicitly configuring it.
 
 ==== Configuration settings
diff --git a/docs/reference/autoscaling/deciders/frozen-existence-decider.asciidoc b/docs/reference/autoscaling/deciders/frozen-existence-decider.asciidoc
index 832cf330053aa..0fc9ad444a213 100644
--- a/docs/reference/autoscaling/deciders/frozen-existence-decider.asciidoc
+++ b/docs/reference/autoscaling/deciders/frozen-existence-decider.asciidoc
@@ -2,7 +2,7 @@
 [[autoscaling-frozen-existence-decider]]
 === Frozen existence decider
 
-The frozen existence decider (`frozen_existence`) ensures that once the first
+The {cloud}/ec-autoscaling.html[autoscaling] frozen existence decider (`frozen_existence`) ensures that once the first
 index enters the frozen ILM phase, the frozen tier is scaled into existence.
 
 The frozen existence decider is enabled for all policies governing frozen data
diff --git a/docs/reference/autoscaling/deciders/frozen-shards-decider.asciidoc b/docs/reference/autoscaling/deciders/frozen-shards-decider.asciidoc
index ab11da04c8642..1977f95797ef0 100644
--- a/docs/reference/autoscaling/deciders/frozen-shards-decider.asciidoc
+++ b/docs/reference/autoscaling/deciders/frozen-shards-decider.asciidoc
@@ -2,7 +2,7 @@
 [[autoscaling-frozen-shards-decider]]
 === Frozen shards decider
 
-The frozen shards decider (`frozen_shards`) calculates the memory required to search
+The {cloud}/ec-autoscaling.html[autoscaling] frozen shards decider (`frozen_shards`) calculates the memory required to search
 the current set of partially mounted indices in the frozen tier. Based on a
 required memory amount per shard, it calculates the necessary memory in the frozen tier.
 
diff --git a/docs/reference/autoscaling/deciders/frozen-storage-decider.asciidoc b/docs/reference/autoscaling/deciders/frozen-storage-decider.asciidoc
index 5a10f31f1365b..3a8e7cdb518b3 100644
--- a/docs/reference/autoscaling/deciders/frozen-storage-decider.asciidoc
+++ b/docs/reference/autoscaling/deciders/frozen-storage-decider.asciidoc
@@ -2,7 +2,7 @@
 [[autoscaling-frozen-storage-decider]]
 === Frozen storage decider
 
-The frozen storage decider (`frozen_storage`) calculates the local storage
+The {cloud}/ec-autoscaling.html[autoscaling] frozen storage decider (`frozen_storage`) calculates the local storage
 required to search the current set of partially mounted indices based on a
 percentage of the total data set size of such indices. It signals that
 additional storage capacity is necessary when existing capacity is less than the
diff --git a/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc b/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc
index 26ced6ad7bb26..5432d96a47edb 100644
--- a/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc
+++ b/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc
@@ -2,7 +2,7 @@
 [[autoscaling-machine-learning-decider]]
 === Machine learning decider
 
-The {ml} decider (`ml`) calculates the memory and CPU requirements to run {ml} 
+The {cloud}/ec-autoscaling.html[autoscaling] {ml} decider (`ml`) calculates the memory and CPU requirements to run {ml} 
 jobs and trained models.
 
 The {ml} decider is enabled for policies governing `ml` nodes.
diff --git a/docs/reference/autoscaling/deciders/proactive-storage-decider.asciidoc b/docs/reference/autoscaling/deciders/proactive-storage-decider.asciidoc
index 763f1de96f6b9..33c989f3b12eb 100644
--- a/docs/reference/autoscaling/deciders/proactive-storage-decider.asciidoc
+++ b/docs/reference/autoscaling/deciders/proactive-storage-decider.asciidoc
@@ -2,7 +2,7 @@
 [[autoscaling-proactive-storage-decider]]
 === Proactive storage decider
 
-The proactive storage decider (`proactive_storage`) calculates the storage required to contain
+The {cloud}/ec-autoscaling.html[autoscaling] proactive storage decider (`proactive_storage`) calculates the storage required to contain
 the current data set plus an estimated amount of expected additional data.
 
 The proactive storage decider is enabled for all policies governing nodes with the `data_hot` role.
diff --git a/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc b/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc
index 50897178a88de..7c38df75169fd 100644
--- a/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc
+++ b/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc
@@ -2,7 +2,7 @@
 [[autoscaling-reactive-storage-decider]]
 === Reactive storage decider
 
-The reactive storage decider (`reactive_storage`) calculates the storage required to contain
+The {cloud}/ec-autoscaling.html[autoscaling] reactive storage decider (`reactive_storage`) calculates the storage required to contain
 the current data set. It signals that additional storage capacity is necessary
 when existing capacity has been exceeded (reactively).
 
diff --git a/docs/reference/autoscaling/index.asciidoc b/docs/reference/autoscaling/index.asciidoc
index fbf1a9536973e..e70c464889419 100644
--- a/docs/reference/autoscaling/index.asciidoc
+++ b/docs/reference/autoscaling/index.asciidoc
@@ -4,7 +4,7 @@
 
 NOTE: {cloud-only}
 
-The autoscaling feature enables an operator to configure tiers of nodes that
+The {cloud}/ec-autoscaling.html[autoscaling] feature enables an operator to configure tiers of nodes that
 self-monitor whether or not they need to scale based on an operator-defined
 policy. Then, via the autoscaling API, an Elasticsearch cluster can report
 whether or not it needs additional resources to meet the policy. For example, an
diff --git a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc
index 9b15bcca3fc85..a6894a933b460 100644
--- a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc
+++ b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc
@@ -17,7 +17,7 @@ PUT _application/analytics/my_analytics_collection
 
 ////
 
-Removes an Analytics Collection and its associated data stream.
+Removes a <<behavioral-analytics-overview,Behavioral Analytics>> Collection and its associated data stream.
 
 [[delete-analytics-collection-request]]
 ==== {api-request-title}
diff --git a/docs/reference/behavioral-analytics/apis/index.asciidoc b/docs/reference/behavioral-analytics/apis/index.asciidoc
index 042b50259b1bb..692d3374f89f5 100644
--- a/docs/reference/behavioral-analytics/apis/index.asciidoc
+++ b/docs/reference/behavioral-analytics/apis/index.asciidoc
@@ -9,7 +9,7 @@ beta::[]
 
 ---
 
-Use the following APIs to manage tasks and resources related to Behavioral Analytics:
+Use the following APIs to manage tasks and resources related to <<behavioral-analytics-overview,Behavioral Analytics>>:
 
 * <<put-analytics-collection>>
 * <<delete-analytics-collection>>
diff --git a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc
index 8d2491ff8a6ee..14511a1258278 100644
--- a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc
+++ b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc
@@ -24,7 +24,7 @@ DELETE _application/analytics/my_analytics_collection2
 // TEARDOWN
 ////
 
-Returns information about Analytics Collections.
+Returns information about <<behavioral-analytics-overview,Behavioral Analytics>> Collections.
 
 [[list-analytics-collection-request]]
 ==== {api-request-title}
diff --git a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc
index 84d9cb5351799..f82717e22ed34 100644
--- a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc
+++ b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc
@@ -22,7 +22,7 @@ DELETE _application/analytics/my_analytics_collection
 // TEARDOWN
 ////
 
-Post an event to an Analytics Collection.
+Post an event to a <<behavioral-analytics-overview,Behavioral Analytics>> Collection.
 
 [[post-analytics-collection-event-request]]
 ==== {api-request-title}
diff --git a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc
index 48273fb3906c4..cbbab2ae3e26c 100644
--- a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc
+++ b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc
@@ -16,7 +16,7 @@ DELETE _application/analytics/my_analytics_collection
 // TEARDOWN
 ////
 
-Creates an Analytics Collection.
+Creates a <<behavioral-analytics-overview,Behavioral Analytics>> Collection.
 
 [[put-analytics-collection-request]]
 ==== {api-request-title}
diff --git a/docs/reference/cat/recovery.asciidoc b/docs/reference/cat/recovery.asciidoc
index 058f4e69ae8e3..c3292fc9971ee 100644
--- a/docs/reference/cat/recovery.asciidoc
+++ b/docs/reference/cat/recovery.asciidoc
@@ -39,7 +39,7 @@ The cat recovery API returns information about shard recoveries, both
 ongoing and completed. It is a more compact view of the JSON
 <<indices-recovery,index recovery>> API.
 
-include::{es-ref-dir}/indices/recovery.asciidoc[tag=shard-recovery-desc]
+include::{es-ref-dir}/modules/shard-recovery-desc.asciidoc[]
 
 
 [[cat-recovery-path-params]]
diff --git a/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc
index 1c72fb8742b93..b510163bab50b 100644
--- a/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc
+++ b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Delete auto-follow pattern</titleabbrev>
 ++++
 
-Delete auto-follow patterns.
+Delete {ccr} <<ccr-auto-follow,auto-follow patterns>>.
 
 [[ccr-delete-auto-follow-pattern-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc
index 46ef288b05088..a2969e993ddfb 100644
--- a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc
+++ b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Get auto-follow pattern</titleabbrev>
 ++++
 
-Get auto-follow patterns.
+Get {ccr} <<ccr-auto-follow,auto-follow patterns>>.
 
 [[ccr-get-auto-follow-pattern-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc
index 1e64ab813e2ad..c5ae5a7b4af9d 100644
--- a/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc
+++ b/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Pause auto-follow pattern</titleabbrev>
 ++++
 
-Pauses an auto-follow pattern.
+Pauses a {ccr} <<ccr-auto-follow,auto-follow pattern>>.
 
 [[ccr-pause-auto-follow-pattern-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc
index d08997068f705..6769f21ca5cef 100644
--- a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc
+++ b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Create auto-follow pattern</titleabbrev>
 ++++
 
-Creates an auto-follow pattern.
+Creates a {ccr} <<ccr-auto-follow,auto-follow pattern>>.
 
 [[ccr-put-auto-follow-pattern-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc
index 04da9b4a35ba0..a580bb3838f9b 100644
--- a/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc
+++ b/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Resume auto-follow pattern</titleabbrev>
 ++++
 
-Resumes an auto-follow pattern.
+Resumes a {ccr} <<ccr-auto-follow,auto-follow pattern>>.
 
 [[ccr-resume-auto-follow-pattern-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/ccr-apis.asciidoc b/docs/reference/ccr/apis/ccr-apis.asciidoc
index 0c9f033639eda..ae94e1931af85 100644
--- a/docs/reference/ccr/apis/ccr-apis.asciidoc
+++ b/docs/reference/ccr/apis/ccr-apis.asciidoc
@@ -2,7 +2,7 @@
 [[ccr-apis]]
 == {ccr-cap} APIs
 
-You can use the following APIs to perform {ccr} operations.
+You can use the following APIs to perform <<xpack-ccr,{ccr}>> operations.
 
 [discrete]
 [[ccr-api-top-level]]
diff --git a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc
index 68fd6e210f884..6c049d9c92b59 100644
--- a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc
+++ b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Get follower info</titleabbrev>
 ++++
 
-Retrieves information about all follower indices.
+Retrieves information about all <<xpack-ccr,{ccr}>> follower indices.
 
 [[ccr-get-follow-info-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc
index 72224cc7f51f4..4892f86b3523d 100644
--- a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc
+++ b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Get follower stats</titleabbrev>
 ++++
 
-Get follower stats.
+Get <<xpack-ccr,{ccr}>> follower stats.
 
 [[ccr-get-follow-stats-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc b/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc
index ea7e8640056bf..1917c08d6640d 100644
--- a/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc
+++ b/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Forget follower</titleabbrev>
 ++++
 
-Removes the follower retention leases from the leader.
+Removes the <<xpack-ccr,{ccr}>> follower retention leases from the leader.
 
 [[ccr-post-forget-follower-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc
index a4ab69aba8d84..6d4730d10efe6 100644
--- a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc
+++ b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Pause follower</titleabbrev>
 ++++
 
-Pauses a follower index.
+Pauses a <<xpack-ccr,{ccr}>> follower index.
 
 [[ccr-post-pause-follow-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc
index 47ba51a3fb8a0..b023a8cb5cb70 100644
--- a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc
+++ b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Resume follower</titleabbrev>
 ++++
 
-Resumes a follower index.
+Resumes a <<xpack-ccr,{ccr}>> follower index.
 
 [[ccr-post-resume-follow-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc
index b96777b455d3b..dab11ef9e7a54 100644
--- a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc
+++ b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Unfollow</titleabbrev>
 ++++
 
-Converts a follower index to a regular index.
+Converts a <<xpack-ccr,{ccr}>> follower index to a regular index.
 
 [[ccr-post-unfollow-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/follow/put-follow.asciidoc b/docs/reference/ccr/apis/follow/put-follow.asciidoc
index eb83e2a13dcf1..b7ae9ac987474 100644
--- a/docs/reference/ccr/apis/follow/put-follow.asciidoc
+++ b/docs/reference/ccr/apis/follow/put-follow.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Create follower</titleabbrev>
 ++++
 
-Creates a follower index.
+Creates a <<xpack-ccr,{ccr}>> follower index.
 
 [[ccr-put-follow-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ccr/apis/get-ccr-stats.asciidoc b/docs/reference/ccr/apis/get-ccr-stats.asciidoc
index 128df5e47c777..92e6bae0bdce8 100644
--- a/docs/reference/ccr/apis/get-ccr-stats.asciidoc
+++ b/docs/reference/ccr/apis/get-ccr-stats.asciidoc
@@ -6,7 +6,7 @@
 <titleabbrev>Get {ccr-init} stats</titleabbrev>
 ++++
 
-Get {ccr} stats.
+Get <<xpack-ccr,{ccr}>> stats.
 
 [[ccr-get-stats-request]]
 ==== {api-request-title}
diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc
index 0b0fde6546c29..7547dd74c5ecd 100644
--- a/docs/reference/cluster/allocation-explain.asciidoc
+++ b/docs/reference/cluster/allocation-explain.asciidoc
@@ -4,7 +4,7 @@
 <titleabbrev>Cluster allocation explain</titleabbrev>
 ++++
 
-Provides an explanation for a shard's current allocation.
+Provides an explanation for a shard's current <<index-modules-allocation,allocation>>.
 
 [source,console]
 ----
@@ -81,6 +81,7 @@ you might expect otherwise.
 
 ===== Unassigned primary shard
 
+====== Conflicting settings
 The following request gets an allocation explanation for an unassigned primary
 shard.
 
@@ -158,6 +159,56 @@ node.
 <5> The decider which led to the `no` decision for the node.
 <6> An explanation as to why the decider returned a `no` decision, with a helpful hint pointing to the setting that led to the decision. In this example, a newly created index has <<indices-get-settings,an index setting>> that requires that it only be allocated to a node named `nonexistent_node`, which does not exist, so the index is unable to allocate.
 
+====== Maximum number of retries exceeded
+
+The following response contains an allocation explanation for an unassigned
+primary shard that has reached the maximum number of allocation retry attempts. 
+
+[source,js]
+----
+{
+  "index" : "my-index-000001",
+  "shard" : 0,
+  "primary" : true,
+  "current_state" : "unassigned",
+  "unassigned_info" : {
+    "at" : "2017-01-04T18:03:28.464Z",
+    "failed shard on node [mEKjwwzLT1yJVb8UxT6anw]: failed recovery, failure RecoveryFailedException",
+    "reason": "ALLOCATION_FAILED",
+    "failed_allocation_attempts": 5,
+    "last_allocation_status": "no",
+  },
+  "can_allocate": "no",
+  "allocate_explanation": "cannot allocate because allocation is not permitted to any of the nodes",
+  "node_allocation_decisions" : [
+    {
+      "node_id" : "3sULLVJrRneSg0EfBB-2Ew",
+      "node_name" : "node_t0",
+      "transport_address" : "127.0.0.1:9400",
+      "roles" : ["data_content", "data_hot"],
+      "node_decision" : "no",
+      "store" : {
+        "matching_size" : "4.2kb",
+        "matching_size_in_bytes" : 4325
+      },
+      "deciders" : [
+        {
+          "decider": "max_retry",
+          "decision" : "NO",
+          "explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2024-07-30T21:04:12.166Z], failed_attempts[5], failed_nodes[[mEKjwwzLT1yJVb8UxT6anw]], delayed=false, details[failed shard on node [mEKjwwzLT1yJVb8UxT6anw]: failed recovery, failure RecoveryFailedException], allocation_status[deciders_no]]]"
+        }
+      ]
+    }
+  ]
+}
+----
+// NOTCONSOLE
+
+If decider message indicates a transient allocation issue, use 
+<<cluster-reroute,the cluster reroute API>> to retry allocation. 
+
+====== No valid shard copy
+
 The following response contains an allocation explanation for an unassigned
 primary shard that was previously allocated.
 
@@ -184,6 +235,8 @@ TIP: If a shard is unassigned with an allocation status of `no_valid_shard_copy`
 
 ===== Unassigned replica shard
 
+====== Allocation delayed
+
 The following response contains an allocation explanation for a replica that's
 unassigned due to <<delayed-allocation,delayed allocation>>.
 
@@ -241,8 +294,52 @@ unassigned due to <<delayed-allocation,delayed allocation>>.
 <2> The remaining delay before allocating the replica shard.
 <3> Information about the shard data found on a node.
 
+====== Allocation throttled
+
+The following response contains an allocation explanation for a replica that's
+queued to allocate but currently waiting on other queued shards.
+
+[source,js]
+----
+{
+  "index" : "my-index-000001",
+  "shard" : 0,
+  "primary" : false,
+  "current_state" : "unassigned",
+  "unassigned_info" : {
+    "reason" : "NODE_LEFT",
+    "at" : "2017-01-04T18:53:59.498Z",
+    "details" : "node_left[G92ZwuuaRY-9n8_tc-IzEg]",
+    "last_allocation_status" : "no_attempt"
+  },
+  "can_allocate": "throttled",
+  "allocate_explanation": "Elasticsearch is currently busy with other activities. It expects to be able to allocate this shard when those activities finish. Please wait.",
+  "node_allocation_decisions" : [
+    {
+      "node_id" : "3sULLVJrRneSg0EfBB-2Ew",
+      "node_name" : "node_t0",
+      "transport_address" : "127.0.0.1:9400",
+      "roles" : ["data_content", "data_hot"],
+      "node_decision" : "no",
+      "deciders" : [
+        {
+          "decider": "throttling",
+          "decision": "THROTTLE",
+          "explanation": "reached the limit of incoming shard recoveries [2], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=2] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
+        }
+      ]
+    }
+  ]
+}
+----
+// NOTCONSOLE
+
+This is a transient message that might appear when a large amount of shards are allocating. 
+
 ===== Assigned shard
 
+====== Cannot remain on current node
+
 The following response contains an allocation explanation for an assigned shard.
 The response indicates the shard is not allowed to remain on its current node
 and must be reallocated.
@@ -295,6 +392,8 @@ and must be reallocated.
 <2> The deciders that factored into the decision of why the shard is not allowed to remain on its current node.
 <3> Whether the shard is allowed to be allocated to another node.
 
+====== Must remain on current node
+
 The following response contains an allocation explanation for a shard that must
 remain on its current node. Moving the shard to another node would not improve
 cluster balance.
@@ -338,7 +437,7 @@ cluster balance.
 ===== No arguments
 
 If you call the API with no arguments, {es} retrieves an allocation explanation
-for an arbitrary unassigned primary or replica shard.
+for an arbitrary unassigned primary or replica shard, returning any unassigned primary shards first. 
 
 [source,console]
 ----
diff --git a/docs/reference/cluster/delete-desired-balance.asciidoc b/docs/reference/cluster/delete-desired-balance.asciidoc
index f81dcab011da4..c67834269e505 100644
--- a/docs/reference/cluster/delete-desired-balance.asciidoc
+++ b/docs/reference/cluster/delete-desired-balance.asciidoc
@@ -6,7 +6,7 @@
 
 NOTE: {cloud-only}
 
-Discards the current desired balance and computes a new desired balance starting from the current allocation of shards.
+Discards the current <<shards-rebalancing-heuristics,desired balance>> and computes a new desired balance starting from the current allocation of shards.
 This can sometimes help {es} find a desired balance which needs fewer shard movements to achieve, especially if the
 cluster has experienced changes so substantial that the current desired balance is no longer optimal without {es} having
 detected that the current desired balance will take more shard movements to achieve than needed. However, this API
diff --git a/docs/reference/cluster/get-desired-balance.asciidoc b/docs/reference/cluster/get-desired-balance.asciidoc
index 3fd87dcfedc4f..74afdaa52daf1 100644
--- a/docs/reference/cluster/get-desired-balance.asciidoc
+++ b/docs/reference/cluster/get-desired-balance.asciidoc
@@ -8,7 +8,7 @@ NOTE: {cloud-only}
 
 Exposes:
 
-* the desired balance computation and reconciliation stats
+* the <<shards-rebalancing-heuristics,desired balance>> computation and reconciliation stats
 * balancing stats such as distribution of shards, disk and ingest forecasts
   across nodes and data tiers (based on the current cluster state)
 * routing table with each shard current and desired location
diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc
index 084ff471367ce..61c58cea95b83 100644
--- a/docs/reference/cluster/nodes-stats.asciidoc
+++ b/docs/reference/cluster/nodes-stats.asciidoc
@@ -808,6 +808,14 @@ This is not shown for the `shards` level, since mappings may be shared across th
 `total_estimated_overhead_in_bytes`::
 (integer) Estimated heap overhead, in bytes, of mappings on this node, which allows for 1kiB of heap for every mapped field.
 
+`total_segments`::
+(integer) Estimated number of Lucene segments on this node
+
+`total_segment_fields`::
+(integer) Estimated number of fields at the segment level on this node
+
+`average_fields_per_segment`::
+(integer) Estimated average number of fields per segment on this node
 =======
 
 `dense_vector`::
@@ -834,6 +842,142 @@ This is not shown for the `shards` level, since mappings may be shared across th
 
 =======
 
+`shards`::
+(object) When the `shards` level is requested, contains the aforementioned `indices` statistics for every shard (per
+index, and then per shard ID), as well as the following shard-specific statistics (which are not shown when the
+requested level is higher than `shards`):
++
+.Additional shard-specific statistics for the `shards` level
+[%collapsible%open]
+=======
+
+`routing`::
+(object) Contains routing information about the shard.
++
+.Properties of `routing`
+[%collapsible%open]
+========
+
+`state`::
+(string) State of the shard. Returned values are:
++
+* `INITIALIZING`: The shard is initializing/recovering.
+* `RELOCATING`: The shard is relocating.
+* `STARTED`: The shard has started.
+* `UNASSIGNED`: The shard is not assigned to any node.
+
+`primary`::
+(Boolean) Whether the shard is a primary shard or not.
+
+`node`::
+(string) ID of the node the shard is allocated to.
+
+`relocating_node`::
+(string) ID of the node the shard is either relocating to or relocating from, or null if shard is not relocating.
+
+========
+
+`commit`::
+(object) Contains information regarding the last commit point of the shard.
++
+.Properties of `commit`
+[%collapsible%open]
+========
+
+`id`::
+(string) Base64 version of the commit ID.
+
+`generation`::
+(integer) Lucene generation of the commit.
+
+`user_data`::
+(object) Contains additional technical information about the commit.
+
+`num_docs`::
+(integer) The number of docs in the commit.
+
+========
+
+`seq_no`::
+(object) Contains information about <<optimistic-concurrency-control,sequence numbers>> and checkpoints for the shard.
++
+.Properties of `seq_no`
+[%collapsible%open]
+========
+
+`max_seq_no`::
+(integer) The maximum sequence number issued so far.
+
+`local_checkpoint`::
+(integer) The current local checkpoint of the shard.
+
+`global_checkpoint`::
+(integer) The current global checkpoint of the shard.
+
+========
+
+`retention_leases`::
+(object) Contains information about <<index-modules-history-retention,history retention leases>>.
++
+.Properties of `retention_leases`
+[%collapsible%open]
+========
+
+`primary_term`::
+(integer) The primary term of this retention lease collection.
+
+`version`::
+(integer) The current version of the retention lease collection.
+
+`leases`::
+(array of objects) List of current leases for this shard.
++
+.Properties of `leases`
+[%collapsible%open]
+=========
+
+`id`::
+(string) The ID of the lease.
+
+`retaining_seq_no`::
+(integer) The minimum sequence number to be retained by the lease.
+
+`timestamp`::
+(integer) The timestamp of when the lease was created or renewed.
+Recorded in milliseconds since the {wikipedia}/Unix_time[Unix Epoch].
+
+`source`::
+(string) The source of the lease.
+
+=========
+========
+
+`shard_path`::
+(object)
++
+.Properties of `shard_path`
+[%collapsible%open]
+========
+
+`state_path`::
+(string) The state-path root, without the index name and the shard ID.
+
+`data_path`::
+(string) The data-path root, without the index name and the shard ID.
+
+`is_custom_data_path`::
+(boolean) Whether the data path is a custom data location and therefore outside of the nodes configured data paths.
+
+========
+
+`search_idle`::
+(boolean) Whether the shard is <<dynamic-index-search-idle-after,search idle>> or not.
+
+`search_idle_time`::
+(integer) Time since previous searcher access.
+Recorded in milliseconds.
+
+=======
 ======
 
 [[cluster-nodes-stats-api-response-body-os]]
diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc
index 3b429ef427071..c39bc0dcd2878 100644
--- a/docs/reference/cluster/stats.asciidoc
+++ b/docs/reference/cluster/stats.asciidoc
@@ -1282,6 +1282,31 @@ They are included here for expert users, but should otherwise be ignored.
 
 =====
 
+====
+
+`repositories`::
+(object) Contains statistics about the <<snapshot-restore,snapshot>> repositories defined in the cluster, broken down
+by repository type.
++
+.Properties of `repositories`
+[%collapsible%open]
+=====
+
+`count`:::
+(integer) The number of repositories of this type in the cluster.
+
+`read_only`:::
+(integer) The number of repositories of this type in the cluster which are registered read-only.
+
+`read_write`:::
+(integer) The number of repositories of this type in the cluster which are not registered as read-only.
+
+Each repository type may also include other statistics about the repositories of that type here.
+
+=====
+
+====
+
 [[cluster-stats-api-example]]
 ==== {api-examples-title}
 
@@ -1579,6 +1604,9 @@ The API returns the following response:
    },
    "snapshots": {
      ...
+   },
+   "repositories": {
+     ...
    }
 }
 --------------------------------------------------
@@ -1589,6 +1617,7 @@ The API returns the following response:
 // TESTRESPONSE[s/"count": \{[^\}]*\}/"count": $body.$_path/]
 // TESTRESPONSE[s/"packaging_types": \[[^\]]*\]/"packaging_types": $body.$_path/]
 // TESTRESPONSE[s/"snapshots": \{[^\}]*\}/"snapshots": $body.$_path/]
+// TESTRESPONSE[s/"repositories": \{[^\}]*\}/"repositories": $body.$_path/]
 // TESTRESPONSE[s/"field_types": \[[^\]]*\]/"field_types": $body.$_path/]
 // TESTRESPONSE[s/"runtime_field_types": \[[^\]]*\]/"runtime_field_types": $body.$_path/]
 // TESTRESPONSE[s/"search": \{[^\}]*\}/"search": $body.$_path/]
@@ -1600,7 +1629,7 @@ The API returns the following response:
 //    the plugins that will be in it. And because we figure folks don't need to
 //    see an exhaustive list anyway.
 // 2. Similarly, ignore the contents of `network_types`, `discovery_types`,
-//    `packaging_types` and `snapshots`.
+//    `packaging_types`, `snapshots` and `repositories`.
 // 3. Ignore the contents of the (nodes) count object, as what's shown here
 //    depends on the license. Voting-only nodes are e.g. only shown when this
 //    test runs with a basic license.
diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc
index 076b315558b60..1290f289e5bbd 100644
--- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc
+++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc
@@ -5,7 +5,7 @@
 [[data-streams-change-mappings-and-settings]]
 === Change mappings and settings for a data stream
 
-Each data stream has a <<create-index-template,matching index
+Each <<data-streams,data stream>> has a <<create-index-template,matching index
 template>>. Mappings and index settings from this template are applied to new
 backing indices created for the stream. This includes the stream's first
 backing index, which is auto-generated when the stream is created.
diff --git a/docs/reference/data-streams/downsampling-manual.asciidoc b/docs/reference/data-streams/downsampling-manual.asciidoc
index 771a08d97d949..44ae77d072034 100644
--- a/docs/reference/data-streams/downsampling-manual.asciidoc
+++ b/docs/reference/data-streams/downsampling-manual.asciidoc
@@ -14,7 +14,7 @@ DELETE _ingest/pipeline/my-timestamp-pipeline
 // TEARDOWN
 ////
 
-The recommended way to downsample a time series data stream (TSDS) is
+The recommended way to <<downsampling,downsample>> a <<tsds,time-series data stream (TSDS)>> is
 <<downsampling-ilm,through index lifecycle management (ILM)>>. However, if
 you're not using ILM, you can downsample a TSDS manually. This guide shows you
 how, using typical Kubernetes cluster monitoring data.
@@ -32,7 +32,7 @@ To test out manual downsampling, follow these steps:
 ==== Prerequisites
 
 * Refer to the <<tsds-prereqs,TSDS prerequisites>>.
-* It is not possible to downsample a data stream directly, nor
+* It is not possible to downsample a <<data-streams,data stream>> directly, nor
 multiple indices at once. It's only possible to downsample one time series index
 (TSDS backing index).
 * In order to downsample an index, it needs to be read-only. For a TSDS write
diff --git a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc
index f20c949c2fbc8..315f7fa85e45f 100644
--- a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc
+++ b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc
@@ -4,7 +4,7 @@
 <titleabbrev>Delete Data Stream Lifecycle</titleabbrev>
 ++++
 
-Deletes the lifecycle from a set of data streams.
+Deletes the <<data-stream-lifecycle,lifecycle>> from a set of data streams.
 
 [[delete-lifecycle-api-prereqs]]
 ==== {api-prereq-title}
diff --git a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc
index 7968bb78939e8..2b15886ebe192 100644
--- a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc
+++ b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc
@@ -4,7 +4,7 @@
 <titleabbrev>Explain Data Stream Lifecycle</titleabbrev>
 ++++
 
-Retrieves the current data stream lifecycle status for one or more data stream backing indices.
+Retrieves the current <<data-stream-lifecycle,data stream lifecycle>> status for one or more data stream backing indices.
 
 [[explain-lifecycle-api-prereqs]]
 ==== {api-prereq-title}
diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc
index a99fa19d9db8d..f48fa1eb52daa 100644
--- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc
+++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc
@@ -4,7 +4,7 @@
 <titleabbrev>Get Data Stream Lifecycle</titleabbrev>
 ++++
 
-Gets stats about the execution of data stream lifecycle.
+Gets stats about the execution of <<data-stream-lifecycle,data stream lifecycle>>.
 
 [[get-lifecycle-stats-api-prereqs]]
 ==== {api-prereq-title}
diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc
index 331285af395b6..6bac1c7f7cc75 100644
--- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc
+++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc
@@ -4,7 +4,7 @@
 <titleabbrev>Get Data Stream Lifecycle</titleabbrev>
 ++++
 
-Gets the lifecycle of a set of data streams.
+Gets the <<data-stream-lifecycle,lifecycle>> of a set of <<data-streams,data streams>>.
 
 [[get-lifecycle-api-prereqs]]
 ==== {api-prereq-title}
@@ -128,14 +128,18 @@ The response will look like the following:
       "name": "my-data-stream-1",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d"
+        "data_retention": "7d",
+        "effective_retention": "7d",
+        "retention_determined_by": "data_stream_configuration"
       }
     },
     {
       "name": "my-data-stream-2",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d"
+        "data_retention": "7d",
+        "effective_retention": "7d",
+        "retention_determined_by": "data_stream_configuration"
       }
     }
   ]
diff --git a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc
index 7d33a5b5f880c..c60c105e818ab 100644
--- a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc
+++ b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc
@@ -4,7 +4,7 @@
 <titleabbrev>Put Data Stream Lifecycle</titleabbrev>
 ++++
 
-Configures the data stream lifecycle for the targeted data streams.
+Configures the data stream <<data-stream-lifecycle,lifecycle>> for the targeted <<data-streams,data streams>>.
 
 [[put-lifecycle-api-prereqs]]
 ==== {api-prereq-title}
diff --git a/docs/reference/data-streams/lifecycle/index.asciidoc b/docs/reference/data-streams/lifecycle/index.asciidoc
index 16ccf2ef82391..e4d5acfb704d3 100644
--- a/docs/reference/data-streams/lifecycle/index.asciidoc
+++ b/docs/reference/data-streams/lifecycle/index.asciidoc
@@ -14,10 +14,11 @@ To achieve that, it supports:
 * Automatic <<index-rollover,rollover>>, which chunks your incoming data in smaller pieces to facilitate better performance
 and backwards incompatible mapping changes.
 * Configurable retention, which allows you to configure the time period for which your data is guaranteed to be stored.
-{es} is allowed at a later time to delete data older than this time period.
+{es} is allowed at a later time to delete data older than this time period. Retention can be configured on the data stream level
+or on a global level. Read more about the different options in this <<tutorial-manage-data-stream-retention,tutorial>>.
 
 A data stream lifecycle also supports downsampling the data stream backing indices.
-See <<data-streams-put-lifecycle-downsampling-example, the downsampling example>> for 
+See <<data-streams-put-lifecycle-downsampling-example, the downsampling example>> for
 more details.
 
 [discrete]
@@ -33,16 +34,17 @@ each data stream and performs the following steps:
 3. After an index is not the write index anymore (i.e. the data stream has been rolled over),
 automatically tail merges the index. Data stream lifecycle executes a merge operation that only targets
 the long tail of small segments instead of the whole shard. As the segments are organised
-into tiers of exponential sizes, merging the long tail of small segments is only a 
+into tiers of exponential sizes, merging the long tail of small segments is only a
 fraction of the cost of force merging to a single segment. The small segments would usually
 hold the most recent data so tail merging will focus the merging resources on the higher-value
 data that is most likely to keep being queried.
-4. If <<data-streams-put-lifecycle-downsampling-example, downsampling>> is configured it will execute 
+4. If <<data-streams-put-lifecycle-downsampling-example, downsampling>> is configured it will execute
 all the configured downsampling rounds.
 5. Applies retention to the remaining backing indices. This means deleting the backing indices whose
-`generation_time` is longer than the configured retention period. The `generation_time` is only applicable to rolled over backing
-indices and it is either the time since the backing index got rolled over, or the time optionally configured in the
-<<index-data-stream-lifecycle-origination-date,`index.lifecycle.origination_date`>> setting.
+`generation_time` is longer than the effective retention period (read more about the
+<<effective-retention-calculation, effective retention calculation>>). The `generation_time` is only applicable to rolled
+over backing indices and it is either the time since the backing index got rolled over, or the time optionally configured
+in the <<index-data-stream-lifecycle-origination-date,`index.lifecycle.origination_date`>> setting.
 
 IMPORTANT: We use the `generation_time` instead of the creation time because this ensures that all data in the backing
 index have passed the retention period. As a result, the retention period is not the exact time data gets deleted, but
@@ -75,4 +77,6 @@ include::tutorial-manage-new-data-stream.asciidoc[]
 
 include::tutorial-manage-existing-data-stream.asciidoc[]
 
+include::tutorial-manage-data-stream-retention.asciidoc[]
+
 include::tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc[]
diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc
new file mode 100644
index 0000000000000..1b2996c62e2df
--- /dev/null
+++ b/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc
@@ -0,0 +1,218 @@
+[role="xpack"]
+[[tutorial-manage-data-stream-retention]]
+=== Tutorial: Data stream retention
+
+In this tutorial, we are going to go over the data stream lifecycle retention; we will define it, go over how it can be configured
+and how it can gets applied. Keep in mind, the following options apply only to data streams that are managed by the data stream lifecycle.
+
+. <<what-is-retention>>
+. <<retention-configuration>>
+. <<effective-retention-calculation>>
+. <<effective-retention-application>>
+
+You can verify if a data steam is managed by the data stream lifecycle via the <<data-streams-get-lifecycle,get data stream lifecycle API>>:
+
+////
+[source,console]
+----
+PUT /_index_template/template
+{
+  "index_patterns": ["my-data-stream*"],
+  "template": {
+    "lifecycle": {}
+  },
+  "data_stream": { }
+}
+
+PUT /_data_stream/my-data-stream
+----
+// TESTSETUP
+////
+
+////
+[source,console]
+----
+DELETE /_data_stream/my-data-stream*
+DELETE /_index_template/template
+PUT /_cluster/settings
+{
+  "persistent" : {
+    "data_streams.lifecycle.retention.*" : null
+  }
+}
+----
+// TEARDOWN
+////
+
+[source,console]
+--------------------------------------------------
+GET _data_stream/my-data-stream/_lifecycle
+--------------------------------------------------
+
+The result should look like this:
+
+[source,console-result]
+--------------------------------------------------
+{
+  "data_streams": [
+    {
+      "name": "my-data-stream",                                   <1>
+      "lifecycle": {
+        "enabled": true                                           <2>
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// TESTRESPONSE[skip:the result is for illustrating purposes only]
+<1> The name of your data stream.
+<2> Ensure that the lifecycle is enabled, meaning this should be `true`.
+
+[discrete]
+[[what-is-retention]]
+==== What is data stream retention?
+
+We define retention as the least amount of time the data of a data stream are going to be kept in {es}. After this time period
+has passed, {es} is allowed to remove these data to free up space and/or manage costs.
+
+NOTE: Retention does not define the period that the data will be removed, but the minimum time period they will be kept.
+
+We define 4 different types of retention:
+
+* The data stream retention, or `data_retention`, which is the retention configured on the data stream level. It can be
+set via an <<index-templates,index template>> for future data streams or via the <<data-streams-put-lifecycle, PUT data
+stream lifecycle API>> for an existing data stream. When the data stream retention is not set, it implies that the data
+need to be kept forever.
+* The global default retention, let's call it `default_retention`, which is a retention configured via the cluster setting
+<<data-streams-lifecycle-retention-default, `data_streams.lifecycle.retention.default`>> and will be
+applied to all data streams managed by data stream lifecycle that do not have `data_retention` configured. Effectively,
+it ensures that there will be no data streams keeping their data forever. This can be set via the
+<<cluster-update-settings, update cluster settings API>>.
+* The global max retention, let's call it `max_retention`, which is a retention configured via the cluster setting
+<<data-streams-lifecycle-retention-max, `data_streams.lifecycle.retention.max`>> and will be applied to
+all data streams managed by data stream lifecycle. Effectively, it ensures that there will be no data streams whose retention
+will exceed this time period. This can be set via the <<cluster-update-settings, update cluster settings API>>.
+* The effective retention, or `effective_retention`, which is the retention applied at a data stream on a given moment.
+Effective retention cannot be set, it is derived by taking into account all the configured retention listed above and is
+calculated as it is described <<effective-retention-calculation,here>>.
+
+NOTE: Global default and max retention do not apply to data streams internal to elastic. Internal data streams are recognised
+ either by having the `system` flag set to `true` or if their name is prefixed with a dot (`.`).
+
+[discrete]
+[[retention-configuration]]
+==== How to configure retention?
+
+- By setting the `data_retention` on the data stream level. This retention can be configured in two ways:
++
+-- For new data streams, it can be defined in the index template that would be applied during the data stream's creation.
+You can use the <<indices-put-template,create index template API>>, for example:
++
+[source,console]
+--------------------------------------------------
+PUT _index_template/template
+{
+  "index_patterns": ["my-data-stream*"],
+  "data_stream": { },
+  "priority": 500,
+  "template": {
+    "lifecycle": {
+      "data_retention": "7d"
+    }
+  },
+  "_meta": {
+    "description": "Template with data stream lifecycle"
+  }
+}
+--------------------------------------------------
+-- For an existing data stream, it can be set via the <<data-streams-put-lifecycle, PUT lifecycle API>>.
++
+[source,console]
+----
+PUT _data_stream/my-data-stream/_lifecycle
+{
+  "data_retention": "30d" <1>
+}
+----
+// TEST[continued]
+<1> The retention period of this data stream is set to 30 days.
+
+- By setting the global retention via the `data_streams.lifecycle.retention.default` and/or `data_streams.lifecycle.retention.max`
+that are set on a cluster level. You can be set via the <<cluster-update-settings, update cluster settings API>>. For example:
++
+[source,console]
+--------------------------------------------------
+PUT /_cluster/settings
+{
+  "persistent" : {
+    "data_streams.lifecycle.retention.default" : "7d",
+    "data_streams.lifecycle.retention.max" : "90d"
+  }
+}
+--------------------------------------------------
+// TEST[continued]
+
+[discrete]
+[[effective-retention-calculation]]
+==== How is the effective retention calculated?
+The effective is calculated in the following way:
+
+- The `effective_retention` is the `default_retention`, when `default_retention` is defined and the data stream does not
+have `data_retention`.
+- The `effective_retention` is the `data_retention`, when `data_retention` is defined and if `max_retention` is defined,
+it is less than the `max_retention`.
+- The `effective_retention` is the `max_retention`, when `max_retention` is defined, and the data stream has either no
+`data_retention` or its `data_retention` is greater than the `max_retention`.
+
+The above is demonstrated in the examples below:
+
+|===
+|`default_retention`    |`max_retention`    |`data_retention`   |`effective_retention`  |Retention determined by
+
+|Not set                |Not set            |Not set            |Infinite               |N/A
+|Not relevant           |12 months          |**30 days**        |30 days                |`data_retention`
+|Not relevant           |Not set            |**30 days**        |30 days                |`data_retention`
+|**30 days**            |12 months          |Not set            |30 days                |`default_retention`
+|**30 days**            |30 days            |Not set            |30 days                |`default_retention`
+|Not relevant           |**30 days**        |12 months          |30 days                |`max_retention`
+|Not set                |**30 days**        |Not set            |30 days                |`max_retention`
+|===
+
+Considering our example, if we retrieve the lifecycle of `my-data-stream`:
+[source,console]
+----
+GET _data_stream/my-data-stream/_lifecycle
+----
+// TEST[continued]
+
+We see that it will remain the same with what the user configured:
+[source,console-result]
+----
+{
+  "data_streams": [
+    {
+      "name": "my-data-stream",
+      "lifecycle": {
+        "enabled": true,
+        "data_retention": "30d",
+        "effective_retention": "30d",
+        "retention_determined_by": "data_stream_configuration"
+      }
+    }
+  ]
+}
+----
+
+[discrete]
+[[effective-retention-application]]
+==== How is the effective retention applied?
+
+Retention is applied to the remaining backing indices of a data stream as the last step of
+<<data-streams-lifecycle-how-it-works, a data stream lifecycle run>>. Data stream lifecycle will retrieve the backing indices
+whose `generation_time` is longer than the effective retention period and delete them. The `generation_time` is only
+applicable to rolled over backing indices and it is either the time since the backing index got rolled over, or the time
+optionally configured in the <<index-data-stream-lifecycle-origination-date,`index.lifecycle.origination_date`>> setting.
+
+IMPORTANT: We use the `generation_time` instead of the creation time because this ensures that all data in the backing
+index have passed the retention period. As a result, the retention period is not the exact time data get deleted, but
+the minimum time data will be stored.
diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc
index c34340a096046..01d51cdde3167 100644
--- a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc
+++ b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc
@@ -91,10 +91,12 @@ The result will look like this:
 {
   "data_streams": [
     {
-      "name": "my-data-stream",<1>
+      "name": "my-data-stream",                                     <1>
       "lifecycle": {
-        "enabled": true,       <2>
-        "data_retention": "7d" <3>
+        "enabled": true,                                            <2>
+        "data_retention": "7d",                                     <3>
+        "effective_retention": "7d",                                <4>
+        "retention_determined_by": "data_stream_configuration"
       }
     }
   ]
@@ -102,8 +104,9 @@ The result will look like this:
 --------------------------------------------------
 <1> The name of your data stream.
 <2> Shows if the data stream lifecycle is enabled for this data stream.
-<3> The retention period of the data indexed in this data stream, this means that the data in this data stream will
-be kept at least for 7 days. After that {es} can delete it at its own discretion.
+<3> The retention period of the data indexed in this data stream, as configured by the user.
+<4> The retention period that will be applied by the data stream lifecycle. This means that the data in this data stream will
+    be kept at least for 7 days. After that {es} can delete it at its own discretion.
 
 If you want to see more information about how the data stream lifecycle is applied on individual backing indices use the
 <<data-streams-explain-lifecycle,explain data stream lifecycle API>>:
diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc
index 5b2e2a1ec70a2..a2c12466b7f2b 100644
--- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc
+++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc
@@ -1,14 +1,14 @@
 [role="xpack"]
 [[tutorial-migrate-data-stream-from-ilm-to-dsl]]
-=== Tutorial: Migrate ILM managed data stream to data stream lifecycle 
+=== Tutorial: Migrate ILM managed data stream to data stream lifecycle
 
-In this tutorial we'll look at migrating an existing data stream from Index Lifecycle Management ({ilm-init}) to
-data stream lifecycle. The existing {ilm-init} managed backing indices will continue 
+In this tutorial we'll look at migrating an existing data stream from <<index-lifecycle-management,Index Lifecycle Management ({ilm-init})>> to
+<<data-stream-lifecycle,data stream lifecycle>>. The existing {ilm-init} managed backing indices will continue
 to be managed by {ilm-init} until they age out and get deleted by {ilm-init}; however,
-the new backing indices will be managed by data stream lifecycle. 
-This way, a data stream is gradually migrated away from being managed by {ilm-init} to 
+the new backing indices will be managed by data stream lifecycle.
+This way, a data stream is gradually migrated away from being managed by {ilm-init} to
 being managed by data stream lifecycle. As we'll see, {ilm-init} and data stream lifecycle
-can co-manage a data stream; however, an index can only be managed by one system at 
+can co-manage a data stream; however, an index can only be managed by one system at
 a time.
 
 [discrete]
@@ -17,7 +17,7 @@ a time.
 To migrate a data stream from {ilm-init} to data stream lifecycle we'll have to execute
 two steps:
 
-1. Update the index template that's backing the data stream to set <<index-lifecycle-prefer-ilm, prefer_ilm>> 
+1. Update the index template that's backing the data stream to set <<index-lifecycle-prefer-ilm, prefer_ilm>>
 to `false`, and to configure data stream lifecycle.
 2. Configure the data stream lifecycle for the _existing_ data stream using
 the <<data-streams-put-lifecycle, lifecycle API>>.
@@ -174,8 +174,8 @@ in the index template).
 To migrate the `dsl-data-stream` to data stream lifecycle we'll have to execute
 two steps:
 
-1. Update the index template that's backing the data stream to set <<index-lifecycle-prefer-ilm, prefer_ilm>> 
-to `false`, and to configure data stream lifecycle. 
+1. Update the index template that's backing the data stream to set <<index-lifecycle-prefer-ilm, prefer_ilm>>
+to `false`, and to configure data stream lifecycle.
 2. Configure the data stream lifecycle for the _existing_ `dsl-data-stream` using
 the <<data-streams-put-lifecycle, lifecycle API>>.
 
@@ -209,9 +209,9 @@ PUT _index_template/dsl-data-stream-template
 // TEST[continued]
 
 <1> The `prefer_ilm` setting will now be configured on the **new** backing indices
-(created by rolling over the data stream) such that {ilm-init} does _not_ take 
+(created by rolling over the data stream) such that {ilm-init} does _not_ take
 precedence over data stream lifecycle.
-<2> We're configuring the data stream lifecycle so _new_ data streams will be 
+<2> We're configuring the data stream lifecycle so _new_ data streams will be
 managed by data stream lifecycle.
 
 We've now made sure that new data streams will be managed by data stream lifecycle.
@@ -227,7 +227,7 @@ PUT _data_stream/dsl-data-stream/_lifecycle
 ----
 // TEST[continued]
 
-We can inspect the data stream to check that the next generation will indeed be 
+We can inspect the data stream to check that the next generation will indeed be
 managed by data stream lifecycle:
 
 [source,console]
@@ -266,7 +266,9 @@ GET _data_stream/dsl-data-stream
       "template": "dsl-data-stream-template",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d"
+        "data_retention": "7d",
+        "effective_retention": "7d",
+        "retention_determined_by": "data_stream_configuration"
       },
       "ilm_policy": "pre-dsl-ilm-policy",
       "next_generation_managed_by": "Data stream lifecycle",         <3>
@@ -292,7 +294,7 @@ GET _data_stream/dsl-data-stream
 <4> The `prefer_ilm` setting value we configured in the index template is reflected
 and will be configured accordingly for new backing indices.
 
-We'll now rollover the data stream to see the new generation index being managed by 
+We'll now rollover the data stream to see the new generation index being managed by
 data stream lifecycle:
 
 [source,console]
@@ -344,7 +346,9 @@ GET _data_stream/dsl-data-stream
       "template": "dsl-data-stream-template",
       "lifecycle": {
         "enabled": true,
-        "data_retention": "7d"
+        "data_retention": "7d",
+        "effective_retention": "7d",
+        "retention_determined_by": "data_stream_configuration"
       },
       "ilm_policy": "pre-dsl-ilm-policy",
       "next_generation_managed_by": "Data stream lifecycle",
@@ -375,9 +379,9 @@ in the index template
 [discrete]
 [[migrate-from-dsl-to-ilm]]
 ==== Migrate data stream back to ILM
-We can easily change this data stream to be managed by {ilm-init} because we didn't remove 
-the {ilm-init} policy when we <<update-index-template-for-dsl, updated 
-the index template>>. 
+We can easily change this data stream to be managed by {ilm-init} because we didn't remove
+the {ilm-init} policy when we <<update-index-template-for-dsl, updated
+the index template>>.
 
 We can achieve this in two ways:
 
diff --git a/docs/reference/data-streams/modify-data-streams-api.asciidoc b/docs/reference/data-streams/modify-data-streams-api.asciidoc
index f05e76e67c32f..2da869083df22 100644
--- a/docs/reference/data-streams/modify-data-streams-api.asciidoc
+++ b/docs/reference/data-streams/modify-data-streams-api.asciidoc
@@ -4,7 +4,7 @@
 <titleabbrev>Modify data streams</titleabbrev>
 ++++
 
-Performs one or more data stream modification actions in a single atomic
+Performs one or more <<data-streams,data stream>> modification actions in a single atomic
 operation.
 
 [source,console]
diff --git a/docs/reference/data-streams/promote-data-stream-api.asciidoc b/docs/reference/data-streams/promote-data-stream-api.asciidoc
index 281e9b549abcb..111c7a2256f8a 100644
--- a/docs/reference/data-streams/promote-data-stream-api.asciidoc
+++ b/docs/reference/data-streams/promote-data-stream-api.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Promote data stream</titleabbrev>
 ++++
 
-The purpose of the promote data stream api is to turn
+The purpose of the promote <<data-streams,data stream>> API is to turn
 a data stream that is replicated by CCR into a regular
 data stream.
 
diff --git a/docs/reference/data-streams/tsds-reindex.asciidoc b/docs/reference/data-streams/tsds-reindex.asciidoc
index ea4ba16df5c4a..9d6594db4e779 100644
--- a/docs/reference/data-streams/tsds-reindex.asciidoc
+++ b/docs/reference/data-streams/tsds-reindex.asciidoc
@@ -9,7 +9,7 @@
 [[tsds-reindex-intro]]
 ==== Introduction
 
-With reindexing, you can copy documents from an old time-series data stream (TSDS) to a new one. Data streams support
+With reindexing, you can copy documents from an old <<tsds,time-series data stream (TSDS)>> to a new one. Data streams support
 reindexing in general, with a few <<reindex-with-a-data-stream, restrictions>>. Still, time-series data streams
 introduce additional challenges due to tight control on the accepted timestamp range for each backing index they
 contain. Direct use of the reindex API would likely error out due to attempting to insert documents with timestamps that are
diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc
index de89fa1ca3f31..01573658c33d0 100644
--- a/docs/reference/data-streams/tsds.asciidoc
+++ b/docs/reference/data-streams/tsds.asciidoc
@@ -107,6 +107,7 @@ parameter:
 * <<number,`integer`>>
 * <<number,`long`>>
 * <<number,`unsigned_long`>>
+* <<boolean,`boolean`>>
 
 For a flattened field, use the `time_series_dimensions` parameter to configure an array of fields as dimensions. For details refer to <<flattened-params,`flattened`>>.
 
diff --git a/docs/reference/eql/eql-apis.asciidoc b/docs/reference/eql/eql-apis.asciidoc
index d3f591ccfe6c1..e8cc2b21492ae 100644
--- a/docs/reference/eql/eql-apis.asciidoc
+++ b/docs/reference/eql/eql-apis.asciidoc
@@ -1,7 +1,7 @@
 [[eql-apis]]
 == EQL APIs
 
-Event Query Language (EQL) is a query language for event-based time series data,
+<<eql,Event Query Language (EQL)>> is a query language for event-based time series data,
 such as logs, metrics, and traces. For an overview of EQL and related tutorials,
 see <<eql>>.
 
diff --git a/docs/reference/esql/esql-across-clusters.asciidoc b/docs/reference/esql/esql-across-clusters.asciidoc
index 8bc1e2a83fc19..d13b3db1c73ea 100644
--- a/docs/reference/esql/esql-across-clusters.asciidoc
+++ b/docs/reference/esql/esql-across-clusters.asciidoc
@@ -54,11 +54,6 @@ Refer to <<remote-clusters-cert, TLS certificate authentication>> for prerequisi
 [[esql-ccs-security-model-api-key]]
 ===== API key authentication
 
-[NOTE]
-====
-`ENRICH` is *not supported* in this version when using {esql} with the API key based security model.
-====
-
 The following information pertains to using {esql} across clusters with the <<remote-clusters-api-key, *API key based security model*>>. You'll need to follow the steps on that page for the *full setup instructions*. This page only contains additional information specific to {esql}.
 
 API key based cross-cluster search (CCS) enables more granular control over allowed actions between clusters.
@@ -71,6 +66,7 @@ You will need to:
 
 Using {esql} with the API key based security model requires some additional permissions that may not be needed when using the traditional query DSL based search.
 The following example API call creates a role that can query remote indices using {esql} when using the API key based security model.
+The final privilege, `remote_cluster`, is required to allow remote enrich operations.
 
 [source,console]
 ----
@@ -89,7 +85,17 @@ POST /_security/role/remote1
       "privileges": [ "read","read_cross_cluster" ], <4>
       "clusters" : ["my_remote_cluster"] <5>
     }
-  ]
+  ], 
+   "remote_cluster": [ <6>
+        {
+            "privileges": [
+                "monitor_enrich"
+            ],
+            "clusters": [
+                "my_remote_cluster"
+            ]
+        }
+    ]
 }
 ----
 
@@ -100,6 +106,7 @@ POST /_security/role/remote1
 <5> The remote clusters to which these privileges apply.
 This remote cluster must be configured with a <<security-api-create-cross-cluster-api-key,cross-cluster API key>> and connected to the remote cluster before the remote index can be queried.
 Verify connection using the <<cluster-remote-info, Remote cluster info>> API.
+<6> Required to allow remote enrichment. Without this, the user cannot read from the `.enrich` indices on the remote cluster. The `remote_cluster` security privilege was introduced in version *8.15.0*.
 
 You will then need a user or API key with the permissions you created above. The following example API call creates a user with the `remote1` role.
 
@@ -114,6 +121,11 @@ POST /_security/user/remote_user
 
 Remember that all cross-cluster requests from the local cluster are bound by the cross cluster API key’s privileges, which are controlled by the remote cluster's administrator.
 
+[TIP]
+====
+Cross cluster API keys created in versions prior to 8.15.0 will need to replaced or updated to add the new permissions required for {esql} with ENRICH.
+====
+
 [discrete]
 [[ccq-remote-cluster-setup]]
 ==== Remote cluster setup
@@ -174,9 +186,11 @@ clusters, aiming to minimize computation or inter-cluster data transfer. Ensurin
 the policy exists with consistent data on both the local cluster and the remote
 clusters is critical for ES|QL to produce a consistent query result.
 
-[NOTE]
+[TIP]
 ====
-Enrich across clusters is *not supported* in this version when using {esql} with the <<remote-clusters-api-key, *API key based security model*>>.
+Enrich in {esql} across clusters using the API key based security model was introduced in version *8.15.0*.
+Cross cluster API keys created in versions prior to 8.15.0 will need to replaced or updated to use the new required permissions.
+Refer to the example in the <<esql-ccs-security-model-api-key,API key authentication>> section.
 ====
 
 In the following example, the enrich with `hosts` policy can be executed on
diff --git a/docs/reference/esql/esql-apis.asciidoc b/docs/reference/esql/esql-apis.asciidoc
index 686a71506bc14..8586cd1ae6bce 100644
--- a/docs/reference/esql/esql-apis.asciidoc
+++ b/docs/reference/esql/esql-apis.asciidoc
@@ -1,7 +1,7 @@
 [[esql-apis]]
 == {esql} APIs
 
-The {es} Query Language ({esql}) provides a powerful way to filter, transform,
+The <<esql,{es} Query Language ({esql})>> provides a powerful way to filter, transform,
 and analyze data stored in {es}, and in the future in other runtimes. For an
 overview of {esql} and related tutorials, see <<esql>>.
 
diff --git a/docs/reference/esql/esql-async-query-delete-api.asciidoc b/docs/reference/esql/esql-async-query-delete-api.asciidoc
index 90f8c06b9124a..5cad566f7f9c0 100644
--- a/docs/reference/esql/esql-async-query-delete-api.asciidoc
+++ b/docs/reference/esql/esql-async-query-delete-api.asciidoc
@@ -4,7 +4,7 @@
 <titleabbrev>{esql} async query delete API</titleabbrev>
 ++++
 
-The {esql} async query delete API is used to manually delete an async query
+The <<esql,{esql}>> async query delete API is used to manually delete an async query
 by ID. If the query is still running, the query will be cancelled. Otherwise,
 the stored results are deleted.
 
diff --git a/docs/reference/esql/esql-commands.asciidoc b/docs/reference/esql/esql-commands.asciidoc
index bed79299b1cc1..235113ac1394a 100644
--- a/docs/reference/esql/esql-commands.asciidoc
+++ b/docs/reference/esql/esql-commands.asciidoc
@@ -37,6 +37,9 @@ image::images/esql/processing-command.svg[A processing command changing an input
 * <<esql-enrich>>
 * <<esql-eval>>
 * <<esql-grok>>
+ifeval::["{release-state}"=="unreleased"]
+* experimental:[] <<esql-inlinestats-by>>
+endif::[]
 * <<esql-keep>>
 * <<esql-limit>>
 ifeval::["{release-state}"=="unreleased"]
@@ -59,6 +62,9 @@ include::processing-commands/drop.asciidoc[]
 include::processing-commands/enrich.asciidoc[]
 include::processing-commands/eval.asciidoc[]
 include::processing-commands/grok.asciidoc[]
+ifeval::["{release-state}"=="unreleased"]
+include::processing-commands/inlinestats.asciidoc[]
+endif::[]
 include::processing-commands/keep.asciidoc[]
 include::processing-commands/limit.asciidoc[]
 ifeval::["{release-state}"=="unreleased"]
diff --git a/docs/reference/esql/esql-multi-index.asciidoc b/docs/reference/esql/esql-multi-index.asciidoc
index 41ff6a27417b1..25874a132d93d 100644
--- a/docs/reference/esql/esql-multi-index.asciidoc
+++ b/docs/reference/esql/esql-multi-index.asciidoc
@@ -97,13 +97,12 @@ In addition, if the query refers to this unsupported field directly, the query f
 [source.merge.styled,esql]
 ----
 FROM events_*
-| KEEP @timestamp, client_ip, event_duration, message
-| SORT @timestamp DESC
+| SORT client_ip DESC
 ----
 
 [source,bash]
 ----
-Cannot use field [client_ip] due to ambiguities being mapped as 
+Cannot use field [client_ip] due to ambiguities being mapped as
 [2] incompatible types:
     [ip] in [events_ip],
     [keyword] in [events_keyword]
@@ -113,12 +112,13 @@ Cannot use field [client_ip] due to ambiguities being mapped as
 [[esql-multi-index-union-types]]
 === Union types
 
+experimental::[]
+
 {esql} has a way to handle <<esql-multi-index-invalid-mapping, field type mismatches>>. When the same field is mapped to multiple types in multiple indices,
 the type of the field is understood to be a _union_ of the various types in the index mappings.
 As seen in the preceding examples, this _union type_ cannot be used in the results,
-and cannot be referred to by the query
--- except when it's passed to a type conversion function that accepts all the types in the _union_ and converts the field
-to a single type. {esql} offers a suite of <<esql-type-conversion-functions,type conversion functions>> to achieve this. 
+and cannot be referred to by the query -- except in `KEEP`, `DROP` or when it's passed to a type conversion function that accepts all the types in
+the _union_ and converts the field to a single type. {esql} offers a suite of <<esql-type-conversion-functions,type conversion functions>> to achieve this.
 
 In the above examples, the query can use a command like `EVAL client_ip = TO_IP(client_ip)` to resolve
 the union of `ip` and `keyword` to just `ip`.
diff --git a/docs/reference/esql/esql-query-api.asciidoc b/docs/reference/esql/esql-query-api.asciidoc
index e8cfa03e3ee88..c8c735b73d2a4 100644
--- a/docs/reference/esql/esql-query-api.asciidoc
+++ b/docs/reference/esql/esql-query-api.asciidoc
@@ -102,7 +102,7 @@ Column `name` and `type` for each column returned in `values`. Each object is a
 Column `name` and `type` for each queried column. Each object is a single column. This is only
 returned if `drop_null_columns` is sent with the request.
 
-`rows`::
+`values`::
 (array of arrays)
 Values for the search results.
 
diff --git a/docs/reference/esql/esql-rest.asciidoc b/docs/reference/esql/esql-rest.asciidoc
index 5b90e96d7a734..2c8c5e81e273d 100644
--- a/docs/reference/esql/esql-rest.asciidoc
+++ b/docs/reference/esql/esql-rest.asciidoc
@@ -278,6 +278,47 @@ POST /_query
 ----
 // TEST[setup:library]
 
+The parameters can be named parameters or positional parameters.
+
+Named parameters use question mark placeholders (`?`) followed by a string.
+
+[source,console]
+----
+POST /_query
+{
+  "query": """
+    FROM library
+    | EVAL year = DATE_EXTRACT("year", release_date)
+    | WHERE page_count > ?page_count AND author == ?author
+    | STATS count = COUNT(*) by year
+    | WHERE count > ?count
+    | LIMIT 5
+  """,
+  "params": [{"page_count" : 300}, {"author" : "Frank Herbert"}, {"count" : 0}]
+}
+----
+// TEST[setup:library]
+
+Positional parameters use question mark placeholders (`?`) followed by an
+integer.
+
+[source,console]
+----
+POST /_query
+{
+  "query": """
+    FROM library
+    | EVAL year = DATE_EXTRACT("year", release_date)
+    | WHERE page_count > ?1 AND author == ?2
+    | STATS count = COUNT(*) by year
+    | WHERE count > ?3
+    | LIMIT 5
+  """,
+  "params": [300, "Frank Herbert", 0]
+}
+----
+// TEST[setup:library]
+
 [discrete]
 [[esql-rest-async-query]]
 ==== Running an async {esql} query
diff --git a/docs/reference/esql/functions/aggregation-functions.asciidoc b/docs/reference/esql/functions/aggregation-functions.asciidoc
index 4c248704b6385..7cdc42ea6cbf9 100644
--- a/docs/reference/esql/functions/aggregation-functions.asciidoc
+++ b/docs/reference/esql/functions/aggregation-functions.asciidoc
@@ -9,30 +9,30 @@ The <<esql-stats-by>> command supports these aggregate functions:
 
 // tag::agg_list[]
 * <<esql-avg>>
-* <<esql-agg-count>>
-* <<esql-agg-count-distinct>>
+* <<esql-count>>
+* <<esql-count_distinct>>
 * <<esql-max>>
-* <<esql-agg-median>>
-* <<esql-agg-median-absolute-deviation>>
+* <<esql-median>>
+* <<esql-median_absolute_deviation>>
 * <<esql-min>>
 * <<esql-percentile>>
-* experimental:[] <<esql-agg-st-centroid>>
-* <<esql-agg-sum>>
+* experimental:[] <<esql-st_centroid_agg>>
+* <<esql-sum>>
 * <<esql-top>>
-* <<esql-agg-values>>
-* experimental:[] <<esql-agg-weighted-avg>>
+* <<esql-values>>
+* experimental:[] <<esql-weighted_avg>>
 // end::agg_list[]
 
-include::count.asciidoc[]
-include::count-distinct.asciidoc[]
-include::median.asciidoc[]
-include::median-absolute-deviation.asciidoc[]
-include::st_centroid_agg.asciidoc[]
-include::sum.asciidoc[]
 include::layout/avg.asciidoc[]
+include::layout/count.asciidoc[]
+include::layout/count_distinct.asciidoc[]
 include::layout/max.asciidoc[]
+include::layout/median.asciidoc[]
+include::layout/median_absolute_deviation.asciidoc[]
 include::layout/min.asciidoc[]
 include::layout/percentile.asciidoc[]
+include::layout/st_centroid_agg.asciidoc[]
+include::layout/sum.asciidoc[]
 include::layout/top.asciidoc[]
-include::values.asciidoc[]
-include::weighted-avg.asciidoc[]
+include::layout/values.asciidoc[]
+include::layout/weighted_avg.asciidoc[]
diff --git a/docs/reference/esql/functions/appendix/count_distinct.asciidoc b/docs/reference/esql/functions/appendix/count_distinct.asciidoc
new file mode 100644
index 0000000000000..065065cf34e06
--- /dev/null
+++ b/docs/reference/esql/functions/appendix/count_distinct.asciidoc
@@ -0,0 +1,25 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-agg-count-distinct-approximate]]
+==== Counts are approximate
+
+Computing exact counts requires loading values into a set and returning its
+size. This doesn't scale when working on high-cardinality sets and/or large
+values as the required memory usage and the need to communicate those
+per-shard sets between nodes would utilize too many resources of the cluster.
+
+This `COUNT_DISTINCT` function is based on the
+https://static.googleusercontent.com/media/research.google.com/fr//pubs/archive/40671.pdf[HyperLogLog++]
+algorithm, which counts based on the hashes of the values with some interesting
+properties:
+
+include::../../../aggregations/metrics/cardinality-aggregation.asciidoc[tag=explanation]
+
+The `COUNT_DISTINCT` function takes an optional second parameter to configure
+the precision threshold. The precision_threshold options allows to trade memory
+for accuracy, and defines a unique count below which counts are expected to be
+close to accurate. Above this value, counts might become a bit more fuzzy. The
+maximum supported value is 40000, thresholds above this number will have the
+same effect as a threshold of 40000. The default value is `3000`.
+
diff --git a/docs/reference/esql/functions/appendix/median.asciidoc b/docs/reference/esql/functions/appendix/median.asciidoc
new file mode 100644
index 0000000000000..929a4ed0dae2c
--- /dev/null
+++ b/docs/reference/esql/functions/appendix/median.asciidoc
@@ -0,0 +1,7 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[WARNING]
+====
+`MEDIAN` is also {wikipedia}/Nondeterministic_algorithm[non-deterministic].
+This means you can get slightly different results using the same data.
+====
diff --git a/docs/reference/esql/functions/appendix/median_absolute_deviation.asciidoc b/docs/reference/esql/functions/appendix/median_absolute_deviation.asciidoc
new file mode 100644
index 0000000000000..a4f96c800946b
--- /dev/null
+++ b/docs/reference/esql/functions/appendix/median_absolute_deviation.asciidoc
@@ -0,0 +1,7 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[WARNING]
+====
+`MEDIAN_ABSOLUTE_DEVIATION` is also {wikipedia}/Nondeterministic_algorithm[non-deterministic].
+This means you can get slightly different results using the same data.
+====
diff --git a/docs/reference/esql/functions/appendix/values.asciidoc b/docs/reference/esql/functions/appendix/values.asciidoc
new file mode 100644
index 0000000000000..ec3cfff2db6a6
--- /dev/null
+++ b/docs/reference/esql/functions/appendix/values.asciidoc
@@ -0,0 +1,10 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[WARNING]
+====
+This can use a significant amount of memory and ES|QL doesn't yet
+grow aggregations beyond memory. So this aggregation will work until
+it is used to collect more values than can fit into memory. Once it
+collects too many values it will fail the query with
+a <<circuit-breaker-errors, Circuit Breaker Error>>.
+====
diff --git a/docs/reference/esql/functions/case.asciidoc b/docs/reference/esql/functions/case.asciidoc
deleted file mode 100644
index b5fda636135b2..0000000000000
--- a/docs/reference/esql/functions/case.asciidoc
+++ /dev/null
@@ -1,70 +0,0 @@
-[discrete]
-[[esql-case]]
-=== `CASE`
-
-*Syntax*
-
-[source,esql]
-----
-CASE(condition1, value1[, ..., conditionN, valueN][, default_value])
-----
-
-*Parameters*
-
-`conditionX`::
-A condition.
-
-`valueX`:: 
-The value that's returned when the corresponding condition is the first to
-evaluate to `true`.
-
-`default_value`::
-The default value that's is returned when no condition matches.
-
-*Description*
-
-Accepts pairs of conditions and values. The function returns the value that
-belongs to the first condition that evaluates to `true`.
-
-If the number of arguments is odd, the last argument is the default value which
-is returned when no condition matches. If the number of arguments is even, and
-no condition matches, the function returns `null`.
-
-*Example*
-
-Determine whether employees are monolingual, bilingual, or polyglot:
-
-[source,esql]
-[source.merge.styled,esql]
-----
-include::{esql-specs}/docs.csv-spec[tag=case]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/docs.csv-spec[tag=case-result]
-|===
-
-Calculate the total connection success rate based on log messages:
-
-[source,esql]
-[source.merge.styled,esql]
-----
-include::{esql-specs}/conditional.csv-spec[tag=docsCaseSuccessRate]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/conditional.csv-spec[tag=docsCaseSuccessRate-result]
-|===
-
-Calculate an hourly error rate as a percentage of the total number of log
-messages:
-
-[source,esql]
-[source.merge.styled,esql]
-----
-include::{esql-specs}/conditional.csv-spec[tag=docsCaseHourlyErrorRate]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/conditional.csv-spec[tag=docsCaseHourlyErrorRate-result]
-|===
diff --git a/docs/reference/esql/functions/count-distinct.asciidoc b/docs/reference/esql/functions/count-distinct.asciidoc
deleted file mode 100644
index a9f30d24e0e83..0000000000000
--- a/docs/reference/esql/functions/count-distinct.asciidoc
+++ /dev/null
@@ -1,85 +0,0 @@
-[discrete]
-[[esql-agg-count-distinct]]
-=== `COUNT_DISTINCT`
-
-*Syntax*
-
-[source,esql]
-----
-COUNT_DISTINCT(expression[, precision_threshold])
-----
-
-*Parameters*
-
-`expression`::
-Expression that outputs the values on which to perform a distinct count.
-
-`precision_threshold`::
-Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The
-maximum supported value is 40000. Thresholds above this number will have the
-same effect as a threshold of 40000. The default value is 3000.
-
-*Description*
-
-Returns the approximate number of distinct values.
-
-*Supported types*
-
-Can take any field type as input.
-
-*Examples*
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats_count_distinct.csv-spec[tag=count-distinct]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats_count_distinct.csv-spec[tag=count-distinct-result]
-|===
-
-With the optional second parameter to configure the precision threshold:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats_count_distinct.csv-spec[tag=count-distinct-precision]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats_count_distinct.csv-spec[tag=count-distinct-precision-result]
-|===
-
-The expression can use inline functions. This example splits a string into
-multiple values using the `SPLIT` function and counts the unique values:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats_count_distinct.csv-spec[tag=docsCountDistinctWithExpression]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats_count_distinct.csv-spec[tag=docsCountDistinctWithExpression-result]
-|===
-
-[discrete]
-[[esql-agg-count-distinct-approximate]]
-==== Counts are approximate
-
-Computing exact counts requires loading values into a set and returning its
-size. This doesn't scale when working on high-cardinality sets and/or large
-values as the required memory usage and the need to communicate those
-per-shard sets between nodes would utilize too many resources of the cluster.
-
-This `COUNT_DISTINCT` function is based on the
-https://static.googleusercontent.com/media/research.google.com/fr//pubs/archive/40671.pdf[HyperLogLog++]
-algorithm, which counts based on the hashes of the values with some interesting
-properties:
-
-include::../../aggregations/metrics/cardinality-aggregation.asciidoc[tag=explanation]
-
-The `COUNT_DISTINCT` function takes an optional second parameter to configure
-the precision threshold. The precision_threshold options allows to trade memory
-for accuracy, and defines a unique count below which counts are expected to be
-close to accurate. Above this value, counts might become a bit more fuzzy. The
-maximum supported value is 40000, thresholds above this number will have the
-same effect as a threshold of 40000. The default value is `3000`.
\ No newline at end of file
diff --git a/docs/reference/esql/functions/count.asciidoc b/docs/reference/esql/functions/count.asciidoc
deleted file mode 100644
index 66cfe76350cdd..0000000000000
--- a/docs/reference/esql/functions/count.asciidoc
+++ /dev/null
@@ -1,83 +0,0 @@
-[discrete]
-[[esql-agg-count]]
-=== `COUNT`
-
-*Syntax*
-
-[source,esql]
-----
-COUNT([expression])
-----
-
-*Parameters*
-
-`expression`::
-Expression that outputs values to be counted.
-If omitted, equivalent to `COUNT(*)` (the number of rows).
-
-*Description*
-
-Returns the total number (count) of input values.
-
-*Supported types*
-
-Can take any field type as input.
-
-*Examples*
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats.csv-spec[tag=count]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats.csv-spec[tag=count-result]
-|===
-
-To count the number of rows, use `COUNT()` or `COUNT(*)`:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/docs.csv-spec[tag=countAll]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/docs.csv-spec[tag=countAll-result]
-|===
-
-The expression can use inline functions. This example splits a string into
-multiple values using the `SPLIT` function and counts the values:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats.csv-spec[tag=docsCountWithExpression]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats.csv-spec[tag=docsCountWithExpression-result]
-|===
-
-[[esql-agg-count-or-null]]
-To count the number of times an expression returns `TRUE` use
-a <<esql-where>> command to remove rows that shouldn't be included:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats.csv-spec[tag=count-where]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats.csv-spec[tag=count-where-result]
-|===
-
-To count the same stream of data based on two different expressions
-use the pattern `COUNT(<expression> OR NULL)`:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats.csv-spec[tag=count-or-null]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats.csv-spec[tag=count-or-null-result]
-|===
diff --git a/docs/reference/esql/functions/description/count.asciidoc b/docs/reference/esql/functions/description/count.asciidoc
new file mode 100644
index 0000000000000..ee806d65a8ea3
--- /dev/null
+++ b/docs/reference/esql/functions/description/count.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Returns the total number (count) of input values.
diff --git a/docs/reference/esql/functions/description/count_distinct.asciidoc b/docs/reference/esql/functions/description/count_distinct.asciidoc
new file mode 100644
index 0000000000000..d10825bb991f5
--- /dev/null
+++ b/docs/reference/esql/functions/description/count_distinct.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Returns the approximate number of distinct values.
diff --git a/docs/reference/esql/functions/description/median.asciidoc b/docs/reference/esql/functions/description/median.asciidoc
new file mode 100644
index 0000000000000..ff3b7b32ed15e
--- /dev/null
+++ b/docs/reference/esql/functions/description/median.asciidoc
@@ -0,0 +1,7 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+The value that is greater than half of all values and less than half of all values, also known as the 50% <<esql-percentile>>.
+
+NOTE: Like <<esql-percentile>>, `MEDIAN` is <<esql-percentile-approximate,usually approximate>>.
diff --git a/docs/reference/esql/functions/description/median_absolute_deviation.asciidoc b/docs/reference/esql/functions/description/median_absolute_deviation.asciidoc
new file mode 100644
index 0000000000000..1a363920dd422
--- /dev/null
+++ b/docs/reference/esql/functions/description/median_absolute_deviation.asciidoc
@@ -0,0 +1,7 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Returns the median absolute deviation, a measure of variability. It is a robust statistic, meaning that it is useful for describing data that may have outliers, or may not be normally distributed. For such data it can be more descriptive than standard deviation.  It is calculated as the median of each data point's deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`.
+
+NOTE: Like <<esql-percentile>>, `MEDIAN_ABSOLUTE_DEVIATION` is <<esql-percentile-approximate,usually approximate>>.
diff --git a/docs/reference/esql/functions/description/mv_percentile.asciidoc b/docs/reference/esql/functions/description/mv_percentile.asciidoc
new file mode 100644
index 0000000000000..3e731f6525cec
--- /dev/null
+++ b/docs/reference/esql/functions/description/mv_percentile.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Converts a multivalued field into a single valued field containing the value at which a certain percentage of observed values occur.
diff --git a/docs/reference/esql/functions/description/mv_pseries_weighted_sum.asciidoc b/docs/reference/esql/functions/description/mv_pseries_weighted_sum.asciidoc
new file mode 100644
index 0000000000000..d464689f40a01
--- /dev/null
+++ b/docs/reference/esql/functions/description/mv_pseries_weighted_sum.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Converts a multivalued expression into a single-valued column by multiplying every element on the input list by its corresponding term in P-Series and computing the sum.
diff --git a/docs/reference/esql/functions/description/st_centroid_agg.asciidoc b/docs/reference/esql/functions/description/st_centroid_agg.asciidoc
new file mode 100644
index 0000000000000..740accf02c33f
--- /dev/null
+++ b/docs/reference/esql/functions/description/st_centroid_agg.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Calculate the spatial centroid over a field with spatial point geometry type.
diff --git a/docs/reference/esql/functions/description/sum.asciidoc b/docs/reference/esql/functions/description/sum.asciidoc
new file mode 100644
index 0000000000000..e3956567b8656
--- /dev/null
+++ b/docs/reference/esql/functions/description/sum.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+The sum of a numeric expression.
diff --git a/docs/reference/esql/functions/description/to_datetime.asciidoc b/docs/reference/esql/functions/description/to_datetime.asciidoc
index b37bd6b22ac2f..91cbfa0b5fe1e 100644
--- a/docs/reference/esql/functions/description/to_datetime.asciidoc
+++ b/docs/reference/esql/functions/description/to_datetime.asciidoc
@@ -3,3 +3,5 @@
 *Description*
 
 Converts an input value to a date value. A string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use <<esql-date_parse>>.
+
+NOTE: Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is truncated, not rounded.
diff --git a/docs/reference/esql/functions/description/values.asciidoc b/docs/reference/esql/functions/description/values.asciidoc
new file mode 100644
index 0000000000000..b3cebcce955f0
--- /dev/null
+++ b/docs/reference/esql/functions/description/values.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed. If you need the values returned in order use <<esql-mv_sort>>.
diff --git a/docs/reference/esql/functions/description/weighted_avg.asciidoc b/docs/reference/esql/functions/description/weighted_avg.asciidoc
new file mode 100644
index 0000000000000..a15d5d4ea171d
--- /dev/null
+++ b/docs/reference/esql/functions/description/weighted_avg.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+The weighted average of a numeric expression.
diff --git a/docs/reference/esql/functions/examples/bucket.asciidoc b/docs/reference/esql/functions/examples/bucket.asciidoc
index e1bba0529d7db..4afea30660339 100644
--- a/docs/reference/esql/functions/examples/bucket.asciidoc
+++ b/docs/reference/esql/functions/examples/bucket.asciidoc
@@ -86,10 +86,6 @@ include::{esql-specs}/bucket.csv-spec[tag=docsBucketNumericWithSpan]
 |===
 include::{esql-specs}/bucket.csv-spec[tag=docsBucketNumericWithSpan-result]
 |===
-
-NOTE: When providing the bucket size as the second parameter, it must be
-of a floating point type.
-
 Create hourly buckets for the last 24 hours, and calculate the number of events per hour:
 [source.merge.styled,esql]
 ----
diff --git a/docs/reference/esql/functions/examples/count.asciidoc b/docs/reference/esql/functions/examples/count.asciidoc
new file mode 100644
index 0000000000000..fb696b51e054c
--- /dev/null
+++ b/docs/reference/esql/functions/examples/count.asciidoc
@@ -0,0 +1,49 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Examples*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats.csv-spec[tag=count]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats.csv-spec[tag=count-result]
+|===
+To count the number of rows, use `COUNT()` or `COUNT(*)`
+[source.merge.styled,esql]
+----
+include::{esql-specs}/docs.csv-spec[tag=countAll]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/docs.csv-spec[tag=countAll-result]
+|===
+The expression can use inline functions. This example splits a string into multiple values using the `SPLIT` function and counts the values
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats.csv-spec[tag=docsCountWithExpression]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats.csv-spec[tag=docsCountWithExpression-result]
+|===
+To count the number of times an expression returns `TRUE` use a <<esql-where>> command to remove rows that shouldn't be included
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats.csv-spec[tag=count-where]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats.csv-spec[tag=count-where-result]
+|===
+To count the same stream of data based on two different expressions use the pattern `COUNT(<expression> OR NULL)`
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats.csv-spec[tag=count-or-null]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats.csv-spec[tag=count-or-null-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/count_distinct.asciidoc b/docs/reference/esql/functions/examples/count_distinct.asciidoc
new file mode 100644
index 0000000000000..44968c0652ec0
--- /dev/null
+++ b/docs/reference/esql/functions/examples/count_distinct.asciidoc
@@ -0,0 +1,31 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Examples*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats_count_distinct.csv-spec[tag=count-distinct]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats_count_distinct.csv-spec[tag=count-distinct-result]
+|===
+With the optional second parameter to configure the precision threshold
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats_count_distinct.csv-spec[tag=count-distinct-precision]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats_count_distinct.csv-spec[tag=count-distinct-precision-result]
+|===
+The expression can use inline functions. This example splits a string into multiple values using the `SPLIT` function and counts the unique values
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats_count_distinct.csv-spec[tag=docsCountDistinctWithExpression]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats_count_distinct.csv-spec[tag=docsCountDistinctWithExpression-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/median.asciidoc b/docs/reference/esql/functions/examples/median.asciidoc
new file mode 100644
index 0000000000000..cb6248dcff148
--- /dev/null
+++ b/docs/reference/esql/functions/examples/median.asciidoc
@@ -0,0 +1,22 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Examples*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats_percentile.csv-spec[tag=median]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats_percentile.csv-spec[tag=median-result]
+|===
+The expression can use inline functions. For example, to calculate the median of the maximum values of a multivalued column, first use `MV_MAX` to get the maximum value per row, and use the result with the `MEDIAN` function
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsMedianNestedExpression]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsMedianNestedExpression-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/median_absolute_deviation.asciidoc b/docs/reference/esql/functions/examples/median_absolute_deviation.asciidoc
new file mode 100644
index 0000000000000..20891126c20fb
--- /dev/null
+++ b/docs/reference/esql/functions/examples/median_absolute_deviation.asciidoc
@@ -0,0 +1,22 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Examples*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats_percentile.csv-spec[tag=median-absolute-deviation]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats_percentile.csv-spec[tag=median-absolute-deviation-result]
+|===
+The expression can use inline functions. For example, to calculate the the median absolute deviation of the maximum values of a multivalued column, first use `MV_MAX` to get the maximum value per row, and use the result with the `MEDIAN_ABSOLUTE_DEVIATION` function
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsMADNestedExpression]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsMADNestedExpression-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/mv_percentile.asciidoc b/docs/reference/esql/functions/examples/mv_percentile.asciidoc
new file mode 100644
index 0000000000000..9b20a5bef5e0d
--- /dev/null
+++ b/docs/reference/esql/functions/examples/mv_percentile.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/mv_percentile.csv-spec[tag=example]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/mv_percentile.csv-spec[tag=example-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/mv_pseries_weighted_sum.asciidoc b/docs/reference/esql/functions/examples/mv_pseries_weighted_sum.asciidoc
new file mode 100644
index 0000000000000..bce4deb1f5225
--- /dev/null
+++ b/docs/reference/esql/functions/examples/mv_pseries_weighted_sum.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/mv_pseries_weighted_sum.csv-spec[tag=example]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/mv_pseries_weighted_sum.csv-spec[tag=example-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/st_centroid_agg.asciidoc b/docs/reference/esql/functions/examples/st_centroid_agg.asciidoc
new file mode 100644
index 0000000000000..69c291b738828
--- /dev/null
+++ b/docs/reference/esql/functions/examples/st_centroid_agg.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/spatial.csv-spec[tag=st_centroid_agg-airports]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/spatial.csv-spec[tag=st_centroid_agg-airports-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/sum.asciidoc b/docs/reference/esql/functions/examples/sum.asciidoc
new file mode 100644
index 0000000000000..1c02ccd784a54
--- /dev/null
+++ b/docs/reference/esql/functions/examples/sum.asciidoc
@@ -0,0 +1,22 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Examples*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats.csv-spec[tag=sum]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats.csv-spec[tag=sum-result]
+|===
+The expression can use inline functions. For example, to calculate the sum of each employee's maximum salary changes, apply the `MV_MAX` function to each row and then sum the results
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats.csv-spec[tag=docsStatsSumNestedExpression]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats.csv-spec[tag=docsStatsSumNestedExpression-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/values.asciidoc b/docs/reference/esql/functions/examples/values.asciidoc
new file mode 100644
index 0000000000000..c013fc39d92ca
--- /dev/null
+++ b/docs/reference/esql/functions/examples/values.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/string.csv-spec[tag=values-grouped]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/string.csv-spec[tag=values-grouped-result]
+|===
+
diff --git a/docs/reference/esql/functions/examples/weighted_avg.asciidoc b/docs/reference/esql/functions/examples/weighted_avg.asciidoc
new file mode 100644
index 0000000000000..e8e8cc3eda006
--- /dev/null
+++ b/docs/reference/esql/functions/examples/weighted_avg.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/stats.csv-spec[tag=weighted-avg]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/stats.csv-spec[tag=weighted-avg-result]
+|===
+
diff --git a/docs/reference/esql/functions/kibana/definition/add.json b/docs/reference/esql/functions/kibana/definition/add.json
new file mode 100644
index 0000000000000..0932a76966560
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/add.json
@@ -0,0 +1,296 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "add",
+  "description" : "Add two numbers together. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date_period",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "time_duration",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date_period",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date_period",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date_period",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_period"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "time_duration",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "time_duration",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "time_duration",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "time_duration"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "unsigned_long"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/bucket.json b/docs/reference/esql/functions/kibana/definition/bucket.json
index 7141ca4c27443..94214a3a4f047 100644
--- a/docs/reference/esql/functions/kibana/definition/bucket.json
+++ b/docs/reference/esql/functions/kibana/definition/bucket.json
@@ -8,7 +8,7 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Numeric or date expression from which to derive buckets."
         },
@@ -16,17 +16,17 @@
           "name" : "buckets",
           "type" : "date_period",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Numeric or date expression from which to derive buckets."
         },
@@ -34,29 +34,269 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
+        },
+        {
+          "name" : "from",
+          "type" : "date",
+          "optional" : true,
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
+        },
+        {
+          "name" : "to",
+          "type" : "keyword",
+          "optional" : true,
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
+        },
+        {
+          "name" : "from",
+          "type" : "date",
+          "optional" : true,
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
+        },
+        {
+          "name" : "to",
+          "type" : "text",
+          "optional" : true,
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
+        },
+        {
+          "name" : "from",
+          "type" : "keyword",
+          "optional" : true,
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
+        },
+        {
+          "name" : "to",
+          "type" : "date",
+          "optional" : true,
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
+        },
+        {
+          "name" : "from",
+          "type" : "keyword",
+          "optional" : true,
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
+        },
+        {
+          "name" : "to",
+          "type" : "keyword",
+          "optional" : true,
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
+        },
+        {
+          "name" : "from",
+          "type" : "keyword",
+          "optional" : true,
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
+        },
+        {
+          "name" : "to",
+          "type" : "text",
+          "optional" : true,
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
+        },
+        {
+          "name" : "from",
+          "type" : "text",
+          "optional" : true,
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
+        },
+        {
+          "name" : "to",
+          "type" : "date",
+          "optional" : true,
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
+        },
+        {
+          "name" : "from",
+          "type" : "text",
+          "optional" : true,
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
+        },
+        {
+          "name" : "to",
+          "type" : "keyword",
+          "optional" : true,
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
+        },
+        {
+          "name" : "from",
+          "type" : "text",
+          "optional" : true,
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
+        },
+        {
+          "name" : "to",
+          "type" : "text",
+          "optional" : true,
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
           "optional" : false,
           "description" : "Numeric or date expression from which to derive buckets."
         },
@@ -64,11 +304,11 @@
           "name" : "buckets",
           "type" : "time_duration",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -82,7 +322,25 @@
           "name" : "buckets",
           "type" : "double",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         }
       ],
       "variadic" : false,
@@ -100,19 +358,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "double",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "double",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -130,19 +388,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "double",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "integer",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -160,19 +418,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "double",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "long",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -190,19 +448,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "integer",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "double",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -220,19 +478,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "integer",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "integer",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -250,19 +508,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "integer",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "long",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -280,19 +538,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "long",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "double",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -310,19 +568,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "long",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "integer",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -340,19 +598,37 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "long",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "long",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         }
       ],
       "variadic" : false,
@@ -370,7 +646,25 @@
           "name" : "buckets",
           "type" : "double",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         }
       ],
       "variadic" : false,
@@ -388,19 +682,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "double",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "double",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -418,19 +712,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "double",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "integer",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -448,19 +742,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "double",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "long",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -478,19 +772,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "integer",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "double",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -508,19 +802,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "integer",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "integer",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -538,19 +832,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "integer",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "long",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -568,19 +862,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "long",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "double",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -598,19 +892,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "long",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "integer",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -628,19 +922,37 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "long",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "long",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         }
       ],
       "variadic" : false,
@@ -658,7 +970,25 @@
           "name" : "buckets",
           "type" : "double",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         }
       ],
       "variadic" : false,
@@ -676,19 +1006,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "double",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "double",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -706,19 +1036,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "double",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "integer",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -736,19 +1066,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "double",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "long",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -766,19 +1096,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "integer",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "double",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -796,19 +1126,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "integer",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "integer",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -826,19 +1156,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "integer",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "long",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -856,19 +1186,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "long",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "double",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -886,19 +1216,19 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "long",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "integer",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
         }
       ],
       "variadic" : false,
@@ -916,19 +1246,37 @@
           "name" : "buckets",
           "type" : "integer",
           "optional" : false,
-          "description" : "Target number of buckets."
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         },
         {
           "name" : "from",
           "type" : "long",
           "optional" : true,
-          "description" : "Start of the range. Can be a number or a date expressed as a string."
+          "description" : "Start of the range. Can be a number, a date or a date expressed as a string."
         },
         {
           "name" : "to",
           "type" : "long",
           "optional" : true,
-          "description" : "End of the range. Can be a number or a date expressed as a string."
+          "description" : "End of the range. Can be a number, a date or a date expressed as a string."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Numeric or date expression from which to derive buckets."
+        },
+        {
+          "name" : "buckets",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         }
       ],
       "variadic" : false,
diff --git a/docs/reference/esql/functions/kibana/definition/case.json b/docs/reference/esql/functions/kibana/definition/case.json
index 5959eed62d37b..27705cd3897f9 100644
--- a/docs/reference/esql/functions/kibana/definition/case.json
+++ b/docs/reference/esql/functions/kibana/definition/case.json
@@ -50,13 +50,13 @@
         },
         {
           "name" : "trueValue",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "The value that's returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches."
         }
       ],
       "variadic" : true,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/coalesce.json b/docs/reference/esql/functions/kibana/definition/coalesce.json
index f00f471e63ecc..2459a4d51bb2d 100644
--- a/docs/reference/esql/functions/kibana/definition/coalesce.json
+++ b/docs/reference/esql/functions/kibana/definition/coalesce.json
@@ -74,19 +74,19 @@
       "params" : [
         {
           "name" : "first",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Expression to evaluate."
         },
         {
           "name" : "rest",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : true,
           "description" : "Other expression to evaluate."
         }
       ],
       "variadic" : true,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/count.json b/docs/reference/esql/functions/kibana/definition/count.json
new file mode 100644
index 0000000000000..2a15fb3bdd335
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/count.json
@@ -0,0 +1,159 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "agg",
+  "name" : "count",
+  "description" : "Returns the total number (count) of input values.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "boolean",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "cartesian_point",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "double",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "geo_point",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "ip",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "long",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "unsigned_long",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "version",
+          "optional" : true,
+          "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    }
+  ],
+  "examples" : [
+    "FROM employees\n| STATS COUNT(height)",
+    "FROM employees \n| STATS count = COUNT(*) BY languages \n| SORT languages DESC",
+    "ROW words=\"foo;bar;baz;qux;quux;foo\"\n| STATS word_count = COUNT(SPLIT(words, \";\"))",
+    "ROW n=1\n| WHERE n < 0\n| STATS COUNT(n)",
+    "ROW n=1\n| STATS COUNT(n > 0 OR NULL), COUNT(n < 0 OR NULL)"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/count_distinct.json b/docs/reference/esql/functions/kibana/definition/count_distinct.json
new file mode 100644
index 0000000000000..f6a148783ba42
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/count_distinct.json
@@ -0,0 +1,607 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "agg",
+  "name" : "count_distinct",
+  "description" : "Returns the approximate number of distinct values.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "unsigned_long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "unsigned_long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "unsigned_long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "unsigned_long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "unsigned_long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "unsigned_long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "unsigned_long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "unsigned_long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "version",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "version",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "version",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "version",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "unsigned_long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    }
+  ],
+  "examples" : [
+    "FROM hosts\n| STATS COUNT_DISTINCT(ip0), COUNT_DISTINCT(ip1)",
+    "FROM hosts\n| STATS COUNT_DISTINCT(ip0, 80000), COUNT_DISTINCT(ip1, 5)",
+    "ROW words=\"foo;bar;baz;qux;quux;foo\"\n| STATS distinct_word_count = COUNT_DISTINCT(SPLIT(words, \";\"))"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/date_diff.json b/docs/reference/esql/functions/kibana/definition/date_diff.json
index 7995d3c6d32b6..d6589f041075d 100644
--- a/docs/reference/esql/functions/kibana/definition/date_diff.json
+++ b/docs/reference/esql/functions/kibana/definition/date_diff.json
@@ -14,13 +14,13 @@
         },
         {
           "name" : "startTimestamp",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "A string representing a start timestamp"
         },
         {
           "name" : "endTimestamp",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "A string representing an end timestamp"
         }
@@ -38,13 +38,13 @@
         },
         {
           "name" : "startTimestamp",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "A string representing a start timestamp"
         },
         {
           "name" : "endTimestamp",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "A string representing an end timestamp"
         }
diff --git a/docs/reference/esql/functions/kibana/definition/date_extract.json b/docs/reference/esql/functions/kibana/definition/date_extract.json
index 75cedcc191b50..557f0e0a47e54 100644
--- a/docs/reference/esql/functions/kibana/definition/date_extract.json
+++ b/docs/reference/esql/functions/kibana/definition/date_extract.json
@@ -14,7 +14,7 @@
         },
         {
           "name" : "date",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Date expression. If `null`, the function returns `null`."
         }
@@ -32,7 +32,7 @@
         },
         {
           "name" : "date",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Date expression. If `null`, the function returns `null`."
         }
diff --git a/docs/reference/esql/functions/kibana/definition/date_format.json b/docs/reference/esql/functions/kibana/definition/date_format.json
index 5e8587c046d70..7bd01d7f4ef31 100644
--- a/docs/reference/esql/functions/kibana/definition/date_format.json
+++ b/docs/reference/esql/functions/kibana/definition/date_format.json
@@ -14,7 +14,7 @@
         },
         {
           "name" : "date",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Date expression. If `null`, the function returns `null`."
         }
@@ -32,7 +32,7 @@
         },
         {
           "name" : "date",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Date expression. If `null`, the function returns `null`."
         }
diff --git a/docs/reference/esql/functions/kibana/definition/date_parse.json b/docs/reference/esql/functions/kibana/definition/date_parse.json
index 890179143bef8..9400340750c2a 100644
--- a/docs/reference/esql/functions/kibana/definition/date_parse.json
+++ b/docs/reference/esql/functions/kibana/definition/date_parse.json
@@ -20,7 +20,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -38,7 +38,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -56,7 +56,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -74,7 +74,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     }
   ],
   "examples" : [
diff --git a/docs/reference/esql/functions/kibana/definition/date_trunc.json b/docs/reference/esql/functions/kibana/definition/date_trunc.json
index 3d8658c496529..bd3f362d1670b 100644
--- a/docs/reference/esql/functions/kibana/definition/date_trunc.json
+++ b/docs/reference/esql/functions/kibana/definition/date_trunc.json
@@ -14,13 +14,13 @@
         },
         {
           "name" : "date",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Date expression"
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -32,13 +32,13 @@
         },
         {
           "name" : "date",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Date expression"
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     }
   ],
   "examples" : [
diff --git a/docs/reference/esql/functions/kibana/definition/div.json b/docs/reference/esql/functions/kibana/definition/div.json
new file mode 100644
index 0000000000000..8bd2c33720d5f
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/div.json
@@ -0,0 +1,189 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "div",
+  "description" : "Divide one number by another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+  "note" : "Division of two integer types will yield an integer result, rounding towards 0. If you need floating point division, <<esql-cast-operator>> one of the arguments to a `DOUBLE`.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "unsigned_long"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/ends_with.json b/docs/reference/esql/functions/kibana/definition/ends_with.json
index b43181817ef9e..754d2c965287b 100644
--- a/docs/reference/esql/functions/kibana/definition/ends_with.json
+++ b/docs/reference/esql/functions/kibana/definition/ends_with.json
@@ -22,6 +22,42 @@
       "variadic" : false,
       "returnType" : "boolean"
     },
+    {
+      "params" : [
+        {
+          "name" : "str",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "String expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "suffix",
+          "type" : "text",
+          "optional" : false,
+          "description" : "String expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "str",
+          "type" : "text",
+          "optional" : false,
+          "description" : "String expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "suffix",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "String expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/equals.json b/docs/reference/esql/functions/kibana/definition/equals.json
new file mode 100644
index 0000000000000..eca80ccdbf657
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/equals.json
@@ -0,0 +1,405 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "equals",
+  "description" : "Check if two fields are equal. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+  "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "cartesian_point",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "cartesian_point",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "cartesian_shape",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "cartesian_shape",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "geo_point",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "geo_point",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "geo_shape",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "geo_shape",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/greater_than.json b/docs/reference/esql/functions/kibana/definition/greater_than.json
new file mode 100644
index 0000000000000..7831b0f41cd9d
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/greater_than.json
@@ -0,0 +1,315 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "greater_than",
+  "description" : "Check if one field is greater than another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+  "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json
new file mode 100644
index 0000000000000..b6a40a838c393
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json
@@ -0,0 +1,315 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "greater_than_or_equal",
+  "description" : "Check if one field is greater than or equal to another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+  "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/in.json b/docs/reference/esql/functions/kibana/definition/in.json
new file mode 100644
index 0000000000000..abf3bd64e2822
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/in.json
@@ -0,0 +1,263 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "in",
+  "description" : "The `IN` operator allows testing whether a field or expression equals an element in a list of literals, fields or expressions.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "cartesian_point",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "cartesian_point",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "cartesian_shape",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "cartesian_shape",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "geo_point",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "geo_point",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "geo_shape",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "geo_shape",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "text",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "text",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "inlist",
+          "type" : "version",
+          "optional" : false,
+          "description" : "A list of items."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    }
+  ],
+  "examples" : [
+    "ROW a = 1, b = 4, c = 3\n| WHERE c-a IN (3, b / 2, a)"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/less_than.json b/docs/reference/esql/functions/kibana/definition/less_than.json
new file mode 100644
index 0000000000000..bf6b9c5c08774
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/less_than.json
@@ -0,0 +1,315 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "less_than",
+  "description" : "Check if one field is less than another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+  "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json
new file mode 100644
index 0000000000000..4e57161887141
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json
@@ -0,0 +1,315 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "less_than_or_equal",
+  "description" : "Check if one field is less than or equal to another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+  "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/like.json b/docs/reference/esql/functions/kibana/definition/like.json
new file mode 100644
index 0000000000000..9a215ff88e399
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/like.json
@@ -0,0 +1,47 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "like",
+  "description" : "Use `LIKE` to filter data based on string patterns using wildcards. `LIKE`\nusually acts on a field placed on the left-hand side of the operator, but it can\nalso act on a constant (literal) expression. The right-hand side of the operator\nrepresents the pattern.\n\nThe following wildcard characters are supported:\n\n* `*` matches zero or more characters.\n* `?` matches one character.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "str",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "A literal expression."
+        },
+        {
+          "name" : "pattern",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Pattern."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "str",
+          "type" : "text",
+          "optional" : false,
+          "description" : "A literal expression."
+        },
+        {
+          "name" : "pattern",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Pattern."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    }
+  ],
+  "examples" : [
+    "FROM employees\n| WHERE first_name LIKE \"?b*\"\n| KEEP first_name, last_name"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/max.json b/docs/reference/esql/functions/kibana/definition/max.json
index 853cb9f9a97c3..b13d367d37345 100644
--- a/docs/reference/esql/functions/kibana/definition/max.json
+++ b/docs/reference/esql/functions/kibana/definition/max.json
@@ -20,13 +20,13 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : ""
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -64,6 +64,18 @@
       "variadic" : false,
       "returnType" : "ip"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
     {
       "params" : [
         {
@@ -75,6 +87,30 @@
       ],
       "variadic" : false,
       "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "text"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "version",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "version"
     }
   ],
   "examples" : [
diff --git a/docs/reference/esql/functions/kibana/definition/median.json b/docs/reference/esql/functions/kibana/definition/median.json
new file mode 100644
index 0000000000000..4887a4497e813
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/median.json
@@ -0,0 +1,49 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "agg",
+  "name" : "median",
+  "description" : "The value that is greater than half of all values and less than half of all values, also known as the 50% <<esql-percentile>>.",
+  "note" : "Like <<esql-percentile>>, `MEDIAN` is <<esql-percentile-approximate,usually approximate>>.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "double",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "integer",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "long",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    }
+  ],
+  "examples" : [
+    "FROM employees\n| STATS MEDIAN(salary), PERCENTILE(salary, 50)",
+    "FROM employees\n| STATS median_max_salary_change = MEDIAN(MV_MAX(salary_change))"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/median_absolute_deviation.json b/docs/reference/esql/functions/kibana/definition/median_absolute_deviation.json
new file mode 100644
index 0000000000000..4a8b1cd30611f
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/median_absolute_deviation.json
@@ -0,0 +1,49 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "agg",
+  "name" : "median_absolute_deviation",
+  "description" : "Returns the median absolute deviation, a measure of variability. It is a robust statistic, meaning that it is useful for describing data that may have outliers, or may not be normally distributed. For such data it can be more descriptive than standard deviation.\n\nIt is calculated as the median of each data point's deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`.",
+  "note" : "Like <<esql-percentile>>, `MEDIAN_ABSOLUTE_DEVIATION` is <<esql-percentile-approximate,usually approximate>>.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "double",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "integer",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "long",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    }
+  ],
+  "examples" : [
+    "FROM employees\n| STATS MEDIAN(salary), MEDIAN_ABSOLUTE_DEVIATION(salary)",
+    "FROM employees\n| STATS m_a_d_max_salary_change = MEDIAN_ABSOLUTE_DEVIATION(MV_MAX(salary_change))"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/min.json b/docs/reference/esql/functions/kibana/definition/min.json
index 1c0c02eb9860f..338ed10d67b2e 100644
--- a/docs/reference/esql/functions/kibana/definition/min.json
+++ b/docs/reference/esql/functions/kibana/definition/min.json
@@ -20,13 +20,13 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : ""
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -64,6 +64,18 @@
       "variadic" : false,
       "returnType" : "ip"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
     {
       "params" : [
         {
@@ -75,6 +87,30 @@
       ],
       "variadic" : false,
       "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "text"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "version",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "version"
     }
   ],
   "examples" : [
diff --git a/docs/reference/esql/functions/kibana/definition/mod.json b/docs/reference/esql/functions/kibana/definition/mod.json
new file mode 100644
index 0000000000000..c43f697127249
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/mod.json
@@ -0,0 +1,188 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "mod",
+  "description" : "Divide one number by another and return the remainder. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "unsigned_long"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/mul.json b/docs/reference/esql/functions/kibana/definition/mul.json
new file mode 100644
index 0000000000000..9e42a55bdffef
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/mul.json
@@ -0,0 +1,188 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "mul",
+  "description" : "Multiply two numbers together. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "A numeric value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "unsigned_long"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/mv_append.json b/docs/reference/esql/functions/kibana/definition/mv_append.json
index 8ee4e7297cc3a..3365226141f8f 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_append.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_append.json
@@ -62,19 +62,19 @@
       "params" : [
         {
           "name" : "field1",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : ""
         },
         {
           "name" : "field2",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : ""
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/mv_count.json b/docs/reference/esql/functions/kibana/definition/mv_count.json
index d414e5b957495..f125327314f4e 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_count.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_count.json
@@ -44,7 +44,7 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Multivalue expression."
         }
diff --git a/docs/reference/esql/functions/kibana/definition/mv_dedupe.json b/docs/reference/esql/functions/kibana/definition/mv_dedupe.json
index 7ab287bc94d34..7d66e3dcc0b9b 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_dedupe.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_dedupe.json
@@ -45,13 +45,13 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Multivalue expression."
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/mv_first.json b/docs/reference/esql/functions/kibana/definition/mv_first.json
index e3141e800e4ad..de6e642068517 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_first.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_first.json
@@ -44,13 +44,13 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Multivalue expression."
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/mv_last.json b/docs/reference/esql/functions/kibana/definition/mv_last.json
index e55d66dbf8b93..ea1293e7acfec 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_last.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_last.json
@@ -44,13 +44,13 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Multivalue expression."
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/mv_max.json b/docs/reference/esql/functions/kibana/definition/mv_max.json
index 0783f6d6d5cbc..eb25369f78f77 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_max.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_max.json
@@ -20,13 +20,13 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Multivalue expression."
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/mv_min.json b/docs/reference/esql/functions/kibana/definition/mv_min.json
index cc23df386356e..87ad94338492e 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_min.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_min.json
@@ -20,13 +20,13 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Multivalue expression."
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/mv_percentile.json b/docs/reference/esql/functions/kibana/definition/mv_percentile.json
new file mode 100644
index 0000000000000..dad611122f0db
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/mv_percentile.json
@@ -0,0 +1,173 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "mv_percentile",
+  "description" : "Converts a multivalued field into a single valued field containing the value at which a certain percentage of observed values occur.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "percentile",
+          "type" : "double",
+          "optional" : false,
+          "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "percentile",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "percentile",
+          "type" : "long",
+          "optional" : false,
+          "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "percentile",
+          "type" : "double",
+          "optional" : false,
+          "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "percentile",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "percentile",
+          "type" : "long",
+          "optional" : false,
+          "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "percentile",
+          "type" : "double",
+          "optional" : false,
+          "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "percentile",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "percentile",
+          "type" : "long",
+          "optional" : false,
+          "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    }
+  ],
+  "examples" : [
+    "ROW values = [5, 5, 10, 12, 5000]\n| EVAL p50 = MV_PERCENTILE(values, 50), median = MV_MEDIAN(values)"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/mv_pseries_weighted_sum.json b/docs/reference/esql/functions/kibana/definition/mv_pseries_weighted_sum.json
new file mode 100644
index 0000000000000..626f7befbb12e
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/mv_pseries_weighted_sum.json
@@ -0,0 +1,29 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "mv_pseries_weighted_sum",
+  "description" : "Converts a multivalued expression into a single-valued column by multiplying every element on the input list by its corresponding term in P-Series and computing the sum.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "p",
+          "type" : "double",
+          "optional" : false,
+          "description" : "It is a constant number that represents the 'p' parameter in the P-Series. It impacts every element's contribution to the weighted sum."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    }
+  ],
+  "examples" : [
+    "ROW a = [70.0, 45.0, 21.0, 21.0, 21.0]\n| EVAL sum = MV_PSERIES_WEIGHTED_SUM(a, 1.5)\n| KEEP sum"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/mv_slice.json b/docs/reference/esql/functions/kibana/definition/mv_slice.json
index 30d0e1179dc89..ff52467b7d84a 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_slice.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_slice.json
@@ -80,7 +80,7 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Multivalue expression. If `null`, the function returns `null`."
         },
@@ -98,7 +98,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/mv_sort.json b/docs/reference/esql/functions/kibana/definition/mv_sort.json
index 28b4c9e8d6fea..d2bbd2c0fdbf4 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_sort.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_sort.json
@@ -26,7 +26,7 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Multivalue expression. If `null`, the function returns `null`."
         },
@@ -38,7 +38,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
diff --git a/docs/reference/esql/functions/kibana/definition/neg.json b/docs/reference/esql/functions/kibana/definition/neg.json
new file mode 100644
index 0000000000000..6a7e86fd8e1a8
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/neg.json
@@ -0,0 +1,68 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "neg",
+  "description" : "Returns the negation of the argument.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_period",
+          "optional" : false,
+          "description" : "A numeric value or a date time interval."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_period"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time interval."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time interval."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time interval."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "time_duration",
+          "optional" : false,
+          "description" : "A numeric value or a date time interval."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "time_duration"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/not_equals.json b/docs/reference/esql/functions/kibana/definition/not_equals.json
new file mode 100644
index 0000000000000..4b4d22a5abef4
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/not_equals.json
@@ -0,0 +1,405 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "not_equals",
+  "description" : "Check if two fields are unequal. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+  "note" : "This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "cartesian_point",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "cartesian_point",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "cartesian_shape",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "cartesian_shape",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "geo_point",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "geo_point",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "geo_shape",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "geo_shape",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "text",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "version",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/now.json b/docs/reference/esql/functions/kibana/definition/now.json
index 9cdb4945afa2e..1a2fc3a1dc42a 100644
--- a/docs/reference/esql/functions/kibana/definition/now.json
+++ b/docs/reference/esql/functions/kibana/definition/now.json
@@ -6,7 +6,7 @@
   "signatures" : [
     {
       "params" : [ ],
-      "returnType" : "datetime"
+      "returnType" : "date"
     }
   ],
   "examples" : [
diff --git a/docs/reference/esql/functions/kibana/definition/rlike.json b/docs/reference/esql/functions/kibana/definition/rlike.json
new file mode 100644
index 0000000000000..09bd1e033fe10
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/rlike.json
@@ -0,0 +1,47 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "rlike",
+  "description" : "Use `RLIKE` to filter data based on string patterns using using\n<<regexp-syntax,regular expressions>>. `RLIKE` usually acts on a field placed on\nthe left-hand side of the operator, but it can also act on a constant (literal)\nexpression. The right-hand side of the operator represents the pattern.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "str",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "A literal value."
+        },
+        {
+          "name" : "pattern",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "A regular expression."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "str",
+          "type" : "text",
+          "optional" : false,
+          "description" : "A literal value."
+        },
+        {
+          "name" : "pattern",
+          "type" : "text",
+          "optional" : false,
+          "description" : "A regular expression."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "boolean"
+    }
+  ],
+  "examples" : [
+    "FROM employees\n| WHERE first_name RLIKE \".leja.*\"\n| KEEP first_name, last_name"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/st_centroid_agg.json b/docs/reference/esql/functions/kibana/definition/st_centroid_agg.json
new file mode 100644
index 0000000000000..b01f91526709a
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/st_centroid_agg.json
@@ -0,0 +1,35 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "agg",
+  "name" : "st_centroid_agg",
+  "description" : "Calculate the spatial centroid over a field with spatial point geometry type.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "cartesian_point",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "cartesian_point"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "geo_point",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "geo_point"
+    }
+  ],
+  "examples" : [
+    "FROM airports\n| STATS centroid=ST_CENTROID_AGG(location)"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/starts_with.json b/docs/reference/esql/functions/kibana/definition/starts_with.json
index b04e9c6837d46..bfa3e7b0e5e42 100644
--- a/docs/reference/esql/functions/kibana/definition/starts_with.json
+++ b/docs/reference/esql/functions/kibana/definition/starts_with.json
@@ -22,6 +22,42 @@
       "variadic" : false,
       "returnType" : "boolean"
     },
+    {
+      "params" : [
+        {
+          "name" : "str",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "String expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "prefix",
+          "type" : "text",
+          "optional" : false,
+          "description" : "String expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "str",
+          "type" : "text",
+          "optional" : false,
+          "description" : "String expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "prefix",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "String expression. If `null`, the function returns `null`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/sub.json b/docs/reference/esql/functions/kibana/definition/sub.json
new file mode 100644
index 0000000000000..37e3852865e7f
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/sub.json
@@ -0,0 +1,260 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "operator",
+  "name" : "sub",
+  "description" : "Subtract one number from another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date_period",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "time_duration",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date_period",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date_period",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_period"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "time_duration",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "time_duration",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "time_duration"
+    },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        },
+        {
+          "name" : "rhs",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "A numeric value or a date time value."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "unsigned_long"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/sum.json b/docs/reference/esql/functions/kibana/definition/sum.json
new file mode 100644
index 0000000000000..b9235b6ba04de
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/sum.json
@@ -0,0 +1,48 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "agg",
+  "name" : "sum",
+  "description" : "The sum of a numeric expression.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "double",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "integer",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "long",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    }
+  ],
+  "examples" : [
+    "FROM employees\n| STATS SUM(languages)",
+    "FROM employees\n| STATS total_salary_changes = SUM(MV_MAX(salary_change))"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/to_datetime.json b/docs/reference/esql/functions/kibana/definition/to_datetime.json
index 10fcf8b22e8b0..032e8e1cbda34 100644
--- a/docs/reference/esql/functions/kibana/definition/to_datetime.json
+++ b/docs/reference/esql/functions/kibana/definition/to_datetime.json
@@ -3,18 +3,19 @@
   "type" : "eval",
   "name" : "to_datetime",
   "description" : "Converts an input value to a date value.\nA string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`.\nTo convert dates in other formats, use <<esql-date_parse>>.",
+  "note" : "Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is truncated, not rounded.",
   "signatures" : [
     {
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Input value. The input can be a single- or multi-valued column or an expression."
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -26,7 +27,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -38,7 +39,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -50,7 +51,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -62,7 +63,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -74,7 +75,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -86,7 +87,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     }
   ],
   "examples" : [
diff --git a/docs/reference/esql/functions/kibana/definition/to_double.json b/docs/reference/esql/functions/kibana/definition/to_double.json
index f4e414068db61..ae7e4832bfb3c 100644
--- a/docs/reference/esql/functions/kibana/definition/to_double.json
+++ b/docs/reference/esql/functions/kibana/definition/to_double.json
@@ -56,7 +56,7 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Input value. The input can be a single- or multi-valued column or an expression."
         }
diff --git a/docs/reference/esql/functions/kibana/definition/to_integer.json b/docs/reference/esql/functions/kibana/definition/to_integer.json
index 2776d8b29c412..5150d12936711 100644
--- a/docs/reference/esql/functions/kibana/definition/to_integer.json
+++ b/docs/reference/esql/functions/kibana/definition/to_integer.json
@@ -32,7 +32,7 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Input value. The input can be a single- or multi-valued column or an expression."
         }
diff --git a/docs/reference/esql/functions/kibana/definition/to_long.json b/docs/reference/esql/functions/kibana/definition/to_long.json
index e3218eba9642a..5fd4bce34e7e0 100644
--- a/docs/reference/esql/functions/kibana/definition/to_long.json
+++ b/docs/reference/esql/functions/kibana/definition/to_long.json
@@ -44,7 +44,7 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Input value. The input can be a single- or multi-valued column or an expression."
         }
diff --git a/docs/reference/esql/functions/kibana/definition/to_string.json b/docs/reference/esql/functions/kibana/definition/to_string.json
index ef03cc06ea636..ea94171834908 100644
--- a/docs/reference/esql/functions/kibana/definition/to_string.json
+++ b/docs/reference/esql/functions/kibana/definition/to_string.json
@@ -44,7 +44,7 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Input value. The input can be a single- or multi-valued column or an expression."
         }
diff --git a/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json b/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json
index d9cba641573fb..5521241224d61 100644
--- a/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json
+++ b/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json
@@ -20,7 +20,7 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "Input value. The input can be a single- or multi-valued column or an expression."
         }
diff --git a/docs/reference/esql/functions/kibana/definition/top.json b/docs/reference/esql/functions/kibana/definition/top.json
index 7502886e51c6c..c688bf5ea77c8 100644
--- a/docs/reference/esql/functions/kibana/definition/top.json
+++ b/docs/reference/esql/functions/kibana/definition/top.json
@@ -32,7 +32,7 @@
       "params" : [
         {
           "name" : "field",
-          "type" : "datetime",
+          "type" : "date",
           "optional" : false,
           "description" : "The field to collect the top values for."
         },
@@ -50,7 +50,7 @@
         }
       ],
       "variadic" : false,
-      "returnType" : "datetime"
+      "returnType" : "date"
     },
     {
       "params" : [
@@ -100,6 +100,30 @@
       "variadic" : false,
       "returnType" : "integer"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "The field to collect the top values for."
+        },
+        {
+          "name" : "limit",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "The maximum number of values to collect."
+        },
+        {
+          "name" : "order",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "The order to calculate the top values. Either `asc` or `desc`."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "ip"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/values.json b/docs/reference/esql/functions/kibana/definition/values.json
new file mode 100644
index 0000000000000..d9f37cd1ac83d
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/values.json
@@ -0,0 +1,119 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "agg",
+  "name" : "values",
+  "description" : "Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed. If you need the values returned in order use <<esql-mv_sort>>.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "double",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "integer",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "ip",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "ip"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "long",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "text"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "version",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "version"
+    }
+  ],
+  "examples" : [
+    "  FROM employees\n| EVAL first_letter = SUBSTRING(first_name, 0, 1)\n| STATS first_name=MV_SORT(VALUES(first_name)) BY first_letter\n| SORT first_letter"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/definition/weighted_avg.json b/docs/reference/esql/functions/kibana/definition/weighted_avg.json
new file mode 100644
index 0000000000000..a04a5bff62bbb
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/weighted_avg.json
@@ -0,0 +1,173 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "agg",
+  "name" : "weighted_avg",
+  "description" : "The weighted average of a numeric expression.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "weight",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric weight."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "weight",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric weight."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "weight",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric weight."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "weight",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric weight."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "weight",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric weight."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "weight",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric weight."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "weight",
+          "type" : "double",
+          "optional" : false,
+          "description" : "A numeric weight."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "weight",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "A numeric weight."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "number",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric value."
+        },
+        {
+          "name" : "weight",
+          "type" : "long",
+          "optional" : false,
+          "description" : "A numeric weight."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    }
+  ],
+  "examples" : [
+    "FROM employees\n| STATS w_avg = WEIGHTED_AVG(salary, height) by languages\n| EVAL w_avg = ROUND(w_avg)\n| KEEP w_avg, languages\n| SORT languages"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/docs/add.md b/docs/reference/esql/functions/kibana/docs/add.md
new file mode 100644
index 0000000000000..3f99bd4c77551
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/add.md
@@ -0,0 +1,7 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### ADD
+Add two numbers together. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.
+
diff --git a/docs/reference/esql/functions/kibana/docs/count.md b/docs/reference/esql/functions/kibana/docs/count.md
new file mode 100644
index 0000000000000..dc9c356a847ed
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/count.md
@@ -0,0 +1,11 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### COUNT
+Returns the total number (count) of input values.
+
+```
+FROM employees
+| STATS COUNT(height)
+```
diff --git a/docs/reference/esql/functions/kibana/docs/count_distinct.md b/docs/reference/esql/functions/kibana/docs/count_distinct.md
new file mode 100644
index 0000000000000..a6b451bf9d38d
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/count_distinct.md
@@ -0,0 +1,11 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### COUNT_DISTINCT
+Returns the approximate number of distinct values.
+
+```
+FROM hosts
+| STATS COUNT_DISTINCT(ip0), COUNT_DISTINCT(ip1)
+```
diff --git a/docs/reference/esql/functions/kibana/docs/div.md b/docs/reference/esql/functions/kibana/docs/div.md
new file mode 100644
index 0000000000000..a8b7b4e58f376
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/div.md
@@ -0,0 +1,8 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### DIV
+Divide one number by another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.
+
+Note: Division of two integer types will yield an integer result, rounding towards 0. If you need floating point division, <<esql-cast-operator>> one of the arguments to a `DOUBLE`.
diff --git a/docs/reference/esql/functions/kibana/docs/equals.md b/docs/reference/esql/functions/kibana/docs/equals.md
new file mode 100644
index 0000000000000..b8fcb72c2ccd5
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/equals.md
@@ -0,0 +1,8 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### EQUALS
+Check if two fields are equal. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.
+
+Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
diff --git a/docs/reference/esql/functions/kibana/docs/greater_than.md b/docs/reference/esql/functions/kibana/docs/greater_than.md
new file mode 100644
index 0000000000000..67f99eda3aed7
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/greater_than.md
@@ -0,0 +1,8 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### GREATER_THAN
+Check if one field is greater than another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.
+
+Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
diff --git a/docs/reference/esql/functions/kibana/docs/greater_than_or_equal.md b/docs/reference/esql/functions/kibana/docs/greater_than_or_equal.md
new file mode 100644
index 0000000000000..73d3ac6371b07
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/greater_than_or_equal.md
@@ -0,0 +1,8 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### GREATER_THAN_OR_EQUAL
+Check if one field is greater than or equal to another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.
+
+Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
diff --git a/docs/reference/esql/functions/kibana/docs/in.md b/docs/reference/esql/functions/kibana/docs/in.md
new file mode 100644
index 0000000000000..e096146374f38
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/in.md
@@ -0,0 +1,11 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### IN
+The `IN` operator allows testing whether a field or expression equals an element in a list of literals, fields or expressions.
+
+```
+ROW a = 1, b = 4, c = 3
+| WHERE c-a IN (3, b / 2, a)
+```
diff --git a/docs/reference/esql/functions/kibana/docs/less_than.md b/docs/reference/esql/functions/kibana/docs/less_than.md
new file mode 100644
index 0000000000000..0d171d06c68d3
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/less_than.md
@@ -0,0 +1,8 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### LESS_THAN
+Check if one field is less than another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.
+
+Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
diff --git a/docs/reference/esql/functions/kibana/docs/less_than_or_equal.md b/docs/reference/esql/functions/kibana/docs/less_than_or_equal.md
new file mode 100644
index 0000000000000..acb92288c2c46
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/less_than_or_equal.md
@@ -0,0 +1,8 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### LESS_THAN_OR_EQUAL
+Check if one field is less than or equal to another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.
+
+Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
diff --git a/docs/reference/esql/functions/kibana/docs/like.md b/docs/reference/esql/functions/kibana/docs/like.md
new file mode 100644
index 0000000000000..4c400bdc65479
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/like.md
@@ -0,0 +1,20 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### LIKE
+Use `LIKE` to filter data based on string patterns using wildcards. `LIKE`
+usually acts on a field placed on the left-hand side of the operator, but it can
+also act on a constant (literal) expression. The right-hand side of the operator
+represents the pattern.
+
+The following wildcard characters are supported:
+
+* `*` matches zero or more characters.
+* `?` matches one character.
+
+```
+FROM employees
+| WHERE first_name LIKE "?b*"
+| KEEP first_name, last_name
+```
diff --git a/docs/reference/esql/functions/kibana/docs/median.md b/docs/reference/esql/functions/kibana/docs/median.md
new file mode 100644
index 0000000000000..7a4370b4d2551
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/median.md
@@ -0,0 +1,12 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### MEDIAN
+The value that is greater than half of all values and less than half of all values, also known as the 50% <<esql-percentile>>.
+
+```
+FROM employees
+| STATS MEDIAN(salary), PERCENTILE(salary, 50)
+```
+Note: Like <<esql-percentile>>, `MEDIAN` is <<esql-percentile-approximate,usually approximate>>.
diff --git a/docs/reference/esql/functions/kibana/docs/median_absolute_deviation.md b/docs/reference/esql/functions/kibana/docs/median_absolute_deviation.md
new file mode 100644
index 0000000000000..8db113deb2c49
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/median_absolute_deviation.md
@@ -0,0 +1,14 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### MEDIAN_ABSOLUTE_DEVIATION
+Returns the median absolute deviation, a measure of variability. It is a robust statistic, meaning that it is useful for describing data that may have outliers, or may not be normally distributed. For such data it can be more descriptive than standard deviation.
+
+It is calculated as the median of each data point's deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`.
+
+```
+FROM employees
+| STATS MEDIAN(salary), MEDIAN_ABSOLUTE_DEVIATION(salary)
+```
+Note: Like <<esql-percentile>>, `MEDIAN_ABSOLUTE_DEVIATION` is <<esql-percentile-approximate,usually approximate>>.
diff --git a/docs/reference/esql/functions/kibana/docs/mod.md b/docs/reference/esql/functions/kibana/docs/mod.md
new file mode 100644
index 0000000000000..e6b3c92406072
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/mod.md
@@ -0,0 +1,7 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### MOD
+Divide one number by another and return the remainder. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.
+
diff --git a/docs/reference/esql/functions/kibana/docs/mul.md b/docs/reference/esql/functions/kibana/docs/mul.md
new file mode 100644
index 0000000000000..3f24f3b1a67bb
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/mul.md
@@ -0,0 +1,7 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### MUL
+Multiply two numbers together. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.
+
diff --git a/docs/reference/esql/functions/kibana/docs/mv_percentile.md b/docs/reference/esql/functions/kibana/docs/mv_percentile.md
new file mode 100644
index 0000000000000..560a0aefa1dc3
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/mv_percentile.md
@@ -0,0 +1,11 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### MV_PERCENTILE
+Converts a multivalued field into a single valued field containing the value at which a certain percentage of observed values occur.
+
+```
+ROW values = [5, 5, 10, 12, 5000]
+| EVAL p50 = MV_PERCENTILE(values, 50), median = MV_MEDIAN(values)
+```
diff --git a/docs/reference/esql/functions/kibana/docs/mv_pseries_weighted_sum.md b/docs/reference/esql/functions/kibana/docs/mv_pseries_weighted_sum.md
new file mode 100644
index 0000000000000..fbeb310449b9b
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/mv_pseries_weighted_sum.md
@@ -0,0 +1,12 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### MV_PSERIES_WEIGHTED_SUM
+Converts a multivalued expression into a single-valued column by multiplying every element on the input list by its corresponding term in P-Series and computing the sum.
+
+```
+ROW a = [70.0, 45.0, 21.0, 21.0, 21.0]
+| EVAL sum = MV_PSERIES_WEIGHTED_SUM(a, 1.5)
+| KEEP sum
+```
diff --git a/docs/reference/esql/functions/kibana/docs/neg.md b/docs/reference/esql/functions/kibana/docs/neg.md
new file mode 100644
index 0000000000000..2d1c65487343f
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/neg.md
@@ -0,0 +1,7 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### NEG
+Returns the negation of the argument.
+
diff --git a/docs/reference/esql/functions/kibana/docs/not_equals.md b/docs/reference/esql/functions/kibana/docs/not_equals.md
new file mode 100644
index 0000000000000..cff2130e766ed
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/not_equals.md
@@ -0,0 +1,8 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### NOT_EQUALS
+Check if two fields are unequal. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.
+
+Note: This is pushed to the underlying search index if one side of the comparison is constant and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
diff --git a/docs/reference/esql/functions/kibana/docs/rlike.md b/docs/reference/esql/functions/kibana/docs/rlike.md
new file mode 100644
index 0000000000000..ed94553e7e44f
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/rlike.md
@@ -0,0 +1,15 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### RLIKE
+Use `RLIKE` to filter data based on string patterns using using
+<<regexp-syntax,regular expressions>>. `RLIKE` usually acts on a field placed on
+the left-hand side of the operator, but it can also act on a constant (literal)
+expression. The right-hand side of the operator represents the pattern.
+
+```
+FROM employees
+| WHERE first_name RLIKE ".leja.*"
+| KEEP first_name, last_name
+```
diff --git a/docs/reference/esql/functions/kibana/docs/st_centroid_agg.md b/docs/reference/esql/functions/kibana/docs/st_centroid_agg.md
new file mode 100644
index 0000000000000..306a32a309a64
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/st_centroid_agg.md
@@ -0,0 +1,11 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### ST_CENTROID_AGG
+Calculate the spatial centroid over a field with spatial point geometry type.
+
+```
+FROM airports
+| STATS centroid=ST_CENTROID_AGG(location)
+```
diff --git a/docs/reference/esql/functions/kibana/docs/sub.md b/docs/reference/esql/functions/kibana/docs/sub.md
new file mode 100644
index 0000000000000..10746ed81cfe3
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/sub.md
@@ -0,0 +1,7 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### SUB
+Subtract one number from another. If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.
+
diff --git a/docs/reference/esql/functions/kibana/docs/sum.md b/docs/reference/esql/functions/kibana/docs/sum.md
new file mode 100644
index 0000000000000..eb72ddb0dece1
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/sum.md
@@ -0,0 +1,11 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### SUM
+The sum of a numeric expression.
+
+```
+FROM employees
+| STATS SUM(languages)
+```
diff --git a/docs/reference/esql/functions/kibana/docs/to_datetime.md b/docs/reference/esql/functions/kibana/docs/to_datetime.md
index 5e8f9c72adc2c..c194dfd17871a 100644
--- a/docs/reference/esql/functions/kibana/docs/to_datetime.md
+++ b/docs/reference/esql/functions/kibana/docs/to_datetime.md
@@ -11,3 +11,4 @@ To convert dates in other formats, use <<esql-date_parse>>.
 ROW string = ["1953-09-02T00:00:00.000Z", "1964-06-02T00:00:00.000Z", "1964-06-02 00:00:00"]
 | EVAL datetime = TO_DATETIME(string)
 ```
+Note: Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is truncated, not rounded.
diff --git a/docs/reference/esql/functions/kibana/docs/values.md b/docs/reference/esql/functions/kibana/docs/values.md
new file mode 100644
index 0000000000000..cba62fc27255e
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/values.md
@@ -0,0 +1,13 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### VALUES
+Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed. If you need the values returned in order use <<esql-mv_sort>>.
+
+```
+  FROM employees
+| EVAL first_letter = SUBSTRING(first_name, 0, 1)
+| STATS first_name=MV_SORT(VALUES(first_name)) BY first_letter
+| SORT first_letter
+```
diff --git a/docs/reference/esql/functions/kibana/docs/weighted_avg.md b/docs/reference/esql/functions/kibana/docs/weighted_avg.md
new file mode 100644
index 0000000000000..6b0b2cc8cd287
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/weighted_avg.md
@@ -0,0 +1,14 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### WEIGHTED_AVG
+The weighted average of a numeric expression.
+
+```
+FROM employees
+| STATS w_avg = WEIGHTED_AVG(salary, height) by languages
+| EVAL w_avg = ROUND(w_avg)
+| KEEP w_avg, languages
+| SORT languages
+```
diff --git a/docs/reference/esql/functions/layout/count.asciidoc b/docs/reference/esql/functions/layout/count.asciidoc
new file mode 100644
index 0000000000000..8c16d74cde9a7
--- /dev/null
+++ b/docs/reference/esql/functions/layout/count.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-count]]
+=== `COUNT`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/count.svg[Embedded,opts=inline]
+
+include::../parameters/count.asciidoc[]
+include::../description/count.asciidoc[]
+include::../types/count.asciidoc[]
+include::../examples/count.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/count_distinct.asciidoc b/docs/reference/esql/functions/layout/count_distinct.asciidoc
new file mode 100644
index 0000000000000..2c9848186e806
--- /dev/null
+++ b/docs/reference/esql/functions/layout/count_distinct.asciidoc
@@ -0,0 +1,16 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-count_distinct]]
+=== `COUNT_DISTINCT`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/count_distinct.svg[Embedded,opts=inline]
+
+include::../parameters/count_distinct.asciidoc[]
+include::../description/count_distinct.asciidoc[]
+include::../types/count_distinct.asciidoc[]
+include::../examples/count_distinct.asciidoc[]
+include::../appendix/count_distinct.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/median.asciidoc b/docs/reference/esql/functions/layout/median.asciidoc
new file mode 100644
index 0000000000000..c03e73523983d
--- /dev/null
+++ b/docs/reference/esql/functions/layout/median.asciidoc
@@ -0,0 +1,16 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-median]]
+=== `MEDIAN`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/median.svg[Embedded,opts=inline]
+
+include::../parameters/median.asciidoc[]
+include::../description/median.asciidoc[]
+include::../types/median.asciidoc[]
+include::../examples/median.asciidoc[]
+include::../appendix/median.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/median_absolute_deviation.asciidoc b/docs/reference/esql/functions/layout/median_absolute_deviation.asciidoc
new file mode 100644
index 0000000000000..b558e6f11d9d0
--- /dev/null
+++ b/docs/reference/esql/functions/layout/median_absolute_deviation.asciidoc
@@ -0,0 +1,16 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-median_absolute_deviation]]
+=== `MEDIAN_ABSOLUTE_DEVIATION`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/median_absolute_deviation.svg[Embedded,opts=inline]
+
+include::../parameters/median_absolute_deviation.asciidoc[]
+include::../description/median_absolute_deviation.asciidoc[]
+include::../types/median_absolute_deviation.asciidoc[]
+include::../examples/median_absolute_deviation.asciidoc[]
+include::../appendix/median_absolute_deviation.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/mv_percentile.asciidoc b/docs/reference/esql/functions/layout/mv_percentile.asciidoc
new file mode 100644
index 0000000000000..a86c4a136b5cd
--- /dev/null
+++ b/docs/reference/esql/functions/layout/mv_percentile.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-mv_percentile]]
+=== `MV_PERCENTILE`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/mv_percentile.svg[Embedded,opts=inline]
+
+include::../parameters/mv_percentile.asciidoc[]
+include::../description/mv_percentile.asciidoc[]
+include::../types/mv_percentile.asciidoc[]
+include::../examples/mv_percentile.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/mv_pseries_weighted_sum.asciidoc b/docs/reference/esql/functions/layout/mv_pseries_weighted_sum.asciidoc
new file mode 100644
index 0000000000000..7c14ecbc3c935
--- /dev/null
+++ b/docs/reference/esql/functions/layout/mv_pseries_weighted_sum.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-mv_pseries_weighted_sum]]
+=== `MV_PSERIES_WEIGHTED_SUM`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/mv_pseries_weighted_sum.svg[Embedded,opts=inline]
+
+include::../parameters/mv_pseries_weighted_sum.asciidoc[]
+include::../description/mv_pseries_weighted_sum.asciidoc[]
+include::../types/mv_pseries_weighted_sum.asciidoc[]
+include::../examples/mv_pseries_weighted_sum.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/st_centroid_agg.asciidoc b/docs/reference/esql/functions/layout/st_centroid_agg.asciidoc
new file mode 100644
index 0000000000000..6626c162f3b06
--- /dev/null
+++ b/docs/reference/esql/functions/layout/st_centroid_agg.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-st_centroid_agg]]
+=== `ST_CENTROID_AGG`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/st_centroid_agg.svg[Embedded,opts=inline]
+
+include::../parameters/st_centroid_agg.asciidoc[]
+include::../description/st_centroid_agg.asciidoc[]
+include::../types/st_centroid_agg.asciidoc[]
+include::../examples/st_centroid_agg.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/sum.asciidoc b/docs/reference/esql/functions/layout/sum.asciidoc
new file mode 100644
index 0000000000000..abac1fdd27b6e
--- /dev/null
+++ b/docs/reference/esql/functions/layout/sum.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-sum]]
+=== `SUM`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/sum.svg[Embedded,opts=inline]
+
+include::../parameters/sum.asciidoc[]
+include::../description/sum.asciidoc[]
+include::../types/sum.asciidoc[]
+include::../examples/sum.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/values.asciidoc b/docs/reference/esql/functions/layout/values.asciidoc
new file mode 100644
index 0000000000000..7d90d4314699a
--- /dev/null
+++ b/docs/reference/esql/functions/layout/values.asciidoc
@@ -0,0 +1,18 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-values]]
+=== `VALUES`
+
+preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/values.svg[Embedded,opts=inline]
+
+include::../parameters/values.asciidoc[]
+include::../description/values.asciidoc[]
+include::../types/values.asciidoc[]
+include::../examples/values.asciidoc[]
+include::../appendix/values.asciidoc[]
diff --git a/docs/reference/esql/functions/layout/weighted_avg.asciidoc b/docs/reference/esql/functions/layout/weighted_avg.asciidoc
new file mode 100644
index 0000000000000..84db8f1269b94
--- /dev/null
+++ b/docs/reference/esql/functions/layout/weighted_avg.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-weighted_avg]]
+=== `WEIGHTED_AVG`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/weighted_avg.svg[Embedded,opts=inline]
+
+include::../parameters/weighted_avg.asciidoc[]
+include::../description/weighted_avg.asciidoc[]
+include::../types/weighted_avg.asciidoc[]
+include::../examples/weighted_avg.asciidoc[]
diff --git a/docs/reference/esql/functions/median-absolute-deviation.asciidoc b/docs/reference/esql/functions/median-absolute-deviation.asciidoc
deleted file mode 100644
index b4f80cced06f6..0000000000000
--- a/docs/reference/esql/functions/median-absolute-deviation.asciidoc
+++ /dev/null
@@ -1,60 +0,0 @@
-[discrete]
-[[esql-agg-median-absolute-deviation]]
-=== `MEDIAN_ABSOLUTE_DEVIATION`
-
-*Syntax*
-
-[source,esql]
-----
-MEDIAN_ABSOLUTE_DEVIATION(expression)
-----
-
-*Parameters*
-
-`expression`::
-Expression from which to return the median absolute deviation.
-
-*Description*
-
-Returns the median absolute deviation, a measure of variability. It is a robust
-statistic, meaning that it is useful for describing data that may have outliers,
-or may not be normally distributed. For such data it can be more descriptive
-than standard deviation.
-
-It is calculated as the median of each data point's deviation from the median of
-the entire sample. That is, for a random variable `X`, the median absolute
-deviation is `median(|median(X) - X|)`.
-
-NOTE: Like <<esql-percentile>>, `MEDIAN_ABSOLUTE_DEVIATION` is
-      <<esql-percentile-approximate,usually approximate>>.
-
-[WARNING]
-====
-`MEDIAN_ABSOLUTE_DEVIATION` is also {wikipedia}/Nondeterministic_algorithm[non-deterministic].
-This means you can get slightly different results using the same data.
-====
-
-*Example*
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats_percentile.csv-spec[tag=median-absolute-deviation]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats_percentile.csv-spec[tag=median-absolute-deviation-result]
-|===
-
-The expression can use inline functions. For example, to calculate the the
-median absolute deviation of the maximum values of a multivalued column, first
-use `MV_MAX` to get the maximum value per row, and use the result with the
-`MEDIAN_ABSOLUTE_DEVIATION` function:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsMADNestedExpression]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsMADNestedExpression-result]
-|===
diff --git a/docs/reference/esql/functions/median.asciidoc b/docs/reference/esql/functions/median.asciidoc
deleted file mode 100644
index 2f7d70775e38e..0000000000000
--- a/docs/reference/esql/functions/median.asciidoc
+++ /dev/null
@@ -1,52 +0,0 @@
-[discrete]
-[[esql-agg-median]]
-=== `MEDIAN`
-
-*Syntax*
-
-[source,esql]
-----
-MEDIAN(expression)
-----
-
-*Parameters*
-
-`expression`::
-Expression from which to return the median value.
-
-*Description*
-
-Returns the value that is greater than half of all values and less than half of
-all values, also known as the 50% <<esql-percentile>>.
-
-NOTE: Like <<esql-percentile>>, `MEDIAN` is <<esql-percentile-approximate,usually approximate>>.
-
-[WARNING]
-====
-`MEDIAN` is also {wikipedia}/Nondeterministic_algorithm[non-deterministic].
-This means you can get slightly different results using the same data.
-====
-
-*Example*
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats_percentile.csv-spec[tag=median]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats_percentile.csv-spec[tag=median-result]
-|===
-
-The expression can use inline functions. For example, to calculate the median of
-the maximum values of a multivalued column, first use `MV_MAX` to get the
-maximum value per row, and use the result with the `MEDIAN` function:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsMedianNestedExpression]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsMedianNestedExpression-result]
-|===
diff --git a/docs/reference/esql/functions/mv-functions.asciidoc b/docs/reference/esql/functions/mv-functions.asciidoc
index 0f4f6233d446c..bd5f14cdd3557 100644
--- a/docs/reference/esql/functions/mv-functions.asciidoc
+++ b/docs/reference/esql/functions/mv-functions.asciidoc
@@ -18,6 +18,7 @@
 * <<esql-mv_max>>
 * <<esql-mv_median>>
 * <<esql-mv_min>>
+* <<esql-mv_pseries_weighted_sum>>
 * <<esql-mv_sort>>
 * <<esql-mv_slice>>
 * <<esql-mv_sum>>
@@ -34,6 +35,7 @@ include::layout/mv_last.asciidoc[]
 include::layout/mv_max.asciidoc[]
 include::layout/mv_median.asciidoc[]
 include::layout/mv_min.asciidoc[]
+include::layout/mv_pseries_weighted_sum.asciidoc[]
 include::layout/mv_slice.asciidoc[]
 include::layout/mv_sort.asciidoc[]
 include::layout/mv_sum.asciidoc[]
diff --git a/docs/reference/esql/functions/parameters/bucket.asciidoc b/docs/reference/esql/functions/parameters/bucket.asciidoc
index 39aac14aaa36d..09c720d6095f3 100644
--- a/docs/reference/esql/functions/parameters/bucket.asciidoc
+++ b/docs/reference/esql/functions/parameters/bucket.asciidoc
@@ -6,10 +6,10 @@
 Numeric or date expression from which to derive buckets.
 
 `buckets`::
-Target number of buckets.
+Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted.
 
 `from`::
-Start of the range. Can be a number or a date expressed as a string.
+Start of the range. Can be a number, a date or a date expressed as a string.
 
 `to`::
-End of the range. Can be a number or a date expressed as a string.
+End of the range. Can be a number, a date or a date expressed as a string.
diff --git a/docs/reference/esql/functions/parameters/count.asciidoc b/docs/reference/esql/functions/parameters/count.asciidoc
new file mode 100644
index 0000000000000..d470061a83e2e
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/count.asciidoc
@@ -0,0 +1,6 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`field`::
+Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows).
diff --git a/docs/reference/esql/functions/parameters/count_distinct.asciidoc b/docs/reference/esql/functions/parameters/count_distinct.asciidoc
new file mode 100644
index 0000000000000..f84cf27c3e075
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/count_distinct.asciidoc
@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`field`::
+Column or literal for which to count the number of distinct values.
+
+`precision`::
+Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000.
diff --git a/docs/reference/esql/functions/parameters/median.asciidoc b/docs/reference/esql/functions/parameters/median.asciidoc
new file mode 100644
index 0000000000000..91c56709d182a
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/median.asciidoc
@@ -0,0 +1,6 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`number`::
+
diff --git a/docs/reference/esql/functions/parameters/median_absolute_deviation.asciidoc b/docs/reference/esql/functions/parameters/median_absolute_deviation.asciidoc
new file mode 100644
index 0000000000000..91c56709d182a
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/median_absolute_deviation.asciidoc
@@ -0,0 +1,6 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`number`::
+
diff --git a/docs/reference/esql/functions/parameters/mv_percentile.asciidoc b/docs/reference/esql/functions/parameters/mv_percentile.asciidoc
new file mode 100644
index 0000000000000..57804185e191a
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/mv_percentile.asciidoc
@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`number`::
+Multivalue expression.
+
+`percentile`::
+The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead.
diff --git a/docs/reference/esql/functions/parameters/mv_pseries_weighted_sum.asciidoc b/docs/reference/esql/functions/parameters/mv_pseries_weighted_sum.asciidoc
new file mode 100644
index 0000000000000..3a828f1464824
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/mv_pseries_weighted_sum.asciidoc
@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`number`::
+Multivalue expression.
+
+`p`::
+It is a constant number that represents the 'p' parameter in the P-Series. It impacts every element's contribution to the weighted sum.
diff --git a/docs/reference/esql/functions/parameters/st_centroid_agg.asciidoc b/docs/reference/esql/functions/parameters/st_centroid_agg.asciidoc
new file mode 100644
index 0000000000000..8903aa1a472a3
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/st_centroid_agg.asciidoc
@@ -0,0 +1,6 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`field`::
+
diff --git a/docs/reference/esql/functions/parameters/sum.asciidoc b/docs/reference/esql/functions/parameters/sum.asciidoc
new file mode 100644
index 0000000000000..91c56709d182a
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/sum.asciidoc
@@ -0,0 +1,6 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`number`::
+
diff --git a/docs/reference/esql/functions/parameters/values.asciidoc b/docs/reference/esql/functions/parameters/values.asciidoc
new file mode 100644
index 0000000000000..8903aa1a472a3
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/values.asciidoc
@@ -0,0 +1,6 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`field`::
+
diff --git a/docs/reference/esql/functions/parameters/weighted_avg.asciidoc b/docs/reference/esql/functions/parameters/weighted_avg.asciidoc
new file mode 100644
index 0000000000000..280dd8ca347d4
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/weighted_avg.asciidoc
@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`number`::
+A numeric value.
+
+`weight`::
+A numeric weight.
diff --git a/docs/reference/esql/functions/round.asciidoc b/docs/reference/esql/functions/round.asciidoc
deleted file mode 100644
index e792db6c1ed69..0000000000000
--- a/docs/reference/esql/functions/round.asciidoc
+++ /dev/null
@@ -1,34 +0,0 @@
-[discrete]
-[[esql-round]]
-=== `ROUND`
-*Syntax*
-
-[.text-center]
-image::esql/functions/signature/round.svg[Embedded,opts=inline]
-
-*Parameters*
-
-`value`::
-Numeric expression. If `null`, the function returns `null`.
-
-`decimals`::
-Numeric expression. If `null`, the function returns `null`.
-
-*Description*
-
-Rounds a number to the closest number with the specified number of digits.
-Defaults to 0 digits if no number of digits is provided. If the specified number
-of digits is negative, rounds to the number of digits left of the decimal point.
-
-include::types/round.asciidoc[]
-
-*Example*
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/docs.csv-spec[tag=round]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/docs.csv-spec[tag=round-result]
-|===
diff --git a/docs/reference/esql/functions/signature/count.svg b/docs/reference/esql/functions/signature/count.svg
new file mode 100644
index 0000000000000..9b19652b98788
--- /dev/null
+++ b/docs/reference/esql/functions/signature/count.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="264" height="46" viewbox="0 0 264 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m80 0h10m32 0h10m80 0h10m32 0h5"/><rect class="s" x="5" y="5" width="80" height="36"/><text class="k" x="15" y="31">COUNT</text><rect class="s" x="95" y="5" width="32" height="36" rx="7"/><text class="syn" x="105" y="31">(</text><rect class="s" x="137" y="5" width="80" height="36" rx="7"/><text class="k" x="147" y="31">field</text><rect class="s" x="227" y="5" width="32" height="36" rx="7"/><text class="syn" x="237" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/count_distinct.svg b/docs/reference/esql/functions/signature/count_distinct.svg
new file mode 100644
index 0000000000000..a5b77da7c555a
--- /dev/null
+++ b/docs/reference/esql/functions/signature/count_distinct.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="552" height="46" viewbox="0 0 552 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m188 0h10m32 0h10m80 0h10m32 0h10m128 0h10m32 0h5"/><rect class="s" x="5" y="5" width="188" height="36"/><text class="k" x="15" y="31">COUNT_DISTINCT</text><rect class="s" x="203" y="5" width="32" height="36" rx="7"/><text class="syn" x="213" y="31">(</text><rect class="s" x="245" y="5" width="80" height="36" rx="7"/><text class="k" x="255" y="31">field</text><rect class="s" x="335" y="5" width="32" height="36" rx="7"/><text class="syn" x="345" y="31">,</text><rect class="s" x="377" y="5" width="128" height="36" rx="7"/><text class="k" x="387" y="31">precision</text><rect class="s" x="515" y="5" width="32" height="36" rx="7"/><text class="syn" x="525" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/median.svg b/docs/reference/esql/functions/signature/median.svg
new file mode 100644
index 0000000000000..c61b3a9e77817
--- /dev/null
+++ b/docs/reference/esql/functions/signature/median.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="288" height="46" viewbox="0 0 288 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m92 0h10m32 0h10m92 0h10m32 0h5"/><rect class="s" x="5" y="5" width="92" height="36"/><text class="k" x="15" y="31">MEDIAN</text><rect class="s" x="107" y="5" width="32" height="36" rx="7"/><text class="syn" x="117" y="31">(</text><rect class="s" x="149" y="5" width="92" height="36" rx="7"/><text class="k" x="159" y="31">number</text><rect class="s" x="251" y="5" width="32" height="36" rx="7"/><text class="syn" x="261" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/median_absolute_deviation.svg b/docs/reference/esql/functions/signature/median_absolute_deviation.svg
new file mode 100644
index 0000000000000..bcf01de52ac12
--- /dev/null
+++ b/docs/reference/esql/functions/signature/median_absolute_deviation.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="516" height="46" viewbox="0 0 516 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m320 0h10m32 0h10m92 0h10m32 0h5"/><rect class="s" x="5" y="5" width="320" height="36"/><text class="k" x="15" y="31">MEDIAN_ABSOLUTE_DEVIATION</text><rect class="s" x="335" y="5" width="32" height="36" rx="7"/><text class="syn" x="345" y="31">(</text><rect class="s" x="377" y="5" width="92" height="36" rx="7"/><text class="k" x="387" y="31">number</text><rect class="s" x="479" y="5" width="32" height="36" rx="7"/><text class="syn" x="489" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/mv_percentile.svg b/docs/reference/esql/functions/signature/mv_percentile.svg
new file mode 100644
index 0000000000000..b4d623636572f
--- /dev/null
+++ b/docs/reference/esql/functions/signature/mv_percentile.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="564" height="46" viewbox="0 0 564 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m176 0h10m32 0h10m92 0h10m32 0h10m140 0h10m32 0h5"/><rect class="s" x="5" y="5" width="176" height="36"/><text class="k" x="15" y="31">MV_PERCENTILE</text><rect class="s" x="191" y="5" width="32" height="36" rx="7"/><text class="syn" x="201" y="31">(</text><rect class="s" x="233" y="5" width="92" height="36" rx="7"/><text class="k" x="243" y="31">number</text><rect class="s" x="335" y="5" width="32" height="36" rx="7"/><text class="syn" x="345" y="31">,</text><rect class="s" x="377" y="5" width="140" height="36" rx="7"/><text class="k" x="387" y="31">percentile</text><rect class="s" x="527" y="5" width="32" height="36" rx="7"/><text class="syn" x="537" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/mv_pseries_weighted_sum.svg b/docs/reference/esql/functions/signature/mv_pseries_weighted_sum.svg
new file mode 100644
index 0000000000000..7e3b42161e52c
--- /dev/null
+++ b/docs/reference/esql/functions/signature/mv_pseries_weighted_sum.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="576" height="46" viewbox="0 0 576 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m296 0h10m32 0h10m92 0h10m32 0h10m32 0h10m32 0h5"/><rect class="s" x="5" y="5" width="296" height="36"/><text class="k" x="15" y="31">MV_PSERIES_WEIGHTED_SUM</text><rect class="s" x="311" y="5" width="32" height="36" rx="7"/><text class="syn" x="321" y="31">(</text><rect class="s" x="353" y="5" width="92" height="36" rx="7"/><text class="k" x="363" y="31">number</text><rect class="s" x="455" y="5" width="32" height="36" rx="7"/><text class="syn" x="465" y="31">,</text><rect class="s" x="497" y="5" width="32" height="36" rx="7"/><text class="k" x="507" y="31">p</text><rect class="s" x="539" y="5" width="32" height="36" rx="7"/><text class="syn" x="549" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/st_centroid_agg.svg b/docs/reference/esql/functions/signature/st_centroid_agg.svg
new file mode 100644
index 0000000000000..45d509614a526
--- /dev/null
+++ b/docs/reference/esql/functions/signature/st_centroid_agg.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="384" height="46" viewbox="0 0 384 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m200 0h10m32 0h10m80 0h10m32 0h5"/><rect class="s" x="5" y="5" width="200" height="36"/><text class="k" x="15" y="31">ST_CENTROID_AGG</text><rect class="s" x="215" y="5" width="32" height="36" rx="7"/><text class="syn" x="225" y="31">(</text><rect class="s" x="257" y="5" width="80" height="36" rx="7"/><text class="k" x="267" y="31">field</text><rect class="s" x="347" y="5" width="32" height="36" rx="7"/><text class="syn" x="357" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/sum.svg b/docs/reference/esql/functions/signature/sum.svg
new file mode 100644
index 0000000000000..d1024edc2a5b9
--- /dev/null
+++ b/docs/reference/esql/functions/signature/sum.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="252" height="46" viewbox="0 0 252 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m56 0h10m32 0h10m92 0h10m32 0h5"/><rect class="s" x="5" y="5" width="56" height="36"/><text class="k" x="15" y="31">SUM</text><rect class="s" x="71" y="5" width="32" height="36" rx="7"/><text class="syn" x="81" y="31">(</text><rect class="s" x="113" y="5" width="92" height="36" rx="7"/><text class="k" x="123" y="31">number</text><rect class="s" x="215" y="5" width="32" height="36" rx="7"/><text class="syn" x="225" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/values.svg b/docs/reference/esql/functions/signature/values.svg
new file mode 100644
index 0000000000000..0fa116ce1eb14
--- /dev/null
+++ b/docs/reference/esql/functions/signature/values.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="276" height="46" viewbox="0 0 276 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m92 0h10m32 0h10m80 0h10m32 0h5"/><rect class="s" x="5" y="5" width="92" height="36"/><text class="k" x="15" y="31">VALUES</text><rect class="s" x="107" y="5" width="32" height="36" rx="7"/><text class="syn" x="117" y="31">(</text><rect class="s" x="149" y="5" width="80" height="36" rx="7"/><text class="k" x="159" y="31">field</text><rect class="s" x="239" y="5" width="32" height="36" rx="7"/><text class="syn" x="249" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/signature/weighted_avg.svg b/docs/reference/esql/functions/signature/weighted_avg.svg
new file mode 100644
index 0000000000000..581e8cd79b0c2
--- /dev/null
+++ b/docs/reference/esql/functions/signature/weighted_avg.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="504" height="46" viewbox="0 0 504 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m164 0h10m32 0h10m92 0h10m32 0h10m92 0h10m32 0h5"/><rect class="s" x="5" y="5" width="164" height="36"/><text class="k" x="15" y="31">WEIGHTED_AVG</text><rect class="s" x="179" y="5" width="32" height="36" rx="7"/><text class="syn" x="189" y="31">(</text><rect class="s" x="221" y="5" width="92" height="36" rx="7"/><text class="k" x="231" y="31">number</text><rect class="s" x="323" y="5" width="32" height="36" rx="7"/><text class="syn" x="333" y="31">,</text><rect class="s" x="365" y="5" width="92" height="36" rx="7"/><text class="k" x="375" y="31">weight</text><rect class="s" x="467" y="5" width="32" height="36" rx="7"/><text class="syn" x="477" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/st_centroid_agg.asciidoc b/docs/reference/esql/functions/st_centroid_agg.asciidoc
deleted file mode 100644
index c980560f8f198..0000000000000
--- a/docs/reference/esql/functions/st_centroid_agg.asciidoc
+++ /dev/null
@@ -1,25 +0,0 @@
-[discrete]
-[[esql-agg-st-centroid]]
-=== `ST_CENTROID_AGG`
-
-experimental::[]
-
-Calculate the spatial centroid over a field with spatial point geometry type.
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/spatial.csv-spec[tag=st_centroid_agg-airports]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/spatial.csv-spec[tag=st_centroid_agg-airports-result]
-|===
-
-Supported types:
-
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-v | result
-geo_point | geo_point
-cartesian_point | cartesian_point
-|===
diff --git a/docs/reference/esql/functions/sum.asciidoc b/docs/reference/esql/functions/sum.asciidoc
deleted file mode 100644
index efe65d5503ec6..0000000000000
--- a/docs/reference/esql/functions/sum.asciidoc
+++ /dev/null
@@ -1,41 +0,0 @@
-[discrete]
-[[esql-agg-sum]]
-=== `SUM`
-
-*Syntax*
-
-[source,esql]
-----
-SUM(expression)
-----
-
-`expression`::
-Numeric expression.
-
-*Description*
-
-Returns the sum of a numeric expression.
-
-*Example*
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats.csv-spec[tag=sum]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats.csv-spec[tag=sum-result]
-|===
-
-The expression can use inline functions. For example, to calculate
-the sum of each employee's maximum salary changes, apply the
-`MV_MAX` function to each row and then sum the results:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats.csv-spec[tag=docsStatsSumNestedExpression]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats.csv-spec[tag=docsStatsSumNestedExpression-result]
-|===
diff --git a/docs/reference/esql/functions/types/add.asciidoc b/docs/reference/esql/functions/types/add.asciidoc
index a0215a803d4e3..54d1aec463c1a 100644
--- a/docs/reference/esql/functions/types/add.asciidoc
+++ b/docs/reference/esql/functions/types/add.asciidoc
@@ -5,10 +5,10 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 lhs | rhs | result
+date | date_period | date
+date | time_duration | date
+date_period | date | date
 date_period | date_period | date_period
-date_period | datetime | datetime
-datetime | date_period | datetime
-datetime | time_duration | datetime
 double | double | double
 double | integer | double
 double | long | double
@@ -18,7 +18,7 @@ integer | long | long
 long | double | double
 long | integer | long
 long | long | long
-time_duration | datetime | datetime
+time_duration | date | date
 time_duration | time_duration | time_duration
 unsigned_long | unsigned_long | unsigned_long
 |===
diff --git a/docs/reference/esql/functions/types/bucket.asciidoc b/docs/reference/esql/functions/types/bucket.asciidoc
index d1ce8e499eb07..172e84b6f7860 100644
--- a/docs/reference/esql/functions/types/bucket.asciidoc
+++ b/docs/reference/esql/functions/types/bucket.asciidoc
@@ -5,9 +5,17 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 field | buckets | from | to | result
-datetime | date_period | | | datetime
-datetime | integer | datetime | datetime | datetime
-datetime | time_duration | | | datetime
+date | date_period | | | date
+date | integer | date | date | date
+date | integer | date | keyword | date
+date | integer | date | text | date
+date | integer | keyword | date | date
+date | integer | keyword | keyword | date
+date | integer | keyword | text | date
+date | integer | text | date | date
+date | integer | text | keyword | date
+date | integer | text | text | date
+date | time_duration | | | date
 double | double | | | double
 double | integer | double | double | double
 double | integer | double | integer | double
@@ -18,6 +26,8 @@ double | integer | integer | long | double
 double | integer | long | double | double
 double | integer | long | integer | double
 double | integer | long | long | double
+double | integer | | | double
+double | long | | | double
 integer | double | | | double
 integer | integer | double | double | double
 integer | integer | double | integer | double
@@ -28,6 +38,8 @@ integer | integer | integer | long | double
 integer | integer | long | double | double
 integer | integer | long | integer | double
 integer | integer | long | long | double
+integer | integer | | | double
+integer | long | | | double
 long | double | | | double
 long | integer | double | double | double
 long | integer | double | integer | double
@@ -38,4 +50,6 @@ long | integer | integer | long | double
 long | integer | long | double | double
 long | integer | long | integer | double
 long | integer | long | long | double
+long | integer | | | double
+long | long | | | double
 |===
diff --git a/docs/reference/esql/functions/types/case.asciidoc b/docs/reference/esql/functions/types/case.asciidoc
index 85e4193b5bf2f..f6c8cfe9361d1 100644
--- a/docs/reference/esql/functions/types/case.asciidoc
+++ b/docs/reference/esql/functions/types/case.asciidoc
@@ -7,7 +7,7 @@
 condition | trueValue | result
 boolean | boolean | boolean
 boolean | cartesian_point | cartesian_point
-boolean | datetime | datetime
+boolean | date | date
 boolean | double | double
 boolean | geo_point | geo_point
 boolean | integer | integer
diff --git a/docs/reference/esql/functions/types/coalesce.asciidoc b/docs/reference/esql/functions/types/coalesce.asciidoc
index 841d836f6837e..368a12db0dca4 100644
--- a/docs/reference/esql/functions/types/coalesce.asciidoc
+++ b/docs/reference/esql/functions/types/coalesce.asciidoc
@@ -9,7 +9,7 @@ boolean | boolean | boolean
 boolean | | boolean
 cartesian_point | cartesian_point | cartesian_point
 cartesian_shape | cartesian_shape | cartesian_shape
-datetime | datetime | datetime
+date | date | date
 geo_point | geo_point | geo_point
 geo_shape | geo_shape | geo_shape
 integer | integer | integer
diff --git a/docs/reference/esql/functions/types/count.asciidoc b/docs/reference/esql/functions/types/count.asciidoc
new file mode 100644
index 0000000000000..959c94c1ec358
--- /dev/null
+++ b/docs/reference/esql/functions/types/count.asciidoc
@@ -0,0 +1,20 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+field | result
+boolean | long
+cartesian_point | long
+date | long
+double | long
+geo_point | long
+integer | long
+ip | long
+keyword | long
+long | long
+text | long
+unsigned_long | long
+version | long
+|===
diff --git a/docs/reference/esql/functions/types/count_distinct.asciidoc b/docs/reference/esql/functions/types/count_distinct.asciidoc
new file mode 100644
index 0000000000000..c365c8814573c
--- /dev/null
+++ b/docs/reference/esql/functions/types/count_distinct.asciidoc
@@ -0,0 +1,44 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+field | precision | result
+boolean | integer | long
+boolean | long | long
+boolean | unsigned_long | long
+boolean | | long
+date | integer | long
+date | long | long
+date | unsigned_long | long
+date | | long
+double | integer | long
+double | long | long
+double | unsigned_long | long
+double | | long
+integer | integer | long
+integer | long | long
+integer | unsigned_long | long
+integer | | long
+ip | integer | long
+ip | long | long
+ip | unsigned_long | long
+ip | | long
+keyword | integer | long
+keyword | long | long
+keyword | unsigned_long | long
+keyword | | long
+long | integer | long
+long | long | long
+long | unsigned_long | long
+long | | long
+text | integer | long
+text | long | long
+text | unsigned_long | long
+text | | long
+version | integer | long
+version | long | long
+version | unsigned_long | long
+version | | long
+|===
diff --git a/docs/reference/esql/functions/types/date_diff.asciidoc b/docs/reference/esql/functions/types/date_diff.asciidoc
index 98adcef51e75c..b0a4818f412ac 100644
--- a/docs/reference/esql/functions/types/date_diff.asciidoc
+++ b/docs/reference/esql/functions/types/date_diff.asciidoc
@@ -5,6 +5,6 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 unit | startTimestamp | endTimestamp | result
-keyword | datetime | datetime | integer
-text | datetime | datetime | integer
+keyword | date | date | integer
+text | date | date | integer
 |===
diff --git a/docs/reference/esql/functions/types/date_extract.asciidoc b/docs/reference/esql/functions/types/date_extract.asciidoc
index 43702ef0671a7..ec9bf70c221cc 100644
--- a/docs/reference/esql/functions/types/date_extract.asciidoc
+++ b/docs/reference/esql/functions/types/date_extract.asciidoc
@@ -5,6 +5,6 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 datePart | date | result
-keyword | datetime | long
-text | datetime | long
+keyword | date | long
+text | date | long
 |===
diff --git a/docs/reference/esql/functions/types/date_format.asciidoc b/docs/reference/esql/functions/types/date_format.asciidoc
index a76f38653b9b8..b2e97dfa8835a 100644
--- a/docs/reference/esql/functions/types/date_format.asciidoc
+++ b/docs/reference/esql/functions/types/date_format.asciidoc
@@ -5,6 +5,6 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 dateFormat | date | result
-keyword | datetime | keyword
-text | datetime | keyword
+keyword | date | keyword
+text | date | keyword
 |===
diff --git a/docs/reference/esql/functions/types/date_parse.asciidoc b/docs/reference/esql/functions/types/date_parse.asciidoc
index 314d02eb06271..f3eab18309dd8 100644
--- a/docs/reference/esql/functions/types/date_parse.asciidoc
+++ b/docs/reference/esql/functions/types/date_parse.asciidoc
@@ -5,8 +5,8 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 datePattern | dateString | result
-keyword | keyword | datetime
-keyword | text | datetime
-text | keyword | datetime
-text | text | datetime
+keyword | keyword | date
+keyword | text | date
+text | keyword | date
+text | text | date
 |===
diff --git a/docs/reference/esql/functions/types/date_trunc.asciidoc b/docs/reference/esql/functions/types/date_trunc.asciidoc
index 8df45cfef54a8..aa7dee99c6c44 100644
--- a/docs/reference/esql/functions/types/date_trunc.asciidoc
+++ b/docs/reference/esql/functions/types/date_trunc.asciidoc
@@ -5,6 +5,6 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 interval | date | result
-date_period | datetime | datetime
-time_duration | datetime | datetime
+date_period | date | date
+time_duration | date | date
 |===
diff --git a/docs/reference/esql/functions/types/ends_with.asciidoc b/docs/reference/esql/functions/types/ends_with.asciidoc
index a0236634bbf01..3e8e77448ce89 100644
--- a/docs/reference/esql/functions/types/ends_with.asciidoc
+++ b/docs/reference/esql/functions/types/ends_with.asciidoc
@@ -6,5 +6,7 @@
 |===
 str | suffix | result
 keyword | keyword | boolean
+keyword | text | boolean
+text | keyword | boolean
 text | text | boolean
 |===
diff --git a/docs/reference/esql/functions/types/equals.asciidoc b/docs/reference/esql/functions/types/equals.asciidoc
index 497c9319fedb3..ad0e46ef4b8da 100644
--- a/docs/reference/esql/functions/types/equals.asciidoc
+++ b/docs/reference/esql/functions/types/equals.asciidoc
@@ -8,7 +8,7 @@ lhs | rhs | result
 boolean | boolean | boolean
 cartesian_point | cartesian_point | boolean
 cartesian_shape | cartesian_shape | boolean
-datetime | datetime | boolean
+date | date | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/greater_than.asciidoc b/docs/reference/esql/functions/types/greater_than.asciidoc
index 771daf1a953b2..c506328126a94 100644
--- a/docs/reference/esql/functions/types/greater_than.asciidoc
+++ b/docs/reference/esql/functions/types/greater_than.asciidoc
@@ -5,7 +5,7 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 lhs | rhs | result
-datetime | datetime | boolean
+date | date | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc b/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc
index 771daf1a953b2..c506328126a94 100644
--- a/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc
+++ b/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc
@@ -5,7 +5,7 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 lhs | rhs | result
-datetime | datetime | boolean
+date | date | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/in.asciidoc b/docs/reference/esql/functions/types/in.asciidoc
new file mode 100644
index 0000000000000..6ed2c250ef0ac
--- /dev/null
+++ b/docs/reference/esql/functions/types/in.asciidoc
@@ -0,0 +1,22 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+field | inlist | result
+boolean | boolean | boolean
+cartesian_point | cartesian_point | boolean
+cartesian_shape | cartesian_shape | boolean
+double | double | boolean
+geo_point | geo_point | boolean
+geo_shape | geo_shape | boolean
+integer | integer | boolean
+ip | ip | boolean
+keyword | keyword | boolean
+keyword | text | boolean
+long | long | boolean
+text | keyword | boolean
+text | text | boolean
+version | version | boolean
+|===
diff --git a/docs/reference/esql/functions/types/less_than.asciidoc b/docs/reference/esql/functions/types/less_than.asciidoc
index 771daf1a953b2..c506328126a94 100644
--- a/docs/reference/esql/functions/types/less_than.asciidoc
+++ b/docs/reference/esql/functions/types/less_than.asciidoc
@@ -5,7 +5,7 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 lhs | rhs | result
-datetime | datetime | boolean
+date | date | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/less_than_or_equal.asciidoc b/docs/reference/esql/functions/types/less_than_or_equal.asciidoc
index 771daf1a953b2..c506328126a94 100644
--- a/docs/reference/esql/functions/types/less_than_or_equal.asciidoc
+++ b/docs/reference/esql/functions/types/less_than_or_equal.asciidoc
@@ -5,7 +5,7 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 lhs | rhs | result
-datetime | datetime | boolean
+date | date | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/max.asciidoc b/docs/reference/esql/functions/types/max.asciidoc
index 5b7293d4a4293..35ce5811e0cd0 100644
--- a/docs/reference/esql/functions/types/max.asciidoc
+++ b/docs/reference/esql/functions/types/max.asciidoc
@@ -6,9 +6,12 @@
 |===
 field | result
 boolean | boolean
-datetime | datetime
+date | date
 double | double
 integer | integer
 ip | ip
+keyword | keyword
 long | long
+text | text
+version | version
 |===
diff --git a/docs/reference/esql/functions/types/median.asciidoc b/docs/reference/esql/functions/types/median.asciidoc
new file mode 100644
index 0000000000000..273dae4af76c2
--- /dev/null
+++ b/docs/reference/esql/functions/types/median.asciidoc
@@ -0,0 +1,11 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+number | result
+double | double
+integer | double
+long | double
+|===
diff --git a/docs/reference/esql/functions/types/median_absolute_deviation.asciidoc b/docs/reference/esql/functions/types/median_absolute_deviation.asciidoc
new file mode 100644
index 0000000000000..273dae4af76c2
--- /dev/null
+++ b/docs/reference/esql/functions/types/median_absolute_deviation.asciidoc
@@ -0,0 +1,11 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+number | result
+double | double
+integer | double
+long | double
+|===
diff --git a/docs/reference/esql/functions/types/min.asciidoc b/docs/reference/esql/functions/types/min.asciidoc
index 5b7293d4a4293..35ce5811e0cd0 100644
--- a/docs/reference/esql/functions/types/min.asciidoc
+++ b/docs/reference/esql/functions/types/min.asciidoc
@@ -6,9 +6,12 @@
 |===
 field | result
 boolean | boolean
-datetime | datetime
+date | date
 double | double
 integer | integer
 ip | ip
+keyword | keyword
 long | long
+text | text
+version | version
 |===
diff --git a/docs/reference/esql/functions/types/mv_append.asciidoc b/docs/reference/esql/functions/types/mv_append.asciidoc
index 49dcef6dc8860..a1894e429ae82 100644
--- a/docs/reference/esql/functions/types/mv_append.asciidoc
+++ b/docs/reference/esql/functions/types/mv_append.asciidoc
@@ -8,7 +8,7 @@ field1 | field2 | result
 boolean | boolean | boolean
 cartesian_point | cartesian_point | cartesian_point
 cartesian_shape | cartesian_shape | cartesian_shape
-datetime | datetime | datetime
+date | date | date
 double | double | double
 geo_point | geo_point | geo_point
 geo_shape | geo_shape | geo_shape
diff --git a/docs/reference/esql/functions/types/mv_count.asciidoc b/docs/reference/esql/functions/types/mv_count.asciidoc
index 8af6b76591acb..260c531731f04 100644
--- a/docs/reference/esql/functions/types/mv_count.asciidoc
+++ b/docs/reference/esql/functions/types/mv_count.asciidoc
@@ -8,7 +8,7 @@ field | result
 boolean | integer
 cartesian_point | integer
 cartesian_shape | integer
-datetime | integer
+date | integer
 double | integer
 geo_point | integer
 geo_shape | integer
diff --git a/docs/reference/esql/functions/types/mv_dedupe.asciidoc b/docs/reference/esql/functions/types/mv_dedupe.asciidoc
index a6b78f781f17a..68e546451c8cb 100644
--- a/docs/reference/esql/functions/types/mv_dedupe.asciidoc
+++ b/docs/reference/esql/functions/types/mv_dedupe.asciidoc
@@ -8,7 +8,7 @@ field | result
 boolean | boolean
 cartesian_point | cartesian_point
 cartesian_shape | cartesian_shape
-datetime | datetime
+date | date
 double | double
 geo_point | geo_point
 geo_shape | geo_shape
diff --git a/docs/reference/esql/functions/types/mv_first.asciidoc b/docs/reference/esql/functions/types/mv_first.asciidoc
index e077c57971a4a..35633544d99a0 100644
--- a/docs/reference/esql/functions/types/mv_first.asciidoc
+++ b/docs/reference/esql/functions/types/mv_first.asciidoc
@@ -8,7 +8,7 @@ field | result
 boolean | boolean
 cartesian_point | cartesian_point
 cartesian_shape | cartesian_shape
-datetime | datetime
+date | date
 double | double
 geo_point | geo_point
 geo_shape | geo_shape
diff --git a/docs/reference/esql/functions/types/mv_last.asciidoc b/docs/reference/esql/functions/types/mv_last.asciidoc
index e077c57971a4a..35633544d99a0 100644
--- a/docs/reference/esql/functions/types/mv_last.asciidoc
+++ b/docs/reference/esql/functions/types/mv_last.asciidoc
@@ -8,7 +8,7 @@ field | result
 boolean | boolean
 cartesian_point | cartesian_point
 cartesian_shape | cartesian_shape
-datetime | datetime
+date | date
 double | double
 geo_point | geo_point
 geo_shape | geo_shape
diff --git a/docs/reference/esql/functions/types/mv_max.asciidoc b/docs/reference/esql/functions/types/mv_max.asciidoc
index 4e5f0a5e0ae89..8ea36aebbad37 100644
--- a/docs/reference/esql/functions/types/mv_max.asciidoc
+++ b/docs/reference/esql/functions/types/mv_max.asciidoc
@@ -6,7 +6,7 @@
 |===
 field | result
 boolean | boolean
-datetime | datetime
+date | date
 double | double
 integer | integer
 ip | ip
diff --git a/docs/reference/esql/functions/types/mv_min.asciidoc b/docs/reference/esql/functions/types/mv_min.asciidoc
index 4e5f0a5e0ae89..8ea36aebbad37 100644
--- a/docs/reference/esql/functions/types/mv_min.asciidoc
+++ b/docs/reference/esql/functions/types/mv_min.asciidoc
@@ -6,7 +6,7 @@
 |===
 field | result
 boolean | boolean
-datetime | datetime
+date | date
 double | double
 integer | integer
 ip | ip
diff --git a/docs/reference/esql/functions/types/mv_percentile.asciidoc b/docs/reference/esql/functions/types/mv_percentile.asciidoc
new file mode 100644
index 0000000000000..99a58b9c3d2e2
--- /dev/null
+++ b/docs/reference/esql/functions/types/mv_percentile.asciidoc
@@ -0,0 +1,17 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+number | percentile | result
+double | double | double
+double | integer | double
+double | long | double
+integer | double | integer
+integer | integer | integer
+integer | long | integer
+long | double | long
+long | integer | long
+long | long | long
+|===
diff --git a/docs/reference/esql/functions/types/mv_pseries_weighted_sum.asciidoc b/docs/reference/esql/functions/types/mv_pseries_weighted_sum.asciidoc
new file mode 100644
index 0000000000000..f28e61f17aa33
--- /dev/null
+++ b/docs/reference/esql/functions/types/mv_pseries_weighted_sum.asciidoc
@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+number | p | result
+double | double | double
+|===
diff --git a/docs/reference/esql/functions/types/mv_slice.asciidoc b/docs/reference/esql/functions/types/mv_slice.asciidoc
index 568de10f53d32..0a9dc073370c7 100644
--- a/docs/reference/esql/functions/types/mv_slice.asciidoc
+++ b/docs/reference/esql/functions/types/mv_slice.asciidoc
@@ -8,7 +8,7 @@ field | start | end | result
 boolean | integer | integer | boolean
 cartesian_point | integer | integer | cartesian_point
 cartesian_shape | integer | integer | cartesian_shape
-datetime | integer | integer | datetime
+date | integer | integer | date
 double | integer | integer | double
 geo_point | integer | integer | geo_point
 geo_shape | integer | integer | geo_shape
diff --git a/docs/reference/esql/functions/types/mv_sort.asciidoc b/docs/reference/esql/functions/types/mv_sort.asciidoc
index 24925ca8a6587..93965187482ac 100644
--- a/docs/reference/esql/functions/types/mv_sort.asciidoc
+++ b/docs/reference/esql/functions/types/mv_sort.asciidoc
@@ -6,7 +6,7 @@
 |===
 field | order | result
 boolean | keyword | boolean
-datetime | keyword | datetime
+date | keyword | date
 double | keyword | double
 integer | keyword | integer
 ip | keyword | ip
diff --git a/docs/reference/esql/functions/types/neg.asciidoc b/docs/reference/esql/functions/types/neg.asciidoc
index 28d3b2a512dec..c0d0a21711552 100644
--- a/docs/reference/esql/functions/types/neg.asciidoc
+++ b/docs/reference/esql/functions/types/neg.asciidoc
@@ -4,7 +4,7 @@
 
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-v | result
+field | result
 date_period | date_period
 double | double
 integer | integer
diff --git a/docs/reference/esql/functions/types/not_equals.asciidoc b/docs/reference/esql/functions/types/not_equals.asciidoc
index 497c9319fedb3..ad0e46ef4b8da 100644
--- a/docs/reference/esql/functions/types/not_equals.asciidoc
+++ b/docs/reference/esql/functions/types/not_equals.asciidoc
@@ -8,7 +8,7 @@ lhs | rhs | result
 boolean | boolean | boolean
 cartesian_point | cartesian_point | boolean
 cartesian_shape | cartesian_shape | boolean
-datetime | datetime | boolean
+date | date | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/now.asciidoc b/docs/reference/esql/functions/types/now.asciidoc
index 5737d98f2f7db..b474ab1042050 100644
--- a/docs/reference/esql/functions/types/now.asciidoc
+++ b/docs/reference/esql/functions/types/now.asciidoc
@@ -5,5 +5,5 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 result
-datetime
+date
 |===
diff --git a/docs/reference/esql/functions/types/rlike.asciidoc b/docs/reference/esql/functions/types/rlike.asciidoc
index 436333fddf5ee..46532f2af3bf3 100644
--- a/docs/reference/esql/functions/types/rlike.asciidoc
+++ b/docs/reference/esql/functions/types/rlike.asciidoc
@@ -4,7 +4,7 @@
 
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-str | pattern | caseInsensitive | result
-keyword | keyword | boolean | boolean
-text | text | boolean | boolean
+str | pattern | result
+keyword | keyword | boolean
+text | text | boolean
 |===
diff --git a/docs/reference/esql/functions/types/st_centroid_agg.asciidoc b/docs/reference/esql/functions/types/st_centroid_agg.asciidoc
new file mode 100644
index 0000000000000..da95e0b9bec1a
--- /dev/null
+++ b/docs/reference/esql/functions/types/st_centroid_agg.asciidoc
@@ -0,0 +1,10 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+field | result
+cartesian_point | cartesian_point
+geo_point | geo_point
+|===
diff --git a/docs/reference/esql/functions/types/starts_with.asciidoc b/docs/reference/esql/functions/types/starts_with.asciidoc
index a6c77014966b4..c88378044f0db 100644
--- a/docs/reference/esql/functions/types/starts_with.asciidoc
+++ b/docs/reference/esql/functions/types/starts_with.asciidoc
@@ -6,5 +6,7 @@
 |===
 str | prefix | result
 keyword | keyword | boolean
+keyword | text | boolean
+text | keyword | boolean
 text | text | boolean
 |===
diff --git a/docs/reference/esql/functions/types/sub.asciidoc b/docs/reference/esql/functions/types/sub.asciidoc
index d309f651705f0..c3ded301ebe68 100644
--- a/docs/reference/esql/functions/types/sub.asciidoc
+++ b/docs/reference/esql/functions/types/sub.asciidoc
@@ -5,9 +5,9 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 lhs | rhs | result
+date | date_period | date
+date | time_duration | date
 date_period | date_period | date_period
-datetime | date_period | datetime
-datetime | time_duration | datetime
 double | double | double
 double | integer | double
 double | long | double
diff --git a/docs/reference/esql/functions/types/sum.asciidoc b/docs/reference/esql/functions/types/sum.asciidoc
new file mode 100644
index 0000000000000..aa4c3ad0d7dd8
--- /dev/null
+++ b/docs/reference/esql/functions/types/sum.asciidoc
@@ -0,0 +1,11 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+number | result
+double | double
+integer | long
+long | long
+|===
diff --git a/docs/reference/esql/functions/types/to_datetime.asciidoc b/docs/reference/esql/functions/types/to_datetime.asciidoc
index 52c4cebb661cf..80c986efca794 100644
--- a/docs/reference/esql/functions/types/to_datetime.asciidoc
+++ b/docs/reference/esql/functions/types/to_datetime.asciidoc
@@ -5,11 +5,11 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 field | result
-datetime | datetime
-double | datetime
-integer | datetime
-keyword | datetime
-long | datetime
-text | datetime
-unsigned_long | datetime
+date | date
+double | date
+integer | date
+keyword | date
+long | date
+text | date
+unsigned_long | date
 |===
diff --git a/docs/reference/esql/functions/types/to_double.asciidoc b/docs/reference/esql/functions/types/to_double.asciidoc
index cff686c7bc4ca..d5f5833cd7249 100644
--- a/docs/reference/esql/functions/types/to_double.asciidoc
+++ b/docs/reference/esql/functions/types/to_double.asciidoc
@@ -9,7 +9,7 @@ boolean | double
 counter_double | double
 counter_integer | double
 counter_long | double
-datetime | double
+date | double
 double | double
 integer | double
 keyword | double
diff --git a/docs/reference/esql/functions/types/to_integer.asciidoc b/docs/reference/esql/functions/types/to_integer.asciidoc
index 974f3c9c82d88..d67f8f07affd9 100644
--- a/docs/reference/esql/functions/types/to_integer.asciidoc
+++ b/docs/reference/esql/functions/types/to_integer.asciidoc
@@ -7,7 +7,7 @@
 field | result
 boolean | integer
 counter_integer | integer
-datetime | integer
+date | integer
 double | integer
 integer | integer
 keyword | integer
diff --git a/docs/reference/esql/functions/types/to_long.asciidoc b/docs/reference/esql/functions/types/to_long.asciidoc
index b3959c5444e34..a07990cb1cfbf 100644
--- a/docs/reference/esql/functions/types/to_long.asciidoc
+++ b/docs/reference/esql/functions/types/to_long.asciidoc
@@ -8,7 +8,7 @@ field | result
 boolean | long
 counter_integer | long
 counter_long | long
-datetime | long
+date | long
 double | long
 integer | long
 keyword | long
diff --git a/docs/reference/esql/functions/types/to_string.asciidoc b/docs/reference/esql/functions/types/to_string.asciidoc
index f14cfbb39929f..26a5b31a2a589 100644
--- a/docs/reference/esql/functions/types/to_string.asciidoc
+++ b/docs/reference/esql/functions/types/to_string.asciidoc
@@ -8,7 +8,7 @@ field | result
 boolean | keyword
 cartesian_point | keyword
 cartesian_shape | keyword
-datetime | keyword
+date | keyword
 double | keyword
 geo_point | keyword
 geo_shape | keyword
diff --git a/docs/reference/esql/functions/types/to_unsigned_long.asciidoc b/docs/reference/esql/functions/types/to_unsigned_long.asciidoc
index a271e1a19321d..87b21f3948dad 100644
--- a/docs/reference/esql/functions/types/to_unsigned_long.asciidoc
+++ b/docs/reference/esql/functions/types/to_unsigned_long.asciidoc
@@ -6,7 +6,7 @@
 |===
 field | result
 boolean | unsigned_long
-datetime | unsigned_long
+date | unsigned_long
 double | unsigned_long
 integer | unsigned_long
 keyword | unsigned_long
diff --git a/docs/reference/esql/functions/types/top.asciidoc b/docs/reference/esql/functions/types/top.asciidoc
index 6cfee3dfaf941..0eb329c10b9ed 100644
--- a/docs/reference/esql/functions/types/top.asciidoc
+++ b/docs/reference/esql/functions/types/top.asciidoc
@@ -6,8 +6,9 @@
 |===
 field | limit | order | result
 boolean | integer | keyword | boolean
-datetime | integer | keyword | datetime
+date | integer | keyword | date
 double | integer | keyword | double
 integer | integer | keyword | integer
+ip | integer | keyword | ip
 long | integer | keyword | long
 |===
diff --git a/docs/reference/esql/functions/types/values.asciidoc b/docs/reference/esql/functions/types/values.asciidoc
new file mode 100644
index 0000000000000..35ce5811e0cd0
--- /dev/null
+++ b/docs/reference/esql/functions/types/values.asciidoc
@@ -0,0 +1,17 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+field | result
+boolean | boolean
+date | date
+double | double
+integer | integer
+ip | ip
+keyword | keyword
+long | long
+text | text
+version | version
+|===
diff --git a/docs/reference/esql/functions/types/weighted_avg.asciidoc b/docs/reference/esql/functions/types/weighted_avg.asciidoc
new file mode 100644
index 0000000000000..55cc14eb453c3
--- /dev/null
+++ b/docs/reference/esql/functions/types/weighted_avg.asciidoc
@@ -0,0 +1,17 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+number | weight | result
+double | double | double
+double | integer | double
+double | long | double
+integer | double | double
+integer | integer | double
+integer | long | double
+long | double | double
+long | integer | double
+long | long | double
+|===
diff --git a/docs/reference/esql/functions/values.asciidoc b/docs/reference/esql/functions/values.asciidoc
deleted file mode 100644
index f13338a572b36..0000000000000
--- a/docs/reference/esql/functions/values.asciidoc
+++ /dev/null
@@ -1,38 +0,0 @@
-[discrete]
-[[esql-agg-values]]
-=== `VALUES`
-
-preview::["Do not use `VALUES` on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]
-
-*Syntax*
-
-[source,esql]
-----
-VALUES(expression)
-----
-
-`expression`::
-Expression of any type except `geo_point`, `cartesian_point`, `geo_shape`, or `cartesian_shape`.
-
-*Description*
-
-Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed.
-If you need the values returned in order use <<esql-mv_sort>>.
-
-WARNING: This can use a significant amount of memory and ES|QL doesn't yet
-         grow aggregations beyond memory. So this aggregation will work until
-         it is used to collect more values than can fit into memory. Once it
-         collects too many values it will fail the query with
-         a <<circuit-breaker-errors, Circuit Breaker Error>>.
-
-*Example*
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/string.csv-spec[tag=values-grouped]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/string.csv-spec[tag=values-grouped-result]
-|===
-
diff --git a/docs/reference/esql/functions/weighted-avg.asciidoc b/docs/reference/esql/functions/weighted-avg.asciidoc
deleted file mode 100644
index 4f166801641df..0000000000000
--- a/docs/reference/esql/functions/weighted-avg.asciidoc
+++ /dev/null
@@ -1,35 +0,0 @@
-[discrete]
-[[esql-agg-weighted-avg]]
-=== `WEIGHTED_AVG`
-
-*Syntax*
-
-[source,esql]
-----
-WEIGHTED_AVG(expression, weight)
-----
-
-`expression`::
-Numeric expression.
-
-`weight`::
-Numeric weight.
-
-*Description*
-
-The weighted average of a numeric expression.
-
-*Supported types*
-
-The result is always a `double` no matter the input type.
-
-*Examples*
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/stats.csv-spec[tag=weighted-avg]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/stats.csv-spec[tag=weighted-avg-result]
-|===
diff --git a/docs/reference/esql/processing-commands/inlinestats.asciidoc b/docs/reference/esql/processing-commands/inlinestats.asciidoc
new file mode 100644
index 0000000000000..0b8d38c7b280f
--- /dev/null
+++ b/docs/reference/esql/processing-commands/inlinestats.asciidoc
@@ -0,0 +1,102 @@
+[discrete]
+[[esql-inlinestats-by]]
+=== `INLINESTATS ... BY`
+
+experimental::["INLINESTATS is highly experimental and only available in SNAPSHOT versions."]
+
+The `INLINESTATS` command calculates an aggregate result and adds new columns
+with the result to the stream of input data.
+
+**Syntax**
+
+[source,esql]
+----
+INLINESTATS [column1 =] expression1[, ..., [columnN =] expressionN]
+[BY grouping_expression1[, ..., grouping_expressionN]]
+----
+
+*Parameters*
+
+`columnX`::
+The name by which the aggregated value is returned. If omitted, the name is
+equal to the corresponding expression (`expressionX`). If multiple columns
+have the same name, all but the rightmost column with this name will be ignored.
+
+`expressionX`::
+An expression that computes an aggregated value. If its name coincides with one
+of the computed columns, that column will be ignored.
+
+`grouping_expressionX`::
+An expression that outputs the values to group by.
+
+NOTE: Individual `null` values are skipped when computing aggregations.
+
+*Description*
+
+The `INLINESTATS` command calculates an aggregate result and merges that result
+back into the stream of input data. Without the optional `BY` clause this will
+produce a single result which is appended to each row. With a `BY` clause this
+will produce one result per grouping and merge the result into the stream based on
+matching group keys.
+
+All of the <<esql-agg-functions,aggregation functions>> are supported.
+
+*Examples*
+
+Find the employees that speak the most languages (it's a tie!):
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/inlinestats.csv-spec[tag=max-languages]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/inlinestats.csv-spec[tag=max-languages-result]
+|===
+
+Find the longest tenured employee who's last name starts with each letter of the alphabet:
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/inlinestats.csv-spec[tag=longest-tenured-by-first]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/inlinestats.csv-spec[tag=longest-tenured-by-first-result]
+|===
+
+Find the northern and southern most airports:
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/inlinestats.csv-spec[tag=extreme-airports]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/inlinestats.csv-spec[tag=extreme-airports-result]
+|===
+
+NOTE: Our test data doesn't have many "small" airports.
+
+If a `BY` field is multivalued then `INLINESTATS` will put the row in *each*
+bucket like <<esql-stats-by>>:
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/inlinestats.csv-spec[tag=mv-group]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/inlinestats.csv-spec[tag=mv-group-result]
+|===
+
+To treat each group key as its own row use <<esql-mv_expand>> before `INLINESTATS`:
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/inlinestats.csv-spec[tag=mv-expand]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/inlinestats.csv-spec[tag=mv-expand-result]
+|===
diff --git a/docs/reference/esql/processing-commands/lookup.asciidoc b/docs/reference/esql/processing-commands/lookup.asciidoc
index 7bb3a5791deef..ca456d8e70eed 100644
--- a/docs/reference/esql/processing-commands/lookup.asciidoc
+++ b/docs/reference/esql/processing-commands/lookup.asciidoc
@@ -2,7 +2,7 @@
 [[esql-lookup]]
 === `LOOKUP`
 
-experimental::["LOOKUP is a highly experimental and only available in SNAPSHOT versions."]
+experimental::["LOOKUP is highly experimental and only available in SNAPSHOT versions."]
 
 `LOOKUP` matches values from the input against a `table` provided in the request,
 adding the other fields from the `table` to the output.
diff --git a/docs/reference/geospatial-analysis.asciidoc b/docs/reference/geospatial-analysis.asciidoc
index 7577bb222127f..6760040e14bc7 100644
--- a/docs/reference/geospatial-analysis.asciidoc
+++ b/docs/reference/geospatial-analysis.asciidoc
@@ -2,7 +2,7 @@
 [[geospatial-analysis]]
 = Geospatial analysis
 
-Did you know that {es} has geospatial capabilities? https://www.elastic.co/blog/geo-location-and-search[{es} and geo] go way back, to 2010. A lot has happened since then and today {es} provides robust geospatial capabilities with speed, all with a stack that scales automatically. 
+Did you know that {es} has geospatial capabilities? https://www.elastic.co/blog/geo-location-and-search[{es} and geo] go way back, to 2010. A lot has happened since then and today {es} provides robust geospatial capabilities with speed, all with a stack that scales automatically.
 
 Not sure where to get started with {es} and geo? Then, you have come to the right place.
 
@@ -18,8 +18,10 @@ Have an index with lat/lon pairs but no geo_point mapping? Use <<runtime-mapping
 [[geospatial-ingest]]
 === Ingest
 
-Data is often messy and incomplete. <<ingest,Ingest pipelines>> lets you clean, transform, and augment your data before indexing. 
+Data is often messy and incomplete. <<ingest,Ingest pipelines>> lets you clean, transform, and augment your data before indexing.
 
+* Use <<csv-processor>> together with <<explicit-mapping,explicit mapping>> to index CSV files with geo data.
+  Kibana's {kibana-ref}/import-geospatial-data.html[Import CSV] feature can help with this.
 * Use <<geoip-processor>> to add geographical location of an IPv4 or IPv6 address.
 * Use <<ingest-geo-grid-processor,geo-grid processor>> to convert grid tiles or hexagonal cell ids to bounding boxes or polygons which describe their shape.
 * Use <<geo-match-enrich-policy-type,geo_match enrich policy>> for reverse geocoding. For example, use {kibana-ref}/reverse-geocoding-tutorial.html[reverse geocoding] to visualize metropolitan areas by web traffic.
@@ -30,6 +32,18 @@ Data is often messy and incomplete. <<ingest,Ingest pipelines>> lets you clean,
 
 <<geo-queries,Geo queries>> answer location-driven questions. Find documents that intersect with, are within, are contained by, or do not intersect your query geometry. Combine geospatial queries with full text search queries for unparalleled searching experience. For example, "Show me all subscribers that live within 5 miles of our new gym location, that joined in the last year and have running mentioned in their profile".
 
+[discrete]
+[[esql-query]]
+=== ES|QL
+
+<<esql,ES|QL>> has support for <<esql-spatial-functions, Geospatial Search>> functions, enabling efficient index searching for documents that intersect with, are within, are contained by, or are disjoint from a query geometry. In addition, the `ST_DISTANCE` function calculates the distance between two points.
+
+* experimental:[] <<esql-st_intersects>>
+* experimental:[] <<esql-st_disjoint>>
+* experimental:[] <<esql-st_contains>>
+* experimental:[] <<esql-st_within>>
+* experimental:[] <<esql-st_distance>>
+
 [discrete]
 [[geospatial-aggregate]]
 === Aggregate
@@ -42,12 +56,12 @@ Geospatial bucket aggregations:
 * <<search-aggregations-bucket-geohashgrid-aggregation,Geohash grid aggregation>> groups geo_point and geo_shape values into buckets that represent a grid.
 * <<search-aggregations-bucket-geohexgrid-aggregation,Geohex grid aggregation>> groups geo_point and geo_shape values into buckets that represent an H3 hexagonal cell.
 * <<search-aggregations-bucket-geotilegrid-aggregation,Geotile grid aggregation>> groups geo_point and geo_shape values into buckets that represent a grid. Each cell corresponds to a {wikipedia}/Tiled_web_map[map tile] as used by many online map sites.
- 
+
 Geospatial metric aggregations:
 
 * <<search-aggregations-metrics-geobounds-aggregation, Geo-bounds aggregation>> computes the geographic bounding box containing all values for a Geopoint or Geoshape field.
 * <<search-aggregations-metrics-geocentroid-aggregation, Geo-centroid aggregation>> computes the weighted centroid from all coordinate values for geo fields.
-* <<search-aggregations-metrics-geo-line,Geo-line aggregation>> aggregates all geo_point values within a bucket into a LineString ordered by the chosen sort field. Use geo_line aggregation to create {kibana-ref}/asset-tracking-tutorial.html[vehicle tracks]. 
+* <<search-aggregations-metrics-geo-line,Geo-line aggregation>> aggregates all geo_point values within a bucket into a LineString ordered by the chosen sort field. Use geo_line aggregation to create {kibana-ref}/asset-tracking-tutorial.html[vehicle tracks].
 
 Combine aggregations to perform complex geospatial analysis. For example, to calculate the most recent GPS tracks per flight, use a <<search-aggregations-bucket-terms-aggregation,terms aggregation>> to group documents into buckets per aircraft. Then use geo-line aggregation to compute a track for each aircraft. In another example, use geotile grid aggregation to group documents into a grid. Then use geo-centroid aggregation to find the weighted centroid of each grid cell.
 
@@ -79,4 +93,4 @@ Put machine learning to work for you and find the data that should stand out wit
 
 Let your location data drive insights and action with {kibana-ref}/geo-alerting.html[geographic alerts]. Commonly referred to as geo-fencing, track moving objects as they enter or exit a boundary to receive notifications through common business systems (email, Slack, Teams, PagerDuty, and more).
 
-Interested in learning more? Follow {kibana-ref}/asset-tracking-tutorial.html[step-by-step instructions] for setting up tracking containment alerts to monitor moving vehicles.
\ No newline at end of file
+Interested in learning more? Follow {kibana-ref}/asset-tracking-tutorial.html[step-by-step instructions] for setting up tracking containment alerts to monitor moving vehicles.
diff --git a/docs/reference/health/health.asciidoc b/docs/reference/health/health.asciidoc
index 6ac7bd2001d45..34714e80e1b18 100644
--- a/docs/reference/health/health.asciidoc
+++ b/docs/reference/health/health.asciidoc
@@ -204,9 +204,8 @@ for health status set `verbose` to `false` to disable the more expensive analysi
     `help_url` field.
 
 `affected_resources`::
-    (Optional, array of strings) If the root cause pertains to multiple resources in the
-    cluster (like indices, shards, nodes, etc...) this will hold all resources that this
-    diagnosis is applicable for.
+    (Optional, object) An object where the keys represent resource types (for example, indices, shards), 
+    and the values are lists of the specific resources affected by the issue.
 
 `help_url`::
     (string) A link to the troubleshooting guide that'll fix the health problem.
diff --git a/docs/reference/high-availability/cluster-design.asciidoc b/docs/reference/high-availability/cluster-design.asciidoc
index 6c17a494f36ae..105c8b236b0b1 100644
--- a/docs/reference/high-availability/cluster-design.asciidoc
+++ b/docs/reference/high-availability/cluster-design.asciidoc
@@ -246,7 +246,8 @@ accumulate into a noticeable performance penalty. An unreliable network may
 have frequent network partitions. {es} will automatically recover from a
 network partition as quickly as it can but your cluster may be partly
 unavailable during a partition and will need to spend time and resources to
-resynchronize any missing data and rebalance itself once the partition heals.
+<<shard-recovery,resynchronize any missing data>> and <<shards-rebalancing-settings,rebalance>> 
+itself once the partition heals.
 Recovering from a failure may involve copying a large amount of data between
 nodes so the recovery time is often determined by the available bandwidth.
 
diff --git a/docs/reference/how-to/size-your-shards.asciidoc b/docs/reference/how-to/size-your-shards.asciidoc
index 31f4039bcfaca..5f67014d5bb4a 100644
--- a/docs/reference/how-to/size-your-shards.asciidoc
+++ b/docs/reference/how-to/size-your-shards.asciidoc
@@ -152,16 +152,18 @@ same data. However, very large shards can also cause slower searches and will
 take longer to recover after a failure.
 
 There is no hard limit on the physical size of a shard, and each shard can in
-theory contain up to just over two billion documents. However, experience shows
-that shards between 10GB and 50GB typically work well for many use cases, as
-long as the per-shard document count is kept below 200 million.
+theory contain up to <<troubleshooting-max-docs-limit,just over two billion 
+documents>>. However, experience shows that shards between 10GB and 50GB 
+typically work well for many use cases, as long as the per-shard document count 
+is kept below 200 million.
 
 You may be able to use larger shards depending on your network and use case,
 and smaller shards may be appropriate for
 {enterprise-search-ref}/index.html[Enterprise Search] and similar use cases.
 
 If you use {ilm-init}, set the <<ilm-rollover,rollover action>>'s
-`max_primary_shard_size` threshold to `50gb` to avoid shards larger than 50GB.
+`max_primary_shard_size` threshold to `50gb` to avoid shards larger than 50GB 
+and `min_primary_shard_size` threshold to `10gb` to avoid shards smaller than 10GB.
 
 To see the current size of your shards, use the <<cat-shards,cat shards API>>.
 
@@ -184,6 +186,29 @@ index                                 prirep shard store
 // TESTRESPONSE[s/\.ds-my-data-stream-2099\.05\.06-000001/my-index-000001/]
 // TESTRESPONSE[s/50gb/.*/]
 
+If an index's shard is experiencing degraded performance from surpassing the 
+recommended 50GB size, you may consider fixing the index's shards' sizing. 
+Shards are immutable and therefore their size is fixed in place, 
+so indices must be copied with corrected settings. This requires first ensuring 
+sufficient disk to copy the data. Afterwards, you can copy the index's data 
+with corrected settings via one of the following options:
+
+* running <<indices-split-index,Split Index>> to increase number of primary 
+shards 
+
+* creating a destination index with corrected settings and then running 
+<<docs-reindex,Reindex>> 
+
+Kindly note performing a <<restore-snapshot-api,Restore Snapshot>> and/or 
+<<indices-clone-index,Clone Index>> would be insufficient to resolve shards' 
+sizing. 
+
+Once a source index's data is copied into its destination index, the source 
+index can be <<indices-delete-index,removed>>. You may then consider setting 
+<<indices-add-alias,Create Alias>> against the destination index for the source 
+index's name to point to it for continuity. 
+
+
 [discrete]
 [[shard-count-recommendation]]
 ==== Master-eligible nodes should have at least 1GB of heap per 3000 indices
@@ -501,6 +526,7 @@ POST _reindex
 Here’s how to resolve common shard-related errors.
 
 [discrete]
+[[troubleshooting-max-shards-open]]
 ==== this action would add [x] total shards, but this cluster currently has [y]/[z] maximum shards open;
 
 The <<cluster-max-shards-per-node,`cluster.max_shards_per_node`>> cluster
diff --git a/docs/reference/ilm/actions/ilm-delete.asciidoc b/docs/reference/ilm/actions/ilm-delete.asciidoc
index eac3b9804709a..beed60105ed96 100644
--- a/docs/reference/ilm/actions/ilm-delete.asciidoc
+++ b/docs/reference/ilm/actions/ilm-delete.asciidoc
@@ -15,6 +15,18 @@ Deletes the searchable snapshot created in a previous phase.
 Defaults to `true`.
 This option is applicable when the <<ilm-searchable-snapshot,searchable
 snapshot>> action is used in any previous phase.
++
+If you set this option to `false`, use the <<delete-snapshot-api,Delete
+snapshots API>> to remove {search-snaps} from your snapshot repository when
+they are no longer needed.
++
+If you manually delete an index before the {ilm-cap} delete phase runs, then
+{ilm-init} will not delete the underlying {search-snap}. Use the
+<<delete-snapshot-api,Delete snapshots API>> to remove the {search-snap} from
+your snapshot repository when it is no longer needed.
++
+See <<searchable-snapshots-reliability,Reliability of {search-snaps}>> for
+further information about deleting {search-snaps}.
 
 WARNING: If a policy with a searchable snapshot action is applied on an existing searchable snapshot index,
 the snapshot backing this index will NOT be deleted because it was not created by this policy. If you want
diff --git a/docs/reference/ilm/apis/delete-lifecycle.asciidoc b/docs/reference/ilm/apis/delete-lifecycle.asciidoc
index 632cb982b3968..fc9a35e4ef570 100644
--- a/docs/reference/ilm/apis/delete-lifecycle.asciidoc
+++ b/docs/reference/ilm/apis/delete-lifecycle.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Delete policy</titleabbrev>
 ++++
 
-Deletes an index lifecycle policy.
+Deletes an index <<index-lifecycle-management,lifecycle>> policy.
 
 [[ilm-delete-lifecycle-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ilm/apis/explain.asciidoc b/docs/reference/ilm/apis/explain.asciidoc
index 348a9e7f99e78..a1ddde8c9f2d9 100644
--- a/docs/reference/ilm/apis/explain.asciidoc
+++ b/docs/reference/ilm/apis/explain.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Explain lifecycle</titleabbrev>
 ++++
 
-Retrieves the current lifecycle status for one or more indices. For data
+Retrieves the current <<index-lifecycle-management,lifecycle>> status for one or more indices. For data
 streams, the API retrieves the current lifecycle status for the stream's backing
 indices.
 
diff --git a/docs/reference/ilm/apis/get-lifecycle.asciidoc b/docs/reference/ilm/apis/get-lifecycle.asciidoc
index 7443610065487..b4e07389a9fb7 100644
--- a/docs/reference/ilm/apis/get-lifecycle.asciidoc
+++ b/docs/reference/ilm/apis/get-lifecycle.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Get policy</titleabbrev>
 ++++
 
-Retrieves a lifecycle policy.
+Retrieves a <<index-lifecycle-management,lifecycle>> policy.
 
 [[ilm-get-lifecycle-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ilm/apis/get-status.asciidoc b/docs/reference/ilm/apis/get-status.asciidoc
index 7e9e963f6f369..f2ab8d65ec9a1 100644
--- a/docs/reference/ilm/apis/get-status.asciidoc
+++ b/docs/reference/ilm/apis/get-status.asciidoc
@@ -7,7 +7,7 @@
 <titleabbrev>Get {ilm} status</titleabbrev>
 ++++
 
-Retrieves the current {ilm} ({ilm-init}) status.
+Retrieves the current <<index-lifecycle-management,{ilm}>> ({ilm-init}) status.
 
 You can start or stop {ilm-init} with the <<ilm-start,start {ilm-init}>> and
 <<ilm-stop,stop {ilm-init}>> APIs.
diff --git a/docs/reference/ilm/apis/move-to-step.asciidoc b/docs/reference/ilm/apis/move-to-step.asciidoc
index 19cc9f7088867..f3441fa997cff 100644
--- a/docs/reference/ilm/apis/move-to-step.asciidoc
+++ b/docs/reference/ilm/apis/move-to-step.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Move to step</titleabbrev>
 ++++
 
-Triggers execution of a specific step in the lifecycle policy.
+Triggers execution of a specific step in the <<index-lifecycle-management,lifecycle>> policy.
 
 [[ilm-move-to-step-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ilm/apis/put-lifecycle.asciidoc b/docs/reference/ilm/apis/put-lifecycle.asciidoc
index ffd59a14d8c25..390f6b1bb4d15 100644
--- a/docs/reference/ilm/apis/put-lifecycle.asciidoc
+++ b/docs/reference/ilm/apis/put-lifecycle.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Create or update lifecycle policy</titleabbrev>
 ++++
 
-Creates or updates lifecycle policy. See <<ilm-index-lifecycle>> for
+Creates or updates <<index-lifecycle-management,lifecycle>> policy. See <<ilm-index-lifecycle>> for
 definitions of policy components.
 
 [[ilm-put-lifecycle-request]]
diff --git a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc
index 711eccc298df1..107cab4d5aa19 100644
--- a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc
+++ b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Remove policy</titleabbrev>
 ++++
 
-Removes assigned lifecycle policies from an index or a data stream's backing
+Removes assigned <<index-lifecycle-management,lifecycle>> policies from an index or a data stream's backing
 indices.
 
 [[ilm-remove-policy-request]]
diff --git a/docs/reference/ilm/apis/retry-policy.asciidoc b/docs/reference/ilm/apis/retry-policy.asciidoc
index cb2587fbb151b..8f01f15e0c3ad 100644
--- a/docs/reference/ilm/apis/retry-policy.asciidoc
+++ b/docs/reference/ilm/apis/retry-policy.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Retry policy</titleabbrev>
 ++++
 
-Retry executing the policy for an index that is in the ERROR step.
+Retry executing the <<index-lifecycle-management,lifecycle>> policy for an index that is in the ERROR step.
 
 [[ilm-retry-policy-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ilm/apis/start.asciidoc b/docs/reference/ilm/apis/start.asciidoc
index 32db585c6b14c..c38b3d9ca8831 100644
--- a/docs/reference/ilm/apis/start.asciidoc
+++ b/docs/reference/ilm/apis/start.asciidoc
@@ -7,7 +7,7 @@
 <titleabbrev>Start {ilm}</titleabbrev>
 ++++
 
-Start the {ilm} ({ilm-init}) plugin.
+Start the <<index-lifecycle-management,{ilm}>> ({ilm-init}) plugin.
 
 [[ilm-start-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ilm/apis/stop.asciidoc b/docs/reference/ilm/apis/stop.asciidoc
index 1e9cfb94d0b1f..a6100d794c2d3 100644
--- a/docs/reference/ilm/apis/stop.asciidoc
+++ b/docs/reference/ilm/apis/stop.asciidoc
@@ -7,7 +7,7 @@
 <titleabbrev>Stop {ilm}</titleabbrev>
 ++++
 
-Stop the {ilm} ({ilm-init}) plugin.
+Stop the <<index-lifecycle-management,{ilm}>> ({ilm-init}) plugin.
 
 [[ilm-stop-request]]
 ==== {api-request-title}
diff --git a/docs/reference/ilm/error-handling.asciidoc b/docs/reference/ilm/error-handling.asciidoc
index d922fa6687823..f810afc6c2b5f 100644
--- a/docs/reference/ilm/error-handling.asciidoc
+++ b/docs/reference/ilm/error-handling.asciidoc
@@ -2,7 +2,7 @@
 [[index-lifecycle-error-handling]]
 == Troubleshooting {ilm} errors
 
-When {ilm-init} executes a lifecycle policy, it's possible for errors to occur
+When <<index-lifecycle-management,{ilm-init}>> executes a lifecycle policy, it's possible for errors to occur
 while performing the necessary index operations for a step. 
 When this happens, {ilm-init} moves the index to an `ERROR` step. 
 If {ilm-init} cannot resolve the error automatically, execution is halted  
diff --git a/docs/reference/ilm/ilm-index-lifecycle.asciidoc b/docs/reference/ilm/ilm-index-lifecycle.asciidoc
index acf59645dae13..040e02742f5e7 100644
--- a/docs/reference/ilm/ilm-index-lifecycle.asciidoc
+++ b/docs/reference/ilm/ilm-index-lifecycle.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Index lifecycle</titleabbrev>
 ++++
 
-{ilm-init} defines five index lifecycle _phases_:
+<<index-lifecycle-management,{ilm-init}>> defines five index lifecycle _phases_:
 
 * **Hot**: The index is actively being updated and queried.
 * **Warm**: The index is no longer being updated but is still being queried.
diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index 24149afe802a2..7232de12c8c50 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -113,7 +113,7 @@ Index mode supports the following values:
 
 `time_series`::: Index mode optimized for storage of metrics documented in <<tsds-index-settings,TSDS Settings>>.
 
-`logs`::: Index mode optimized for storage of logs. It applies default sort settings on the `hostname` and `timestamp` fields and uses <<synthetic-source,synthetic `_source`>>. <<index-modules-index-sorting,Index sorting>> on different fields is still allowed.
+`logsdb`::: Index mode optimized for storage of logs. It applies default sort settings on the `hostname` and `timestamp` fields and uses <<synthetic-source,synthetic `_source`>>. <<index-modules-index-sorting,Index sorting>> on different fields is still allowed.
 preview:[]
 
 [[routing-partition-size]] `index.routing_partition_size`::
diff --git a/docs/reference/indices/recovery.asciidoc b/docs/reference/indices/recovery.asciidoc
index b4e4bd33f819a..06b4d9d92e49f 100644
--- a/docs/reference/indices/recovery.asciidoc
+++ b/docs/reference/indices/recovery.asciidoc
@@ -35,21 +35,7 @@ index, or alias.
 Use the index recovery API to get information about ongoing and completed shard
 recoveries.
 
-// tag::shard-recovery-desc[]
-Shard recovery is the process of initializing a shard copy, such as restoring a
-primary shard from a snapshot or syncing a replica shard from a primary shard.
-When a shard recovery completes, the recovered shard is available for search
-and indexing.
-
-Recovery automatically occurs during the following processes:
-
-* Node startup. This type of recovery is called a local store recovery.
-* Primary shard replication.
-* Relocation of a shard to a different node in the same cluster.
-* <<snapshots-restore-snapshot,Snapshot restore>> operation.
-* <<indices-clone-index,Clone>>, <<indices-shrink-index,shrink>>, or
-<<indices-split-index,split>> operation.
-// end::shard-recovery-desc[]
+include::{es-ref-dir}/modules/shard-recovery-desc.asciidoc[]
 
 The index recovery API reports information about completed recoveries only for
 shard copies that currently exist in the cluster. It only reports the last
@@ -360,7 +346,7 @@ The API returns the following response:
   "index1" : {
     "shards" : [ {
       "id" : 0,
-      "type" : "STORE",
+      "type" : "EXISTING_STORE",
       "stage" : "DONE",
       "primary" : true,
       "start_time" : "2014-02-24T12:38:06.349",
diff --git a/docs/reference/inference/images/inference-landscape.png b/docs/reference/inference/images/inference-landscape.png
new file mode 100644
index 0000000000000..a35d1370fd09b
Binary files /dev/null and b/docs/reference/inference/images/inference-landscape.png differ
diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc
index 9c75820a8f92b..8fdf8aecc2ae5 100644
--- a/docs/reference/inference/inference-apis.asciidoc
+++ b/docs/reference/inference/inference-apis.asciidoc
@@ -5,14 +5,16 @@
 experimental[]
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio or
-Hugging Face. For built-in models and models uploaded through Eland, the {infer}
-APIs offer an alternative way to use and manage trained models. However, if you
-do not plan to use the {infer} APIs to use these models or if you want to use
-non-NLP models, use the <<ml-df-trained-models-apis>>.
+{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure,
+Google AI Studio or Hugging Face. For built-in models and models uploaded
+through Eland, the {infer} APIs offer an alternative way to use and manage
+trained models. However, if you do not plan to use the {infer} APIs to use these
+models or if you want to use non-NLP models, use the
+<<ml-df-trained-models-apis>>.
 
 The {infer} APIs enable you to create {infer} endpoints and use {ml} models of
-different providers - such as Cohere, OpenAI, or HuggingFace - as a service. Use
+different providers - such as Amazon Bedrock, Anthropic, Azure AI Studio,
+Cohere, Google AI, Mistral, OpenAI, or HuggingFace - as a service. Use
 the following APIs to manage {infer} models and perform {infer}:
 
 * <<delete-inference-api>>
@@ -20,12 +22,26 @@ the following APIs to manage {infer} models and perform {infer}:
 * <<post-inference-api>>
 * <<put-inference-api>>
 
+[[inference-landscape]]
+.A representation of the Elastic inference landscape
+image::images/inference-landscape.png[A representation of the Elastic inference landscape,align="center"]
+
+An {infer} endpoint enables you to use the corresponding {ml} model without
+manual deployment and apply it to your data at ingestion time through
+<<semantic-search-semantic-text, semantic text>>. 
+
+Choose a model from your provider or use ELSER – a retrieval model trained by 
+Elastic –, then create an {infer} endpoint by the <<put-inference-api>>.
+Now use <<semantic-search-semantic-text, semantic text>> to perform
+<<semantic-search, semantic search>> on your data.
 
 include::delete-inference.asciidoc[]
 include::get-inference.asciidoc[]
 include::post-inference.asciidoc[]
 include::put-inference.asciidoc[]
+include::service-alibabacloud-ai-search.asciidoc[]
 include::service-amazon-bedrock.asciidoc[]
+include::service-anthropic.asciidoc[]
 include::service-azure-ai-studio.asciidoc[]
 include::service-azure-openai.asciidoc[]
 include::service-cohere.asciidoc[]
diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc
index b809a96b8f81a..ba26a563541fc 100644
--- a/docs/reference/inference/put-inference.asciidoc
+++ b/docs/reference/inference/put-inference.asciidoc
@@ -6,10 +6,17 @@ experimental[]
 
 Creates an {infer} endpoint to perform an {infer} task.
 
-IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Mistral, Azure OpenAI, Google AI Studio, Google Vertex AI or Hugging Face.
-For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models.
-However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
+[IMPORTANT]
+====
+* The {infer} APIs enable you to use certain services, such as built-in
+{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Mistral,
+Azure OpenAI, Google AI Studio, Google Vertex AI, Anthropic or Hugging Face.
+* For built-in models and models uploaded through Eland, the {infer} APIs offer an
+alternative way to use and manage trained models. However, if you do not plan to
+use the {infer} APIs to use these models or if you want to use non-NLP models,
+use the <<ml-df-trained-models-apis>>.
+====
+
 
 [discrete]
 [[put-inference-api-request]]
@@ -32,7 +39,9 @@ The create {infer} API enables you to create an {infer} endpoint and configure a
 
 The following services are available through the {infer} API, click the links to review the configuration details of the services:
 
+* <<infer-service-alibabacloud-ai-search,AlibabaCloud AI Search>>
 * <<infer-service-amazon-bedrock,Amazon Bedrock>>
+* <<infer-service-anthropic,Anthropic>>
 * <<infer-service-azure-ai-studio,Azure AI Studio>>
 * <<infer-service-azure-openai,Azure OpenAI>>
 * <<infer-service-cohere,Cohere>>
@@ -44,10 +53,5 @@ The following services are available through the {infer} API, click the links to
 * <<infer-service-mistral,Mistral>>
 * <<infer-service-openai,OpenAI>>
 
-[NOTE]
-====
-You might see a 502 bad gateway error in the response when using the {kib} Console.
-This error usually just reflects a timeout, while the model downloads in the background.
-You can check the download progress in the {ml-app} UI.
-If using the Python client, you can set the `timeout` parameter to a higher value.
-====
\ No newline at end of file
+The {es} and ELSER services run on a {ml} node in your {es} cluster. The rest of
+the services connect to external providers.
\ No newline at end of file
diff --git a/docs/reference/inference/service-alibabacloud-ai-search.asciidoc b/docs/reference/inference/service-alibabacloud-ai-search.asciidoc
new file mode 100644
index 0000000000000..23a3d532635ac
--- /dev/null
+++ b/docs/reference/inference/service-alibabacloud-ai-search.asciidoc
@@ -0,0 +1,184 @@
+[[infer-service-alibabacloud-ai-search]]
+=== AlibabaCloud AI Search {infer} service
+
+Creates an {infer} endpoint to perform an {infer} task with the `alibabacloud-ai-search` service.
+
+[discrete]
+[[infer-service-alibabacloud-ai-search-api-request]]
+==== {api-request-title}
+
+`PUT /_inference/<task_type>/<inference_id>`
+
+[discrete]
+[[infer-service-alibabacloud-ai-search-api-path-params]]
+==== {api-path-parms-title}
+
+`<inference_id>`::
+(Required, string)
+include::inference-shared.asciidoc[tag=inference-id]
+
+`<task_type>`::
+(Required, string)
+include::inference-shared.asciidoc[tag=task-type]
++
+--
+Available task types:
+
+* `text_embedding`,
+* `sparse_embedding`,
+* `rerank`.
+--
+
+[discrete]
+[[infer-service-alibabacloud-ai-search-api-request-body]]
+==== {api-request-body-title}
+
+`service`::
+(Required, string) The type of service supported for the specified task type.
+In this case,
+`alibabacloud-ai-search`.
+
+`service_settings`::
+(Required, object)
+include::inference-shared.asciidoc[tag=service-settings]
++
+--
+These settings are specific to the `alibabacloud-ai-search` service.
+--
+
+`api_key`:::
+(Required, string)
+A valid API key for the AlibabaCloud AI Search API.
+
+`service_id`:::
+(Required, string)
+The name of the model service to use for the {infer} task.
++
+--
+Available service_ids for the `text_embedding` task:
+
+* `ops-text-embedding-001`
+* `ops-text-embedding-zh-001`
+* `ops-text-embedding-en-001`
+* `ops-text-embedding-002`
+
+For the supported `text_embedding` service_ids, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details[documentation].
+
+Available service_id for the `sparse_embedding` task:
+
+* `ops-text-sparse-embedding-001`
+
+For the supported `sparse_embedding` service_id, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-sparse-embedding-api-details[documentation].
+
+Available service_id for the `rerank` task is:
+
+* `ops-bge-reranker-larger`
+
+For the supported `rerank` service_id, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/ranker-api-details[documentation].
+--
+
+`host`:::
+(Required, string)
+The name of the host address used for the {infer} task. You can find the host address at https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] of the documentation.
+
+`workspace`:::
+(Required, string)
+The name of the workspace used for the {infer} task.
+
+`rate_limit`:::
+(Optional, object)
+By default, the `alibabacloud-ai-search` service sets the number of requests allowed per minute to `1000`.
+This helps to minimize the number of rate limit errors returned from AlibabaCloud AI Search.
+To modify this, set the `requests_per_minute` setting of this object in your service settings:
++
+--
+include::inference-shared.asciidoc[tag=request-per-minute-example]
+--
+
+
+`task_settings`::
+(Optional, object)
+include::inference-shared.asciidoc[tag=task-settings]
++
+.`task_settings` for the `text_embedding` task type
+[%collapsible%closed]
+=====
+`input_type`:::
+(Optional, string)
+Specifies the type of input passed to the model.
+Valid values are:
+* `ingest`: for storing document embeddings in a vector database.
+* `search`: for storing embeddings of search queries run against a vector database to find relevant documents.
+=====
++
+.`task_settings` for the `sparse_embedding` task type
+[%collapsible%closed]
+=====
+`input_type`:::
+(Optional, string)
+Specifies the type of input passed to the model.
+Valid values are:
+* `ingest`: for storing document embeddings in a vector database.
+* `search`: for storing embeddings of search queries run against a vector database to find relevant documents.
+
+`return_token`:::
+(Optional, boolean)
+If `true`, the token name will be returned in the response. Defaults to `false` which means only the token ID will be returned in the response.
+=====
+
+[discrete]
+[[inference-example-alibabacloud-ai-search]]
+==== AlibabaCloud AI Search service examples
+
+The following example shows how to create an {infer} endpoint called `alibabacloud_ai_search_embeddings` to perform a `text_embedding` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/alibabacloud_ai_search_embeddings
+{
+    "service": "alibabacloud-ai-search",
+    "service_settings": {
+        "api_key": "<api_key>",
+        "service_id": "ops-text-embedding-001",
+        "host": "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com",
+        "workspace": "default"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+The following example shows how to create an {infer} endpoint called
+`alibabacloud_ai_search_sparse` to perform a `sparse_embedding` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/sparse_embedding/alibabacloud_ai_search_sparse
+{
+    "service": "alibabacloud-ai-search",
+    "service_settings": {
+        "api_key": "<api_key>",
+        "service_id": "ops-text-sparse-embedding-001",
+        "host": "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com",
+        "workspace": "default"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+The next example shows how to create an {infer} endpoint called
+`alibabacloud_ai_search_rerank` to perform a `rerank` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/rerank/alibabacloud_ai_search_rerank
+{
+    "service": "alibabacloud-ai-search",
+    "service_settings": {
+        "api_key": "<api_key>",
+        "service_id": "ops-bge-reranker-larger",
+        "host": "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com",
+        "workspace": "default"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
diff --git a/docs/reference/inference/service-amazon-bedrock.asciidoc b/docs/reference/inference/service-amazon-bedrock.asciidoc
index 4ffa368613a0e..dbffd5c26fbcc 100644
--- a/docs/reference/inference/service-amazon-bedrock.asciidoc
+++ b/docs/reference/inference/service-amazon-bedrock.asciidoc
@@ -122,14 +122,6 @@ Only available for `anthropic`, `cohere`, and `mistral` providers.
 Alternative to `temperature`. Limits samples to the top-K most likely words, balancing coherence and variability.
 Should not be used if `temperature` is specified.
 
-=====
-+
-.`task_settings` for the `text_embedding` task type
-[%collapsible%closed]
-=====
-
-There are no `task_settings` available for the `text_embedding` task type.
-
 =====
 
 [discrete]
diff --git a/docs/reference/inference/service-cohere.asciidoc b/docs/reference/inference/service-cohere.asciidoc
index 52d71e0bc02a5..84eae6e880617 100644
--- a/docs/reference/inference/service-cohere.asciidoc
+++ b/docs/reference/inference/service-cohere.asciidoc
@@ -131,6 +131,7 @@ Specify whether to return doc text within the results.
 `top_n`::
 (Optional, integer)
 The number of most relevant documents to return, defaults to the number of the documents.
+If this {infer} endpoint is used in a `text_similarity_reranker` retriever query and `top_n` is set, it must be greater than or equal to `rank_window_size` in the query.
 =====
 +
 .`task_settings` for the `text_embedding` task type
diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc
index 3b9b5b1928d7b..572cad591fba6 100644
--- a/docs/reference/inference/service-elasticsearch.asciidoc
+++ b/docs/reference/inference/service-elasticsearch.asciidoc
@@ -1,7 +1,12 @@
 [[infer-service-elasticsearch]]
 === Elasticsearch {infer} service
 
-Creates an {infer} endpoint to perform an {infer} task with the `elasticsearch` service.
+Creates an {infer} endpoint to perform an {infer} task with the `elasticsearch`
+service.
+
+NOTE: If you use the E5 model through the `elasticsearch` service, the API
+request will automatically download and deploy the model if it isn't downloaded
+yet.
 
 
 [discrete]
@@ -26,6 +31,7 @@ include::inference-shared.asciidoc[tag=task-type]
 Available task types:
 
 * `rerank`,
+* `sparse_embedding`,
 * `text_embedding`.
 --
 
@@ -35,7 +41,7 @@ Available task types:
 
 `service`::
 (Required, string)
-The type of service supported for the specified task type. In this case, 
+The type of service supported for the specified task type. In this case,
 `elasticsearch`.
 
 `service_settings`::
@@ -46,6 +52,22 @@ include::inference-shared.asciidoc[tag=service-settings]
 These settings are specific to the `elasticsearch` service.
 --
 
+`adaptive_allocations`:::
+(Optional, object)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation]
+
+`enabled`::::
+(Optional, Boolean)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-enabled]
+
+`max_number_of_allocations`::::
+(Optional, integer)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-max-number]
+
+`min_number_of_allocations`::::
+(Optional, integer)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-min-number]
+
 `model_id`:::
 (Required, string)
 The name of the model to use for the {infer} task.
@@ -54,11 +76,13 @@ It can be the ID of either a built-in model (for example, `.multilingual-e5-smal
 
 `num_allocations`:::
 (Required, integer)
-The total number of allocations this model is assigned across machine learning nodes. Increasing this value generally increases the throughput.
+The total number of allocations this model is assigned across machine learning nodes.
+Increasing this value generally increases the throughput.
+If `adaptive_allocations` is enabled, do not set this value, because it's automatically set.
 
 `num_threads`:::
 (Required, integer)
-Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node. 
+Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node.
 Must be a power of 2. Max allowed value is 32.
 
 `task_settings`::
@@ -81,6 +105,9 @@ Returns the document instead of only the index. Defaults to `true`.
 The following example shows how to create an {infer} endpoint called
 `my-e5-model` to perform a `text_embedding` task type.
 
+The API request below will automatically download the E5 model if it isn't
+already downloaded and then deploy the model.
+
 [source,console]
 ------------------------------------------------------------
 PUT _inference/text_embedding/my-e5-model
@@ -98,6 +125,14 @@ PUT _inference/text_embedding/my-e5-model
 Valid values are `.multilingual-e5-small` and `.multilingual-e5-small_linux-x86_64`.
 For further details, refer to the {ml-docs}/ml-nlp-e5.html[E5 model documentation].
 
+[NOTE]
+====
+You might see a 502 bad gateway error in the response when using the {kib} Console.
+This error usually just reflects a timeout, while the model downloads in the background.
+You can check the download progress in the {ml-app} UI.
+If using the Python client, you can set the `timeout` parameter to a higher value.
+====
+
 [discrete]
 [[inference-example-eland]]
 ==== Models uploaded by Eland via the elasticsearch service
@@ -107,16 +142,45 @@ The following example shows how to create an {infer} endpoint called
 
 [source,console]
 ------------------------------------------------------------
-PUT _inference/text_embedding/my-msmarco-minilm-model
+PUT _inference/text_embedding/my-msmarco-minilm-model <1>
 {
   "service": "elasticsearch",
   "service_settings": {
     "num_allocations": 1,
     "num_threads": 1,
-    "model_id": "msmarco-MiniLM-L12-cos-v5" <1>
+    "model_id": "msmarco-MiniLM-L12-cos-v5" <2>
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> Provide an unique identifier for the inference endpoint. The `inference_id` must be unique and must not match the `model_id`.
+<2> The `model_id` must be the ID of a text embedding model which has already been
+{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland].
+
+[discrete]
+[[inference-example-adaptive-allocation]]
+==== Setting adaptive allocation for E5 via the `elasticsearch` service
+
+The following example shows how to create an {infer} endpoint called
+`my-e5-model` to perform a `text_embedding` task type and configure adaptive
+allocations.
+
+The API request below will automatically download the E5 model if it isn't
+already downloaded and then deploy the model.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/my-e5-model
+{
+  "service": "elasticsearch",
+  "service_settings": {
+    "adaptive_allocations": {
+      "enabled": true,
+      "min_number_of_allocations": 3,
+      "max_number_of_allocations": 10
+    },
+    "model_id": ".multilingual-e5-small"
   }
 }
 ------------------------------------------------------------
 // TEST[skip:TBD]
-<1> The `model_id` must be the ID of a text embedding model which has already been
-{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland].
\ No newline at end of file
diff --git a/docs/reference/inference/service-elser.asciidoc b/docs/reference/inference/service-elser.asciidoc
index 829ff4968c5be..fdce94901984b 100644
--- a/docs/reference/inference/service-elser.asciidoc
+++ b/docs/reference/inference/service-elser.asciidoc
@@ -3,6 +3,9 @@
 
 Creates an {infer} endpoint to perform an {infer} task with the `elser` service.
 
+NOTE: The API request will automatically download and deploy the ELSER model if
+it isn't already downloaded.
+
 
 [discrete]
 [[infer-service-elser-api-request]]
@@ -34,7 +37,7 @@ Available task types:
 
 `service`::
 (Required, string)
-The type of service supported for the specified task type. In this case, 
+The type of service supported for the specified task type. In this case,
 `elser`.
 
 `service_settings`::
@@ -45,13 +48,31 @@ include::inference-shared.asciidoc[tag=service-settings]
 These settings are specific to the `elser` service.
 --
 
+`adaptive_allocations`:::
+(Optional, object)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation]
+
+`enabled`::::
+(Optional, Boolean)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-enabled]
+
+`max_number_of_allocations`::::
+(Optional, integer)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-max-number]
+
+`min_number_of_allocations`::::
+(Optional, integer)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-min-number] 
+
 `num_allocations`:::
 (Required, integer)
-The total number of allocations this model is assigned across machine learning nodes. Increasing this value generally increases the throughput.
+The total number of allocations this model is assigned across machine learning nodes.
+Increasing this value generally increases the throughput.
+If `adaptive_allocations` is enabled, do not set this value, because it's automatically set.
 
 `num_threads`:::
 (Required, integer)
-Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node. 
+Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node.
 Must be a power of 2. Max allowed value is 32.
 
 
@@ -63,6 +84,9 @@ The following example shows how to create an {infer} endpoint called
 `my-elser-model` to perform a `sparse_embedding` task type.
 Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more info.
 
+The request below will automatically download the ELSER model if it isn't
+already downloaded and then deploy the model.
+
 [source,console]
 ------------------------------------------------------------
 PUT _inference/sparse_embedding/my-elser-model
@@ -92,4 +116,39 @@ Example response:
   "task_settings": {}
 }
 ------------------------------------------------------------
-// NOTCONSOLE
\ No newline at end of file
+// NOTCONSOLE
+
+[NOTE]
+====
+You might see a 502 bad gateway error in the response when using the {kib} Console.
+This error usually just reflects a timeout, while the model downloads in the background.
+You can check the download progress in the {ml-app} UI.
+If using the Python client, you can set the `timeout` parameter to a higher value.
+====
+
+[discrete]
+[[inference-example-elser-adaptive-allocation]]
+==== Setting adaptive allocation for the ELSER service
+
+The following example shows how to create an {infer} endpoint called
+`my-elser-model` to perform a `sparse_embedding` task type and configure
+adaptive allocations.
+
+The request below will automatically download the ELSER model if it isn't
+already downloaded and then deploy the model.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/sparse_embedding/my-elser-model
+{
+  "service": "elser",
+  "service_settings": {
+    "adaptive_allocations": {
+      "enabled": true,
+      "min_number_of_allocations": 3,
+      "max_number_of_allocations": 10
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
\ No newline at end of file
diff --git a/docs/reference/ingest/apis/delete-geoip-database.asciidoc b/docs/reference/ingest/apis/delete-geoip-database.asciidoc
new file mode 100644
index 0000000000000..957e59f0f0de4
--- /dev/null
+++ b/docs/reference/ingest/apis/delete-geoip-database.asciidoc
@@ -0,0 +1,55 @@
+[[delete-geoip-database-api]]
+=== Delete geoip database configuration  API
+++++
+<titleabbrev>Delete geoip database configuration</titleabbrev>
+++++
+
+Deletes a geoip database configuration.
+
+[source,console]
+----
+DELETE /_ingest/geoip/database/my-database-id
+----
+// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.]
+
+
+[[delete-geoip-database-api-request]]
+==== {api-request-title}
+
+`DELETE /_ingest/geoip/database/<database>`
+
+[[delete-geoip-database-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the
+`manage` <<privileges-list-cluster,cluster privilege>> to use this API.
+
+[[delete-geoip-database-api-path-params]]
+==== {api-path-parms-title}
+
+`<database>`::
++
+--
+(Required, string) Database configuration ID used to limit the request.
+
+--
+
+
+[[delete-geoip-database-api-query-params]]
+==== {api-query-parms-title}
+
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms]
+
+
+[[delete-geoip-database-api-example]]
+==== {api-examples-title}
+
+
+[[delete-geoip-database-api-specific-ex]]
+===== Delete a specific geoip database configuration
+
+[source,console]
+----
+DELETE /_ingest/geoip/database/example-database-id
+----
+// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.]
diff --git a/docs/reference/ingest/apis/delete-pipeline.asciidoc b/docs/reference/ingest/apis/delete-pipeline.asciidoc
index 6f50251dbf1cd..94ac87c61b56b 100644
--- a/docs/reference/ingest/apis/delete-pipeline.asciidoc
+++ b/docs/reference/ingest/apis/delete-pipeline.asciidoc
@@ -62,7 +62,7 @@ use a value of `*`.
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms]
 
 
-[[delete-pipeline-api-api-example]]
+[[delete-pipeline-api-example]]
 ==== {api-examples-title}
 
 
diff --git a/docs/reference/ingest/apis/geoip-stats-api.asciidoc b/docs/reference/ingest/apis/geoip-stats.asciidoc
similarity index 100%
rename from docs/reference/ingest/apis/geoip-stats-api.asciidoc
rename to docs/reference/ingest/apis/geoip-stats.asciidoc
diff --git a/docs/reference/ingest/apis/get-geoip-database.asciidoc b/docs/reference/ingest/apis/get-geoip-database.asciidoc
new file mode 100644
index 0000000000000..f055e3e759db8
--- /dev/null
+++ b/docs/reference/ingest/apis/get-geoip-database.asciidoc
@@ -0,0 +1,80 @@
+[[get-geoip-database-api]]
+=== Get geoip database configuration API
+++++
+<titleabbrev>Get geoip database configuration</titleabbrev>
+++++
+
+Returns information about one or more geoip database configurations.
+
+[source,console]
+----
+GET /_ingest/geoip/database/my-database-id
+----
+// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.]
+
+
+
+[[get-geoip-database-api-request]]
+==== {api-request-title}
+
+`GET /_ingest/geoip/database/<database>`
+
+`GET /_ingest/geoip/database`
+
+[[get-geoip-database-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the
+ `manage` <<privileges-list-cluster,cluster privilege>> to use this API.
+
+[[get-geoip-database-api-path-params]]
+==== {api-path-parms-title}
+
+`<database>`::
+(Optional, string)
+Comma-separated list of database configuration IDs to retrieve. Wildcard (`*`) expressions are
+supported.
++
+To get all database configurations, omit this parameter or use `*`.
+
+
+[[get-geoip-database-api-query-params]]
+==== {api-query-parms-title}
+
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
+
+
+[[get-geoip-database-api-example]]
+==== {api-examples-title}
+
+
+[[get-geoip-database-api-specific-ex]]
+===== Get information for a specific geoip database configuration
+
+[source,console]
+----
+GET /_ingest/geoip/database/my-database-id
+----
+// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.]
+
+The API returns the following response:
+
+[source,console-result]
+----
+{
+  "databases" : [
+    {
+      "id" : "my-database-id",
+      "version" : 1,
+      "modified_date_millis" : 1723040276114,
+      "database" : {
+        "name" : "GeoIP2-Domain",
+        "maxmind" : {
+          "account_id" : "1234567"
+        }
+      }
+    }
+  ]
+}
+----
+// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.]
diff --git a/docs/reference/ingest/apis/get-pipeline.asciidoc b/docs/reference/ingest/apis/get-pipeline.asciidoc
index 71a261d97bdeb..f2a1155bca12b 100644
--- a/docs/reference/ingest/apis/get-pipeline.asciidoc
+++ b/docs/reference/ingest/apis/get-pipeline.asciidoc
@@ -65,7 +65,7 @@ To get all ingest pipelines, omit this parameter or use `*`.
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 
 
-[[get-pipeline-api-api-example]]
+[[get-pipeline-api-example]]
 ==== {api-examples-title}
 
 
diff --git a/docs/reference/ingest/apis/index.asciidoc b/docs/reference/ingest/apis/index.asciidoc
index 04fcd500a9721..e068f99ea0ad3 100644
--- a/docs/reference/ingest/apis/index.asciidoc
+++ b/docs/reference/ingest/apis/index.asciidoc
@@ -13,7 +13,7 @@ Use the following APIs to create, manage, and test ingest pipelines:
 * <<put-pipeline-api>> to create or update a pipeline
 * <<get-pipeline-api>> to retrieve a pipeline configuration
 * <<delete-pipeline-api>> to delete a pipeline
-* <<simulate-pipeline-api>> to test a pipeline
+* <<simulate-pipeline-api>> and <<simulate-ingest-api>> to test ingest pipelines
 
 [discrete]
 [[ingest-stat-apis]]
@@ -21,12 +21,27 @@ Use the following APIs to create, manage, and test ingest pipelines:
 
 Use the following APIs to get statistics about ingest processing:
 
-* <<geoip-stats-api>> to get download statistics for GeoIP2 databases used with
+* <<geoip-stats-api>> to get download statistics for IP geolocation databases used with
 the <<geoip-processor,`geoip` processor>>.
 
+[discrete]
+[[ingest-geoip-database-apis]]
+=== Ingest GeoIP Database APIs
+
+preview::["The commercial IP geolocation database download management APIs are in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but this feature is not subject to the support SLA of official GA features."]
+
+Use the following APIs to configure and manage commercial IP geolocation database downloads:
+
+* <<put-geoip-database-api>> to create or update a database configuration
+* <<get-geoip-database-api>> to retrieve a database configuration
+* <<delete-geoip-database-api>> to delete a database configuration
+
 include::put-pipeline.asciidoc[]
-include::delete-pipeline.asciidoc[]
-include::geoip-stats-api.asciidoc[]
 include::get-pipeline.asciidoc[]
+include::delete-pipeline.asciidoc[]
 include::simulate-pipeline.asciidoc[]
 include::simulate-ingest.asciidoc[]
+include::geoip-stats.asciidoc[]
+include::put-geoip-database.asciidoc[]
+include::get-geoip-database.asciidoc[]
+include::delete-geoip-database.asciidoc[]
diff --git a/docs/reference/ingest/apis/put-geoip-database.asciidoc b/docs/reference/ingest/apis/put-geoip-database.asciidoc
new file mode 100644
index 0000000000000..311c303002387
--- /dev/null
+++ b/docs/reference/ingest/apis/put-geoip-database.asciidoc
@@ -0,0 +1,72 @@
+[[put-geoip-database-api]]
+=== Create or update geoip database configuration API
+++++
+<titleabbrev>Create or update geoip database configuration</titleabbrev>
+++++
+
+Creates or updates an IP geolocation database configuration.
+
+IMPORTANT: The Maxmind `account_id` shown below requires a license key. Because the license key is sensitive information,
+it is stored as a <<secure-settings,secure setting>> in {es} named `ingest.geoip.downloader.maxmind.license_key`. Only
+one Maxmind license key is currently allowed per {es} cluster. A valid license key must be in the secure settings in order
+to download from Maxmind. The license key setting does not take effect until all nodes are restarted.
+
+[source,console]
+----
+PUT _ingest/geoip/database/my-database-id
+{
+  "name": "GeoIP2-Domain",
+  "maxmind": {
+    "account_id": "1025402"
+  }
+}
+----
+// TEST[skip:we don't want to leak the enterprise-geoip-downloader task, which touching these APIs would cause. Therefore, skip this test.]
+
+[[put-geoip-database-api-request]]
+==== {api-request-title}
+
+`PUT /_ingest/geoip/database/<database>`
+
+[[put-geoip-database-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the
+`manage` <<privileges-list-cluster,cluster privilege>> to use this API.
+
+
+[[put-geoip-database-api-path-params]]
+==== {api-path-parms-title}
+
+`<database>`::
++
+__
+(Required, string) ID of the database configuration to create or update.
+
+[[put-geoip-database-api-query-params]]
+==== {api-query-parms-title}
+
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms]
+
+[[put-geoip-database-api-request-body]]
+==== {api-request-body-title}
+
+// tag::geoip-database-object[]
+`name`::
+(Required, string)
+The provider-assigned name of the IP geolocation database to download.
+
+`<provider>`::
+(Required, a provider object and its associated configuration)
+The configuration necessary to identify which IP geolocation provider to use to download
+the database, as well as any provider-specific configuration necessary for such downloading.
++
+At present, the only supported provider is `maxmind`, and the maxmind provider
+requires that an `account_id` (string) is configured.
+// end::geoip-database-object[]
+
+[[geoip-database-configuration-licensing]]
+==== Licensing
+
+Downloading databases from third party providers is a commercial feature that requires an
+appropriate license. For more information, refer to https://www.elastic.co/subscriptions.
diff --git a/docs/reference/ingest/enrich.asciidoc b/docs/reference/ingest/enrich.asciidoc
index 6642cdc2a74ce..4bd50641149c0 100644
--- a/docs/reference/ingest/enrich.asciidoc
+++ b/docs/reference/ingest/enrich.asciidoc
@@ -230,12 +230,12 @@ Instead, you can:
 [[ingest-enrich-components]]
 ==== Enrich components
 
-The enrich coordinator is a component that manages and performs the searches 
+The enrich coordinator is a component that manages and performs the searches
 required to enrich documents on each ingest node. It combines searches from all enrich
 processors in all pipelines into bulk <<search-multi-search,multi-searches>>.
 
-The enrich policy executor is a component that manages the executions of all 
-enrich policies. When an enrich policy is executed, this component creates 
+The enrich policy executor is a component that manages the executions of all
+enrich policies. When an enrich policy is executed, this component creates
 a new enrich index and removes the previous enrich index. The enrich policy
 executions are managed from the elected master node. The execution of these
 policies occurs on a different node.
@@ -249,9 +249,15 @@ enrich policy executor.
 The enrich coordinator supports the following node settings:
 
 `enrich.cache_size`::
-Maximum number of searches to cache for enriching documents. Defaults to `1000`.
-There is a single cache for all enrich processors in the cluster. This setting
-determines the size of that cache.
+Maximum size of the cache that caches searches for enriching documents.
+The size can be specified in three units: the raw number of
+cached searches (e.g. `1000`), an absolute size in bytes (e.g. `100Mb`),
+or a percentage of the max heap space of the node (e.g. `1%`).
+Both for the absolute byte size and the percentage of heap space,
+{es} does not guarantee that the enrich cache size will adhere exactly to that maximum,
+as {es} uses the byte size of the serialized search response
+which is is a good representation of the used space on the heap, but not an exact match.
+Defaults to `1%`. There is a single cache for all enrich processors in the cluster.
 
 `enrich.coordinator_proxy.max_concurrent_requests`::
 Maximum number of concurrent <<search-multi-search,multi-search requests>> to
@@ -280,4 +286,4 @@ Maximum number of enrich policies to execute concurrently. Defaults to `50`.
 
 include::geo-match-enrich-policy-type-ex.asciidoc[]
 include::match-enrich-policy-type-ex.asciidoc[]
-include::range-enrich-policy-type-ex.asciidoc[]
\ No newline at end of file
+include::range-enrich-policy-type-ex.asciidoc[]
diff --git a/docs/reference/ingest/processors/community-id.asciidoc b/docs/reference/ingest/processors/community-id.asciidoc
index 03e65ac04a209..2d86bd21fa1e9 100644
--- a/docs/reference/ingest/processors/community-id.asciidoc
+++ b/docs/reference/ingest/processors/community-id.asciidoc
@@ -23,11 +23,12 @@ configuration is required.
 | `source_port`      | no       | `source.port` | Field containing the source port.
 | `destination_ip`   | no       | `destination.ip` | Field containing the destination IP address.
 | `destination_port` | no       | `destination.port` | Field containing the destination port.
-| `iana_number` | no | `network.iana_number` | Field containing the IANA number. The following protocol numbers are currently supported: `1` ICMP, `2` IGMP, `6` TCP, `17` UDP, `47` GRE, `58` ICMP IPv6, `88` EIGRP, `89` OSPF, `103` PIM, and `132` SCTP.
+| `iana_number` | no | `network.iana_number` | Field containing the IANA number.
 | `icmp_type`        | no       | `icmp.type`   | Field containing the ICMP type.
 | `icmp_code`        | no       | `icmp.code`   | Field containing the ICMP code.
-| `transport`        | no       | `network.transport` | Field containing the transport protocol.
-Used only when the `iana_number` field is not present.
+| `transport`        | no       | `network.transport` | Field containing the transport protocol name or number.
+Used only when the `iana_number` field is not present. The following protocol names are currently supported:
+`ICMP`, `IGMP`, `TCP`, `UDP`, `GRE`, `ICMP IPv6`, `EIGRP`, `OSPF`, `PIM`, and `SCTP`.
 | `target_field`     | no       | `network.community_id` | Output field for the community ID.
 | `seed`             | no       | `0`           | Seed for the community ID hash. Must be between
 0 and 65535 (inclusive). The seed can prevent hash collisions between network domains, such as
diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc
index 738ac234d6162..3a9ba58dedbf0 100644
--- a/docs/reference/ingest/processors/geoip.asciidoc
+++ b/docs/reference/ingest/processors/geoip.asciidoc
@@ -24,6 +24,9 @@ stats API>>.
 If your cluster can't connect to the Elastic GeoIP endpoint or you want to
 manage your own updates, see <<manage-geoip-database-updates>>.
 
+If you would like to have {es} download database files directly from Maxmind using your own provided
+license key, see <<put-geoip-database-api>>.
+
 If {es} can't connect to the endpoint for 30 days all updated databases will become
 invalid. {es} will stop enriching documents with geoip data and will add `tags: ["_geoip_expired_database"]`
 field instead.
@@ -36,9 +39,9 @@ field instead.
 [options="header"]
 |======
 | Name                                     | Required  | Default                                                                            | Description
-| `field`                                  | yes       | -                                                                                  | The field to get the ip address from for the geographical lookup.
+| `field`                                  | yes       | -                                                                                  | The field to get the IP address from for the geographical lookup.
 | `target_field`                           | no        | geoip                                                                              | The field that will hold the geographical information looked up from the MaxMind database.
-| `database_file`                          | no        | GeoLite2-City.mmdb                                                                 | The database filename referring to one of the automatically downloaded GeoLite2 databases (GeoLite2-City.mmdb, GeoLite2-Country.mmdb, or GeoLite2-ASN.mmdb) or the name of a supported database file in the `ingest-geoip` config directory.
+| `database_file`                          | no        | GeoLite2-City.mmdb                                                                 | The database filename referring to one of the automatically downloaded GeoLite2 databases (GeoLite2-City.mmdb, GeoLite2-Country.mmdb, or GeoLite2-ASN.mmdb), or the name of a supported database file in the `ingest-geoip` config directory, or the name of a <<get-geoip-database-api, configured database>> (with the `.mmdb` suffix appended).
 | `properties`                             | no        | [`continent_name`, `country_iso_code`, `country_name`, `region_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
 | `ignore_missing`                         | no        | `false`                                                                            | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 | `first_only`                             | no        | `true`                                                                             | If `true` only first found geoip data will be returned, even if `field` contains array
@@ -64,12 +67,12 @@ depend on what has been found and which properties were configured in `propertie
 * If the GeoIP2 Domain database is used, then the following fields may be added under the `target_field`: `ip`, and `domain`.
 The fields actually added depend on what has been found and which properties were configured in `properties`.
 * If the GeoIP2 ISP database is used, then the following fields may be added under the `target_field`: `ip`, `asn`,
-`organization_name`, `network`, `isp`, `isp_organization`, `mobile_country_code`, and `mobile_network_code`. The fields actually added
+`organization_name`, `network`, `isp`, `isp_organization_name`, `mobile_country_code`, and `mobile_network_code`. The fields actually added
 depend on what has been found and which properties were configured in `properties`.
 * If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`,
 `country_iso_code`, `country_name`, `continent_code`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`,
 `location`, `asn`, `organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`,
-`residential_proxy`, `domain`, `isp`, `isp_organization`, `mobile_country_code`, `mobile_network_code`, `user_type`, and
+`residential_proxy`, `domain`, `isp`, `isp_organization_name`, `mobile_country_code`, `mobile_network_code`, `user_type`, and
 `connection_type`. The fields actually added  depend on what has been found and which properties were configured in `properties`.
 
 preview::["Do not use the GeoIP2 Anonymous IP, GeoIP2 Connection Type, GeoIP2 Domain, GeoIP2 ISP, and GeoIP2 Enterprise databases in production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]
diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc
index 88d97d9422d5e..982da1fe17f7a 100644
--- a/docs/reference/ingest/processors/inference.asciidoc
+++ b/docs/reference/ingest/processors/inference.asciidoc
@@ -40,6 +40,11 @@ include::common-options.asciidoc[]
 Select the `content` field for inference and write the result to 
 `content_embedding`.
 
+IMPORTANT: If the specified `output_field` already exists in the ingest document, it won't be overwritten.
+The {infer} results will be appended to the existing fields within `output_field`, which could lead to duplicate fields and potential errors.
+To avoid this, use an unique `output_field` field name that does not clash with any existing fields.
+
+
 [source,js]
 --------------------------------------------------
 {
diff --git a/docs/reference/intro.asciidoc b/docs/reference/intro.asciidoc
index 3fc23b44994a7..cd9c126e7b1fd 100644
--- a/docs/reference/intro.asciidoc
+++ b/docs/reference/intro.asciidoc
@@ -1,42 +1,70 @@
 [[elasticsearch-intro]]
 == What is {es}?
-_**You know, for search (and analysis)**_
-
-{es} is the distributed search and analytics engine at the heart of
-the {stack}. {ls} and {beats} facilitate collecting, aggregating, and
-enriching your data and storing it in {es}. {kib} enables you to
-interactively explore, visualize, and share insights into your data and manage
-and monitor the stack. {es} is where the indexing, search, and analysis
-magic happens.
-
-{es} provides near real-time search and analytics for all types of data. Whether you
-have structured or unstructured text, numerical data, or geospatial data,
-{es} can efficiently store and index it in a way that supports fast searches.
-You can go far beyond simple data retrieval and aggregate information to discover
-trends and patterns in your data. And as your data and query volume grows, the
-distributed nature of {es} enables your deployment to grow seamlessly right
-along with it.
-
-While not _every_ problem is a search problem, {es} offers speed and flexibility
-to handle data in a wide variety of use cases:
-
-* Add a search box to an app or website
-* Store and analyze logs, metrics, and security event data
-* Use machine learning to automatically model the behavior of your data in real
-  time
-* Use {es} as a vector database to create, store, and search vector embeddings
-* Automate business workflows using {es} as a storage engine
-* Manage, integrate, and analyze spatial information using {es} as a geographic
-  information system (GIS)
-* Store and process genetic data using {es} as a bioinformatics research tool
-
-We’re continually amazed by the novel ways people use search. But whether
-your use case is similar to one of these, or you're using {es} to tackle a new
-problem, the way you work with your data, documents, and indices in {es} is
-the same.
+
+{es-repo}[{es}] is a distributed search and analytics engine, scalable data store, and vector database built on Apache Lucene.
+It's optimized for speed and relevance on production-scale workloads.
+Use {es} to search, index, store, and analyze data of all shapes and sizes in near real time.
+
+[TIP]
+====
+{es} has a lot of features. Explore the full list on the https://www.elastic.co/elasticsearch/features[product webpage^].
+====
+
+{es} is the heart of the {estc-welcome-current}/stack-components.html[Elastic Stack] and powers the Elastic https://www.elastic.co/enterprise-search[Search], https://www.elastic.co/observability[Observability] and https://www.elastic.co/security[Security] solutions.
+
+{es} is used for a wide and growing range of use cases. Here are a few examples:
+
+* *Monitor log and event data*. Store logs, metrics, and event data for observability and security information and event management (SIEM).
+* *Build search applications*. Add search capabilities to apps or websites, or build enterprise search engines over your organization's internal data sources.
+* *Vector database*. Store and search vectorized data, and create vector embeddings with built-in and third-party natural language processing (NLP) models.
+* *Retrieval augmented generation (RAG)*. Use {es} as a retrieval engine to augment Generative AI models.
+* *Application and security monitoring*. Monitor and analyze application performance and security data effectively.
+* *Machine learning*. Use {ml} to automatically model the behavior of your data in real-time.
+
+This is just a sample of search, observability, and security use cases enabled by {es}. 
+Refer to our https://www.elastic.co/customers/success-stories[customer success stories] for concrete examples across a range of industries.
+// Link to demos, search labs chatbots
+
+[discrete]
+[[elasticsearch-intro-elastic-stack]]
+.What is the Elastic Stack?
+*******************************
+{es} is the core component of the Elastic Stack, a suite of products for collecting, storing, searching, and visualizing data.
+https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current/stack-components.html[Learn more about the Elastic Stack].
+*******************************
+// TODO: Remove once we've moved Stack Overview to a subpage?
+
+[discrete]
+[[elasticsearch-intro-deploy]]
+=== Deployment options
+
+To use {es}, you need a running instance of the {es} service.
+You can deploy {es} in various ways:
+
+* <<run-elasticsearch-locally,*Local dev*>>. Get started quickly with a minimal local Docker setup. 
+* {cloud}/ec-getting-started-trial.html[*Elastic Cloud*]. {es} is available as part of our hosted Elastic Stack offering, deployed in the cloud with your provider of choice. Sign up for a https://cloud.elastic.co/registration[14 day free trial].
+* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless* (technical preview)]. Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14 day free trial].
+
+**Advanced deployment options**
+
+* <<elasticsearch-deployment-options,*Self-managed*>>. Install, configure, and run {es} on your own premises.
+* {ece-ref}/Elastic-Cloud-Enterprise-overview.html[*Elastic Cloud Enterprise*]. Deploy Elastic Cloud on public or private clouds, virtual machines, or your own premises.
+* {eck-ref}/k8s-overview.html[*Elastic Cloud on Kubernetes*]. Deploy Elastic Cloud on Kubernetes.
+
+[discrete]
+[[elasticsearch-next-steps]]
+=== Learn more
+
+Here are some resources to help you get started:
+
+* <<getting-started, Quickstart>>. A beginner's guide to deploying your first {es} instance, indexing data, and running queries.
+* https://elastic.co/webinars/getting-started-elasticsearch[Webinar: Introduction to {es}]. Register for our live webinars to learn directly from {es} experts.
+* https://www.elastic.co/search-labs[Elastic Search Labs]. Tutorials and blogs that explore AI-powered search using the latest {es} features.
+** Follow our tutorial https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[to build a hybrid search solution in Python].
+** Check out the https://github.com/elastic/elasticsearch-labs?tab=readme-ov-file#elasticsearch-examples--apps[`elasticsearch-labs` repository] for a range of Python notebooks and apps for various use cases.
 
 [[documents-indices]]
-=== Data in: documents and indices
+=== Documents and indices
 
 {es} is a distributed document store. Instead of storing information as rows of
 columnar data, {es} stores complex data structures that have been serialized
@@ -65,8 +93,7 @@ behavior makes it easy to index and explore your data--just start
 indexing documents and {es} will detect and map booleans, floating point and
 integer values, dates, and strings to the appropriate {es} data types.
 
-Ultimately, however, you know more about your data and how you want to use it
-than {es} can. You can define rules to control dynamic mapping and explicitly
+You can define rules to control dynamic mapping and explicitly
 define mappings to take full control of how fields are stored and indexed.
 
 Defining your own mappings enables you to:
@@ -89,7 +116,7 @@ used at search time. When you query a full-text field, the query text undergoes
 the same analysis before the terms are looked up in the index.
 
 [[search-analyze]]
-=== Information out: search and analyze
+=== Search and analyze
 
 While you can use {es} as a document store and retrieve documents and their
 metadata, the real power comes from being able to easily access the full suite
@@ -160,27 +187,8 @@ size 70 needles, you’re displaying a count of the size 70 needles
 that match your users' search criteria--for example, all size 70 _non-stick
 embroidery_ needles.
 
-[discrete]
-[[more-features]]
-===== But wait, there’s more
-
-Want to automate the analysis of your time series data? You can use
-{ml-docs}/ml-ad-overview.html[machine learning] features to create accurate
-baselines of normal behavior in your data and identify anomalous patterns. With
-machine learning, you can detect:
-
-* Anomalies related to temporal deviations in values, counts, or frequencies
-* Statistical rarity
-* Unusual behaviors for a member of a population
-
-And the best part? You can do this without having to specify algorithms, models,
-or other data science-related configurations.
-
 [[scalability]]
-=== Scalability and resilience: clusters, nodes, and shards
-++++
-<titleabbrev>Scalability and resilience</titleabbrev>
-++++
+=== Scalability and resilience
 
 {es} is built to be always available and to scale with your needs. It does this
 by being distributed by nature. You can add servers (nodes) to a cluster to
@@ -209,7 +217,7 @@ interrupting indexing or query operations.
 
 [discrete]
 [[it-depends]]
-==== It depends...
+==== Shard size and number of shards
 
 There are a number of performance considerations and trade offs with respect
 to shard size and the number of primary shards configured for an index. The more
@@ -237,7 +245,7 @@ testing with your own data and queries].
 
 [discrete]
 [[disaster-ccr]]
-==== In case of disaster
+==== Disaster recovery
 
 A cluster's nodes need good, reliable connections to each other. To provide
 better connections, you typically co-locate the nodes in the same data center or
@@ -257,7 +265,7 @@ secondary clusters are read-only followers.
 
 [discrete]
 [[admin]]
-==== Care and feeding
+==== Security, management, and monitoring
 
 As with any enterprise system, you need tools to secure, manage, and
 monitor your {es} clusters. Security, monitoring, and administrative features
@@ -265,3 +273,5 @@ that are integrated into {es} enable you to use {kibana-ref}/introduction.html[{
 as a control center for managing a cluster. Features like <<downsampling,
 downsampling>> and <<index-lifecycle-management, index lifecycle management>>
 help you intelligently manage your data over time.
+
+Refer to <<monitor-elasticsearch-cluster>> for more information.
\ No newline at end of file
diff --git a/docs/reference/mapping/params/copy-to.asciidoc b/docs/reference/mapping/params/copy-to.asciidoc
index 10eebfb027736..b26ceac349a3e 100644
--- a/docs/reference/mapping/params/copy-to.asciidoc
+++ b/docs/reference/mapping/params/copy-to.asciidoc
@@ -64,16 +64,104 @@ Some important points:
 * It is the field _value_ which is copied, not the terms (which result from the analysis process).
 * The original <<mapping-source-field,`_source`>> field will not be modified to show the copied values.
 * The same value can be copied to multiple fields, with `"copy_to": [ "field_1", "field_2" ]`
-* You cannot copy recursively via intermediary fields such as a `copy_to` on 
-`field_1` to `field_2` and `copy_to` on `field_2` to `field_3` expecting 
-indexing into `field_1` will eventuate in `field_3`, instead use copy_to 
-directly to multiple fields from the originating field.
+* You cannot copy recursively using intermediary fields.
+The following configuration will not copy data from `field_1` to `field_3`:
++
+[source,console]
+----
+PUT bad_example_index
+{
+  "mappings": {
+    "properties": {
+      "field_1": {
+        "type": "text",
+        "copy_to": "field_2"
+      },
+      "field_2": {
+        "type": "text",
+        "copy_to": "field_3"
+      },
+      "field_3": {
+        "type": "text"
+      }
+    }
+  }
+}
+----
+Instead, copy to multiple fields from the source field:
++
+[source,console]
+----
+PUT good_example_index
+{
+  "mappings": {
+    "properties": {
+      "field_1": {
+        "type": "text",
+        "copy_to": ["field_2", "field_3"]
+      },
+      "field_2": {
+        "type": "text"
+      },
+      "field_3": {
+        "type": "text"
+      }
+    }
+  }
+}
+----
+
+NOTE: `copy_to` is not supported for field types where values take the form of objects, e.g. `date_range`.
+
+[float]
+[[copy-to-dynamic-mapping]]
+==== Dynamic mapping
+
+Consider the following points when using `copy_to` with dynamic mappings:
+
 * If the target field does not exist in the index mappings, the usual
 <<dynamic-mapping,dynamic mapping>> behavior applies. By default, with
 <<dynamic,`dynamic`>> set to `true`, a non-existent target field will be
-dynamically added to the index mappings. If `dynamic` is set to `false`, the
+dynamically added to the index mappings.
+* If `dynamic` is set to `false`, the
 target field will not be added to the index mappings, and the value will not be
-copied. If `dynamic` is set to `strict`, copying to a non-existent field will
+copied.
+* If `dynamic` is set to `strict`, copying to a non-existent field will
 result in an error.
++
+** If the target field is nested, then `copy_to` fields must specify the full path to the nested field.
+Omitting the full path will lead to a `strict_dynamic_mapping_exception`.
+Use `"copy_to": ["parent_field.child_field"]` to correctly target a nested field.
++
+For example:
++
+[source,console]
+--------------------------------------------------
+PUT /test_index
+{
+  "mappings": {
+    "dynamic": "strict",
+    "properties": {
+      "description": {
+        "properties": {
+          "notes": {
+            "type": "text",
+            "copy_to": [ "description.notes_raw"], <1>
+            "analyzer": "standard",
+            "search_analyzer": "standard"
+          },
+          "notes_raw": {
+            "type": "keyword"
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
 
-NOTE: `copy_to` is _not_ supported for field types where values take the form of objects, e.g. `date_range`
\ No newline at end of file
+<1> The `notes` field is copied to the `notes_raw` field. Targeting `notes_raw` alone instead of `description.notes_raw`
+would lead to a `strict_dynamic_mapping_exception`.
++
+In this example, `notes_raw` is not defined at the root of the mapping, but under the `description` field.
+Without the fully qualified path, {es} would interpret the `copy_to` target as a root-level field, not as a nested field under `description`. 
\ No newline at end of file
diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc
index e081b122355bb..32f3d13edf581 100644
--- a/docs/reference/mapping/types/boolean.asciidoc
+++ b/docs/reference/mapping/types/boolean.asciidoc
@@ -223,6 +223,13 @@ The following parameters are accepted by `boolean` fields:
 
     Metadata about the field.
 
+`time_series_dimension`::
+(Optional, Boolean)
++
+--
+include::keyword.asciidoc[tag=dimension]
+--
+
 [[boolean-synthetic-source]]
 ==== Synthetic `_source`
 
diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc
index f2f0b3ae8bb23..0cd9ee0578b70 100644
--- a/docs/reference/mapping/types/dense-vector.asciidoc
+++ b/docs/reference/mapping/types/dense-vector.asciidoc
@@ -448,3 +448,63 @@ POST /my-bit-vectors/_search?filter_path=hits.hits
 }
 ----
 
+==== Updatable field type
+
+To better accommodate scaling and performance needs, updating the `type` setting in `index_options` is possible with the <<indices-put-mapping,Update Mapping API>>, according to the following graph (jumps allowed):
+
+[source,txt]
+----
+flat --> int8_flat --> int4_flat --> hnsw --> int8_hnsw --> int4_hnsw
+----
+
+For updating all HNSW types (`hnsw`, `int8_hnsw`, `int4_hnsw`) the number of connections `m` must either stay the same or increase. For scalar quantized formats  (`int8_flat`, `int4_flat`, `int8_hnsw`, `int4_hnsw`) the `confidence_interval` must always be consistent (once defined, it cannot change).
+
+Updating `type` in `index_options` will fail in all other scenarios.
+
+Switching `types` won't re-index vectors that have already been indexed (they will keep using their original `type`), vectors being indexed after the change will use the new `type` instead.
+
+For example, it's possible to define a dense vector field that utilizes the `flat` type (raw float32 arrays) for a first batch of data to be indexed.
+
+[source,console]
+--------------------------------------------------
+PUT my-index-000001
+{
+    "mappings": {
+        "properties": {
+            "text_embedding": {
+                "type": "dense_vector",
+                "dims": 384,
+                "index_options": {
+                    "type": "flat"
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+Changing the `type` to `int4_hnsw` makes sure vectors indexed after the change will use an int4 scalar quantized representation and HNSW (e.g., for KNN queries).
+That includes new segments created by <<index-modules-merge,merging>> previously created segments.
+
+[source,console]
+--------------------------------------------------
+PUT /my-index-000001/_mapping
+{
+    "properties": {
+        "text_embedding": {
+            "type": "dense_vector",
+            "dims": 384,
+            "index_options": {
+                "type": "int4_hnsw"
+            }
+        }
+    }
+}
+--------------------------------------------------
+// TEST[setup:my_index]
+
+Vectors indexed before this change will keep using the `flat` type (raw float32 representation and brute force search for KNN queries).
+
+In order to have all the vectors updated to the new type, either reindexing or force merging should be used.
+
+For debugging purposes, it's possible to inspect how many segments (and docs) exist for each `type` with the <<index-segments-api-request,Index Segments API>>.
diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc
index 6ee30e6b9f831..a006f288dc66d 100644
--- a/docs/reference/mapping/types/semantic-text.asciidoc
+++ b/docs/reference/mapping/types/semantic-text.asciidoc
@@ -7,8 +7,8 @@
 
 beta[]
 
-The `semantic_text` field type automatically generates embeddings for text
-content using an inference endpoint.
+The `semantic_text` field type automatically generates embeddings for text content using an inference endpoint.
+Long passages are <<auto-text-chunking, automatically chunked>> to smaller sections to enable the processing of larger corpuses of text.
 
 The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings.
 You can create the inference endpoint by using the <<put-inference-api>>.
@@ -52,8 +52,8 @@ Use the <<put-inference-api>> to create the endpoint.
 The `inference_id` will not be validated when the mapping is created, but when documents are ingested into the index.
 When the first document is indexed, the `inference_id` will be used to generate underlying indexing structures for the field.
 
-WARNING: Removing an inference endpoint will cause ingestion of documents and semantic queries to fail on indices that define `semantic_text` fields with that inference endpoint as their `inference_id`.
-Please check that inference endpoints are not used in `semantic_text` fields before removal.
+WARNING: Removing an {infer} endpoint will cause ingestion of documents and semantic queries to fail on indices that define `semantic_text` fields with that {infer} endpoint as their `inference_id`.
+Trying to <<delete-inference-api,delete an {infer} endpoint>> that is used on a `semantic_text` field will result in an error.
 
 [discrete]
 [[auto-text-chunking]]
@@ -65,6 +65,9 @@ To allow for large amounts of text to be used in semantic search, `semantic_text
 Each chunk will include the text subpassage and the corresponding embedding generated from it.
 When querying, the individual passages will be automatically searched for each document, and the most relevant passage will be used to compute a score.
 
+Documents are split into 250-word sections with a 100-word overlap so that each section shares 100 words with the previous section.
+This overlap ensures continuity and prevents vital contextual information in the input text from being lost by a hard break.
+
 
 [discrete]
 [[semantic-text-structure]]
@@ -118,13 +121,19 @@ In case you want to customize data indexing, use the
 <<sparse-vector,`sparse_vector`>> or <<dense-vector,`dense_vector`>> field
 types and create an ingest pipeline with an
 <<inference-processor, {infer} processor>> to generate the embeddings.
-<<semantic-search-inference,This tutorial>> walks you through the process.
+<<semantic-search-inference,This tutorial>> walks you through the process. In
+these cases - when you use `sparse_vector` or `dense_vector` field types instead
+of the `semantic_text` field type to customize indexing - using the 
+<<query-dsl-semantic-query,`semantic_query`>> is not supported for querying the 
+field data.
+
 
 [discrete]
 [[update-script]]
 ==== Updates to `semantic_text` fields
 
-Updates that use scripts are not supported when the index contains a `semantic_text` field.
+Updates that use scripts are not supported for an index contains a `semantic_text` field.
+Even if the script targets non-`semantic_text` fields, the update will fail when the index contains a `semantic_text` field.
 
 
 [discrete]
diff --git a/docs/reference/migration/migrate_8_15.asciidoc b/docs/reference/migration/migrate_8_15.asciidoc
index a183e68a50693..1961230da1bbf 100644
--- a/docs/reference/migration/migrate_8_15.asciidoc
+++ b/docs/reference/migration/migrate_8_15.asciidoc
@@ -16,5 +16,125 @@ coming::[8.15.0]
 [[breaking-changes-8.15]]
 === Breaking changes
 
-There are no breaking changes in {es} 8.15.
+The following changes in {es} 8.15 might affect your applications
+and prevent them from operating normally.
+Before upgrading to 8.15, review these changes and take the described steps
+to mitigate the impact.
+
+[discrete]
+[[breaking_815_cluster_and_node_setting_changes]]
+==== Cluster and node setting changes
+
+[[change_skip_unavailable_remote_cluster_setting_default_value_to_true]]
+.Change `skip_unavailable` remote cluster setting default value to true
+[%collapsible]
+====
+*Details* +
+The default value of the `skip_unavailable` setting is now set to true. All existing and future remote clusters that do not define this setting will use the new default. This setting only affects cross-cluster searches using the _search or _async_search API.
+
+*Impact* +
+Unavailable remote clusters in a cross-cluster search will no longer cause the search to fail unless skip_unavailable is configured to be `false` in elasticsearch.yml or via the `_cluster/settings` API. Unavailable clusters with `skip_unavailable`=`true` (either explicitly or by using the new default) are marked as SKIPPED in the search response metadata section and do not fail the entire search. If users want to ensure that a search returns a failure when a particular remote cluster is not available, `skip_unavailable` must be now be set explicitly.
+====
+
+[discrete]
+[[breaking_815_rollup_changes]]
+==== Rollup changes
+
+[[disallow_new_rollup_jobs_in_clusters_with_no_rollup_usage]]
+.Disallow new rollup jobs in clusters with no rollup usage
+[%collapsible]
+====
+*Details* +
+The put rollup API will fail with an error when a rollup job is created in a cluster with no rollup usage
+
+*Impact* +
+Clusters with no rollup usage (either no rollup job or index) can not create new rollup jobs
+====
+
+[discrete]
+[[breaking_815_rest_api_changes]]
+==== REST API changes
+
+[[interpret_timeout_1_as_infinite_ack_timeout]]
+.Interpret `?timeout=-1` as infinite ack timeout
+[%collapsible]
+====
+*Details* +
+Today {es} accepts the parameter `?timeout=-1` in many APIs, but interprets
+this to mean the same as `?timeout=0`. From 8.15 onwards `?timeout=-1` will
+mean to wait indefinitely, aligning the behaviour of this parameter with
+other similar parameters such as `?master_timeout`.
+
+*Impact* +
+Use `?timeout=0` to force relevant operations to time out immediately
+instead of `?timeout=-1`
+====
+
+[[replace_model_id_with_inference_id]]
+.Replace `model_id` with `inference_id` in GET inference API
+[%collapsible]
+====
+*Details* +
+From 8.15 onwards the <<get-inference-api>> response will return an
+`inference_id` field instead of a `model_id`.
+
+*Impact* +
+If your application uses the `model_id` in a GET inference API response,
+switch it to use `inference_id` instead.
+====
+
+
+[discrete]
+[[deprecated-8.15]]
+=== Deprecations
+
+The following functionality has been deprecated in {es} 8.15
+and will be removed in a future version.
+While this won't have an immediate impact on your applications,
+we strongly encourage you to take the described steps to update your code
+after upgrading to 8.15.
+
+To find out if you are using any deprecated functionality,
+enable <<deprecation-logging, deprecation logging>>.
+
+[discrete]
+[[deprecations_815_cluster_and_node_setting]]
+==== Cluster and node setting deprecations
+
+[[deprecate_absolute_size_values_for_indices_breaker_total_limit_setting]]
+.Deprecate absolute size values for `indices.breaker.total.limit` setting
+[%collapsible]
+====
+*Details* +
+Previously, the value of `indices.breaker.total.limit` could be specified as an absolute size in bytes. This setting controls the overal amount of memory the server is allowed to use before taking remedial actions. Setting this to a specific number of bytes led to strange behaviour when the node maximum heap size changed because the circut breaker limit would remain unchanged. This would either leave the value too low, causing part of the heap to remain unused; or it would leave the value too high, causing the circuit breaker to be ineffective at preventing OOM errors.  The only reasonable behaviour for this setting is that it scales with the size of the heap, and so absolute byte limits are now deprecated.
+
+*Impact* +
+Users must change their configuration to specify a percentage instead of an absolute number of bytes for `indices.breaker.total.limit`, or else accept the default, which is already specified as a percentage.
+====
+
+[discrete]
+[[deprecations_815_rest_api]]
+==== REST API deprecations
+
+[[deprecate_text_expansion_weighted_tokens_queries]]
+.Deprecate `text_expansion` and `weighted_tokens` queries
+[%collapsible]
+====
+*Details* +
+The `text_expansion` and `weighted_tokens` queries have been replaced by `sparse_vector`.
+
+*Impact* +
+Please update your existing `text_expansion` and `weighted_tokens` queries to use `sparse_vector.`
+====
+
+[[deprecate_using_slm_privileges_to_access_ilm]]
+.Deprecate using slm privileges to access ilm
+[%collapsible]
+====
+*Details* +
+The `read_slm` privilege can get the ILM status, and the `manage_slm` privilege can start and stop ILM. Access to these APIs should be granted using the `read_ilm` and `manage_ilm` privileges instead. Access to ILM APIs will be removed from SLM privileges in a future major release, and is now deprecated.
+
+*Impact* +
+Users that need access to the ILM status API should now use the `read_ilm` privilege. Users that need to start and stop ILM, should use the `manage_ilm` privilege.
+====
 
diff --git a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc
index 012904a9affa7..7bf02e7a0dd6e 100644
--- a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc
@@ -105,6 +105,20 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules]
 (array)
 include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-actions]
 
+//Begin analysis_config.detectors.custom_rules.params
+`params`:::
+(object)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-params]
++
+.Properties of `params`
+[%collapsible%open]
+=======
+`force_time_shift`::::
+(object)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-params-force-time-shift]
+=======
+//End analysis_config.detectors.custom_rules.params
+
 //Begin analysis_config.detectors.custom_rules.conditions
 `conditions`:::
 (array)
diff --git a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc
index 6953235c854cb..ee13247fc8838 100644
--- a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc
@@ -15,7 +15,7 @@ Updates certain properties of an {anomaly-job}.
 [[ml-update-job-prereqs]]
 == {api-prereq-title}
 
-Requires the `manage_ml` cluster privilege. This privilege is included in the 
+Requires the `manage_ml` cluster privilege. This privilege is included in the
 `machine_learning_admin` built-in role.
 
 [[ml-update-job-path-parms]]
@@ -51,7 +51,7 @@ You can update the `analysis_limits` only while the job is closed.
 [%collapsible%open]
 ====
 `model_memory_limit`:::
-(long or string) 
+(long or string)
 include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-ad]
 +
 --
@@ -61,8 +61,8 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-ad]
 determine the current usage, refer to the `model_bytes` value in
 the <<ml-get-job-stats,get job stats>> API.
 * If the `memory_status` property in the
-<<modelsizestats,`model_size_stats` object>> has a value of 
-`hard_limit`, this means that it was unable to process some data. You might want 
+<<modelsizestats,`model_size_stats` object>> has a value of
+`hard_limit`, this means that it was unable to process some data. You might want
 to re-run the job with an increased `model_memory_limit`.
 =======
 --
@@ -111,6 +111,21 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules]
 (array)
 include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-actions]
 
+//Begin analysis_config.detectors.custom_rules.params
+`params`:::
+(object)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-params]
++
+.Properties of `params`
+[%collapsible%open]
+=======
+[[force-time-shift-params]]
+`force_time_shift`::::
+(object)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-params-force-time-shift]
+=======
+//End analysis_config.detectors.custom_rules.params
+
 // Begin detectors.custom_rules.conditions
 `conditions`:::
 (array)
diff --git a/docs/reference/ml/anomaly-detection/functions/ml-geo-functions.asciidoc b/docs/reference/ml/anomaly-detection/functions/ml-geo-functions.asciidoc
index 5c061daa1cd44..63a0f047db647 100644
--- a/docs/reference/ml/anomaly-detection/functions/ml-geo-functions.asciidoc
+++ b/docs/reference/ml/anomaly-detection/functions/ml-geo-functions.asciidoc
@@ -52,6 +52,12 @@ detects anomalies where the geographic location of a credit card transaction is
 unusual for a particular customer’s credit card. An anomaly might indicate 
 fraud.
 
+A "typical" value indicates a centroid of a cluster of previously observed 
+locations that is closest to the "actual" location at that time. For example, 
+there may be one centroid near the person's home that is associated with the 
+cluster of local grocery stores and restaurants, and another centroid near the 
+person's work associated with the cluster of lunch and coffee places.   
+
 IMPORTANT: The `field_name` that you supply must be a single string that 
 contains two comma-separated numbers of the form `latitude,longitude`, a 
 `geo_point` field, a `geo_shape` field that contains point values, or a 
diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc
index a69fd2f1812e9..44c2012f502e1 100644
--- a/docs/reference/ml/ml-shared.asciidoc
+++ b/docs/reference/ml/ml-shared.asciidoc
@@ -1,3 +1,27 @@
+tag::adaptive-allocation[]
+Adaptive allocations configuration object.
+If enabled, the number of allocations of the model is set based on the current load the process gets.
+When the load is high, a new model allocation is automatically created (respecting the value of `max_number_of_allocations` if it's set).
+When the load is low, a model allocation is automatically removed (respecting the value of `min_number_of_allocations` if it's set).
+The number of model allocations cannot be scaled down to less than `1` this way.
+If `adaptive_allocations` is enabled, do not set the number of allocations manually.
+end::adaptive-allocation[]
+
+tag::adaptive-allocation-enabled[]
+If `true`, `adaptive_allocations` is enabled.
+Defaults to `false`.
+end::adaptive-allocation-enabled[]
+
+tag::adaptive-allocation-max-number[]
+Specifies the maximum number of allocations to scale to.
+If set, it must be greater than or equal to `min_number_of_allocations`.
+end::adaptive-allocation-max-number[]
+
+tag::adaptive-allocation-min-number[]
+Specifies the minimum number of allocations to scale to.
+If set, it must be greater than or equal to `1`.
+end::adaptive-allocation-min-number[]
+
 tag::aggregations[]
 If set, the {dfeed} performs aggregation searches. Support for aggregations is
 limited and should be used only with low cardinality data. For more information,
@@ -360,6 +384,10 @@ model. Unless you also specify `skip_result`, the results will be created as
 usual. This action is suitable when certain values are expected to be
 consistently anomalous and they affect the model in a way that negatively
 impacts the rest of the results.
+* `force_time_shift`: This action will shift the time inside the anomaly detector by a specified
+amount. This is useful, e.g. to quickly adapt to the daylight saving time events that
+are known beforehand. This action requires a `force_time_shift` parameter
+in the `params` object.
 end::custom-rules-actions[]
 
 tag::custom-rules-scope[]
@@ -404,6 +432,16 @@ tag::custom-rules-conditions-value[]
 The value that is compared against the `applies_to` field using the `operator`.
 end::custom-rules-conditions-value[]
 
+tag::custom-rules-params[]
+A set of parameter objects that customize the actions defined in the custom rules
+actions array. The available parameters (depending on the specified actions) include:
+`force_time_shift`.
+end::custom-rules-params[]
+
+tag::custom-rules-params-force-time-shift[]
+Set `time_shift_amount` to the signed number of seconds by which you want to shift the time.
+end::custom-rules-params-force-time-shift[]
+
 tag::custom-settings[]
 Advanced configuration option. Contains custom metadata about the job. For
 example, it can contain custom URL information as shown in
diff --git a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc
index f1b3fffb8a9a2..6f7e2a4d9f988 100644
--- a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc
+++ b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc
@@ -30,7 +30,10 @@ must be unique and should not match any other deployment ID or model ID, unless
 it is the same as the ID of the model being deployed. If `deployment_id` is not
 set, it defaults to the `model_id`.
 
-Scaling inference performance can be achieved by setting the parameters
+You can enable adaptive allocations to automatically scale model allocations up
+and down based on the actual resource requirement of the processes.
+
+Manually scaling inference performance can be achieved by setting the parameters
 `number_of_allocations` and `threads_per_allocation`.
 
 Increasing `threads_per_allocation` means more threads are used when an
@@ -58,6 +61,46 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-id]
 [[start-trained-model-deployment-query-params]]
 == {api-query-parms-title}
 
+`deployment_id`::
+(Optional, string)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=deployment-id]
++
+--
+Defaults to `model_id`.
+--
+
+`timeout`::
+(Optional, time)
+Controls the amount of time to wait for the model to deploy. Defaults to 30
+seconds.
+
+`wait_for`::
+(Optional, string)
+Specifies the allocation status to wait for before returning. Defaults to
+`started`. The value `starting` indicates deployment is starting but not yet on
+any node. The value `started` indicates the model has started on at least one
+node. The value `fully_allocated` indicates the deployment has started on all
+valid nodes.
+
+[[start-trained-model-deployment-request-body]]
+== {api-request-body-title}
+
+`adaptive_allocations`::
+(Optional, object)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation]
+
+`enabled`:::
+(Optional, Boolean)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-enabled]
+
+`max_number_of_allocations`:::
+(Optional, integer)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-max-number]
+
+`min_number_of_allocations`:::
+(Optional, integer)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-min-number]
+
 `cache_size`::
 (Optional, <<byte-units,byte value>>)
 The inference cache size (in memory outside the JVM heap) per node for the
@@ -65,15 +108,11 @@ model. In serverless, the cache is disabled by default. Otherwise, the default v
 `model_size_bytes` field in the <<get-trained-models-stats>>. To disable the
 cache, `0b` can be provided.
 
-`deployment_id`::
-(Optional, string)
-include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=deployment-id]
-Defaults to `model_id`.
-
 `number_of_allocations`::
 (Optional, integer)
 The total number of allocations this model is assigned across {ml} nodes.
-Increasing this value generally increases the throughput. Defaults to 1.
+Increasing this value generally increases the throughput. Defaults to `1`.
+If `adaptive_allocations` is enabled, do not set this value, because it's automatically set.
 
 `priority`::
 (Optional, string)
@@ -110,18 +149,6 @@ compute-bound process; `threads_per_allocations` must not exceed the number of
 available allocated processors per node. Defaults to 1. Must be a power of 2.
 Max allowed value is 32.
 
-`timeout`::
-(Optional, time)
-Controls the amount of time to wait for the model to deploy. Defaults to 30
-seconds.
-
-`wait_for`::
-(Optional, string)
-Specifies the allocation status to wait for before returning. Defaults to
-`started`. The value `starting` indicates deployment is starting but not yet on
-any node. The value `started` indicates the model has started on at least one
-node. The value `fully_allocated` indicates the deployment has started on all
-valid nodes.
 
 [[start-trained-model-deployment-example]]
 == {api-examples-title}
@@ -182,3 +209,24 @@ The `my_model` trained model can be deployed again with a different ID:
 POST _ml/trained_models/my_model/deployment/_start?deployment_id=my_model_for_search
 --------------------------------------------------
 // TEST[skip:TBD]
+
+
+[[start-trained-model-deployment-adaptive-allocation-example]]
+=== Setting adaptive allocations
+
+The following example starts a new deployment of the `my_model` trained model
+with the ID `my_model_for_search` and enables adaptive allocations with the
+minimum number of 3 allocations and the maximum number of 10. 
+
+[source,console]
+--------------------------------------------------
+POST _ml/trained_models/my_model/deployment/_start?deployment_id=my_model_for_search
+{
+  "adaptive_allocations": {
+    "enabled": true,
+    "min_number_of_allocations": 3,
+    "max_number_of_allocations": 10
+  }
+}
+--------------------------------------------------
+// TEST[skip:TBD]
\ No newline at end of file
diff --git a/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc
index ea5508fac26dd..d49ee3c6e872c 100644
--- a/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc
+++ b/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc
@@ -25,7 +25,11 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the
 == {api-description-title}
 
 You can update a trained model deployment whose `assignment_state` is `started`.
-You can either increase or decrease the number of allocations of such a deployment.
+You can enable adaptive allocations to automatically scale model allocations up
+and down based on the actual resource requirement of the processes.
+Or you can manually increase or decrease the number of allocations of a model
+deployment.
+
 
 [[update-trained-model-deployments-path-parms]]
 == {api-path-parms-title}
@@ -37,17 +41,34 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=deployment-id]
 [[update-trained-model-deployment-request-body]]
 == {api-request-body-title}
 
+`adaptive_allocations`::
+(Optional, object)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation]
+
+`enabled`:::
+(Optional, Boolean)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-enabled]
+
+`max_number_of_allocations`:::
+(Optional, integer)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-max-number]
+
+`min_number_of_allocations`:::
+(Optional, integer)
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-min-number]
+
 `number_of_allocations`::
 (Optional, integer)
 The total number of allocations this model is assigned across {ml} nodes.
 Increasing this value generally increases the throughput.
+If `adaptive_allocations` is enabled, do not set this value, because it's automatically set.
 
 
 [[update-trained-model-deployment-example]]
 == {api-examples-title}
 
 The following example updates the deployment for a
- `elastic__distilbert-base-uncased-finetuned-conll03-english` trained model to have 4 allocations:
+`elastic__distilbert-base-uncased-finetuned-conll03-english` trained model to have 4 allocations:
 
 [source,console]
 --------------------------------------------------
@@ -84,3 +105,21 @@ The API returns the following results:
     }
 }
 ----
+
+The following example updates the deployment for a
+`elastic__distilbert-base-uncased-finetuned-conll03-english` trained model to
+enable adaptive allocations with the minimum number of 3 allocations and the
+maximum number of 10:
+
+[source,console]
+--------------------------------------------------
+POST _ml/trained_models/elastic__distilbert-base-uncased-finetuned-conll03-english/deployment/_update
+{
+  "adaptive_allocations": {
+    "enabled": true,
+    "min_number_of_allocations": 3,
+    "max_number_of_allocations": 10
+  }
+}
+--------------------------------------------------
+// TEST[skip:TBD]
\ No newline at end of file
diff --git a/docs/reference/modules/cluster.asciidoc b/docs/reference/modules/cluster.asciidoc
index 4b9ede5450683..b3eaa5b47c238 100644
--- a/docs/reference/modules/cluster.asciidoc
+++ b/docs/reference/modules/cluster.asciidoc
@@ -1,9 +1,7 @@
 [[modules-cluster]]
 === Cluster-level shard allocation and routing settings
 
-_Shard allocation_ is the process of allocating shards to nodes. This can
-happen during initial recovery, replica allocation, rebalancing, or
-when nodes are added or removed.
+include::{es-ref-dir}/modules/shard-allocation-desc.asciidoc[]
 
 One of the main roles of the master is to decide which shards to allocate to
 which nodes, and when to move shards between nodes in order to rebalance the
diff --git a/docs/reference/modules/cluster/remote-clusters-settings.asciidoc b/docs/reference/modules/cluster/remote-clusters-settings.asciidoc
index 2308ec259da48..537783ef6ff01 100644
--- a/docs/reference/modules/cluster/remote-clusters-settings.asciidoc
+++ b/docs/reference/modules/cluster/remote-clusters-settings.asciidoc
@@ -6,7 +6,10 @@ mode are described separately.
 
 `cluster.remote.<cluster_alias>.mode`::
   The mode used for a remote cluster connection. The only supported modes are
-  `sniff` and `proxy`.
+  `sniff` and `proxy`. The default is `sniff`. See <<sniff-proxy-modes>> for
+  further information about these modes, and <<remote-cluster-sniff-settings>>
+  and <<remote-cluster-proxy-settings>> for further information about their
+  settings.
 
 `cluster.remote.initial_connect_timeout`::
 
@@ -97,6 +100,11 @@ you configure the remotes.
 [[remote-cluster-sniff-settings]]
 ==== Sniff mode remote cluster settings
 
+To use <<sniff-mode,sniff mode>> to connect to a remote cluster, set
+`cluster.remote.<cluster_alias>.mode: sniff` and then configure the following
+settings. You may also leave `cluster.remote.<cluster_alias>.mode` unset since
+`sniff` is the default mode.
+
 `cluster.remote.<cluster_alias>.seeds`::
 
   The list of seed nodes used to sniff the remote cluster state.
@@ -117,6 +125,10 @@ you configure the remotes.
 [[remote-cluster-proxy-settings]]
 ==== Proxy mode remote cluster settings
 
+To use <<proxy-mode,proxy mode>> to connect to a remote cluster, set
+`cluster.remote.<cluster_alias>.mode: proxy` and then configure the following
+settings.
+
 `cluster.remote.<cluster_alias>.proxy_address`::
 
   The address used for all remote connections.
diff --git a/docs/reference/modules/cluster/shards_allocation.asciidoc b/docs/reference/modules/cluster/shards_allocation.asciidoc
index dc53837125ee9..dab5d61a792cb 100644
--- a/docs/reference/modules/cluster/shards_allocation.asciidoc
+++ b/docs/reference/modules/cluster/shards_allocation.asciidoc
@@ -15,7 +15,8 @@ Enable or disable allocation for specific kinds of shards:
 * `new_primaries` -   Allows shard allocation only for primary shards for new indices.
 * `none` -            No shard allocations of any kind are allowed for any indices.
 
-This setting does not affect the recovery of local primary shards when
+This setting only affects future allocations, and does not re-allocate or un-allocate currently allocated shards.
+It also does not affect the recovery of local primary shards when
 restarting a node. A restarted node that has a copy of an unassigned primary
 shard will recover that primary immediately, assuming that its allocation id matches
 one of the active allocation ids in the cluster state.
diff --git a/docs/reference/modules/discovery/fault-detection.asciidoc b/docs/reference/modules/discovery/fault-detection.asciidoc
index dfa49e5b0d9af..21f4ae2317e6a 100644
--- a/docs/reference/modules/discovery/fault-detection.asciidoc
+++ b/docs/reference/modules/discovery/fault-detection.asciidoc
@@ -35,268 +35,30 @@ starting from the beginning of the cluster state update. Refer to
 
 [[cluster-fault-detection-troubleshooting]]
 ==== Troubleshooting an unstable cluster
-//tag::troubleshooting[]
-Normally, a node will only leave a cluster if deliberately shut down. If a node
-leaves the cluster unexpectedly, it's important to address the cause. A cluster
-in which nodes leave unexpectedly is unstable and can create several issues.
-For instance:
 
-* The cluster health may be yellow or red.
-
-* Some shards will be initializing and other shards may be failing.
-
-* Search, indexing, and monitoring operations may fail and report exceptions in
-logs.
-
-* The `.security` index may be unavailable, blocking access to the cluster.
-
-* The master may appear busy due to frequent cluster state updates.
-
-To troubleshoot a cluster in this state, first ensure the cluster has a
-<<discovery-troubleshooting,stable master>>. Next, focus on the nodes
-unexpectedly leaving the cluster ahead of all other issues. It will not be
-possible to solve other issues until the cluster has a stable master node and
-stable node membership.
-
-Diagnostics and statistics are usually not useful in an unstable cluster. These
-tools only offer a view of the state of the cluster at a single point in time.
-Instead, look at the cluster logs to see the pattern of behaviour over time.
-Focus particularly on logs from the elected master. When a node leaves the
-cluster, logs for the elected master include a message like this (with line
-breaks added to make it easier to read):
-
-[source,text]
-----
-[2022-03-21T11:02:35,513][INFO ][o.e.c.c.NodeLeftExecutor] [instance-0000000000]
-    node-left: [{instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{aNlyORLASam1ammv2DzYXA}{172.27.47.21}{172.27.47.21:19054}{m}]
-    with reason [disconnected]
-----
-
-This message says that the `NodeLeftExecutor` on the elected master
-(`instance-0000000000`) processed a `node-left` task, identifying the node that
-was removed and the reason for its removal. When the node joins the cluster
-again, logs for the elected master will include a message like this (with line
-breaks added to make it easier to read):
-
-[source,text]
-----
-[2022-03-21T11:02:59,892][INFO ][o.e.c.c.NodeJoinExecutor] [instance-0000000000]
-    node-join: [{instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{UNw_RuazQCSBskWZV8ID_w}{172.27.47.21}{172.27.47.21:19054}{m}]
-    with reason [joining after restart, removed [24s] ago with reason [disconnected]]
-----
-
-This message says that the `NodeJoinExecutor` on the elected master
-(`instance-0000000000`) processed a `node-join` task, identifying the node that
-was added to the cluster and the reason for the task.
-
-Other nodes may log similar messages, but report fewer details:
-
-[source,text]
-----
-[2020-01-29T11:02:36,985][INFO ][o.e.c.s.ClusterApplierService]
-    [instance-0000000001] removed {
-        {instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{aNlyORLASam1ammv2DzYXA}{172.27.47.21}{172.27.47.21:19054}{m}
-        {tiebreaker-0000000003}{UNw_RuazQCSBskWZV8ID_w}{bltyVOQ-RNu20OQfTHSLtA}{172.27.161.154}{172.27.161.154:19251}{mv}
-    }, term: 14, version: 1653415, reason: Publication{term=14, version=1653415}
-----
-
-These messages are not especially useful for troubleshooting, so focus on the
-ones from the `NodeLeftExecutor` and `NodeJoinExecutor` which are only emitted
-on the elected master and which contain more details. If you don't see the
-messages from the `NodeLeftExecutor` and `NodeJoinExecutor`, check that:
-
-* You're looking at the logs for the elected master node.
-
-* The logs cover the correct time period.
-
-* Logging is enabled at `INFO` level.
-
-Nodes will also log a message containing `master node changed` whenever they
-start or stop following the elected master. You can use these messages to
-determine each node's view of the state of the master over time.
-
-If a node restarts, it will leave the cluster and then join the cluster again.
-When it rejoins, the `NodeJoinExecutor` will log that it processed a
-`node-join` task indicating that the node is `joining after restart`. If a node
-is unexpectedly restarting, look at the node's logs to see why it is shutting
-down.
-
-The <<health-api>> API on the affected node will also provide some useful
-information about the situation.
-
-If the node did not restart then you should look at the reason for its
-departure more closely. Each reason has different troubleshooting steps,
-described below. There are three possible reasons:
-
-* `disconnected`: The connection from the master node to the removed node was
-closed.
-
-* `lagging`: The master published a cluster state update, but the removed node
-did not apply it within the permitted timeout. By default, this timeout is 2
-minutes. Refer to <<modules-discovery-settings>> for information about the
-settings which control this mechanism.
-
-* `followers check retry count exceeded`: The master sent a number of
-consecutive health checks to the removed node. These checks were rejected or
-timed out. By default, each health check times out after 10 seconds and {es}
-removes the node removed after three consecutively failed health checks. Refer
-to <<modules-discovery-settings>> for information about the settings which
-control this mechanism.
+See <<troubleshooting-unstable-cluster>>.
 
 [discrete]
 ===== Diagnosing `disconnected` nodes
 
-Nodes typically leave the cluster with reason `disconnected` when they shut
-down, but if they rejoin the cluster without restarting then there is some
-other problem.
-
-{es} is designed to run on a fairly reliable network. It opens a number of TCP
-connections between nodes and expects these connections to remain open forever.
-If a connection is closed then {es} will try and reconnect, so the occasional
-blip should have limited impact on the cluster even if the affected node
-briefly leaves the cluster. In contrast, repeatedly-dropped connections will
-severely affect its operation.
-
-The connections from the elected master node to every other node in the cluster
-are particularly important. The elected master never spontaneously closes its
-outbound connections to other nodes. Similarly, once a connection is fully
-established, a node never spontaneously close its inbound connections unless
-the node is shutting down.
-
-If you see a node unexpectedly leave the cluster with the `disconnected`
-reason, something other than {es} likely caused the connection to close. A
-common cause is a misconfigured firewall with an improper timeout or another
-policy that's <<long-lived-connections,incompatible with {es}>>. It could also
-be caused by general connectivity issues, such as packet loss due to faulty
-hardware or network congestion. If you're an advanced user, you can get more
-detailed information about network exceptions by configuring the following
-loggers:
-
-[source,yaml]
-----
-logger.org.elasticsearch.transport.TcpTransport: DEBUG
-logger.org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport: DEBUG
-----
-
-In extreme cases, you may need to take packet captures using `tcpdump` to
-determine whether messages between nodes are being dropped or rejected by some
-other device on the network.
+See <<troubleshooting-unstable-cluster-disconnected>>.
 
 [discrete]
 ===== Diagnosing `lagging` nodes
 
-{es} needs every node to process cluster state updates reasonably quickly. If a
-node takes too long to process a cluster state update, it can be harmful to the
-cluster. The master will remove these nodes with the `lagging` reason. Refer to
-<<modules-discovery-settings>> for information about the settings which control
-this mechanism.
-
-Lagging is typically caused by performance issues on the removed node. However,
-a node may also lag due to severe network delays. To rule out network delays,
-ensure that `net.ipv4.tcp_retries2` is <<system-config-tcpretries,configured
-properly>>. Log messages that contain `warn threshold` may provide more
-information about the root cause.
-
-If you're an advanced user, you can get more detailed information about what
-the node was doing when it was removed by configuring the following logger:
-
-[source,yaml]
-----
-logger.org.elasticsearch.cluster.coordination.LagDetector: DEBUG
-----
-
-When this logger is enabled, {es} will attempt to run the
-<<cluster-nodes-hot-threads>> API on the faulty node and report the results in
-the logs on the elected master. The results are compressed, encoded, and split
-into chunks to avoid truncation:
-
-[source,text]
-----
-[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 1]: H4sIAAAAAAAA/x...
-[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 2]: p7x3w1hmOQVtuV...
-[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 3]: v7uTboMGDbyOy+...
-[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 4]: 4tse0RnPnLeDNN...
-[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] (gzip compressed, base64-encoded, and split into 4 parts on preceding log lines)
-----
-
-To reconstruct the output, base64-decode the data and decompress it using
-`gzip`. For instance, on Unix-like systems:
-
-[source,sh]
-----
-cat lagdetector.log | sed -e 's/.*://' | base64 --decode | gzip --decompress
-----
+See <<troubleshooting-unstable-cluster-lagging>>.
 
 [discrete]
 ===== Diagnosing `follower check retry count exceeded` nodes
 
-Nodes sometimes leave the cluster with reason `follower check retry count
-exceeded` when they shut down, but if they rejoin the cluster without
-restarting then there is some other problem.
-
-{es} needs every node to respond to network messages successfully and
-reasonably quickly. If a node rejects requests or does not respond at all then
-it can be harmful to the cluster. If enough consecutive checks fail then the
-master will remove the node with reason `follower check retry count exceeded`
-and will indicate in the `node-left` message how many of the consecutive
-unsuccessful checks failed and how many of them timed out. Refer to
-<<modules-discovery-settings>> for information about the settings which control
-this mechanism.
-
-Timeouts and failures may be due to network delays or performance problems on
-the affected nodes. Ensure that `net.ipv4.tcp_retries2` is
-<<system-config-tcpretries,configured properly>> to eliminate network delays as
-a possible cause for this kind of instability. Log messages containing
-`warn threshold` may give further clues about the cause of the instability.
-
-If the last check failed with an exception then the exception is reported, and
-typically indicates the problem that needs to be addressed. If any of the
-checks timed out then narrow down the problem as follows.
-
-include::../../troubleshooting/network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-gc-vm]
-
-include::../../troubleshooting/network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-packet-capture-fault-detection]
-
-include::../../troubleshooting/network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-threads]
-
-By default the follower checks will time out after 30s, so if node departures
-are unpredictable then capture stack dumps every 15s to be sure that at least
-one stack dump was taken at the right time.
+See <<troubleshooting-unstable-cluster-follower-check>>.
 
 [discrete]
 ===== Diagnosing `ShardLockObtainFailedException` failures
 
-If a node leaves and rejoins the cluster then {es} will usually shut down and
-re-initialize its shards. If the shards do not shut down quickly enough then
-{es} may fail to re-initialize them due to a `ShardLockObtainFailedException`.
+See <<troubleshooting-unstable-cluster-shardlockobtainfailedexception>>.
 
-To gather more information about the reason for shards shutting down slowly,
-configure the following logger:
-
-[source,yaml]
-----
-logger.org.elasticsearch.env.NodeEnvironment: DEBUG
-----
-
-When this logger is enabled, {es} will attempt to run the
-<<cluster-nodes-hot-threads>> API whenever it encounters a
-`ShardLockObtainFailedException`. The results are compressed, encoded, and
-split into chunks to avoid truncation:
-
-[source,text]
-----
-[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 1]: H4sIAAAAAAAA/x...
-[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 2]: p7x3w1hmOQVtuV...
-[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 3]: v7uTboMGDbyOy+...
-[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 4]: 4tse0RnPnLeDNN...
-[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] (gzip compressed, base64-encoded, and split into 4 parts on preceding log lines)
-----
-
-To reconstruct the output, base64-decode the data and decompress it using
-`gzip`. For instance, on Unix-like systems:
+[discrete]
+===== Diagnosing other network disconnections
 
-[source,sh]
-----
-cat shardlock.log | sed -e 's/.*://' | base64 --decode | gzip --decompress
-----
-//end::troubleshooting[]
\ No newline at end of file
+See <<troubleshooting-unstable-cluster-network>>.
diff --git a/docs/reference/modules/gateway.asciidoc b/docs/reference/modules/gateway.asciidoc
index d6ee730d5021c..bf7e6de64f093 100644
--- a/docs/reference/modules/gateway.asciidoc
+++ b/docs/reference/modules/gateway.asciidoc
@@ -4,11 +4,11 @@
 The local gateway stores the cluster state and shard data across full
 cluster restarts.
 
-The following _static_ settings, which must be set on every master node,
+The following _static_ settings, which must be set on every <<master-node,master-eligible node>>,
 control how long a freshly elected master should wait before it tries to
-recover the cluster state and the cluster's data.
+recover the <<cluster-state,cluster state>> and the cluster's data.
 
-NOTE: These settings only take effect on a full cluster restart.
+NOTE: These settings only take effect during a <<restart-cluster-full,full cluster restart>>.
 
 `gateway.expected_data_nodes`::
 (<<static-cluster-setting,Static>>)
diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc
index 593aa79ded4d9..8fdc9f2e4f9cb 100644
--- a/docs/reference/modules/network.asciidoc
+++ b/docs/reference/modules/network.asciidoc
@@ -5,7 +5,9 @@ Each {es} node has two different network interfaces. Clients send requests to
 {es}'s REST APIs using its <<http-settings,HTTP interface>>, but nodes
 communicate with other nodes using the <<transport-settings,transport
 interface>>. The transport interface is also used for communication with
-<<remote-clusters,remote clusters>>.
+<<remote-clusters,remote clusters>>. The transport interface uses a custom
+binary protocol sent over <<long-lived-connections,long-lived>> TCP channels.
+Both interfaces can be configured to use <<secure-cluster,TLS for security>>.
 
 You can configure both of these interfaces at the same time using the
 `network.*` settings. If you have a more complicated network, you might need to
diff --git a/docs/reference/modules/remote-clusters.asciidoc b/docs/reference/modules/remote-clusters.asciidoc
index 25217302b7631..ca1c507aa4ed9 100644
--- a/docs/reference/modules/remote-clusters.asciidoc
+++ b/docs/reference/modules/remote-clusters.asciidoc
@@ -1,7 +1,7 @@
 [[remote-clusters]]
 == Remote clusters
 You can connect a local cluster to other {es} clusters, known as _remote
-clusters_. Remote clusters can be located in different datacenters or 
+clusters_. Remote clusters can be located in different datacenters or
 geographic regions, and contain indices or data streams that can be replicated
 with {ccr} or searched by a local cluster using {ccs}.
 
@@ -30,9 +30,9 @@ capabilities, the local and remote cluster must be on the same
 [discrete]
 === Add remote clusters
 
-NOTE: The instructions that follow describe how to create a remote connection from a 
-self-managed cluster. You can also set up {ccs} and {ccr} from an 
-link:https://www.elastic.co/guide/en/cloud/current/ec-enable-ccs.html[{ess} deployment] 
+NOTE: The instructions that follow describe how to create a remote connection from a
+self-managed cluster. You can also set up {ccs} and {ccr} from an
+link:https://www.elastic.co/guide/en/cloud/current/ec-enable-ccs.html[{ess} deployment]
 or from an link:https://www.elastic.co/guide/en/cloud-enterprise/current/ece-enable-ccs.html[{ece} deployment].
 
 To add remote clusters, you can choose between
@@ -52,7 +52,7 @@ controls. <<remote-clusters-api-key>>.
 
 Certificate based security model::
 Uses mutual TLS authentication for cross-cluster operations. User authentication
-is performed on the local cluster and a user's role names are passed to the 
+is performed on the local cluster and a user's role names are passed to the
 remote cluster. In this model, a superuser on the local cluster gains total read
 access to the remote cluster, so it is only suitable for clusters that are in
 the same security domain. <<remote-clusters-cert>>.
@@ -63,13 +63,17 @@ the same security domain. <<remote-clusters-cert>>.
 
 [[sniff-mode]]
 Sniff mode::
-In sniff mode, a cluster is created using a name and a list of seed nodes. When
-a remote cluster is registered, its cluster state is retrieved from one of the
-seed nodes and up to three _gateway nodes_ are selected as part of remote
-cluster requests. This mode requires that the gateway node's publish addresses
-are accessible by the local cluster.
+In sniff mode, a cluster alias is registered with a name of your choosing and a
+list of addresses of _seed_ nodes specified with the
+`cluster.remote.<cluster_alias>.seeds` setting. When you register a remote
+cluster using sniff mode, {es} retrieves from one of the seed nodes the
+addresses of up to three _gateway nodes_. Each `remote_cluster_client` node in
+the local {es} cluster then opens several TCP connections to the publish
+addresses of the gateway nodes. This mode therefore requires that the gateway
+nodes' publish addresses are accessible to nodes in the local cluster.
 +
-Sniff mode is the default connection mode.
+Sniff mode is the default connection mode. See <<remote-cluster-sniff-settings>>
+for more information about configuring sniff mode.
 +
 [[gateway-nodes-selection]]
 The _gateway nodes_ selection depends on the following criteria:
@@ -84,13 +88,21 @@ However, such nodes still have to satisfy the two above requirements.
 
 [[proxy-mode]]
 Proxy mode::
-In proxy mode, a cluster is created using a name and a single proxy address.
-When you register a remote cluster, a configurable number of socket connections
-are opened to the proxy address. The proxy is required to route those
-connections to the remote cluster. Proxy mode does not require remote cluster
-nodes to have accessible publish addresses.
+In proxy mode, a cluster alias is registered with a name of your choosing and
+the address of a TCP (layer 4) reverse proxy specified with the
+`cluster.remote.<cluster_alias>.proxy_address` setting. You must configure this
+proxy to route connections to one or more nodes of the remote cluster. When you
+register a remote cluster using proxy mode, {es} opens several TCP connections
+to the proxy address and uses these connections to communicate with the remote
+cluster. In proxy mode {es} disregards the publish addresses of the remote
+cluster nodes which means that the publish addresses of the remote cluster
+nodes need not be accessible to the local cluster.
++
+Proxy mode is not the default connection mode, so you must set
+`cluster.remote.<cluster_alias>.mode: proxy` to use it. See
+<<remote-cluster-proxy-settings>> for more information about configuring proxy
+mode.
 +
-The proxy mode is not the default connection mode and must be configured.
 Proxy mode has the same <<gateway-nodes-selection, version compatibility
 requirements>> as sniff mode.
 
diff --git a/docs/reference/modules/shard-allocation-desc.asciidoc b/docs/reference/modules/shard-allocation-desc.asciidoc
new file mode 100644
index 0000000000000..426ad0da72e1b
--- /dev/null
+++ b/docs/reference/modules/shard-allocation-desc.asciidoc
@@ -0,0 +1,2 @@
+Shard allocation is the process of assigning shard copies to nodes. This can
+happen during initial recovery, replica allocation, rebalancing, when nodes are added to or removed from the cluster, or when cluster or index settings that impact allocation are updated.
\ No newline at end of file
diff --git a/docs/reference/modules/shard-ops.asciidoc b/docs/reference/modules/shard-ops.asciidoc
new file mode 100644
index 0000000000000..c0e5ee6a220f0
--- /dev/null
+++ b/docs/reference/modules/shard-ops.asciidoc
@@ -0,0 +1,75 @@
+[[shard-allocation-relocation-recovery]]
+=== Shard allocation, relocation, and recovery
+
+Each <<documents-indices,index>> in Elasticsearch is divided into one or more <<scalability,shards>>. 
+Each document in an index belongs to a single shard.
+
+A cluster can contain multiple copies of a shard. Each shard has one distinguished shard copy called the _primary_, and zero or more non-primary copies called _replicas_. The primary shard copy serves as the main entry point for all indexing operations. The operations on the primary shard copy are then forwarded to its replicas.
+
+Replicas maintain redundant copies of your data across the <<modules-node,nodes>> in your cluster, protecting against hardware failure and increasing capacity to serve read requests like searching or retrieving a document. If the primary shard copy fails, then a replica is promoted to primary and takes over the primary's responsibilities.
+
+Over the course of normal operation, Elasticsearch allocates shard copies to nodes, relocates shard copies across nodes to balance the cluster or satisfy new allocation constraints, and recovers shards to initialize new copies. In this topic, you'll learn how these operations work and how you can control them.
+
+TIP: To learn about optimizing the number and size of shards in your cluster, refer to <<size-your-shards,Size your shards>>. To learn about how read and write operations are replicated across shards and shard copies, refer to <<docs-replication,Reading and writing documents>>.
+
+[[shard-allocation]]
+==== Shard allocation
+
+include::{es-ref-dir}/modules/shard-allocation-desc.asciidoc[]
+
+By default, the primary and replica shard copies for an index can be allocated to any node in the cluster, and may be relocated to rebalance the cluster. 
+
+===== Adjust shard allocation settings
+
+You can control how shard copies are allocated using the following settings:
+
+- <<modules-cluster,Cluster-level shard allocation settings>>: Use these settings to control how shard copies are allocated and balanced across the entire cluster. For example, you might want to allocate nodes availability zones, or prevent certain nodes from being used so you can perform maintenance.
+
+- <<index-modules-allocation,Index-level shard allocation settings>>: Use these settings to control how the shard copies for a specific index are allocated. For example, you might want to allocate an index to a node in a specific data tier, or to an node with specific attributes.
+
+===== Monitor shard allocation
+
+If a shard copy is unassigned, it means that the shard copy is not allocated to any node in the cluster. This can happen if there are not enough nodes in the cluster to allocate the shard copy, or if the shard copy can't be allocated to any node that satisfies the shard allocation filtering rules. When a shard copy is unassigned, your cluster is considered unhealthy and returns a yellow or red cluster health status.
+
+You can use the following APIs to monitor shard allocation:
+
+- <<cluster-allocation-explain,Cluster allocation explain>>
+- <<cat-allocation,cat allocation>>
+- <<cluster-health,cluster health>>
+
+<<red-yellow-cluster-status,Learn more about troubleshooting unassigned shard copies and recovering your cluster health>>.
+
+[[shard-recovery]]
+==== Shard recovery
+
+include::{es-ref-dir}/modules/shard-recovery-desc.asciidoc[]
+
+===== Adjust shard recovery settings
+
+To control how shards are recovered, for example the resources that can be used by recovery operations, and which indices should be prioritized for recovery, you can adjust the following settings: 
+
+- <<recovery,Index recovery settings>>
+- <<modules-cluster,Cluster-level shard allocation settings>>
+- <<index-modules-allocation,Index-level shard allocation settings>>, including <<delayed-allocation,delayed allocation>> and <<recovery-prioritization,index recovery prioritization>> 
+
+Shard recovery operations also respect general shard allocation settings. 
+
+===== Monitor shard recovery
+
+You can use the following APIs to monitor shard allocation:
+
+ - View a list of in-progress and completed recoveries using the <<cat-recovery,cat recovery API>>
+ - View detailed information about a specific recovery using the <<indices-recovery,index recovery API>>
+
+[[shard-relocation]]
+==== Shard relocation
+
+Shard relocation is the process of moving shard copies from one node to another. This can happen when a node joins or leaves the cluster, or when the cluster is rebalancing.
+
+When a shard copy is relocated, it is created as a new shard copy on the target node. When the shard copy is fully allocated and recovered, the old shard copy is deleted. If the shard copy being relocated is a primary, then the new shard copy is marked as primary before the old shard copy is deleted.
+
+===== Adjust shard relocation settings
+
+You can control how and when shard copies are relocated. For example, you can adjust the rebalancing settings that control when shard copies are relocated to balance the cluster, or the high watermark for disk-based shard allocation that can trigger relocation. These settings are part of the <<modules-cluster,cluster-level shard allocation settings>>.
+
+Shard relocation operations also respect shard allocation and recovery settings. 
\ No newline at end of file
diff --git a/docs/reference/modules/shard-recovery-desc.asciidoc b/docs/reference/modules/shard-recovery-desc.asciidoc
new file mode 100644
index 0000000000000..67eaceb528962
--- /dev/null
+++ b/docs/reference/modules/shard-recovery-desc.asciidoc
@@ -0,0 +1,16 @@
+Shard recovery is the process of initializing a shard copy, such as restoring a
+primary shard from a snapshot or creating a replica shard from a primary shard.
+When a shard recovery completes, the recovered shard is available for search
+and indexing.
+
+Recovery automatically occurs during the following processes:
+
+* When creating an index for the first time.
+* When a node rejoins the cluster and starts up any missing primary shard copies using the data that it holds in its data path.
+* Creation of new replica shard copies from the primary.
+* Relocation of a shard copy to a different node in the same cluster.
+* A <<snapshots-restore-snapshot,snapshot restore>> operation.
+* A <<indices-clone-index,clone>>, <<indices-shrink-index,shrink>>, or
+<<indices-split-index,split>> operation.
+
+You can determine the cause of a shard recovery using the <<indices-recovery,recovery>> or  <<cat-recovery,cat recovery>> APIs.
\ No newline at end of file
diff --git a/docs/reference/query-dsl/intervals-query.asciidoc b/docs/reference/query-dsl/intervals-query.asciidoc
index 63ba4046a395d..1e3380389d861 100644
--- a/docs/reference/query-dsl/intervals-query.asciidoc
+++ b/docs/reference/query-dsl/intervals-query.asciidoc
@@ -397,68 +397,3 @@ This query does *not* match a document containing the phrase `hot porridge is
 salty porridge`, because the intervals returned by the match query for `hot
 porridge` only cover the initial two terms in this document, and these do not
 overlap the intervals covering `salty`.
-
-Another restriction to be aware of is the case of `any_of` rules that contain
-sub-rules which overlap. In particular, if one of the rules is a strict
-prefix of the other, then the longer rule can never match, which can
-cause surprises when used in combination with `max_gaps`. Consider the
-following query, searching for `the` immediately followed by `big` or `big bad`,
-immediately followed by `wolf`:
-
-[source,console]
---------------------------------------------------
-POST _search
-{
-  "query": {
-    "intervals" : {
-      "my_text" : {
-        "all_of" : {
-          "intervals" : [
-            { "match" : { "query" : "the" } },
-            { "any_of" : {
-                "intervals" : [
-                    { "match" : { "query" : "big" } },
-                    { "match" : { "query" : "big bad" } }
-                ] } },
-            { "match" : { "query" : "wolf" } }
-          ],
-          "max_gaps" : 0,
-          "ordered" : true
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
-
-Counter-intuitively, this query does *not* match the document `the big bad
-wolf`, because the `any_of` rule in the middle only produces intervals
-for `big` - intervals for `big bad` being longer than those for `big`, while
-starting at the same position, and so being minimized away. In these cases,
-it's better to rewrite the query so that all of the options are explicitly
-laid out at the top level:
-
-[source,console]
---------------------------------------------------
-POST _search
-{
-  "query": {
-    "intervals" : {
-      "my_text" : {
-        "any_of" : {
-          "intervals" : [
-            { "match" : {
-                "query" : "the big bad wolf",
-                "ordered" : true,
-                "max_gaps" : 0 } },
-            { "match" : {
-                "query" : "the big wolf",
-                "ordered" : true,
-                "max_gaps" : 0 } }
-           ]
-        }
-      }
-    }
-  }
-}
---------------------------------------------------
diff --git a/docs/reference/query-dsl/query-string-query.asciidoc b/docs/reference/query-dsl/query-string-query.asciidoc
index 319ede7c4ac05..b45247ace3735 100644
--- a/docs/reference/query-dsl/query-string-query.asciidoc
+++ b/docs/reference/query-dsl/query-string-query.asciidoc
@@ -30,7 +30,7 @@ If you don't need to support a query syntax, consider using the
 syntax, use the <<query-dsl-simple-query-string-query,`simple_query_string`>>
 query, which is less strict.
 ====
- 
+
 
 [[query-string-query-ex-request]]
 ==== Example request
@@ -83,7 +83,7 @@ could be expensive.
 
 There is a limit on the number of fields times terms that can be queried at once.
 It is defined by the `indices.query.bool.max_clause_count`
-<<search-settings,search setting>>, which defaults to 4096.
+<<search-settings,search setting>>.
 ====
 --
 
diff --git a/docs/reference/query-dsl/rule-query.asciidoc b/docs/reference/query-dsl/rule-query.asciidoc
index cc5616c01eecd..dfedc2261bbde 100644
--- a/docs/reference/query-dsl/rule-query.asciidoc
+++ b/docs/reference/query-dsl/rule-query.asciidoc
@@ -13,9 +13,10 @@ The old syntax using `rule_query` and `ruleset_id` is deprecated and will be rem
 ====
 
 Applies <<query-rules-apis,query rules>> to the query before returning results.
-This feature is used to promote documents in the manner of a <<query-dsl-pinned-query>> based on matching defined rules.
+Query rules can be used to promote documents in the manner of a <<query-dsl-pinned-query>> based on matching defined rules, or to identify specific documents to exclude from a contextual result set.
 If no matching query rules are defined, the "organic" matches for the query are returned.
 All matching rules are applied in the order in which they appear in the query ruleset.
+If the same document matches both an `exclude` rule and a `pinned` rule, the document will be excluded.
 
 [NOTE]
 ====
diff --git a/docs/reference/query-dsl/semantic-query.asciidoc b/docs/reference/query-dsl/semantic-query.asciidoc
index d0eb2da95ebc6..22b5e6c5e6aad 100644
--- a/docs/reference/query-dsl/semantic-query.asciidoc
+++ b/docs/reference/query-dsl/semantic-query.asciidoc
@@ -128,6 +128,10 @@ If you want to fine-tune a search on a `semantic_text` field, you need to know t
 You can find the task type using the <<get-inference-api>>, and check the `task_type` associated with the {infer} service.
 Depending on the `task_type`, use either the <<query-dsl-sparse-vector-query,`sparse_vector`>> or the <<query-dsl-knn-query,`knn`>> query for greater flexibility and customization.
 
+NOTE: While it is possible to use the `sparse_vector` query or the `knn` query
+on a `semantic_text` field, it is not supported to use the `semantic_query` on a
+`sparse_vector` or `dense_vector` field type.
+
 
 [discrete]
 [[search-sparse-inference]]
diff --git a/docs/reference/query-dsl/span-multi-term-query.asciidoc b/docs/reference/query-dsl/span-multi-term-query.asciidoc
index aefb3e4b75eb5..5a5f0e1f5ff99 100644
--- a/docs/reference/query-dsl/span-multi-term-query.asciidoc
+++ b/docs/reference/query-dsl/span-multi-term-query.asciidoc
@@ -39,7 +39,8 @@ GET /_search
 --------------------------------------------------
 
 WARNING: `span_multi` queries will hit too many clauses failure if the number of terms that match the query exceeds the
-boolean query limit (defaults to 4096).To avoid an unbounded expansion you can set the <<query-dsl-multi-term-rewrite,
+`indices.query.bool.max_clause_count` <<search-settings,search setting>>.
+To avoid an unbounded expansion you can set the <<query-dsl-multi-term-rewrite,
 rewrite method>> of the multi term query to `top_terms_*` rewrite. Or, if you use `span_multi` on `prefix` query only,
 you can activate the <<index-prefixes,`index_prefixes`>> field option of the `text` field instead. This will
 rewrite any prefix query on the field to a single term query that matches the indexed prefix.
diff --git a/docs/reference/query-rules/apis/put-query-rule.asciidoc b/docs/reference/query-rules/apis/put-query-rule.asciidoc
index 9737673be009c..714ed9b096d1d 100644
--- a/docs/reference/query-rules/apis/put-query-rule.asciidoc
+++ b/docs/reference/query-rules/apis/put-query-rule.asciidoc
@@ -26,7 +26,10 @@ Requires the `manage_search_query_rules` privilege.
 
 `type`::
 (Required, string) The type of rule.
-At this time only `pinned` query rule types are allowed.
+At this time the following query rule types are allowed:
+
+- `pinned` will identify and pin specific documents to the top of search results.
+- `exclude` will exclude specific documents from search results.
 
 `criteria`::
 (Required, array of objects) The criteria that must be met for the rule to be applied.
@@ -80,17 +83,18 @@ Required for all criteria types except `always`.
 The format of this action depends on the rule type.
 
 Actions depend on the rule type.
-For `pinned` rules, actions follow the format specified by the <<query-dsl-pinned-query,Pinned Query>>.
-The following actions are allowed:
+The following actions are allowed for `pinned` or `exclude` rules:
 
-- `ids` (Optional, array of strings) The unique <<mapping-id-field, document IDs>> of the documents to pin.
+- `ids` (Optional, array of strings) The unique <<mapping-id-field, document IDs>> of the documents to apply the rule to.
 Only one of `ids` or `docs` may be specified, and at least one must be specified.
-- `docs` (Optional, array of objects) The documents to pin.
+- `docs` (Optional, array of objects) The documents to apply the rule to.
 Only one of `ids` or `docs` may be specified, and at least one must be specified.
+There is a maximum value of 100 documents in a rule.
 You can specify the following attributes for each document:
 +
 --
-- `_index` (Required, string) The index of the document to pin.
+- `_index` (Required, string) The index of the document.
+If null, all documents with the specified `_id` will be affected across all searched indices.
 - `_id` (Required, string) The unique <<mapping-id-field, document ID>>.
 --
 
@@ -104,7 +108,7 @@ If multiple matching rules pin more than 100 documents, only the first 100 docum
 
 The following example creates a new query rule with the ID `my-rule1` in a query ruleset called `my-ruleset`.
 
-`my-rule1` will pin documents with IDs `id1` and `id2` when `user_query` contains `pugs` _or_ `puggles` **and** `user_country` exactly matches `us`.
+- `my-rule1` will select documents to promote with IDs `id1` and `id2` when `user_query` contains `pugs` _or_ `puggles` **and** `user_country` exactly matches `us`.
 
 [source,console]
 ----
diff --git a/docs/reference/query-rules/apis/put-query-ruleset.asciidoc b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc
index c164e9e140a4e..df7ec100db076 100644
--- a/docs/reference/query-rules/apis/put-query-ruleset.asciidoc
+++ b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc
@@ -34,7 +34,7 @@ Each rule must have the following information:
 
 - `rule_id` (Required, string) A unique identifier for this rule.
 - `type` (Required, string) The type of rule.
-At this time only `pinned` query rule types are allowed.
+At this time only `pinned` and `exclude` query rule types are allowed.
 - `criteria` (Required, array of objects) The criteria that must be met for the rule to be applied.
 If multiple criteria are specified for a rule, all criteria must be met for the rule to be applied.
 - `actions` (Required, object) The actions to take when the rule is matched.
@@ -84,13 +84,13 @@ Only one value must match for the criteria to be met.
 Required for all criteria types except `always`.
 
 Actions depend on the rule type.
-For `pinned` rules, actions follow the format specified by the <<query-dsl-pinned-query,Pinned Query>>.
-The following actions are allowed:
+The following actions are allowed for `pinned` or `exclude` rules:
 
-- `ids` (Optional, array of strings) The unique <<mapping-id-field, document IDs>> of the documents to pin.
+- `ids` (Optional, array of strings) The unique <<mapping-id-field, document IDs>> of the documents to apply the rule to.
 Only one of `ids` or `docs` may be specified, and at least one must be specified.
-- `docs` (Optional, array of objects) The documents to pin.
+- `docs` (Optional, array of objects) The documents to apply the rule to.
 Only one of `ids` or `docs` may be specified, and at least one must be specified.
+There is a maximum value of 100 documents in a rule.
 You can specify the following attributes for each document:
 +
 --
@@ -98,7 +98,7 @@ You can specify the following attributes for each document:
 - `_id` (Required, string) The unique <<mapping-id-field, document ID>>.
 --
 
-IMPORTANT: Due to limitations within <<query-dsl-pinned-query,Pinned queries>>, you can only pin documents using `ids` or `docs`, but cannot use both in single rule.
+IMPORTANT: Due to limitations within <<query-dsl-pinned-query,Pinned queries>>, you can only select documents using `ids` or `docs`, but cannot use both in single rule.
 It is advised to use one or the other in query rulesets, to avoid errors.
 Additionally, pinned queries have a maximum limit of 100 pinned hits.
 If multiple matching rules pin more than 100 documents, only the first 100 documents are pinned in the order they are specified in the ruleset.
@@ -111,7 +111,7 @@ The following example creates a new query ruleset called `my-ruleset`.
 Two rules are associated with `my-ruleset`:
 
 - `my-rule1` will pin documents with IDs `id1` and `id2` when `user_query` contains `pugs` _or_ `puggles` **and** `user_country` exactly matches `us`.
-- `my-rule2` will pin documents from different, specified indices with IDs `id3` and `id4` when the `query_string` fuzzily matches `rescue dogs`.
+- `my-rule2` will exclude documents from different, specified indices with IDs `id3` and `id4` when the `query_string` fuzzily matches `rescue dogs`.
 
 [source,console]
 ----
@@ -142,7 +142,7 @@ PUT _query_rules/my-ruleset
         },
         {
             "rule_id": "my-rule2",
-            "type": "pinned",
+            "type": "exclude",
             "criteria": [
                 {
                     "type": "fuzzy",
diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc
index c13c1c95c09ff..2069c1bd96ff0 100644
--- a/docs/reference/release-notes/8.15.0.asciidoc
+++ b/docs/reference/release-notes/8.15.0.asciidoc
@@ -1,16 +1,532 @@
 [[release-notes-8.15.0]]
 == {es} version 8.15.0
 
-coming[8.15.0]
-
 Also see <<breaking-changes-8.15,Breaking changes in 8.15>>.
 
+
 [[known-issues-8.15.0]]
 [float]
 === Known issues
-
 * The `pytorch_inference` process used to run Machine Learning models can consume large amounts of memory.
 In environments where the available memory is limited, the OS Out of Memory Killer will kill the `pytorch_inference`
 process to reclaim memory. This can cause inference requests to fail.
 Elasticsearch will automatically restart the `pytorch_inference` process
 after it is killed up to four times in 24 hours. (issue: {es-issue}110530[#110530])
+
+* Pipeline aggregations under `time_series` and `categorize_text` aggregations are never
+returned (issue: {es-issue}111679[#111679])
+
+* Elasticsearch will not start on Windows machines if
+[`bootstrap.memory_lock` is set to `true`](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration-memory.html#bootstrap-memory_lock).
+Either downgrade to an earlier version, upgrade to 8.15.1, or else follow the
+recommendation in the manual to entirely disable swap instead of using the
+memory lock feature (issue: {es-issue}111847[#111847])
+
+* The `took` field of the response to the <<docs-bulk>> API is incorrect and may be rather large. Clients which
+<<api-conventions-number-values,incorrectly>> assume that this value will be within a particular range (e.g. that it fits into a 32-bit
+signed integer) may encounter errors (issue: {es-issue}111854[#111854])
+
+[[breaking-8.15.0]]
+[float]
+=== Breaking changes
+
+Cluster Coordination::
+* Interpret `?timeout=-1` as infinite ack timeout {es-pull}107675[#107675]
+
+Inference API::
+* Replace `model_id` with `inference_id` in GET inference API {es-pull}111366[#111366]
+
+Rollup::
+* Disallow new rollup jobs in clusters with no rollup usage {es-pull}108624[#108624] (issue: {es-issue}108381[#108381])
+
+Search::
+* Change `skip_unavailable` remote cluster setting default value to true {es-pull}105792[#105792]
+
+[[bug-8.15.0]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Don't sample calls to `ReduceContext#consumeBucketsAndMaybeBreak` ins `InternalDateHistogram` and `InternalHistogram` during reduction {es-pull}110186[#110186]
+* Fix `ClassCastException` in Significant Terms {es-pull}108429[#108429] (issue: {es-issue}108427[#108427])
+* Run terms concurrently when cardinality is only lower than shard size {es-pull}110369[#110369] (issue: {es-issue}105505[#105505])
+
+Allocation::
+* Fix misc trappy allocation API timeouts {es-pull}109241[#109241]
+* Fix trappy timeout in allocation explain API {es-pull}109240[#109240]
+
+Analysis::
+* Correct positioning for unique token filter {es-pull}109395[#109395]
+
+Authentication::
+* Add comma before charset parameter in WWW-Authenticate response header {es-pull}110906[#110906]
+* Avoid NPE if `users_roles` file does not exist {es-pull}109606[#109606]
+* Improve security-crypto threadpool overflow handling {es-pull}111369[#111369]
+
+Authorization::
+* Fix trailing slash in `security.put_privileges` specification {es-pull}110177[#110177]
+* Fixes cluster state-based role mappings not recovered from disk {es-pull}109167[#109167]
+* Handle unmatching remote cluster wildcards properly for `IndicesRequest.SingleIndexNoWildcards` requests {es-pull}109185[#109185]
+
+Autoscaling::
+* Expose `?master_timeout` in autoscaling APIs {es-pull}108759[#108759]
+
+CRUD::
+* Update checkpoints after post-replication actions, even on failure {es-pull}109908[#109908]
+
+Cluster Coordination::
+* Deserialize publish requests on generic thread-pool {es-pull}108814[#108814] (issue: {es-issue}106352[#106352])
+* Fail cluster state API if blocked {es-pull}109618[#109618] (issue: {es-issue}107503[#107503])
+* Use `scheduleUnlessShuttingDown` in `LeaderChecker` {es-pull}108643[#108643] (issue: {es-issue}108642[#108642])
+
+Data streams::
+* Apm-data: set concrete values for `metricset.interval` {es-pull}109043[#109043]
+* Ecs@mappings: reduce scope for `ecs_geo_point` {es-pull}108349[#108349] (issue: {es-issue}108338[#108338])
+* Include component templates in retention validaiton {es-pull}109779[#109779]
+
+Distributed::
+* Associate restore snapshot task to parent mount task {es-pull}108705[#108705] (issue: {es-issue}105830[#105830])
+* Don't detect `PlainActionFuture` deadlock on concurrent complete {es-pull}110361[#110361] (issues: {es-issue}110181[#110181], {es-issue}110360[#110360])
+* Handle nullable `DocsStats` and `StoresStats` {es-pull}109196[#109196]
+
+Downsampling::
+* Support flattened fields and multi-fields as dimensions in downsampling {es-pull}110066[#110066] (issue: {es-issue}99297[#99297])
+
+ES|QL::
+* ESQL: Change "substring" function to not return null on empty string {es-pull}109174[#109174]
+* ESQL: Fix Join references {es-pull}109989[#109989]
+* ESQL: Fix LOOKUP attribute shadowing {es-pull}109807[#109807] (issue: {es-issue}109392[#109392])
+* ESQL: Fix Max doubles bug with negatives and add tests for Max and Min {es-pull}110586[#110586]
+* ESQL: Fix `IpPrefix` function not handling correctly `ByteRefs` {es-pull}109205[#109205] (issue: {es-issue}109198[#109198])
+* ESQL: Fix equals `hashCode` for functions {es-pull}107947[#107947] (issue: {es-issue}104393[#104393])
+* ESQL: Fix variable shadowing when pushing down past Project {es-pull}108360[#108360] (issue: {es-issue}108008[#108008])
+* ESQL: Validate unique plan attribute names {es-pull}110488[#110488] (issue: {es-issue}110541[#110541])
+* ESQL: change from quoting from backtick to quote {es-pull}108395[#108395]
+* ESQL: make named params objects truly per request {es-pull}110046[#110046] (issue: {es-issue}110028[#110028])
+* ES|QL: Fix DISSECT that overwrites input {es-pull}110201[#110201] (issue: {es-issue}110184[#110184])
+* ES|QL: limit query depth to 500 levels {es-pull}108089[#108089] (issue: {es-issue}107752[#107752])
+* ES|QL: reduce max expression depth to 400 {es-pull}111186[#111186] (issue: {es-issue}109846[#109846])
+* Fix ST_DISTANCE Lucene push-down for complex predicates {es-pull}110391[#110391] (issue: {es-issue}110349[#110349])
+* Fix `ClassCastException` with MV_EXPAND on missing field {es-pull}110096[#110096] (issue: {es-issue}109974[#109974])
+* Fix bug in union-types with type-casting in grouping key of STATS {es-pull}110476[#110476] (issues: {es-issue}109922[#109922], {es-issue}110477[#110477])
+* Fix for union-types for multiple columns with the same name {es-pull}110793[#110793] (issues: {es-issue}110490[#110490], {es-issue}109916[#109916])
+* [ESQL] Count_distinct(_source) should return a 400 {es-pull}110824[#110824]
+* [ESQL] Fix parsing of large magnitude negative numbers {es-pull}110665[#110665] (issue: {es-issue}104323[#104323])
+* [ESQL] Migrate `SimplifyComparisonArithmetics` optimization {es-pull}109256[#109256] (issues: {es-issue}108388[#108388], {es-issue}108743[#108743])
+
+Engine::
+* Async close of `IndexShard` {es-pull}108145[#108145]
+
+Highlighting::
+* Fix issue with returning incomplete fragment for plain highlighter {es-pull}110707[#110707]
+
+ILM+SLM::
+* Allow `read_slm` to call GET /_slm/status {es-pull}108333[#108333]
+
+Indices APIs::
+* Create a new `NodeRequest` for every `NodesDataTiersUsageTransport` use {es-pull}108379[#108379]
+
+Infra/Core::
+* Add a cluster listener to fix missing node features after upgrading from a version prior to 8.13 {es-pull}110710[#110710] (issue: {es-issue}109254[#109254])
+* Add bounds checking to parsing ISO8601 timezone offset values {es-pull}108672[#108672]
+* Fix native preallocate to actually run {es-pull}110851[#110851]
+* Ignore additional cpu.stat fields {es-pull}108019[#108019] (issue: {es-issue}107983[#107983])
+* Specify parse index when error occurs on multiple datetime parses {es-pull}108607[#108607]
+
+Infra/Metrics::
+* Provide document size reporter with `MapperService` {es-pull}109794[#109794]
+
+Infra/Node Lifecycle::
+* Expose `?master_timeout` on get-shutdown API {es-pull}108886[#108886]
+* Fix serialization of put-shutdown request {es-pull}107862[#107862] (issue: {es-issue}107857[#107857])
+* Support wait indefinitely for search tasks to complete on node shutdown {es-pull}107426[#107426]
+
+Infra/REST API::
+* Add some missing timeout params to REST API specs {es-pull}108761[#108761]
+* Consider `error_trace` supported by all endpoints {es-pull}109613[#109613] (issue: {es-issue}109612[#109612])
+
+Ingest Node::
+* Fix Dissect with leading non-ascii characters {es-pull}111184[#111184]
+* Fix enrich policy runner exception handling on empty segments response {es-pull}111290[#111290]
+* GeoIP tasks should wait longer for master {es-pull}108410[#108410]
+* Removing the use of Stream::peek from `GeoIpDownloader::cleanDatabases` {es-pull}110666[#110666]
+* Simulate should succeed if `ignore_missing_pipeline` {es-pull}108106[#108106] (issue: {es-issue}107314[#107314])
+
+Machine Learning::
+* Allow deletion of the ELSER inference service when reference in ingest {es-pull}108146[#108146]
+* Avoid `InferenceRunner` deadlock {es-pull}109551[#109551]
+* Correctly handle duplicate model ids for the `_cat` trained models api and usage statistics {es-pull}109126[#109126]
+* Do not use global ordinals strategy if the leaf reader context cannot be obtained {es-pull}108459[#108459]
+* Fix NPE in trained model assignment updater {es-pull}108942[#108942]
+* Fix serialising inference delete response {es-pull}109384[#109384]
+* Fix "stack use after scope" memory error {ml-pull}2673[#2673]
+* Fix trailing slash in `ml.get_categories` specification {es-pull}110146[#110146]
+* Handle any exception thrown by inference {ml-pull}2680[#2680]
+* Increase response size limit for batched requests {es-pull}110112[#110112]
+* Offload request to generic threadpool {es-pull}109104[#109104] (issue: {es-issue}109100[#109100])
+* Propagate accurate deployment timeout {es-pull}109534[#109534] (issue: {es-issue}109407[#109407])
+* Refactor TextEmbeddingResults to use primitives rather than objects {es-pull}108161[#108161]
+* Require question to be non-null in `QuestionAnsweringConfig` {es-pull}107972[#107972]
+* Start Trained Model Deployment API request query params now override body params {es-pull}109487[#109487]
+* Suppress deprecation warnings from ingest pipelines when deleting trained model {es-pull}108679[#108679] (issue: {es-issue}105004[#105004])
+* Use default translog durability on AD results index {es-pull}108999[#108999]
+* Use the multi node routing action for internal inference services {es-pull}109358[#109358]
+* [Inference API] Extract optional long instead of integer in `RateLimitSettings#of` {es-pull}108602[#108602]
+* [Inference API] Fix serialization for inference delete endpoint response {es-pull}110431[#110431]
+* [Inference API] Replace `model_id` with `inference_id` in inference API except when stored {es-pull}111366[#111366]
+
+Mapping::
+* Fix off by one error when handling null values in range fields {es-pull}107977[#107977] (issue: {es-issue}107282[#107282])
+* Limit number of synonym rules that can be created {es-pull}109981[#109981] (issue: {es-issue}108785[#108785])
+* Propagate mapper builder context flags across nested mapper builder context creation {es-pull}109963[#109963]
+* `DenseVectorFieldMapper` fixed typo {es-pull}108065[#108065]
+
+Network::
+* Use proper executor for failing requests when connection closes {es-pull}109236[#109236] (issue: {es-issue}109225[#109225])
+* `NoSuchRemoteClusterException` should not be thrown when a remote is configured {es-pull}107435[#107435] (issue: {es-issue}107381[#107381])
+
+Packaging::
+* Adding override for lintian false positive on `libvec.so` {es-pull}108521[#108521] (issue: {es-issue}108514[#108514])
+
+Ranking::
+* Fix score count validation in reranker response {es-pull}111424[#111424] (issue: {es-issue}111202[#111202])
+
+Rollup::
+* Fix trailing slash in two rollup specifications {es-pull}110176[#110176]
+
+Search::
+* Adding score from `RankDoc` to `SearchHit` {es-pull}108870[#108870]
+* Better handling of multiple rescorers clauses with LTR {es-pull}109071[#109071]
+* Correct query profiling for conjunctions {es-pull}108122[#108122] (issue: {es-issue}108116[#108116])
+* Fix `DecayFunctions'` `toString` {es-pull}107415[#107415] (issue: {es-issue}100870[#100870])
+* Fix leak in collapsing search results {es-pull}110927[#110927]
+* Fork freeing search/scroll contexts to GENERIC pool {es-pull}109481[#109481]
+
+Security::
+* Add permission to secure access to certain config files {es-pull}107827[#107827]
+* Add permission to secure access to certain config files specified by settings {es-pull}108895[#108895]
+* Fix trappy timeouts in security settings APIs {es-pull}109233[#109233]
+
+Snapshot/Restore::
+* Stricter failure handling in multi-repo get-snapshots request handling {es-pull}107191[#107191]
+
+TSDB::
+* Sort time series indices by time range in `GetDataStreams` API {es-pull}107967[#107967] (issue: {es-issue}102088[#102088])
+
+Transform::
+* Always pick the user `maxPageSize` value {es-pull}109876[#109876] (issue: {es-issue}109844[#109844])
+* Exit gracefully when deleted {es-pull}107917[#107917] (issue: {es-issue}107266[#107266])
+* Fix NPE during destination index creation {es-pull}108891[#108891] (issue: {es-issue}108890[#108890])
+* Forward `indexServiceSafe` exception to listener {es-pull}108517[#108517] (issue: {es-issue}108418[#108418])
+* Halt Indexer on Stop/Abort API {es-pull}107792[#107792]
+* Handle `IndexNotFoundException` {es-pull}108394[#108394] (issue: {es-issue}107263[#107263])
+* Prevent concurrent jobs during cleanup {es-pull}109047[#109047]
+* Redirect `VersionConflict` to reset code {es-pull}108070[#108070]
+* Reset max page size to settings value {es-pull}109449[#109449] (issue: {es-issue}109308[#109308])
+
+Vector Search::
+* Ensure vector similarity correctly limits `inner_hits` returned for nested kNN {es-pull}111363[#111363] (issue: {es-issue}111093[#111093])
+* Ensure we return non-negative scores when scoring scalar dot-products {es-pull}108522[#108522]
+
+Watcher::
+* Avoiding running watch jobs in TickerScheduleTriggerEngine if it is paused {es-pull}110061[#110061] (issue: {es-issue}105933[#105933])
+
+[[deprecation-8.15.0]]
+[float]
+=== Deprecations
+
+ILM+SLM::
+* Deprecate using slm privileges to access ilm {es-pull}110540[#110540]
+
+Infra/Settings::
+* `ParseHeapRatioOrDeprecatedByteSizeValue` for `indices.breaker.total.limit` {es-pull}110236[#110236]
+
+Machine Learning::
+* Deprecate `text_expansion` and `weighted_tokens` queries {es-pull}109880[#109880]
+
+[[enhancement-8.15.0]]
+[float]
+=== Enhancements
+
+Aggregations::
+* Aggs: Scripted metric allow list {es-pull}109444[#109444]
+* Enable inter-segment concurrency for low cardinality numeric terms aggs {es-pull}108306[#108306]
+* Increase size of big arrays only when there is an actual value in the aggregators {es-pull}107764[#107764]
+* Increase size of big arrays only when there is an actual value in the aggregators (Analytics module) {es-pull}107813[#107813]
+* Optimise `BinaryRangeAggregator` for single value fields {es-pull}108016[#108016]
+* Optimise cardinality aggregations for single value fields {es-pull}107892[#107892]
+* Optimise composite aggregations for single value fields {es-pull}107897[#107897]
+* Optimise few metric aggregations for single value fields {es-pull}107832[#107832]
+* Optimise histogram aggregations for single value fields {es-pull}107893[#107893]
+* Optimise multiterms aggregation for single value fields {es-pull}107937[#107937]
+* Optimise terms aggregations for single value fields {es-pull}107930[#107930]
+* Speed up collecting zero document string terms {es-pull}110922[#110922]
+
+Allocation::
+* Log shard movements {es-pull}105829[#105829]
+* Support effective watermark thresholds in node stats API {es-pull}107244[#107244] (issue: {es-issue}106676[#106676])
+
+Application::
+* Add Create or update query rule API call {es-pull}109042[#109042]
+* Rename rule query and add support for multiple rulesets {es-pull}108831[#108831]
+* Support multiple associated groups for TopN {es-pull}108409[#108409] (issue: {es-issue}108018[#108018])
+* [Connector API] Change `UpdateConnectorFiltering` API to have better defaults {es-pull}108612[#108612]
+
+Authentication::
+* Expose API Key cache metrics {es-pull}109078[#109078]
+
+Authorization::
+* Cluster state role mapper file settings service {es-pull}107886[#107886]
+* Cluster-state based Security role mapper {es-pull}107410[#107410]
+* Introduce role description field {es-pull}107088[#107088]
+* [Osquery] Extend `kibana_system` role with an access to new `osquery_manager` index {es-pull}108849[#108849]
+
+Data streams::
+* Add metrics@custom component template to metrics-*-* index template {es-pull}109540[#109540] (issue: {es-issue}109475[#109475])
+* Apm-data: enable plugin by default {es-pull}108860[#108860]
+* Apm-data: ignore malformed fields, and too many dynamic fields {es-pull}108444[#108444]
+* Apm-data: improve default pipeline performance {es-pull}108396[#108396] (issue: {es-issue}108290[#108290])
+* Apm-data: improve indexing resilience {es-pull}108227[#108227]
+* Apm-data: increase priority above Fleet templates {es-pull}108885[#108885]
+* Apm-data: increase version for templates {es-pull}108340[#108340]
+* Apm-data: set codec: best_compression for logs-apm.* data streams {es-pull}108862[#108862]
+* Remove `default_field: message` from metrics index templates {es-pull}110651[#110651]
+
+Distributed::
+* Add `wait_for_completion` parameter to delete snapshot request {es-pull}109462[#109462] (issue: {es-issue}101300[#101300])
+* Improve mechanism for extracting the result of a `PlainActionFuture` {es-pull}110019[#110019] (issue: {es-issue}108125[#108125])
+
+ES|QL::
+* Add `BlockHash` for 3 `BytesRefs` {es-pull}108165[#108165]
+* Allow `LuceneSourceOperator` to early terminate {es-pull}108820[#108820]
+* Check if `CsvTests` required capabilities exist {es-pull}108684[#108684]
+* ESQL: Add aggregates node level reduction {es-pull}107876[#107876]
+* ESQL: Add more time span units {es-pull}108300[#108300]
+* ESQL: Implement LOOKUP, an "inline" enrich {es-pull}107987[#107987] (issue: {es-issue}107306[#107306])
+* ESQL: Renamed `TopList` to Top {es-pull}110347[#110347]
+* ESQL: Union Types Support {es-pull}107545[#107545] (issue: {es-issue}100603[#100603])
+* ESQL: add REPEAT string function {es-pull}109220[#109220]
+* ES|QL Add primitive float support to the Compute Engine {es-pull}109746[#109746] (issue: {es-issue}109178[#109178])
+* ES|QL Add primitive float variants of all aggregators to the compute engine {es-pull}109781[#109781]
+* ES|QL: vectorize eval {es-pull}109332[#109332]
+* Optimize ST_DISTANCE filtering with Lucene circle intersection query {es-pull}110102[#110102] (issue: {es-issue}109972[#109972])
+* Optimize for single value in ordinals grouping {es-pull}108118[#108118]
+* Rewrite away type converting functions that do not convert types {es-pull}108713[#108713] (issue: {es-issue}107716[#107716])
+* ST_DISTANCE Function {es-pull}108764[#108764] (issue: {es-issue}108212[#108212])
+* Support metrics counter types in ESQL {es-pull}107877[#107877]
+* [ESQL] CBRT function {es-pull}108574[#108574]
+* [ES|QL] Convert string to datetime when the other size of an arithmetic operator is `date_period` or `time_duration` {es-pull}108455[#108455]
+* [ES|QL] Support Named and Positional Parameters in `EsqlQueryRequest` {es-pull}108421[#108421] (issue: {es-issue}107029[#107029])
+* [ES|QL] `weighted_avg` {es-pull}109993[#109993]
+
+Engine::
+* Drop shards close timeout when stopping node. {es-pull}107978[#107978] (issue: {es-issue}107938[#107938])
+* Update translog `writeLocation` for `flushListener` after commit {es-pull}109603[#109603]
+
+Geo::
+* Optimize `GeoBounds` and `GeoCentroid` aggregations for single value fields {es-pull}107663[#107663]
+
+Health::
+* Log details of non-green indicators in `HealthPeriodicLogger` {es-pull}108266[#108266]
+
+Highlighting::
+* Unified Highlighter to support matched_fields  {es-pull}107640[#107640] (issue: {es-issue}5172[#5172])
+
+Infra/Core::
+* Add allocation explain output for THROTTLING shards {es-pull}109563[#109563]
+* Create custom parser for ISO-8601 datetimes {es-pull}106486[#106486] (issue: {es-issue}102063[#102063])
+* Extend ISO8601 datetime parser to specify forbidden fields, allowing it to be used on more formats {es-pull}108606[#108606]
+* add Elastic-internal stable bridge api for use by Logstash {es-pull}108171[#108171]
+
+Infra/Metrics::
+* Add auto-sharding APM metrics {es-pull}107593[#107593]
+* Add request metric to `RestController` to track success/failure (by status code) {es-pull}109957[#109957]
+* Allow RA metrics to be reported upon parsing completed or accumulated {es-pull}108726[#108726]
+* Provide the `DocumentSizeReporter` with index mode {es-pull}108947[#108947]
+* Return noop instance `DocSizeObserver` for updates with scripts {es-pull}108856[#108856]
+
+Ingest Node::
+* Add `continent_code` support to the geoip processor {es-pull}108780[#108780] (issue: {es-issue}85820[#85820])
+* Add support for the 'Connection Type' database to the geoip processor {es-pull}108683[#108683]
+* Add support for the 'Domain' database to the geoip processor {es-pull}108639[#108639]
+* Add support for the 'ISP' database to the geoip processor {es-pull}108651[#108651]
+* Adding `hits_time_in_millis` and `misses_time_in_millis` to enrich cache stats {es-pull}107579[#107579]
+* Adding `user_type` support for the enterprise database for the geoip processor {es-pull}108687[#108687]
+* Adding human readable times to geoip stats {es-pull}107647[#107647]
+* Include doc size info in ingest stats {es-pull}107240[#107240] (issue: {es-issue}106386[#106386])
+* Make ingest byte stat names more descriptive {es-pull}108786[#108786]
+* Return ingest byte stats even when 0-valued {es-pull}108796[#108796]
+* Test pipeline run after reroute {es-pull}108693[#108693]
+
+Logs::
+* Introduce a node setting controlling the activation of the `logs` index mode in logs@settings component template {es-pull}109025[#109025] (issue: {es-issue}108762[#108762])
+* Support index sorting with nested fields {es-pull}110251[#110251] (issue: {es-issue}107349[#107349])
+
+Machine Learning::
+* Add Anthropic messages integration to Inference API {es-pull}109893[#109893]
+* Add `sparse_vector` query {es-pull}108254[#108254]
+* Add model download progress to the download task status {es-pull}107676[#107676]
+* Add rate limiting support for the Inference API {es-pull}107706[#107706]
+* Add the rerank task to the Elasticsearch internal inference service {es-pull}108452[#108452]
+* Default the HF service to cosine similarity {es-pull}109967[#109967]
+* GA the update trained model action {es-pull}108868[#108868]
+* Handle the "JSON memory allocator bytes" field {es-pull}109653[#109653]
+* Inference Processor: skip inference when all fields are missing {es-pull}108131[#108131]
+* Log 'No statistics at.. ' message as a warning {ml-pull}2684[#2684]
+* Optimise frequent item sets aggregation for single value fields {es-pull}108130[#108130]
+* Sentence Chunker {es-pull}110334[#110334]
+* [Inference API] Add Amazon Bedrock Support to Inference API {es-pull}110248[#110248]
+* [Inference API] Add Mistral Embeddings Support to Inference API {es-pull}109194[#109194]
+* [Inference API] Check for related pipelines on delete inference endpoint {es-pull}109123[#109123]
+
+Mapping::
+* Add ignored field values to synthetic source {es-pull}107567[#107567]
+* Apply FLS to the contents of `IgnoredSourceFieldMapper` {es-pull}109931[#109931]
+* Binary field enables doc values by default for index mode with synthe… {es-pull}107739[#107739] (issue: {es-issue}107554[#107554])
+* Feature/annotated text store defaults {es-pull}107922[#107922] (issue: {es-issue}107734[#107734])
+* Handle `ignore_above` in synthetic source for flattened fields {es-pull}110214[#110214]
+* Opt in keyword field into fallback synthetic source if needed {es-pull}110016[#110016]
+* Opt in number fields into fallback synthetic source when doc values a… {es-pull}110160[#110160]
+* Reflect latest changes in synthetic source documentation {es-pull}109501[#109501]
+* Store source for fields in objects with `dynamic` override {es-pull}108911[#108911]
+* Store source for nested objects {es-pull}108818[#108818]
+* Support synthetic source for `geo_point` when `ignore_malformed` is used {es-pull}109651[#109651]
+* Support synthetic source for `scaled_float` and `unsigned_long` when `ignore_malformed` is used {es-pull}109506[#109506]
+* Support synthetic source for date fields when `ignore_malformed` is used {es-pull}109410[#109410]
+* Support synthetic source together with `ignore_malformed` in histogram fields {es-pull}109882[#109882]
+* Track source for arrays of objects {es-pull}108417[#108417] (issue: {es-issue}90708[#90708])
+* Track synthetic source for disabled objects {es-pull}108051[#108051]
+
+Network::
+* Detect long-running tasks on network threads {es-pull}109204[#109204]
+
+Ranking::
+* Enabling profiling for `RankBuilders` and adding tests for RRF {es-pull}109470[#109470]
+
+Relevance::
+* [Query Rules] Add API calls to get or delete individual query rules within a ruleset {es-pull}109554[#109554]
+* [Query Rules] Require Enterprise License for Query Rules {es-pull}109634[#109634]
+
+Search::
+* Add AVX-512 optimised vector distance functions for int7 on x64 {es-pull}109084[#109084]
+* Add `SparseVectorStats` {es-pull}108793[#108793]
+* Add `_name` support for top level `knn` clauses {es-pull}107645[#107645] (issues: {es-issue}106254[#106254], {es-issue}107448[#107448])
+* Add a SIMD (AVX2) optimised vector distance function for int7 on x64 {es-pull}108088[#108088]
+* Add min/max range of the `event.ingested` field to cluster state for searchable snapshots {es-pull}106252[#106252]
+* Add per-field KNN vector format to Index Segments API {es-pull}107216[#107216]
+* Add support for hiragana_uppercase & katakana_uppercase token filters in kuromoji analysis plugin {es-pull}106553[#106553]
+* Adding support for explain in rrf {es-pull}108682[#108682]
+* Allow rescorer with field collapsing {es-pull}107779[#107779] (issue: {es-issue}27243[#27243])
+* Cut over stored fields to ZSTD for compression {es-pull}103374[#103374]
+* Limit the value in prefix query {es-pull}108537[#108537] (issue: {es-issue}108486[#108486])
+* Make dense vector field type updatable {es-pull}106591[#106591]
+* Multivalue Sparse Vector Support {es-pull}109007[#109007]
+
+Security::
+* Add bulk delete roles API {es-pull}110383[#110383]
+* Remote cluster - API key security model - cluster privileges {es-pull}107493[#107493]
+
+Snapshot/Restore::
+* Denser in-memory representation of `ShardBlobsToDelete` {es-pull}109848[#109848]
+* Log repo UUID at generation/registration time {es-pull}109672[#109672]
+* Make repository analysis API available to non-operators {es-pull}110179[#110179] (issue: {es-issue}100318[#100318])
+* Track `RequestedRangeNotSatisfiedException` separately in S3 Metrics {es-pull}109657[#109657]
+
+Stats::
+* DocsStats: Add human readable bytesize {es-pull}109720[#109720]
+
+TSDB::
+* Optimise `time_series` aggregation for single value fields {es-pull}107990[#107990]
+* Support `ignore_above` on keyword dimensions {es-pull}110337[#110337]
+
+Vector Search::
+* Adding hamming distance function to painless for `dense_vector` fields {es-pull}109359[#109359]
+* Support k parameter for knn query {es-pull}110233[#110233] (issue: {es-issue}108473[#108473])
+
+[[feature-8.15.0]]
+[float]
+=== New features
+
+Aggregations::
+* Opt `scripted_metric` out of parallelization {es-pull}109597[#109597]
+
+Application::
+* [Connector API] Add claim sync job endpoint {es-pull}109480[#109480]
+
+ES|QL::
+* ESQL: Add `ip_prefix` function {es-pull}109070[#109070] (issue: {es-issue}99064[#99064])
+* ESQL: Introduce a casting operator, `::` {es-pull}107409[#107409]
+* ESQL: `top_list` aggregation {es-pull}109386[#109386] (issue: {es-issue}109213[#109213])
+* ESQL: add Arrow dataframes output format {es-pull}109873[#109873]
+* Reapply "ESQL: Expose "_ignored" metadata field" {es-pull}108871[#108871]
+
+Infra/REST API::
+* Add a capabilities API to check node and cluster capabilities {es-pull}106820[#106820]
+
+Ingest Node::
+* Directly download commercial ip geolocation databases from providers {es-pull}110844[#110844]
+* Mark the Redact processor as Generally Available {es-pull}110395[#110395]
+
+Logs::
+* Introduce logs index mode as Tech Preview {es-pull}108896[#108896] (issue: {es-issue}108896[#108896])
+
+Machine Learning::
+* Add support for Azure AI Studio embeddings and completions to the inference service. {es-pull}108472[#108472]
+
+Mapping::
+* Add `semantic_text` field type and `semantic` query {es-pull}110338[#110338]
+* Add generic fallback implementation for synthetic source {es-pull}108222[#108222]
+* Add synthetic source support for `geo_shape` via fallback implementation {es-pull}108881[#108881]
+* Add synthetic source support for binary fields {es-pull}107549[#107549]
+* Enable fallback synthetic source by default {es-pull}109370[#109370] (issue: {es-issue}106460[#106460])
+* Enable fallback synthetic source for `point` and `shape` {es-pull}109312[#109312]
+* Enable fallback synthetic source for `token_count` {es-pull}109044[#109044]
+* Implement synthetic source support for annotated text field {es-pull}107735[#107735]
+* Implement synthetic source support for range fields {es-pull}107081[#107081]
+* Support arrays in fallback synthetic source implementation {es-pull}108878[#108878]
+* Support synthetic source for `aggregate_metric_double` when ignore_malf… {es-pull}108746[#108746]
+
+Ranking::
+* Add text similarity reranker retriever {es-pull}109813[#109813]
+
+Relevance::
+* Mark Query Rules as GA {es-pull}110004[#110004]
+
+Search::
+* Add new int4 quantization to dense_vector {es-pull}109317[#109317]
+* Adding RankFeature search phase implementation {es-pull}108538[#108538]
+* Adding aggregations support for the `_ignored` field {es-pull}101373[#101373] (issue: {es-issue}59946[#59946])
+* Update Lucene version to 9.11 {es-pull}109219[#109219]
+
+Security::
+* Query Roles API {es-pull}108733[#108733]
+
+Transform::
+* Introduce _transform/_node_stats API {es-pull}107279[#107279]
+
+Vector Search::
+* Adds new `bit` `element_type` for `dense_vectors` {es-pull}110059[#110059]
+
+[[upgrade-8.15.0]]
+[float]
+=== Upgrades
+
+Infra/Plugins::
+* Update ASM to 9.7 for plugin scanner {es-pull}108822[#108822] (issue: {es-issue}108776[#108776])
+
+Ingest Node::
+* Bump Tika dependency to 2.9.2 {es-pull}108144[#108144]
+
+Network::
+* Upgrade to Netty 4.1.109 {es-pull}108155[#108155]
+
+Search::
+* Upgrade to Lucene-9.11.1 {es-pull}110234[#110234]
+
+Security::
+* Upgrade bouncy castle (non-fips) to 1.78.1 {es-pull}108223[#108223]
+
+Snapshot/Restore::
+* Bump jackson version in modules:repository-azure {es-pull}109717[#109717]
+
+
diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc
index 0ed01ff422700..007dd740f34cf 100644
--- a/docs/reference/release-notes/highlights.asciidoc
+++ b/docs/reference/release-notes/highlights.asciidoc
@@ -68,6 +68,31 @@ improves query and merge speed significantly when compared to raw vectors.
 
 {es-pull}109317[#109317]
 
+[discrete]
+[[esql_inlinestats]]
+=== ESQL: INLINESTATS
+This adds the `INLINESTATS` command to ESQL which performs a STATS and
+then enriches the results into the output stream. So, this query:
+
+[source,esql]
+----
+FROM test
+| INLINESTATS m=MAX(a * b) BY b
+| WHERE m == a * b
+| SORT a DESC, b DESC
+| LIMIT 3
+----
+
+Produces output like:
+
+|  a  |  b  |   m   |
+| --- | --- | ----- |
+|  99 | 999 | 98901 |
+|  99 | 998 | 98802 |
+|  99 | 997 | 98703 |
+
+{es-pull}109583[#109583]
+
 [discrete]
 [[mark_query_rules_as_ga]]
 === Mark Query Rules as GA
@@ -112,6 +137,18 @@ The Redact processor uses the Grok rules engine to obscure text in the input doc
 
 {es-pull}110395[#110395]
 
+[discrete]
+[[always_allow_rebalancing_by_default]]
+=== Always allow rebalancing by default
+In earlier versions of {es} the `cluster.routing.allocation.allow_rebalance` setting defaults to
+`indices_all_active` which blocks all rebalancing moves while the cluster is in `yellow` or `red` health. This was
+appropriate for the legacy allocator which might do too many rebalancing moves otherwise. Today's allocator has
+better support for rebalancing a cluster that is not in `green` health, and expects to be able to rebalance some
+shards away from over-full nodes to avoid allocating shards to undesirable locations in the first place. From
+version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy allocator is explicitly enabled.
+
+{es-pull}111015[#111015]
+
 // end::notable-highlights[]
 
 
diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc
index 7c2e42a26b923..fabd495cdc525 100644
--- a/docs/reference/rest-api/common-parms.asciidoc
+++ b/docs/reference/rest-api/common-parms.asciidoc
@@ -649,8 +649,9 @@ tag::level[]
 +
 --
 (Optional, string)
-Indicates whether statistics are aggregated
-at the cluster, index, or shard level.
+Indicates whether statistics are aggregated at the cluster, index, or shard level.
+If the shards level is requested, some additional
+<<cluster-nodes-stats,shard-specific statistics>> are shown.
 
 Valid values are:
 
@@ -1326,13 +1327,21 @@ that lower ranked documents have more influence. This value must be greater than
 equal to `1`. Defaults to `60`.
 end::rrf-rank-constant[]
 
-tag::rrf-window-size[]
-`window_size`::
+tag::rrf-rank-window-size[]
+`rank_window_size`::
 (Optional, integer)
 +
 This value determines the size of the individual result sets per
 query. A higher value will improve result relevance at the cost of performance. The final
 ranked result set is pruned down to the search request's <<search-size-param, size>>.
-`window_size` must be greater than or equal to `size` and greater than or equal to `1`.
+`rank_window_size` must be greater than or equal to `size` and greater than or equal to `1`.
 Defaults to the `size` parameter.
-end::rrf-window-size[]
+end::rrf-rank-window-size[]
+
+tag::rrf-filter[]
+`filter`::
+(Optional, <<query-dsl, query object or list of query objects>>)
++
+Applies the specified <<query-dsl-bool-query, boolean query filter>> to all of the specified sub-retrievers,
+according to each retriever's specifications.
+end::rrf-filter[]
diff --git a/docs/reference/rest-api/security/bulk-create-roles.asciidoc b/docs/reference/rest-api/security/bulk-create-roles.asciidoc
index a8072b7ba549a..e4b6ef7f765c2 100644
--- a/docs/reference/rest-api/security/bulk-create-roles.asciidoc
+++ b/docs/reference/rest-api/security/bulk-create-roles.asciidoc
@@ -315,7 +315,7 @@ The result would then have the `errors` field set to `true` and hold the error f
         "details": {
             "my_admin_role": { <4>
                 "type": "action_request_validation_exception",
-                "reason": "Validation Failed: 1: unknown cluster privilege [bad_cluster_privilege]. a privilege must be either one of the predefined cluster privilege names [manage_own_api_key,none,cancel_task,cross_cluster_replication,cross_cluster_search,delegate_pki,grant_api_key,manage_autoscaling,manage_index_templates,manage_logstash_pipelines,manage_oidc,manage_saml,manage_search_application,manage_search_query_rules,manage_search_synonyms,manage_service_account,manage_token,manage_user_profile,monitor_connector,monitor_data_stream_global_retention,monitor_enrich,monitor_inference,monitor_ml,monitor_rollup,monitor_snapshot,monitor_text_structure,monitor_watcher,post_behavioral_analytics_event,read_ccr,read_connector_secrets,read_fleet_secrets,read_ilm,read_pipeline,read_security,read_slm,transport_client,write_connector_secrets,write_fleet_secrets,create_snapshot,manage_behavioral_analytics,manage_ccr,manage_connector,manage_data_stream_global_retention,manage_enrich,manage_ilm,manage_inference,manage_ml,manage_rollup,manage_slm,manage_watcher,monitor_data_frame_transforms,monitor_transform,manage_api_key,manage_ingest_pipelines,manage_pipeline,manage_data_frame_transforms,manage_transform,manage_security,monitor,manage,all] or a pattern over one of the available cluster actions;"
+                "reason": "Validation Failed: 1: unknown cluster privilege [bad_cluster_privilege]. a privilege must be either one of the predefined cluster privilege names [manage_own_api_key,manage_data_stream_global_retention,monitor_data_stream_global_retention,none,cancel_task,cross_cluster_replication,cross_cluster_search,delegate_pki,grant_api_key,manage_autoscaling,manage_index_templates,manage_logstash_pipelines,manage_oidc,manage_saml,manage_search_application,manage_search_query_rules,manage_search_synonyms,manage_service_account,manage_token,manage_user_profile,monitor_connector,monitor_enrich,monitor_inference,monitor_ml,monitor_rollup,monitor_snapshot,monitor_text_structure,monitor_watcher,post_behavioral_analytics_event,read_ccr,read_connector_secrets,read_fleet_secrets,read_ilm,read_pipeline,read_security,read_slm,transport_client,write_connector_secrets,write_fleet_secrets,create_snapshot,manage_behavioral_analytics,manage_ccr,manage_connector,manage_enrich,manage_ilm,manage_inference,manage_ml,manage_rollup,manage_slm,manage_watcher,monitor_data_frame_transforms,monitor_transform,manage_api_key,manage_ingest_pipelines,manage_pipeline,manage_data_frame_transforms,manage_transform,manage_security,monitor,manage,all] or a pattern over one of the available cluster actions;"
             }
         }
     }
diff --git a/docs/reference/search/point-in-time-api.asciidoc b/docs/reference/search/point-in-time-api.asciidoc
index 2e32324cb44d9..9cd91626c7600 100644
--- a/docs/reference/search/point-in-time-api.asciidoc
+++ b/docs/reference/search/point-in-time-api.asciidoc
@@ -78,6 +78,44 @@ IMPORTANT: The open point in time request and each subsequent search request can
 return different `id`; thus always use the most recently received `id` for the
 next search request.
 
+In addition to the `keep_alive` parameter, the `allow_partial_search_results` parameter
+can also be defined.
+This parameter determines whether the <<point-in-time-api, point in time (PIT)>>
+should tolerate unavailable shards or <<shard-failures, shard failures>> when
+initially creating the PIT.
+If set to true, the PIT will be created with the available shards, along with a
+reference to any missing ones.
+If set to false, the operation will fail if any shard is unavailable.
+The default value is false.
+
+The PIT response includes a summary of the total number of shards, as well as the number
+of successful shards when creating the PIT.
+
+[source,console]
+--------------------------------------------------
+POST /my-index-000001/_pit?keep_alive=1m&allow_partial_search_results=true
+--------------------------------------------------
+// TEST[setup:my_index]
+
+[source,js]
+--------------------------------------------------
+{
+  "id": "46ToAwMDaWR5BXV1aWQyKwZub2RlXzMAAAAAAAAAACoBYwADaWR4BXV1aWQxAgZub2RlXzEAAAAAAAAAAAEBYQADaWR5BXV1aWQyKgZub2RlXzIAAAAAAAAAAAwBYgACBXV1aWQyAAAFdXVpZDEAAQltYXRjaF9hbGw_gAAAAA=",
+  "_shards": {
+    "total": 10,
+    "successful": 10,
+    "skipped": 0,
+    "failed": 0
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+When a PIT that contains shard failures is used in a search request, the missing are
+always reported in the search response as a NoShardAvailableActionException exception.
+To get rid of these exceptions, a new PIT needs to be created so that shards missing
+from the previous PIT can be handled, assuming they become available in the meantime.
+
 [[point-in-time-keep-alive]]
 ==== Keeping point in time alive
 The `keep_alive` parameter, which is passed to a open point in time request and
diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc
index ed39ac786880b..58cc8ce9ef459 100644
--- a/docs/reference/search/retriever.asciidoc
+++ b/docs/reference/search/retriever.asciidoc
@@ -77,23 +77,48 @@ Collapses the top documents by a specified key into a single top document per ke
 When a retriever tree contains a compound retriever (a retriever with two or more child
 retrievers) *only* the query element is allowed.
 
-===== Example
+[discrete]
+[[standard-retriever-example]]
+==== Example
 
 [source,js]
 ----
-GET /index/_search
+GET /restaurants/_search
 {
-    "retriever": {
-        "standard": {
-            "query" { ... },
-            "filter" { ... },
-            "min_score": ...
+  "retriever": { <1>
+    "standard": { <2>
+      "query": { <3>
+        "bool": { <4>
+          "should": [ <5>
+            {
+              "match": { <6>
+                "region": "Austria"
+              }
+            }
+          ],
+          "filter": [ <7>
+            {
+              "term": { <8>
+                "year": "2019" <9>
+              }
+            }
+          ]
         }
-    },
-    "size": ...
+      }
+    }
+  }
 }
 ----
 // NOTCONSOLE
+<1> Opens the `retriever` object.
+<2> The `standard` retriever is used for definining traditional {es} queries.
+<3> The entry point for defining the search query.
+<4> The `bool` object allows for combining multiple query clauses logically.
+<5> The `should` array indicates conditions under which a document will match. Documents matching these conditions will increase their relevancy score.
+<6> The `match` object finds documents where the `region` field contains the word "Austria."
+<7> The `filter` array provides filtering conditions that must be met but do not contribute to the relevancy score.
+<8> The `term` object is used for exact matches, in this case, filtering documents by the `year` field.
+<9> The exact value to match in the `year` field.
 
 [[knn-retriever]]
 ==== kNN Retriever
@@ -142,29 +167,39 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-similarity]
 
 The parameters `query_vector` and `query_vector_builder` cannot be used together.
 
-===== Example:
+[discrete]
+[[knn-retriever-example]]
+==== Example
 
 [source,js]
 ----
-GET /index/_search
+GET my-embeddings/_search
 {
-    "retriever": {
-        "knn": {
-            "field": ...,
-            "query_vector": ...,
-            "k": ...,
-            "num_candidates": ...
-        }
+  "retriever": {
+    "knn": { <1>
+      "field": "vector", <2>
+      "query_vector": [10, 22, 77], <3>
+      "k": 10, <4>
+      "num_candidates": 10 <5>
     }
+  }
 }
 ----
 // NOTCONSOLE
 
+<1> Configuration for k-nearest neighbor (knn) search, which is based on vector similarity.
+<2> Specifies the field name that contains the vectors.
+<3> The query vector against which document vectors are compared in the `knn` search.
+<4> The number of nearest neighbors to return as top hits. This value must be fewer than or equal to `num_candidates`.
+<5> The size of the initial candidate set from which the final `k` nearest neighbors are selected.
+
 [[rrf-retriever]]
 ==== RRF Retriever
 
-An <<rrf, RRF>> retriever returns top documents based on the RRF formula
+An <<rrf, RRF>> retriever returns top documents based on the RRF formula,
 equally weighting two or more child retrievers.
+Reciprocal rank fusion (RRF) is a method for combining multiple result
+sets with different relevance indicators into a single result set.
 
 ===== Parameters
 
@@ -172,7 +207,9 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-retrievers]
 
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-constant]
 
-include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-window-size]
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-window-size]
+
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-filter]
 
 ===== Restrictions
 
@@ -180,26 +217,103 @@ An RRF retriever is a compound retriever. Child retrievers may not use
 elements that are restricted by having a compound retriever as part of
 the retriever tree.
 
-===== Example
+[discrete]
+[[rrf-retriever-example-hybrid]]
+==== Example: Hybrid search
+
+A simple hybrid search example (lexical search + dense vector search) combining a `standard` retriever with a `knn` retriever using RRF:
 
 [source,js]
 ----
-GET /index/_search
+GET /restaurants/_search
+{
+  "retriever": {
+    "rrf": { <1>
+      "retrievers": [ <2>
+        {
+          "standard": { <3>
+            "query": {
+              "multi_match": {
+                "query": "San Francisco",
+                "fields": [
+                  "city",
+                  "region"
+                ]
+              }
+            }
+          }
+        },
+        {
+          "knn": { <4>
+            "field": "vector",
+            "query_vector": [10, 22, 77],
+            "k": 10,
+            "num_candidates": 10
+          }
+        }
+      ],
+      "rank_constant": 1, <5>
+      "rank_window_size": 50  <6>
+    }
+  }
+}
+----
+// NOTCONSOLE
+<1> Defines a retriever tree with an RRF retriever.
+<2> The sub-retriever array.
+<3> The first sub-retriever is a `standard` retriever.
+<4> The second sub-retriever is a `knn` retriever.
+<5> The rank constant for the RRF retriever.
+<6> The rank window size for the RRF retriever.
+
+[discrete]
+[[rrf-retriever-example-hybrid-sparse]]
+==== Example: Hybrid search with sparse vectors
+
+A more complex hybrid search example (lexical search + ELSER sparse vector search + dense vector search) using RRF:
+
+[source,js]
+----
+GET movies/_search
 {
-    "retriever": {
-        "rrf": {
-            "retrievers": [
-                {
-                    "standard" { ... }
-                },
-                {
-                    "knn": { ... }
-                }
-            ],
-            "rank_constant": ...
-            "rank_window_size": ...
+  "retriever": {
+    "rrf": {
+      "retrievers": [
+        {
+          "standard": {
+            "query": {
+              "sparse_vector": {
+                "field": "plot_embedding",
+                "inference_id": "my-elser-model",
+                "query": "films that explore psychological depths"
+              }
+            }
+          }
+        },
+        {
+          "standard": {
+            "query": {
+              "multi_match": {
+                "query": "crime",
+                "fields": [
+                  "plot",
+                  "title"
+                ]
+              }
+            }
+          }
+        },
+        {
+          "knn": {
+            "field": "vector",
+            "query_vector": [10, 22, 77],
+            "k": 10,
+            "num_candidates": 10
+          }
         }
+      ]
     }
+  }
 }
 ----
 // NOTCONSOLE
@@ -207,15 +321,31 @@ GET /index/_search
 [[text-similarity-reranker-retriever]]
 ==== Text Similarity Re-ranker Retriever
 
-The `text_similarity_reranker` is a type of retriever that enhances search results by re-ranking documents based on semantic similarity to a specified inference text, using a machine learning model.
+The `text_similarity_reranker` retriever uses an NLP model to improve search results by reordering the top-k documents based on their semantic similarity to the query.
+
+[TIP]
+====
+Refer to <<semantic-reranking>> for a high level overview of semantic reranking.
+====
 
 ===== Prerequisites
 
 To use `text_similarity_reranker` you must first set up a `rerank` task using the <<put-inference-api, Create {infer} API>>.
-The `rerank` task should be set up with a machine learning model that can compute text similarity.
-Currently you can integrate directly with the Cohere Rerank endpoint using the <<infer-service-cohere,`cohere-rerank`>> task, or upload a model to {es} <<inference-example-eland,using Eland>>.
+The `rerank` task should be set up with a machine learning model that can compute text similarity. Refer to {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-similarity[the Elastic NLP model reference] for a list of third-party text similarity models supported by {es}.
+
+Currently you can:
+
+* Integrate directly with the <<infer-service-cohere,Cohere Rerank inference endpoint>> using the `rerank` task type
+* Integrate directly with the <<infer-service-google-vertex-ai,Google Vertex AI inference endpoint>> using the `rerank` task type
+* Upload a model to {es} with {eland-docs}/machine-learning.html#ml-nlp-pytorch[Eland] using the `text_similarity` NLP task type.
+** Then set up an <<inference-example-eland,{es} service inference endpoint>> with the `rerank` task type
+** Refer to the <<text-similarity-reranker-retriever-example-eland,example>> on this page for a step-by-step guide.
 
 ===== Parameters
+`retriever`::
+(Required, <<retriever, retriever>>)
++
+The child retriever that generates the initial set of top documents to be re-ranked.
 
 `field`::
 (Required, `string`)
@@ -242,31 +372,128 @@ The number of top documents to consider in the re-ranking process. Defaults to `
 +
 Sets a minimum threshold score for including documents in the re-ranked results. Documents with similarity scores below this threshold will be excluded. Note that score calculations vary depending on the model used.
 
+`filter`::
+(Optional, <<query-dsl, query object or list of query objects>>)
++
+Applies the specified <<query-dsl-bool-query, boolean query filter>> to the child  <<retriever, retriever>>.
+If the child retriever already specifies any filters, then this top-level filter is applied in conjuction
+with the filter defined in the child retriever.
+
 ===== Restrictions
 
 A text similarity re-ranker retriever is a compound retriever. Child retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree.
 
-===== Example
+[discrete]
+[[text-similarity-reranker-retriever-example-cohere]]
+==== Example: Cohere Rerank
+
+This example enables out-of-the-box semantic search by reranking top documents using the Cohere Rerank API. This approach eliminate the need to generate and store embeddings for all indexed documents.
+This requires a <<infer-service-cohere,Cohere Rerank inference endpoint>> using the `rerank` task type.
 
 [source,js]
 ----
 GET /index/_search
 {
-    "retriever": {
-        "text_similarity_reranker": {
-            "retriever": {
-                "standard": { ... }
+   "retriever": {
+      "text_similarity_reranker": {
+         "retriever": {
+            "standard": {
+               "query": {
+                  "match_phrase": {
+                     "text": "landmark in Paris"
+                  }
+               }
             }
-        },
-        "field": "text",
-        "inference_id": "my-cohere-rerank-model",
-        "inference_text": "Most famous landmark in Paris",
-        "rank_window_size": 100,
-        "min_score": 0.5
+         },
+         "field": "text",
+         "inference_id": "my-cohere-rerank-model",
+         "inference_text": "Most famous landmark in Paris",
+         "rank_window_size": 100,
+         "min_score": 0.5
+      }
+   }
+}
+----
+// NOTCONSOLE
+
+[discrete]
+[[text-similarity-reranker-retriever-example-eland]]
+==== Example: Semantic reranking with a Hugging Face model
+
+The following example uses the `cross-encoder/ms-marco-MiniLM-L-6-v2` model from Hugging Face to rerank search results based on semantic similarity.
+The model must be uploaded to {es} using https://www.elastic.co/guide/en/elasticsearch/client/eland/current/machine-learning.html#ml-nlp-pytorch[Eland].
+
+[TIP]
+====
+Refer to {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-similarity[the Elastic NLP model reference] for a list of third party text similarity models supported by {es}.
+====
+
+Follow these steps to load the model and create a semantic reranker.
+
+. Install Eland using `pip`
++
+[source,sh]
+----
+python -m pip install eland[pytorch]
+----
++
+. Upload the model to {es} using Eland. This example assumes you have an Elastic Cloud deployment and an API key. Refer to the https://www.elastic.co/guide/en/elasticsearch/client/eland/current/machine-learning.html#ml-nlp-pytorch-auth[Eland documentation] for more authentication options.
++
+[source,sh]
+----
+eland_import_hub_model \
+  --cloud-id $CLOUD_ID \
+  --es-api-key $ES_API_KEY \
+  --hub-model-id cross-encoder/ms-marco-MiniLM-L-6-v2 \
+  --task-type text_similarity \
+  --clear-previous \
+  --start
+----
++
+. Create an inference endpoint for the `rerank` task
++
+[source,js]
+----
+PUT _inference/rerank/my-msmarco-minilm-model
+{
+  "service": "elasticsearch",
+  "service_settings": {
+    "num_allocations": 1,
+    "num_threads": 1,
+    "model_id": "cross-encoder__ms-marco-minilm-l-6-v2"
+  }
+}
+----
+// NOTCONSOLE
++
+. Define a `text_similarity_rerank` retriever.
++
+[source,js]
+----
+POST movies/_search
+{
+  "retriever": {
+    "text_similarity_reranker": {
+      "retriever": {
+        "standard": {
+          "query": {
+            "match": {
+              "genre": "drama"
+            }
+          }
+        }
+      },
+      "field": "plot",
+      "inference_id": "my-msmarco-minilm-model",
+      "inference_text": "films that explore psychological depths"
     }
+  }
 }
 ----
 // NOTCONSOLE
++
+This retriever uses a standard `match` query to search the `movie` index for films tagged with the genre "drama".
+It then re-ranks the results based on semantic similarity to the text in the `inference_text` parameter, using the model we uploaded to {es}.
 
 ==== Using `from` and `size` with a retriever tree
 
diff --git a/docs/reference/search/rrf.asciidoc b/docs/reference/search/rrf.asciidoc
index fb474fe6bf4e6..2525dfff23b94 100644
--- a/docs/reference/search/rrf.asciidoc
+++ b/docs/reference/search/rrf.asciidoc
@@ -1,9 +1,7 @@
 [[rrf]]
 === Reciprocal rank fusion
 
-preview::["This functionality is in technical preview and may be changed or removed in a future release.
-The syntax will likely change before GA.
-Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]
+preview::["This functionality is in technical preview and may be changed or removed in a future release. The syntax will likely change before GA. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]
 
 https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf[Reciprocal rank fusion (RRF)]
 is a method for combining multiple result sets with different relevance indicators into a single result set.
@@ -43,7 +41,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-retrievers]
 
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-constant]
 
-include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-window-size]
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-window-size]
 
 An example request using RRF:
 
diff --git a/docs/reference/search/search-shards.asciidoc b/docs/reference/search/search-shards.asciidoc
index b9646f4d37303..49045acf4c484 100644
--- a/docs/reference/search/search-shards.asciidoc
+++ b/docs/reference/search/search-shards.asciidoc
@@ -63,6 +63,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=preference]
 
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing]
 
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 
 [[search-shards-api-example]]
 ==== {api-examples-title}
diff --git a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
index 4a5efe09ea5a0..6b9b13b124e9f 100644
--- a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
+++ b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
@@ -1,24 +1,24 @@
-[cols="^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^"]
 |====
-| 18+^h| Remote cluster version
+| 19+^h| Remote cluster version
 h| Local cluster version
-            |  6.8        | 7.1–7.16   | 7.17       | 8.0        | 8.1        | 8.2        | 8.3       | 8.4       | 8.5       | 8.6        | 8.7        | 8.8        | 8.9        | 8.10       | 8.11       | 8.12      | 8.13      | 8.14
-| 6.8       |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}
-| 7.1–7.16  |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}
-| 7.17      |  {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.0       |  {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.1       |  {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.2       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.3       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.4       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.5       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.6       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.7       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.8       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.9       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.10      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.11      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.12      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.13      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}
-| 8.14      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}
+            |  6.8        | 7.1–7.16   | 7.17       | 8.0        | 8.1        | 8.2        | 8.3       | 8.4       | 8.5       | 8.6        | 8.7        | 8.8        | 8.9        | 8.10       | 8.11       | 8.12      | 8.13      | 8.14      | 8.15
+| 6.8       |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}
+| 7.1–7.16  |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}
+| 7.17      |  {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.0       |  {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.1       |  {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.2       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.3       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.4       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.5       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.6       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.7       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.8       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.9       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.10      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.11      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.12      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.13      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.14      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.15      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}
 |====
diff --git a/docs/reference/search/search-your-data/learning-to-rank-model-training.asciidoc b/docs/reference/search/search-your-data/learning-to-rank-model-training.asciidoc
index 6525147839412..0f4640ebdf347 100644
--- a/docs/reference/search/search-your-data/learning-to-rank-model-training.asciidoc
+++ b/docs/reference/search/search-your-data/learning-to-rank-model-training.asciidoc
@@ -4,8 +4,6 @@
 <titleabbrev>Deploy and manage LTR models</titleabbrev>
 ++++
 
-preview::["The Learning To Rank feature is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but this feature is not subject to the support SLA of official GA features."]
-
 NOTE: This feature was introduced in version 8.12.0 and is only available to certain subscription levels.
 For more information, see {subscriptions}.
 
diff --git a/docs/reference/search/search-your-data/learning-to-rank-search-usage.asciidoc b/docs/reference/search/search-your-data/learning-to-rank-search-usage.asciidoc
index 2e9693eff0451..f14219e24bc11 100644
--- a/docs/reference/search/search-your-data/learning-to-rank-search-usage.asciidoc
+++ b/docs/reference/search/search-your-data/learning-to-rank-search-usage.asciidoc
@@ -4,8 +4,6 @@
 <titleabbrev>Search using LTR</titleabbrev>
 ++++
 
-preview::["The Learning To Rank feature is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but this feature is not subject to the support SLA of official GA features."]
-
 NOTE: This feature was introduced in version 8.12.0 and is only available to certain subscription levels.
 For more information, see {subscriptions}.
 
diff --git a/docs/reference/search/search-your-data/learning-to-rank.asciidoc b/docs/reference/search/search-your-data/learning-to-rank.asciidoc
index 08fad9db9c0f6..ebd6d67fe42da 100644
--- a/docs/reference/search/search-your-data/learning-to-rank.asciidoc
+++ b/docs/reference/search/search-your-data/learning-to-rank.asciidoc
@@ -1,8 +1,6 @@
 [[learning-to-rank]]
 == Learning To Rank
 
-preview::["The Learning To Rank feature is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but this feature is not subject to the support SLA of official GA features."]
-
 NOTE: This feature was introduced in version 8.12.0 and is only available to certain subscription levels.
 For more information, see {subscriptions}.
 
diff --git a/docs/reference/search/search-your-data/near-real-time.asciidoc b/docs/reference/search/search-your-data/near-real-time.asciidoc
index 46a996c237c38..47618ecd9fd7a 100644
--- a/docs/reference/search/search-your-data/near-real-time.asciidoc
+++ b/docs/reference/search/search-your-data/near-real-time.asciidoc
@@ -2,7 +2,7 @@
 
 [[near-real-time]]
 === Near real-time search
-The overview of <<documents-indices,documents and indices>> indicates that when a document is stored in {es}, it is indexed and fully searchable in _near real-time_--within 1 second. What defines near real-time search?
+When a document is stored in {es}, it is indexed and fully searchable in _near real-time_--within 1 second. What defines near real-time search?
 
 Lucene, the Java libraries on which {es} is based, introduced the concept of per-segment search. A _segment_ is similar to an inverted index, but the word _index_ in Lucene means "a collection of segments plus a commit point". After a commit, a new segment is added to the commit point and the buffer is cleared.
 
diff --git a/docs/reference/search/search-your-data/paginate-search-results.asciidoc b/docs/reference/search/search-your-data/paginate-search-results.asciidoc
index a81598273dfd3..f69fd60be0484 100644
--- a/docs/reference/search/search-your-data/paginate-search-results.asciidoc
+++ b/docs/reference/search/search-your-data/paginate-search-results.asciidoc
@@ -106,9 +106,9 @@ The search response includes an array of `sort` values for each hit:
         "_id" : "654322",
         "_score" : null,
         "_source" : ...,
-        "sort" : [               
+        "sort" : [
           1463538855,
-          "654322"                           
+          "654322"
         ]
       },
       {
@@ -118,7 +118,7 @@ The search response includes an array of `sort` values for each hit:
         "_source" : ...,
         "sort" : [                                <1>
           1463538857,
-          "654323"                        
+          "654323"
         ]
       }
     ]
@@ -150,7 +150,7 @@ GET twitter/_search
 --------------------------------------------------
 //TEST[continued]
 
-Repeat this process by updating the `search_after` array every time you retrieve a 
+Repeat this process by updating the `search_after` array every time you retrieve a
 new page of results. If a <<near-real-time,refresh>> occurs between these requests,
 the order of your results may change, causing inconsistent results across pages. To
 prevent this, you can create a <<point-in-time-api,point in time (PIT)>> to
@@ -167,10 +167,12 @@ The API returns a PIT ID.
 [source,console-result]
 ----
 {
-  "id": "46ToAwMDaWR5BXV1aWQyKwZub2RlXzMAAAAAAAAAACoBYwADaWR4BXV1aWQxAgZub2RlXzEAAAAAAAAAAAEBYQADaWR5BXV1aWQyKgZub2RlXzIAAAAAAAAAAAwBYgACBXV1aWQyAAAFdXVpZDEAAQltYXRjaF9hbGw_gAAAAA=="
+  "id": "46ToAwMDaWR5BXV1aWQyKwZub2RlXzMAAAAAAAAAACoBYwADaWR4BXV1aWQxAgZub2RlXzEAAAAAAAAAAAEBYQADaWR5BXV1aWQyKgZub2RlXzIAAAAAAAAAAAwBYgACBXV1aWQyAAAFdXVpZDEAAQltYXRjaF9hbGw_gAAAAA==",
+  "_shards": ...
 }
 ----
 // TESTRESPONSE[s/"id": "46ToAwMDaWR5BXV1aWQyKwZub2RlXzMAAAAAAAAAACoBYwADaWR4BXV1aWQxAgZub2RlXzEAAAAAAAAAAAEBYQADaWR5BXV1aWQyKgZub2RlXzIAAAAAAAAAAAwBYgACBXV1aWQyAAAFdXVpZDEAAQltYXRjaF9hbGw_gAAAAA=="/"id": $body.id/]
+// TESTRESPONSE[s/"_shards": \.\.\./"_shards": "$body._shards"/]
 
 To get the first page of results, submit a search request with a `sort`
 argument. If using a PIT, specify the PIT ID in the `pit.id` parameter and omit
@@ -362,7 +364,7 @@ Perl::
 
 Python::
 
-    See https://elasticsearch-py.readthedocs.org/en/master/helpers.html[elasticsearch.helpers.*]
+    See https://elasticsearch-py.readthedocs.io/en/stable/helpers.html[elasticsearch.helpers.*]
 
 JavaScript::
 
diff --git a/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc b/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc
index 99659ae76e092..c0fe7471946f3 100644
--- a/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc
+++ b/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc
@@ -13,23 +13,23 @@ For implementation details, including notable restrictions, check out the
 
 [discrete]
 [[retrievers-overview-types]]
-==== Retriever types 
+==== Retriever types
 
 Retrievers come in various types, each tailored for different search operations.
 The following retrievers are currently available:
 
-* <<standard-retriever,*Standard Retriever*>>. Returns top documents from a 
-traditional https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl.html[query]. 
-Mimics a traditional query but in the context of a retriever framework. This 
-ensures backward compatibility as existing `_search` requests remain supported. 
-That way you can transition to the new abstraction at your own pace without 
+* <<standard-retriever,*Standard Retriever*>>. Returns top documents from a
+traditional https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl.html[query].
+Mimics a traditional query but in the context of a retriever framework. This
+ensures backward compatibility as existing `_search` requests remain supported.
+That way you can transition to the new abstraction at your own pace without
 mixing syntaxes.
-* <<knn-retriever,*kNN Retriever*>>. Returns top documents from a <<search-api-knn,knn search>>, 
+* <<knn-retriever,*kNN Retriever*>>. Returns top documents from a <<search-api-knn,knn search>>,
 in the context of a retriever framework.
 * <<rrf-retriever,*RRF Retriever*>>. Combines and ranks multiple first-stage retrievers using
-the reciprocal rank fusion (RRF) algorithm. Allows you to combine multiple result sets 
+the reciprocal rank fusion (RRF) algorithm. Allows you to combine multiple result sets
 with different relevance indicators into a single result set.
-An RRF retriever is a *compound retriever*, where its `filter` element is 
+An RRF retriever is a *compound retriever*, where its `filter` element is
 propagated to its sub retrievers.
 +
 Sub retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree.
@@ -38,7 +38,7 @@ See the <<rrf-using-multiple-standard-retrievers,RRF documentation>> for detaile
 Requires first creating a `rerank` task using the <<put-inference-api,{es} Inference API>>.
 
 [discrete]
-==== What makes retrievers useful? 
+==== What makes retrievers useful?
 
 Here's an overview of what makes retrievers useful and how they differ from regular queries.
 
@@ -140,7 +140,7 @@ GET example-index/_search
   ],
   "rank":{
     "rrf":{
-      "window_size":50,
+      "rank_window_size":50,
       "rank_constant":20
     }
   }
@@ -155,14 +155,14 @@ GET example-index/_search
 
 Here are some important terms:
 
-* *Retrieval Pipeline*. Defines the entire retrieval and ranking logic to 
+* *Retrieval Pipeline*. Defines the entire retrieval and ranking logic to
 produce top hits.
 * *Retriever Tree*. A hierarchical structure that defines how retrievers interact.
 * *First-stage Retriever*. Returns an initial set of candidate documents.
-* *Compound Retriever*. Builds on one or more retrievers, 
+* *Compound Retriever*. Builds on one or more retrievers,
 enhancing document retrieval and ranking logic.
-* *Combiners*. Compound retrievers that merge top hits 
-from multiple sub-retrievers. 
+* *Combiners*. Compound retrievers that merge top hits
+from multiple sub-retrievers.
 * *Rerankers*. Special compound retrievers that reorder hits and may adjust the number of hits, with distinctions between first-stage and second-stage rerankers.
 
 [discrete]
@@ -180,4 +180,4 @@ Refer to the {kibana-ref}/playground.html[Playground documentation] for more inf
 [[retrievers-overview-api-reference]]
 ==== API reference
 
-For implementation details, including notable restrictions, check out the <<retriever,reference documentation>> in the Search API docs.
\ No newline at end of file
+For implementation details, including notable restrictions, check out the <<retriever,reference documentation>> in the Search API docs.
diff --git a/docs/reference/search/search-your-data/retrievers-reranking/semantic-reranking.asciidoc b/docs/reference/search/search-your-data/retrievers-reranking/semantic-reranking.asciidoc
index 75c06aa953302..add2d7455983e 100644
--- a/docs/reference/search/search-your-data/retrievers-reranking/semantic-reranking.asciidoc
+++ b/docs/reference/search/search-your-data/retrievers-reranking/semantic-reranking.asciidoc
@@ -5,7 +5,7 @@ preview::[]
 
 [TIP]
 ====
-This overview focuses more on the high-level concepts and use cases for semantic reranking. For full implementation details on how to set up and use semantic reranking in {es}, see the <<retriever,reference documentation>> in the Search API docs.
+This overview focuses more on the high-level concepts and use cases for semantic reranking. For full implementation details on how to set up and use semantic reranking in {es}, see the <<text-similarity-reranker-retriever,reference documentation>> in the Search API docs.
 ====
 
 Rerankers improve the relevance of results from earlier-stage retrieval mechanisms.
@@ -89,11 +89,16 @@ In {es}, semantic rerankers are implemented using the {es} <<inference-apis,Infe
 
 To use semantic reranking in {es}, you need to:
 
-. Choose a reranking model. In addition to cross-encoder models running on {es} inference nodes, we also expose external models and services via the Inference API to semantic rerankers.
-** This includes cross-encoder models running in https://huggingface.co/inference-endpoints[HuggingFace Inference Endpoints] and the https://cohere.com/rerank[Cohere Rerank API].
-. Create a `rerank` task using the <<put-inference-api,{es} Inference API>>.
+. *Choose a reranking model*.
+Currently you can:
+
+** Integrate directly with the <<infer-service-cohere,Cohere Rerank inference endpoint>> using the `rerank` task type
+** Integrate directly with the <<infer-service-google-vertex-ai,Google Vertex AI inference endpoint>> using the `rerank` task type
+** Upload a model to {es} from Hugging Face with {eland-docs}/machine-learning.html#ml-nlp-pytorch[Eland]. You'll need to use the `text_similarity` NLP task type when loading the model using Eland. Refer to {ml-docs}/ml-nlp-model-ref.html#ml-nlp-model-ref-text-similarity[the Elastic NLP model reference] for a list of third party text similarity models supported by {es} for semantic reranking.
+*** Then set up an <<inference-example-eland,{es} service inference endpoint>> with the `rerank` task type
+. *Create a `rerank` task using the <<put-inference-api,{es} Inference API>>*.
 The Inference API creates an inference endpoint and configures your chosen machine learning model to perform the reranking task.
-. Define a `text_similarity_reranker` retriever in your search request.
+. *Define a `text_similarity_reranker` retriever in your search request*.
 The retriever syntax makes it simple to configure both the retrieval and reranking of search results in a single API call.
 
 .*Example search request* with semantic reranker
@@ -127,20 +132,6 @@ POST _search
 // TEST[skip:TBD]
 ==============
 
-[discrete]
-[[semantic-reranking-types]]
-==== Supported reranking types
-
-The following `text_similarity_reranker` model configuration options are available.
-
-*Text similarity with cross-encoder*
-
-This solution uses a hosted or 3rd party inference service which relies on a cross-encoder model.
-The model receives the text fields from the _top-K_ documents, as well as the search query, and calculates scores directly, which are then used to rerank the documents.
-
-Used with the Cohere inference service rolled out in 8.13, turn on semantic reranking that works out of the box.
-Check out our https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/cohere-elasticsearch.ipynb[Python notebook] for using Cohere with {es}.
-
 [discrete]
 [[semantic-reranking-learn-more]]
 ==== Learn more
diff --git a/docs/reference/search/search-your-data/search-using-query-rules.asciidoc b/docs/reference/search/search-your-data/search-using-query-rules.asciidoc
index 594c22fb65981..18be825d02376 100644
--- a/docs/reference/search/search-your-data/search-using-query-rules.asciidoc
+++ b/docs/reference/search/search-your-data/search-using-query-rules.asciidoc
@@ -37,9 +37,10 @@ When defining a rule, consider the following:
 ===== Rule type
 
 The type of rule we want to apply.
-For the moment there is a single rule type:
+We support the following rule types:
 
 * `pinned` will re-write the query into a <<query-dsl-pinned-query, pinned query>>, pinning specified results matching the query rule at the top of the returned result set.
+* `exclude` will exclude specified results from the returned result set.
 
 [discrete]
 [[query-rule-criteria]]
@@ -91,12 +92,11 @@ Allowed criteria types are:
 
 The actions to take when the rule matches a query:
 
-* `ids` will pin the specified <<mapping-id-field,`_id`>>s.
-* `docs` will pin the specified documents in the specified indices.
+* `ids` will select the specified <<mapping-id-field,`_id`>>s.
+* `docs` will select the specified documents in the specified indices.
 
 Use `ids` when searching over a single index, and `docs` when searching over multiple indices.
 `ids` and `docs` cannot be combined in the same query.
-See <<query-dsl-pinned-query,pinned query>> for details.
 
 [discrete]
 [[add-query-rules]]
@@ -105,10 +105,10 @@ See <<query-dsl-pinned-query,pinned query>> for details.
 You can add query rules using the <<put-query-ruleset>> call.
 This adds a ruleset containing one or more query rules that will be applied to queries that match their specified criteria.
 
-The following command will create a query ruleset called `my-ruleset` with two pinned document rules:
+The following command will create a query ruleset called `my-ruleset` with two query rules:
 
 * The first rule will generate a <<query-dsl-pinned-query>> pinning the <<mapping-id-field,`_id`>>s `id1` and `id2` when the `query_string` metadata value is a fuzzy match to either `puggles` or `pugs` _and_ the user's location is in the US.
-* The second rule will generate a <<query-dsl-pinned-query>> pinning the <<mapping-id-field, `_id`>> of `id3` specifically from the `my-index-000001` index and `id4` from the `my-index-000002` index when the `query_string` metadata value contains `beagles`.
+* The second rule will generate a query that excludes the <<mapping-id-field, `_id`>> `id3` specifically from the `my-index-000001` index and `id4` from the `my-index-000002` index when the `query_string` metadata value contains `beagles`.
 
 ////
 [source,console]
@@ -147,7 +147,7 @@ PUT /_query_rules/my-ruleset
     },
     {
       "rule_id": "rule2",
-      "type": "pinned",
+      "type": "exclude",
       "criteria": [
         {
           "type": "contains",
@@ -222,7 +222,8 @@ This rule query will match against `rule1` in the defined query ruleset, and wil
 Any other matches from the organic query will be returned below the pinned results.
 
 It's possible to have multiple rules in a ruleset match a single <<query-dsl-rule-query, rule query>>.
-In this case, the pinned documents are returned in the following order:
+In this case, the rules are applied in the following order:
 
 - Where the matching rule appears in the ruleset
 - If multiple documents are specified in a single rule, in the order they are specified
+- If a document is matched by both a `pinned` rule and an `exclude` rule, the `exclude` rule will take precedence
diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
index ae27b46d4b876..719aeb070fc7c 100644
--- a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
+++ b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
@@ -11,13 +11,13 @@ IMPORTANT: For the easiest way to perform semantic search in the {stack}, refer
 
 The following examples use Cohere's `embed-english-v3.0` model, the `all-mpnet-base-v2` model from HuggingFace, and OpenAI's `text-embedding-ada-002` second generation embedding model.
 You can use any Cohere and OpenAI models, they are all supported by the {infer} API.
-For a list of supported models available on HuggingFace, refer to
-<<inference-example-hugging-face-supported-models, the supported model list>>.
+For a list of recommended models available on HuggingFace, refer to <<inference-example-hugging-face-supported-models, the supported model list>>.
 
 Azure based examples use models available through https://ai.azure.com/explore/models?selectedTask=embeddings[Azure AI Studio]
 or https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models[Azure OpenAI].
 Mistral examples use the `mistral-embed` model from https://docs.mistral.ai/getting-started/models/[the Mistral API].
 Amazon Bedrock examples use the `amazon.titan-embed-text-v1` model from https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html[the Amazon Bedrock base models].
+AlibabaCloud AI Search examples use the `ops-text-embedding-zh-001` model from https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details[the AlibabaCloud AI Search base models].
 
 Click the name of the service you want to use on any of the widgets below to review the corresponding instructions.
 
@@ -40,8 +40,7 @@ include::{es-ref-dir}/tab-widgets/inference-api/infer-api-task-widget.asciidoc[]
 ==== Create the index mapping
 
 The mapping of the destination index - the index that contains the embeddings that the model will create based on your input text - must be created.
-The destination index must have a field with the <<dense-vector, `dense_vector`>>
-field type to index the output of the used model.
+The destination index must have a field with the <<dense-vector, `dense_vector`>> field type for most models and the <<sparse-vector, `sparse_vector`>> field type for the sparse vector models like in the case of the `elser` service to index the output of the used model.
 
 include::{es-ref-dir}/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc[]
 
@@ -49,8 +48,7 @@ include::{es-ref-dir}/tab-widgets/inference-api/infer-api-mapping-widget.asciido
 [[infer-service-inference-ingest-pipeline]]
 ==== Create an ingest pipeline with an inference processor
 
-Create an <<ingest,ingest pipeline>> with an
-<<inference-processor,{infer} processor>> and use the model you created above to infer against the data that is being ingested in the pipeline.
+Create an <<ingest,ingest pipeline>> with an <<inference-processor,{infer} processor>> and use the model you created above to infer against the data that is being ingested in the pipeline.
 
 include::{es-ref-dir}/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc[]
 
@@ -103,8 +101,8 @@ POST _tasks/<task_id>/_cancel
 ==== Semantic search
 
 After the data set has been enriched with the embeddings, you can query the data using {ref}/knn-search.html#knn-semantic-search[semantic search].
-Pass a
-`query_vector_builder` to the k-nearest neighbor (kNN) vector search API, and provide the query text and the model you have used to create the embeddings.
+In case of dense vector models, pass a `query_vector_builder` to the k-nearest neighbor (kNN) vector search API, and provide the query text and the model you have used to create the embeddings.
+In case of a sparse vector model like ELSER, use a `sparse_vector` query, and provide the query text with the model you have used to create the embeddings.
 
 NOTE: If you cancelled the reindexing process, you run the query only a part of the data which affects the quality of your results.
 
diff --git a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc
index ba25cebcd1e1a..b47bc2370ab10 100644
--- a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc
+++ b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc
@@ -23,6 +23,11 @@ For more information, see <<security-privileges>>.
 [[searchable-snapshots-api-mount-desc]]
 ==== {api-description-title}
 
+This API mounts a snapshot as a searchable snapshot index. 
+
+Don't use this API for snapshots managed by {ilm-init}. Manually mounting
+{ilm-init}-managed snapshots can <<manually-mounting-snapshots,interfere>> with
+<<index-lifecycle-management,{ilm-init} processes>>.
 
 [[searchable-snapshots-api-mount-path-params]]
 ==== {api-path-parms-title}
diff --git a/docs/reference/searchable-snapshots/index.asciidoc b/docs/reference/searchable-snapshots/index.asciidoc
index 12b6f2477a93c..a38971a0bae6a 100644
--- a/docs/reference/searchable-snapshots/index.asciidoc
+++ b/docs/reference/searchable-snapshots/index.asciidoc
@@ -170,6 +170,20 @@ do not have a dedicated frozen tier, you must configure the
 cache on one or more nodes. Partially mounted indices are only allocated to
 nodes that have a shared cache.
 
+[[manually-mounting-snapshots]]
+[WARNING]
+.Manual snapshot mounting
+====
+Manually mounting snapshots captured by an Index Lifecycle Management ({ilm-init}) policy can
+interfere with {ilm-init}'s automatic management. This may lead to issues such as data loss
+or complications with snapshot handling. 
+
+For optimal results, allow {ilm-init} to manage
+snapshots automatically. 
+
+<<ilm-searchable-snapshot,Learn more about {ilm-init} snapshot management>>.
+====
+
 [[searchable-snapshots-shared-cache]]
 `xpack.searchable.snapshot.shared_cache.size`::
 (<<static-cluster-setting,Static>>)
@@ -325,6 +339,11 @@ cluster has write access then you must make sure that the other cluster does not
 delete these snapshots. The snapshot contains the sole full copy of your data.
 If you delete it then the data cannot be recovered from elsewhere.
 
+* The data in a searchable snapshot index are cached in local storage, so if you
+delete the underlying searchable snapshot {es} will continue to operate normally
+until the first cache miss. This may be much later, for instance when a shard
+relocates to a different node, or when the node holding the shard restarts.
+
 * If the repository fails or corrupts the contents of the snapshot and you
 cannot restore it to its previous healthy state then the data is permanently
 lost.
diff --git a/docs/reference/security/authorization/field-and-document-access-control.asciidoc b/docs/reference/security/authorization/field-and-document-access-control.asciidoc
index f4d4fcd49a35f..7c7ea75ece161 100644
--- a/docs/reference/security/authorization/field-and-document-access-control.asciidoc
+++ b/docs/reference/security/authorization/field-and-document-access-control.asciidoc
@@ -54,8 +54,11 @@ specify any field restrictions. If you assign a user both roles, `role_a` gives
 the user access to all documents and `role_b` gives the user access to all
 fields.
 
+[IMPORTANT]
+===========
 If you need to restrict access to both documents and fields, consider splitting
 documents by index instead.
+===========
 
 include::role-templates.asciidoc[]
 include::set-security-user.asciidoc[]
diff --git a/docs/reference/security/authorization/privileges.asciidoc b/docs/reference/security/authorization/privileges.asciidoc
index 145bd8ebc06bb..747b1eef40441 100644
--- a/docs/reference/security/authorization/privileges.asciidoc
+++ b/docs/reference/security/authorization/privileges.asciidoc
@@ -102,9 +102,7 @@ deprecated[7.5] Use `manage_transform` instead.
 This privilege is not available in {serverless-full}.
 
 `manage_data_stream_global_retention`::
-All operations related to managing the data stream global retention settings.
-+
-This privilege is not available in {serverless-full}.
+This privilege has no effect.deprecated[8.16]
 
 `manage_enrich`::
 All operations related to managing and executing enrich policies.
@@ -229,9 +227,7 @@ All cluster read-only operations, like cluster health and state, hot threads,
 node info, node and cluster stats, and pending cluster tasks.
 
 `monitor_data_stream_global_retention`::
-Allows the retrieval of the data stream global retention settings.
-+
-This privilege is not available in {serverless-full}.
+This privilege has no effect.deprecated[8.16]
 
 `monitor_enrich`::
 All read-only operations related to managing and executing enrich policies.
diff --git a/docs/reference/settings/data-stream-lifecycle-settings.asciidoc b/docs/reference/settings/data-stream-lifecycle-settings.asciidoc
index 0f00e956472d0..4b055525d4e6c 100644
--- a/docs/reference/settings/data-stream-lifecycle-settings.asciidoc
+++ b/docs/reference/settings/data-stream-lifecycle-settings.asciidoc
@@ -10,6 +10,18 @@ These are the settings available for configuring <<data-stream-lifecycle, data s
 
 ==== Cluster level settings
 
+[[data-streams-lifecycle-retention-max]]
+`data_streams.lifecycle.retention.max`::
+(<<dynamic-cluster-setting,Dynamic>>, <<time-units, time unit value>>)
+The maximum retention period that will apply to all user data streams managed by the data stream lifecycle. The max retention will also
+override the retention of a data stream whose configured retention exceeds the max retention. It should be greater than `10s`.
+
+[[data-streams-lifecycle-retention-default]]
+`data_streams.lifecycle.retention.default`::
+(<<dynamic-cluster-setting,Dynamic>>, <<time-units, time unit value>>)
+The retention period that will apply to all user data streams managed by the data stream lifecycle that do not have retention configured.
+It should be greater than `10s` and less or equals than <<data-streams-lifecycle-retention-max, `data_streams.lifecycle.retention.max`>>.
+
 [[data-streams-lifecycle-poll-interval]]
 `data_streams.lifecycle.poll_interval`::
 (<<dynamic-cluster-setting,Dynamic>>, <<time-units, time unit value>>)
diff --git a/docs/reference/settings/notification-settings.asciidoc b/docs/reference/settings/notification-settings.asciidoc
index 4a48c26974084..145112ef4d27c 100644
--- a/docs/reference/settings/notification-settings.asciidoc
+++ b/docs/reference/settings/notification-settings.asciidoc
@@ -42,6 +42,11 @@ Specifies the path to a file that contains a key for encrypting sensitive data.
 If `xpack.watcher.encrypt_sensitive_data` is set to `true`, this setting is
 required. For more information, see <<encrypting-data>>.
 
+`xpack.watcher.max.history.record.size`::
+(<<static-cluster-setting,Static>>)
+The maximum size watcher history record that can be written into the watcher history index. Any larger history record will have some of
+its larger fields removed. Defaults to 10mb.
+
 `xpack.http.proxy.host`::
 (<<static-cluster-setting,Static>>)
 Specifies the address of the proxy server to use to connect to HTTP services.
diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc
index 64626aafb2441..b346fddc5e5a1 100644
--- a/docs/reference/setup.asciidoc
+++ b/docs/reference/setup.asciidoc
@@ -33,7 +33,6 @@ include::setup/configuration.asciidoc[]
 
 include::setup/important-settings.asciidoc[]
 
-
 include::setup/secure-settings.asciidoc[]
 
 include::settings/audit-settings.asciidoc[]
@@ -82,6 +81,8 @@ include::modules/indices/search-settings.asciidoc[]
 
 include::settings/security-settings.asciidoc[]
 
+include::modules/shard-ops.asciidoc[]
+
 include::modules/indices/request_cache.asciidoc[]
 
 include::settings/snapshot-settings.asciidoc[]
@@ -93,7 +94,9 @@ include::modules/threadpool.asciidoc[]
 include::settings/notification-settings.asciidoc[]
 
 include::setup/advanced-configuration.asciidoc[]
+
 include::setup/sysconfig.asciidoc[]
+
 include::setup/bootstrap-checks.asciidoc[]
 
 include::setup/bootstrap-checks-xes.asciidoc[]
diff --git a/docs/reference/setup/restart-cluster.asciidoc b/docs/reference/setup/restart-cluster.asciidoc
index 9488c6632836b..a3bf7723cb5a9 100644
--- a/docs/reference/setup/restart-cluster.asciidoc
+++ b/docs/reference/setup/restart-cluster.asciidoc
@@ -11,7 +11,7 @@ time, so the service remains uninterrupted.
 [WARNING]
 ====
 Nodes exceeding the low watermark threshold will be slow to restart. Reduce the disk
-usage below the <<cluster-routing-watermark-low,low watermark>> before to restarting nodes.
+usage below the <<cluster-routing-watermark-low,low watermark>> before restarting nodes.
 ====
 
 [discrete]
diff --git a/docs/reference/slm/apis/slm-put.asciidoc b/docs/reference/slm/apis/slm-put.asciidoc
index be265554deef5..51ad571ee12e7 100644
--- a/docs/reference/slm/apis/slm-put.asciidoc
+++ b/docs/reference/slm/apis/slm-put.asciidoc
@@ -100,13 +100,19 @@ Minimum number of snapshots to retain, even if the snapshots have expired.
 ====
 
 `schedule`::
-(Required, <<api-cron-expressions,Cron syntax>>)
+(Required, <<api-cron-expressions,Cron syntax>> or <<time-units, time units>>)
 Periodic or absolute schedule at which the policy creates snapshots. {slm-init}
 applies `schedule` changes immediately.
+Schedule may be either a Cron schedule or a time unit describing the interval between snapshots.
+When using a time unit interval, the first snapshot is scheduled one interval after the policy modification time, and then again every interval after.
+
 
 [[slm-api-put-example]]
 ==== {api-examples-title}
 
+
+[[slm-api-put-daily-policy]]
+===== Create a policy
 Create a `daily-snapshots` lifecycle policy:
 
 [source,console]
@@ -138,4 +144,25 @@ PUT /_slm/policy/daily-snapshots
 <6> Optional retention configuration
 <7> Keep snapshots for 30 days
 <8> Always keep at least 5 successful snapshots, even if they're more than 30 days old
-<9> Keep no more than 50 successful snapshots, even if they're less than 30 days old
\ No newline at end of file
+<9> Keep no more than 50 successful snapshots, even if they're less than 30 days old
+
+
+[[slm-api-put-hourly-policy]]
+===== Use Interval Scheduling
+Create an `hourly-snapshots` lifecycle policy using interval scheduling:
+
+[source,console]
+--------------------------------------------------
+PUT /_slm/policy/hourly-snapshots
+{
+  "schedule": "1h",
+  "name": "<hourly-snap-{now/d}>",
+  "repository": "my_repository",
+  "config": {
+    "indices": ["data-*", "important"]
+  }
+}
+--------------------------------------------------
+// TEST[setup:setup-repository]
+Creates a snapshot once every hour. The first snapshot will be created one hour after the policy is modified,
+with subsequent snapshots being created every hour afterward.
diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc
index d8b03cbc0e880..e677408da3f25 100644
--- a/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc
@@ -4,7 +4,7 @@
 <titleabbrev>Get snapshot status</titleabbrev>
 ++++
 
-Retrieves a detailed description of the current state for each shard participating in the snapshot.
+Retrieves a detailed description of the current state for each shard participating in the snapshot. Note that this API should only be used to obtain detailed shard-level information for ongoing snapshots. If this detail is not needed, or you want to obtain information about one or more existing snapshots, use the <<get-snapshot-api,get snapshot API>>.
 
 ////
 [source,console]
@@ -172,13 +172,8 @@ Indicates the current snapshot state.
 `STARTED`::
   The snapshot is currently running.
 
-`PARTIAL`::
-  The global cluster state was stored, but data of at least one shard was not stored successfully.
-  The <<get-snapshot-api-response-failures,`failures`>> section of the response contains more detailed information about shards
-  that were not processed correctly.
-
 `SUCCESS`::
-  The snapshot finished and all shards were stored successfully.
+  The snapshot completed.
 ====
 --
 
diff --git a/docs/reference/snapshot-restore/repository-azure.asciidoc b/docs/reference/snapshot-restore/repository-azure.asciidoc
index f3d04159bc025..c361414052e14 100644
--- a/docs/reference/snapshot-restore/repository-azure.asciidoc
+++ b/docs/reference/snapshot-restore/repository-azure.asciidoc
@@ -1,56 +1,72 @@
 [[repository-azure]]
 === Azure repository
 
-You can use https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction[Azure Blob storage] as a repository for <<snapshot-restore,Snapshot/Restore>>.
+You can use
+https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction[Azure
+Blob storage] as a repository for <<snapshot-restore,Snapshot and restore>>.
 
 [[repository-azure-usage]]
 ==== Setup
 
-To enable Azure repositories, you have first to define your Azure storage settings as
-{ref}/secure-settings.html[secure settings].
+To enable Azure repositories, first configure an Azure repository client by
+specifying one or more settings of the form
+`azure.client.CLIENT_NAME.SETTING_NAME`. By default, `azure` repositories use a
+client named `default`, but you may specify a different client name when
+registering each repository.
 
-You can define these settings before the node is started, or call the <<cluster-nodes-reload-secure-settings,Nodes reload secure settings API>> after the settings are defined to apply them to a running node.
+The only mandatory Azure repository client setting is `account`, which is a
+{ref}/secure-settings.html[secure setting] defined in the <<secure-settings,{es}
+keystore>>. To provide this setting, use the `elasticsearch-keystore` tool on
+each node:
 
 [source,sh]
 ----------------------------------------------------------------
 bin/elasticsearch-keystore add azure.client.default.account
-bin/elasticsearch-keystore add azure.client.default.key
 ----------------------------------------------------------------
 
-Note that you can also define more than one account:
+If you adjust this setting after a node has started, call the
+<<cluster-nodes-reload-secure-settings,Nodes reload secure settings API>> to
+reload the new value.
+
+You may define more than one client by setting their `account` values. For
+instance, to set the `default` client and another client called `secondary`, run
+the following commands on each node:
 
 [source,sh]
 ----------------------------------------------------------------
 bin/elasticsearch-keystore add azure.client.default.account
-bin/elasticsearch-keystore add azure.client.default.key
 bin/elasticsearch-keystore add azure.client.secondary.account
-bin/elasticsearch-keystore add azure.client.secondary.sas_token
 ----------------------------------------------------------------
 
-For more information about these settings, see
-<<repository-azure-client-settings>>.
+The `key` and `sas_token` settings are also secure settings and can be set using
+commands like the following:
 
-[IMPORTANT]
-.Supported Azure Storage Account types
-===============================================
-The Azure repository type works with all Standard storage accounts
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore add azure.client.default.key
+bin/elasticsearch-keystore add azure.client.secondary.sas_token
+----------------------------------------------------------------
 
-* Standard Locally Redundant Storage - `Standard_LRS`
-* Standard Zone-Redundant Storage - `Standard_ZRS`
-* Standard Geo-Redundant Storage - `Standard_GRS`
-* Standard Read Access Geo-Redundant Storage - `Standard_RAGRS`
+Other Azure repository client settings must be set in `elasticsearch.yml` before
+the node starts. For example:
 
-https://azure.microsoft.com/en-gb/documentation/articles/storage-premium-storage[Premium Locally Redundant Storage] (`Premium_LRS`) is **not supported** as it is only usable as VM disk storage, not as general storage.
-===============================================
+[source,yaml]
+----
+azure.client.default.timeout: 10s
+azure.client.default.max_retries: 7
+azure.client.default.endpoint_suffix: core.chinacloudapi.cn
+azure.client.secondary.timeout: 30s
+----
 
-[[repository-azure-client-settings]]
-==== Client settings
+In this example, the client side timeout is `10s` per try for repositories which
+use the `default` client, with `7` retries before failing and an endpoint
+suffix of `core.chinacloudapi.cn`. Repositories which use the `secondary` client
+will have a timeout of `30s` per try, but will use the default endpoint and will
+fail after the default number of retries.
 
-The client that you use to connect to Azure has a number of settings available.
-The settings have the form `azure.client.CLIENT_NAME.SETTING_NAME`. By default,
-`azure` repositories use a client named `default`, but this can be modified using
-the <<repository-azure-repository-settings,repository setting>> `client`.
-For example:
+Once an Azure repository client is configured correctly, register an Azure
+repository as follows, providing the client name using the `client`
+<<repository-azure-repository-settings,repository setting>>:
 
 [source,console]
 ----
@@ -64,48 +80,45 @@ PUT _snapshot/my_backup
 ----
 // TEST[skip:we don't have azure setup while testing this]
 
-Most client settings can be added to the `elasticsearch.yml` configuration file.
-For example:
+If you are using the `default` client, you may omit the `client` repository
+setting:
 
-[source,yaml]
+[source,console]
 ----
-azure.client.default.timeout: 10s
-azure.client.default.max_retries: 7
-azure.client.default.endpoint_suffix: core.chinacloudapi.cn
-azure.client.secondary.timeout: 30s
+PUT _snapshot/my_backup
+{
+  "type": "azure"
+}
 ----
-
-In this example, the client side timeout is `10s` per try for the `default`
-account with `7` retries before failing. The endpoint suffix is
-`core.chinacloudapi.cn` and `30s` per try for the `secondary` account with `3`
-retries.
-
-The `account`, `key`, and `sas_token` storage settings are reloadable secure
-settings, which you add to the {es} keystore. For more information about
-creating and updating the {es} keystore, see
-{ref}/secure-settings.html[Secure settings]. After you reload the settings, the
-internal Azure clients, which are used to transfer the snapshot, utilize the
-latest settings from the keystore.
+// TEST[skip:we don't have azure setup while testing this]
 
 NOTE: In progress snapshot or restore jobs will not be preempted by a *reload*
 of the storage secure settings. They will complete using the client as it was
 built when the operation started.
 
-The following list contains the available client settings. Those that must be
-stored in the keystore are marked as "secure"; the other settings belong in the
-`elasticsearch.yml` file.
+[[repository-azure-client-settings]]
+==== Client settings
+
+The following list describes the available client settings. Those that must be
+stored in the keystore are marked as ({ref}/secure-settings.html[Secure],
+{ref}/secure-settings.html#reloadable-secure-settings[reloadable]); the other
+settings must be stored in the `elasticsearch.yml` file. The default
+`CLIENT_NAME` is `default` but you may configure a client with a different name
+and specify that client by name when registering a repository.
 
-`account` ({ref}/secure-settings.html[Secure], {ref}/secure-settings.html#reloadable-secure-settings[reloadable])::
-  The Azure account name, which is used by the repository's internal Azure client.
+`azure.client.CLIENT_NAME.account` ({ref}/secure-settings.html[Secure], {ref}/secure-settings.html#reloadable-secure-settings[reloadable])::
+  The Azure account name, which is used by the repository's internal Azure
+  client. This setting is required for all clients.
 
-`endpoint_suffix`::
+`azure.client.CLIENT_NAME.endpoint_suffix`::
   The Azure endpoint suffix to connect to. The default value is
   `core.windows.net`.
 
-`key` ({ref}/secure-settings.html[Secure], {ref}/secure-settings.html#reloadable-secure-settings[reloadable])::
-  The Azure secret key, which is used by the repository's internal Azure client. Alternatively, use `sas_token`.
+`azure.client.CLIENT_NAME.key` ({ref}/secure-settings.html[Secure], {ref}/secure-settings.html#reloadable-secure-settings[reloadable])::
+  The Azure secret key, which is used by the repository's internal Azure client.
+  Alternatively, use `sas_token`.
 
-`max_retries`::
+`azure.client.CLIENT_NAME.max_retries`::
     The number of retries to use when an Azure request fails. This setting helps
     control the exponential backoff policy. It specifies the number of retries
     that must occur before the snapshot fails. The default value is `3`. The
@@ -113,19 +126,20 @@ stored in the keystore are marked as "secure"; the other settings belong in the
     of wait time before retrying after a first timeout or failure. The maximum
     backoff period is defined by Azure SDK as `90s`.
 
-`proxy.host`::
-  The host name of a proxy to connect to Azure through. For example: `azure.client.default.proxy.host: proxy.host`.
+`azure.client.CLIENT_NAME.proxy.host`::
+  The host name of a proxy to connect to Azure through. By default, no proxy is
+  used.
 
-`proxy.port`::
-  The port of a proxy to connect to Azure through. For example, `azure.client.default.proxy.port: 8888`.
+`azure.client.CLIENT_NAME.proxy.port`::
+  The port of a proxy to connect to Azure through. By default, no proxy is used.
 
-`proxy.type`::
+`azure.client.CLIENT_NAME.proxy.type`::
  Register a proxy type for the client. Supported values are `direct`, `http`,
  and `socks`. For example: `azure.client.default.proxy.type: http`. When
  `proxy.type` is set to `http` or `socks`, `proxy.host` and `proxy.port` must
  also be provided. The default value is `direct`.
 
-`sas_token` ({ref}/secure-settings.html[Secure], {ref}/secure-settings.html#reloadable-secure-settings[reloadable])::
+`azure.client.CLIENT_NAME.sas_token` ({ref}/secure-settings.html[Secure], {ref}/secure-settings.html#reloadable-secure-settings[reloadable])::
   A shared access signatures (SAS) token, which the repository's internal Azure
   client uses for authentication. The SAS token must have read (r), write (w),
   list (l), and delete (d) permissions for the repository base path and all its
@@ -133,28 +147,78 @@ stored in the keystore are marked as "secure"; the other settings belong in the
   to resource types service (s), container (c), and object (o). Alternatively,
   use `key`.
 
-`timeout`::
-  The client side timeout for any single request to Azure. The value should
-  specify the time unit. For example, a value of `5s` specifies a 5 second
+`azure.client.CLIENT_NAME.timeout`::
+  The client side timeout for any single request to Azure, as a
+  <<time-units,time unit>>. For example, a value of `5s` specifies a 5 second
   timeout. There is no default value, which means that {es} uses the
-  https://azure.github.io/azure-storage-java/com/microsoft/azure/storage/RequestOptions.html#setTimeoutIntervalInMs(java.lang.Integer)[default value]
-  set by the Azure client (known as 5 minutes). This setting can be defined
-  globally, per account, or both.
+  https://azure.github.io/azure-storage-java/com/microsoft/azure/storage/RequestOptions.html#setTimeoutIntervalInMs(java.lang.Integer)[default
+  value] set by the Azure client.
+
+`azure.client.CLIENT_NAME.endpoint`::
+  The Azure endpoint to connect to. It must include the protocol used to connect
+  to Azure.
+
+`azure.client.CLIENT_NAME.secondary_endpoint`::
+  The Azure secondary endpoint to connect to. It must include the protocol used
+  to connect to Azure.
+
+[[repository-azure-default-credentials]]
+[NOTE]
+.Obtaining credentials from the environment
+======================================================
+If you specify neither the `key` nor the `sas_token` settings for a client then
+{es} will attempt to automatically obtain credentials from the environment in
+which it is running using mechanisms built into the Azure SDK. This is ideal
+for when running {es} on the Azure platform.
+
+When running {es} on an
+https://azure.microsoft.com/en-gb/products/virtual-machines[Azure Virtual
+Machine], you should use
+https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview[Azure
+Managed Identity] to provide credentials to {es}. To use Azure Managed Identity,
+assign a suitably authorized identity to the Azure Virtual Machine on which {es}
+is running.
+
+When running {es} in
+https://azure.microsoft.com/en-gb/products/kubernetes-service[Azure Kubernetes
+Service], for instance using {eck-ref}[{eck}], you should use
+https://azure.github.io/azure-workload-identity/docs/introduction.html[Azure
+Workload Identity] to provide credentials to {es}. To use Azure Workload
+Identity, mount the `azure-identity-token` volume as a subdirectory of the
+<<config-files-location,{es} config directory>> and set the
+`AZURE_FEDERATED_TOKEN_FILE` environment variable to point to a file called
+`azure-identity-token` within the mounted volume.
+
+The Azure SDK has several other mechanisms to automatically obtain credentials
+from its environment, but the two methods described above are the only ones
+that are tested and supported for use in {es}.
+======================================================
 
-`endpoint`::
-  The Azure endpoint to connect to. It must include the protocol used to connect to Azure.
 
-`secondary_endpoint`::
-  The Azure secondary endpoint to connect to. It must include the protocol used to connect to Azure.
 
 [[repository-azure-repository-settings]]
 ==== Repository settings
 
-The Azure repository supports following settings:
+The Azure repository supports the following settings, which may be specified
+when registering an Azure repository as follows:
+
+[source,console]
+----
+PUT _snapshot/my_backup
+{
+  "type": "azure",
+  "settings": {
+    "client": "secondary",
+    "container": "my_container",
+    "base_path": "snapshots_prefix"
+  }
+}
+----
+// TEST[skip:we don't have azure setup while testing this]
 
 `client`::
 
-    Azure named client to use. Defaults to `default`.
+    The name of the Azure repository client to use. Defaults to `default`.
 
 `container`::
 
@@ -172,13 +236,15 @@ multiple deployments may share the same bucket.
 
 `chunk_size`::
 
-    Big files can be broken down into multiple smaller blobs in the blob store during snapshotting.
-    It is not recommended to change this value from its default unless there is an explicit reason for limiting the
-    size of blobs in the repository. Setting a value lower than the default can result in an increased number of API
-    calls to the Azure blob store during snapshot create as well as restore operations compared to using the default
-    value and thus make both operations slower as well as more costly.
-    Specify the chunk size as a value and unit, for example:
-    `10MB`, `5KB`, `500B`. Defaults to the maximum size of a blob in the Azure blob store which is `5TB`.
+    Big files can be broken down into multiple smaller blobs in the blob store
+    during snapshotting. It is not recommended to change this value from its
+    default unless there is an explicit reason for limiting the size of blobs in
+    the repository. Setting a value lower than the default can result in an
+    increased number of API calls to the Azure blob store during snapshot create
+    as well as restore operations compared to using the default value and thus
+    make both operations slower as well as more costly. Specify the chunk size
+    as a <<byte-units,byte unit>>, for example: `10MB`, `5KB`, `500B`. Defaults
+    to the maximum size of a blob in the Azure blob store which is `5TB`.
 
 `compress`::
 
@@ -193,59 +259,6 @@ include::repository-shared-settings.asciidoc[]
     `primary_only` or `secondary_only`. Defaults to `primary_only`. Note that if you set it
     to `secondary_only`, it will force `readonly` to true.
 
-Some examples, using scripts:
-
-[source,console]
-----
-# The simplest one
-PUT _snapshot/my_backup1
-{
-  "type": "azure"
-}
-
-# With some settings
-PUT _snapshot/my_backup2
-{
-  "type": "azure",
-  "settings": {
-    "container": "backup-container",
-    "base_path": "backups",
-    "chunk_size": "32MB",
-    "compress": true
-  }
-}
-
-
-# With two accounts defined in elasticsearch.yml (my_account1 and my_account2)
-PUT _snapshot/my_backup3
-{
-  "type": "azure",
-  "settings": {
-    "client": "secondary"
-  }
-}
-PUT _snapshot/my_backup4
-{
-  "type": "azure",
-  "settings": {
-    "client": "secondary",
-    "location_mode": "primary_only"
-  }
-}
-----
-// TEST[skip:we don't have azure setup while testing this]
-
-Example using Java:
-
-[source,java]
-----
-client.admin().cluster().preparePutRepository("my_backup_java1")
-    .setType("azure").setSettings(Settings.builder()
-        .put(Storage.CONTAINER, "backup-container")
-        .put(Storage.CHUNK_SIZE, new ByteSizeValue(32, ByteSizeUnit.MB))
-    ).get();
-----
-
 [[repository-azure-validation]]
 ==== Repository validation rules
 
@@ -260,6 +273,19 @@ permitted in container names.
 * All letters in a container name must be lowercase.
 * Container names must be from 3 through 63 characters long.
 
+[IMPORTANT]
+.Supported Azure Storage Account types
+===============================================
+The Azure repository type works with all Standard storage accounts
+
+* Standard Locally Redundant Storage - `Standard_LRS`
+* Standard Zone-Redundant Storage - `Standard_ZRS`
+* Standard Geo-Redundant Storage - `Standard_GRS`
+* Standard Read Access Geo-Redundant Storage - `Standard_RAGRS`
+
+https://azure.microsoft.com/en-gb/documentation/articles/storage-premium-storage[Premium Locally Redundant Storage] (`Premium_LRS`) is **not supported** as it is only usable as VM disk storage, not as general storage.
+===============================================
+
 [[repository-azure-linearizable-registers]]
 ==== Linearizable register implementation
 
diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc
index d757a74110ca9..3a9c12caebad9 100644
--- a/docs/reference/snapshot-restore/repository-s3.asciidoc
+++ b/docs/reference/snapshot-restore/repository-s3.asciidoc
@@ -317,6 +317,15 @@ include::repository-shared-settings.asciidoc[]
     https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html[AWS
     DeleteObjects API].
 
+`max_multipart_upload_cleanup_size`::
+
+    (<<number,numeric>>) Sets the maximum number of possibly-dangling multipart
+    uploads to clean up in each batch of snapshot deletions. Defaults to `1000`
+    which is the maximum number supported by the
+    https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html[AWS
+    ListMultipartUploads API]. If set to `0`, {es} will not attempt to clean up
+    dangling multipart uploads.
+
 NOTE: The option of defining client settings in the repository settings as
 documented below is considered deprecated, and will be removed in a future
 version.
@@ -492,33 +501,6 @@ by the `elasticsearch` user. By default, {es} runs as user `elasticsearch` using
 
 If the symlink exists, it will be used by default by all S3 repositories that don't have explicit `client` credentials.
 
-==== Cleaning up multi-part uploads
-
-{es} uses S3's multi-part upload process to upload larger blobs to the
-repository. The multi-part upload process works by dividing each blob into
-smaller parts, uploading each part independently, and then completing the
-upload in a separate step. This reduces the amount of data that {es} must
-re-send if an upload fails: {es} only needs to re-send the part that failed
-rather than starting from the beginning of the whole blob. The storage for each
-part is charged independently starting from the time at which the part was
-uploaded.
-
-If a multi-part upload cannot be completed then it must be aborted in order to
-delete any parts that were successfully uploaded, preventing further storage
-charges from accumulating. {es} will automatically abort a multi-part upload on
-failure, but sometimes the abort request itself fails. For example, if the
-repository becomes inaccessible or the instance on which {es} is running is
-terminated abruptly then {es} cannot complete or abort any ongoing uploads.
-
-You must make sure that failed uploads are eventually aborted to avoid
-unnecessary storage costs. You can use the
-https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html[List
-multipart uploads API] to list the ongoing uploads and look for any which are
-unusually long-running, or you can
-https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpu-abort-incomplete-mpu-lifecycle-config.html[configure
-a bucket lifecycle policy] to automatically abort incomplete uploads once they
-reach a certain age.
-
 [[repository-s3-aws-vpc]]
 ==== AWS VPC bandwidth settings
 
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc
index 6039d1de5345b..3a686e27cf580 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc
@@ -7,6 +7,12 @@
             id="infer-api-ingest-cohere">
       Cohere
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-ingest-elser-tab"
+            id="infer-api-ingest-elser">
+      ELSER
+    </button>
     <button role="tab"
             aria-selected="false"
             aria-controls="infer-api-ingest-hf-tab"
@@ -43,6 +49,12 @@
             id="infer-api-ingest-amazon-bedrock">
       Amazon Bedrock
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-ingest-alibabacloud-ai-search-tab"
+            id="infer-api-ingest-alibabacloud-ai-search">
+      AlibabaCloud AI Search
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -52,6 +64,17 @@
 
 include::infer-api-ingest-pipeline.asciidoc[tag=cohere]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-ingest-elser-tab"
+       aria-labelledby="infer-api-ingest-elser"
+       hidden="">
+++++
+
+include::infer-api-ingest-pipeline.asciidoc[tag=elser]
+
 ++++
   </div>
   <div tabindex="0"
@@ -118,6 +141,17 @@ include::infer-api-ingest-pipeline.asciidoc[tag=mistral]
 
 include::infer-api-ingest-pipeline.asciidoc[tag=amazon-bedrock]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-ingest-alibabacloud-ai-search-tab"
+       aria-labelledby="infer-api-ingest-alibabacloud-ai-search"
+       hidden="">
+++++
+
+include::infer-api-ingest-pipeline.asciidoc[tag=alibabacloud-ai-search]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc
index f95c4a6dbc8c8..6678b60fabc40 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc
@@ -35,6 +35,32 @@ and the `output_field` that will contain the {infer} results.
 
 // end::cohere[]
 
+// tag::elser[]
+
+[source,console]
+--------------------------------------------------
+PUT _ingest/pipeline/elser_embeddings
+{
+  "processors": [
+    {
+      "inference": {
+        "model_id": "elser_embeddings", <1>
+        "input_output": { <2>
+          "input_field": "content",
+          "output_field": "content_embedding"
+        }
+      }
+    }
+  ]
+}
+--------------------------------------------------
+<1> The name of the inference endpoint you created by using the
+<<put-inference-api>>, it's referred to as `inference_id` in that step.
+<2> Configuration object that defines the `input_field` for the {infer} process
+and the `output_field` that will contain the {infer} results.
+
+// end::elser[]
+
 // tag::hugging-face[]
 
 [source,console]
@@ -190,3 +216,29 @@ PUT _ingest/pipeline/amazon_bedrock_embeddings
 and the `output_field` that will contain the {infer} results.
 
 // end::amazon-bedrock[]
+
+// tag::alibabacloud-ai-search[]
+
+[source,console]
+--------------------------------------------------
+PUT _ingest/pipeline/alibabacloud_ai_search_embeddings
+{
+  "processors": [
+    {
+      "inference": {
+        "model_id": "alibabacloud_ai_search_embeddings", <1>
+        "input_output": { <2>
+          "input_field": "content",
+          "output_field": "content_embedding"
+        }
+      }
+    }
+  ]
+}
+--------------------------------------------------
+<1> The name of the inference endpoint you created by using the
+<<put-inference-api>>, it's referred to as `inference_id` in that step.
+<2> Configuration object that defines the `input_field` for the {infer} process
+and the `output_field` that will contain the {infer} results.
+
+// end::alibabacloud-ai-search[]
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc
index 66b0cde549545..66b790bdd57a5 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc
@@ -7,6 +7,12 @@
             id="infer-api-mapping-cohere">
       Cohere
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-mapping-elser-tab"
+            id="infer-api-mapping-elser">
+      ELSER
+    </button>
     <button role="tab"
             aria-selected="false"
             aria-controls="infer-api-mapping-hf-tab"
@@ -43,6 +49,12 @@
             id="infer-api-mapping-amazon-bedrock">
       Amazon Bedrock
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-mapping-alibabacloud-ai-search-tab"
+            id="infer-api-mapping-alibabacloud-ai-search">
+      AlibabaCloud AI Search
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -52,6 +64,17 @@
 
 include::infer-api-mapping.asciidoc[tag=cohere]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-mapping-elser-tab"
+       aria-labelledby="infer-api-mapping-elser"
+       hidden="">
+++++
+
+include::infer-api-mapping.asciidoc[tag=elser]
+
 ++++
   </div>
   <div tabindex="0"
@@ -118,6 +141,17 @@ include::infer-api-mapping.asciidoc[tag=mistral]
 
 include::infer-api-mapping.asciidoc[tag=amazon-bedrock]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-mapping-alibabacloud-ai-search-tab"
+       aria-labelledby="infer-api-mapping-alibabacloud-ai-search"
+       hidden="">
+++++
+
+include::infer-api-mapping.asciidoc[tag=alibabacloud-ai-search]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc
index 72c648e63871d..c86538ceb9c87 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc
@@ -31,6 +31,34 @@ the {infer} pipeline configuration in the next step.
 
 // end::cohere[]
 
+// tag::elser[]
+
+[source,console]
+--------------------------------------------------
+PUT elser-embeddings
+{
+  "mappings": {
+    "properties": {
+      "content_embedding": { <1>
+        "type": "sparse_vector" <2>
+      },
+      "content": { <3>
+        "type": "text" <4>
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> The name of the field to contain the generated tokens. It must be refrenced
+in the {infer} pipeline configuration in the next step.
+<2> The field to contain the tokens is a `sparse_vector` field for ELSER.
+<3> The name of the field from which to create the dense vector representation.
+In this example, the name of the field is `content`. It must be referenced in
+the {infer} pipeline configuration in the next step.
+<4> The field type which is text in this example.
+
+// end::elser[]
+
 // tag::hugging-face[]
 
 [source,console]
@@ -242,3 +270,35 @@ the {infer} pipeline configuration in the next step.
 <6> The field type which is text in this example.
 
 // end::amazon-bedrock[]
+
+// tag::alibabacloud-ai-search[]
+
+[source,console]
+--------------------------------------------------
+PUT alibabacloud-ai-search-embeddings
+{
+  "mappings": {
+    "properties": {
+      "content_embedding": { <1>
+        "type": "dense_vector", <2>
+        "dims": 1024, <3>
+        "element_type": "float"
+      },
+      "content": { <4>
+        "type": "text" <5>
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> The name of the field to contain the generated tokens. It must be referenced
+in the {infer} pipeline configuration in the next step.
+<2> The field to contain the tokens is a `dense_vector` field.
+<3> The output dimensions of the model. This value may be different depending on the underlying model used.
+See the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details[AlibabaCloud AI Search embedding model] documentation.
+<4> The name of the field from which to create the dense vector representation.
+In this example, the name of the field is `content`. It must be referenced in
+the {infer} pipeline configuration in the next step.
+<5> The field type which is text in this example.
+
+// end::alibabacloud-ai-search[]
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc
index 9a8028e2b3c6c..86f52fee2063c 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc
@@ -7,6 +7,12 @@
             id="infer-api-reindex-cohere">
       Cohere
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-reindex-elser-tab"
+            id="infer-api-reindex-elser">
+      ELSER
+    </button>
     <button role="tab"
             aria-selected="false"
             aria-controls="infer-api-reindex-hf-tab"
@@ -43,6 +49,12 @@
             id="infer-api-reindex-amazon-bedrock">
       Amazon Bedrock
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-reindex-alibabacloud-ai-search-tab"
+            id="infer-api-reindex-alibabacloud-ai-search">
+      AlibabaCloud AI Search
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -52,6 +64,17 @@
 
 include::infer-api-reindex.asciidoc[tag=cohere]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-reindex-elser-tab"
+       aria-labelledby="infer-api-reindex-elser"
+       hidden="">
+++++
+
+include::infer-api-reindex.asciidoc[tag=elser]
+
 ++++
   </div>
   <div tabindex="0"
@@ -118,6 +141,17 @@ include::infer-api-reindex.asciidoc[tag=mistral]
 
 include::infer-api-reindex.asciidoc[tag=amazon-bedrock]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-reindex-alibabacloud-ai-search-tab"
+       aria-labelledby="infer-api-reindex-alibabacloud-ai-search"
+       hidden="">
+++++
+
+include::infer-api-reindex.asciidoc[tag=alibabacloud-ai-search]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc
index 995189f1309aa..25d4023c650c0 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc
@@ -25,6 +25,29 @@ may affect the throughput of the reindexing process.
 
 // end::cohere[]
 
+// tag::elser[]
+
+[source,console]
+----
+POST _reindex?wait_for_completion=false
+{
+  "source": {
+    "index": "test-data",
+    "size": 50 <1>
+  },
+  "dest": {
+    "index": "elser-embeddings",
+    "pipeline": "elser_embeddings"
+  }
+}
+----
+// TEST[skip:TBD]
+<1> The default batch size for reindexing is 1000. Reducing `size` to a smaller
+number makes the update of the reindexing process quicker which enables you to
+follow the progress closely and detect errors early.
+
+// end::elser[]
+
 // tag::hugging-face[]
 
 [source,console]
@@ -177,3 +200,26 @@ number makes the update of the reindexing process quicker which enables you to
 follow the progress closely and detect errors early.
 
 // end::amazon-bedrock[]
+
+// tag::alibabacloud-ai-search[]
+
+[source,console]
+----
+POST _reindex?wait_for_completion=false
+{
+  "source": {
+    "index": "test-data",
+    "size": 50 <1>
+  },
+  "dest": {
+    "index": "alibabacloud-ai-search-embeddings",
+    "pipeline": "alibabacloud_ai_search_embeddings"
+  }
+}
+----
+// TEST[skip:TBD]
+<1> The default batch size for reindexing is 1000. Reducing `size` to a smaller
+number makes the update of the reindexing process quicker which enables you to
+follow the progress closely and detect errors early.
+
+// end::alibabacloud-ai-search[]
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc
index cf2e4994279d9..fb686a2d8be12 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc
@@ -7,6 +7,12 @@
             id="infer-api-requirements-cohere">
       Cohere
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-requirements-elser-tab"
+            id="infer-api-requirements-elser">
+      ELSER
+    </button>
     <button role="tab"
             aria-selected="false"
             aria-controls="infer-api-requirements-hf-tab"
@@ -43,6 +49,12 @@
             id="infer-api-requirements-amazon-bedrock">
       Amazon Bedrock
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-requirements-alibabacloud-ai-search-tab"
+            id="infer-api-requirements-alibabacloud-ai-search">
+      AlibabaCloud AI Search
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -52,6 +64,17 @@
 
 include::infer-api-requirements.asciidoc[tag=cohere]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-requirements-elser-tab"
+       aria-labelledby="infer-api-requirements-elser"
+       hidden="">
+++++
+
+include::infer-api-requirements.asciidoc[tag=elser]
+
 ++++
   </div>
   <div tabindex="0"
@@ -118,6 +141,17 @@ include::infer-api-requirements.asciidoc[tag=mistral]
 
 include::infer-api-requirements.asciidoc[tag=amazon-bedrock]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-requirements-alibabacloud-ai-search-tab"
+       aria-labelledby="infer-api-requirements-alibabacloud-ai-search"
+       hidden="">
+++++
+
+include::infer-api-requirements.asciidoc[tag=alibabacloud-ai-search]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc
index 856e4d5f0fe47..c9e7ca8b80ba6 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc
@@ -5,6 +5,13 @@ the Cohere service.
 
 // end::cohere[]
 
+// tag::elser[]
+
+ELSER is a model trained by Elastic. If you have an {es} deployment, there is no
+further requirement for using the {infer} API with the `elser` service.
+
+// end::elser[]
+
 // tag::hugging-face[]
 
 A https://huggingface.co/[HuggingFace account] is required to use the {infer}
@@ -45,3 +52,9 @@ You can apply for access to Azure OpenAI by completing the form at https://aka.m
 * A pair of access and secret keys used to access Amazon Bedrock
 
 // end::amazon-bedrock[]
+
+// tag::alibabacloud-ai-search[]
+* An AlibabaCloud Account with https://console.aliyun.com[AlibabaCloud] access
+* An API key generated for your account from the https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[API keys section]
+
+// end::alibabacloud-ai-search[]
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc
index 52cf65c4a1509..996148d80a4bd 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc
@@ -7,6 +7,12 @@
             id="infer-api-search-cohere">
       Cohere
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-search-elser-tab"
+            id="infer-api-search-elser">
+      ELSER
+    </button>
     <button role="tab"
             aria-selected="false"
             aria-controls="infer-api-search-hf-tab"
@@ -43,6 +49,12 @@
             id="infer-api-search-amazon-bedrock">
       Amazon Bedrock
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-search-alibabacloud-ai-search-tab"
+            id="infer-api-search-alibabacloud-ai-search">
+      AlibabaCloud AI Search
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -52,6 +64,17 @@
 
 include::infer-api-search.asciidoc[tag=cohere]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-search-elser-tab"
+       aria-labelledby="infer-api-search-elser"
+       hidden="">
+++++
+
+include::infer-api-search.asciidoc[tag=elser]
+
 ++++
   </div>
   <div tabindex="0"
@@ -118,6 +141,17 @@ include::infer-api-search.asciidoc[tag=mistral]
 
 include::infer-api-search.asciidoc[tag=amazon-bedrock]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-search-alibabacloud-ai-search-tab"
+       aria-labelledby="infer-api-search-alibabacloud-ai-search"
+       hidden="">
+++++
+
+include::infer-api-search.asciidoc[tag=alibabacloud-ai-search]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc
index 5e23afeb19a9f..fe1f58b6bd1a9 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc
@@ -72,6 +72,67 @@ query from the `cohere-embeddings` index sorted by their proximity to the query:
 
 // end::cohere[]
 
+// tag::elser[]
+
+[source,console]
+--------------------------------------------------
+GET elser-embeddings/_search
+{
+  "query":{
+    "sparse_vector":{
+      "field": "content_embedding",
+      "inference_id": "elser_embeddings",
+      "query": "How to avoid muscle soreness after running?"
+    }
+  },
+  "_source": [
+    "id",
+    "content"
+  ]
+}
+--------------------------------------------------
+// TEST[skip:TBD]
+
+As a result, you receive the top 10 documents that are closest in meaning to the
+query from the `cohere-embeddings` index sorted by their proximity to the query:
+
+[source,consol-result]
+--------------------------------------------------
+"hits": [
+      {
+        "_index": "elser-embeddings",
+        "_id": "ZLGc_pABZbBmsu5_eCoH",
+        "_score": 21.472063,
+        "_source": {
+          "id": 2258240,
+          "content": "You may notice some muscle aches while you are exercising. This is called acute soreness. More often, you may begin to feel sore about 12 hours after exercising, and the discomfort usually peaks at 48 to 72 hours after exercise. This is called delayed-onset muscle soreness.It is thought that, during this time, your body is repairing the muscle, making it stronger and bigger.You may also notice the muscles feel better if you exercise lightly. This is normal.his is called delayed-onset muscle soreness. It is thought that, during this time, your body is repairing the muscle, making it stronger and bigger. You may also notice the muscles feel better if you exercise lightly. This is normal."
+        }
+      },
+      {
+        "_index": "elser-embeddings",
+        "_id": "ZbGc_pABZbBmsu5_eCoH",
+        "_score": 21.421381,
+        "_source": {
+          "id": 2258242,
+          "content": "Photo Credit Jupiterimages/Stockbyte/Getty Images. That stiff, achy feeling you get in the days after exercise is a normal physiological response known as delayed onset muscle soreness. You can take it as a positive sign that your muscles have felt the workout, but the pain may also turn you off to further exercise.ou are more likely to develop delayed onset muscle soreness if you are new to working out, if you’ve gone a long time without exercising and start up again, if you have picked up a new type of physical activity or if you have recently boosted the intensity, length or frequency of your exercise sessions."
+        }
+      },
+      {
+        "_index": "elser-embeddings",
+        "_id": "ZrGc_pABZbBmsu5_eCoH",
+        "_score": 20.542095,
+        "_source": {
+          "id": 2258248,
+          "content": "They found that stretching before and after exercise has no effect on muscle soreness. Exercise might cause inflammation, which leads to an increase in the production of immune cells (comprised mostly of macrophages and neutrophils). Levels of these immune cells reach a peak 24-48 hours after exercise.These cells, in turn, produce bradykinins and prostaglandins, which make the pain receptors in your body more sensitive. Whenever you move, these pain receptors are stimulated.hey found that stretching before and after exercise has no effect on muscle soreness. Exercise might cause inflammation, which leads to an increase in the production of immune cells (comprised mostly of macrophages and neutrophils). Levels of these immune cells reach a peak 24-48 hours after exercise."
+        }
+      },
+    (...)
+  ]
+--------------------------------------------------
+// NOTCONSOLE
+
+// end::elser[]
+
 // tag::hugging-face[]
 
 [source,console]
@@ -470,3 +531,68 @@ query from the `amazon-bedrock-embeddings` index sorted by their proximity to th
 // NOTCONSOLE
 
 // end::amazon-bedrock[]
+
+// tag::alibabacloud-ai-search[]
+
+[source,console]
+--------------------------------------------------
+GET alibabacloud-ai-search-embeddings/_search
+{
+  "knn": {
+    "field": "content_embedding",
+    "query_vector_builder": {
+      "text_embedding": {
+        "model_id": "alibabacloud_ai_search_embeddings",
+        "model_text": "Calculate fuel cost"
+      }
+    },
+    "k": 10,
+    "num_candidates": 100
+  },
+  "_source": [
+    "id",
+    "content"
+  ]
+}
+--------------------------------------------------
+// TEST[skip:TBD]
+
+As a result, you receive the top 10 documents that are closest in meaning to the
+query from the `alibabacloud-ai-search-embeddings` index sorted by their proximity to the query:
+
+[source,consol-result]
+--------------------------------------------------
+"hits": [
+      {
+        "_index": "alibabacloud-ai-search-embeddings",
+        "_id": "DDd5OowBHxQKHyc3TDSC",
+        "_score": 0.83704096,
+        "_source": {
+          "id": 862114,
+          "body": "How to calculate fuel cost for a road trip. By Tara Baukus Mello • Bankrate.com. Dear Driving for Dollars, My family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost.It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes.y family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost. It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes."
+        }
+      },
+      {
+        "_index": "alibabacloud-ai-search-embeddings",
+        "_id": "ajd5OowBHxQKHyc3TDSC",
+        "_score": 0.8345704,
+        "_source": {
+          "id": 820622,
+          "body": "Home Heating Calculator. Typically, approximately 50% of the energy consumed in a home annually is for space heating. When deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important.This calculator can help you estimate the cost of fuel for different heating appliances.hen deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important. This calculator can help you estimate the cost of fuel for different heating appliances."
+        }
+      },
+      {
+        "_index": "alibabacloud-ai-search-embeddings",
+        "_id": "Djd5OowBHxQKHyc3TDSC",
+        "_score": 0.8327426,
+        "_source": {
+          "id": 8202683,
+          "body": "Fuel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel.If you are paying $4 per gallon, the trip would cost you $200.Most boats have much larger gas tanks than cars.uel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel."
+        }
+      },
+      (...)
+    ]
+--------------------------------------------------
+// NOTCONSOLE
+
+// end::alibabacloud-ai-search[]
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc
index d13301b64a871..1dfa6077553fe 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc
@@ -7,7 +7,13 @@
             id="infer-api-task-cohere">
       Cohere
     </button>
-        <button role="tab"
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-task-elser-tab"
+            id="infer-api-task-elser">
+      ELSER
+    </button>
+    <button role="tab"
             aria-selected="false"
             aria-controls="infer-api-task-hf-tab"
             id="infer-api-task-hf">
@@ -43,6 +49,12 @@
             id="infer-api-task-amazon-bedrock">
       Amazon Bedrock
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-task-alibabacloud-ai-search-tab"
+            id="infer-api-task-alibabacloud-ai-search">
+      AlibabaCloud AI Search
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -52,6 +64,17 @@
 
 include::infer-api-task.asciidoc[tag=cohere]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-task-elser-tab"
+       aria-labelledby="infer-api-task-elser"
+       hidden="">
+++++
+
+include::infer-api-task.asciidoc[tag=elser]
+
 ++++
   </div>
   <div tabindex="0"
@@ -118,6 +141,17 @@ include::infer-api-task.asciidoc[tag=mistral]
 
 include::infer-api-task.asciidoc[tag=amazon-bedrock]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-task-alibabacloud-ai-search-tab"
+       aria-labelledby="infer-api-task-alibabacloud-ai-search"
+       hidden="">
+++++
+
+include::infer-api-task.asciidoc[tag=alibabacloud-ai-search]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc
index c6ef2a46a8731..2b4aa1a200102 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc
@@ -28,9 +28,29 @@ NOTE: When using this model the recommended similarity measure to use in the
 embeddings are normalized to unit length in which case the `dot_product` and
 the `cosine` measures are equivalent.
 
+// end::cohere[]
 
+// tag::elser[]
 
-// end::cohere[]
+[source,console]
+------------------------------------------------------------
+PUT _inference/sparse_embedding/elser_embeddings <1>
+{
+  "service": "elser",
+  "service_settings": {
+    "num_allocations": 1,
+    "num_threads": 1
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The task type is `sparse_embedding` in the path and the `inference_id` which
+is the unique identifier of the {infer} endpoint is `elser_embeddings`.
+
+You don't need to download and deploy the ELSER model upfront, the API request
+above will download the model if it's not downloaded yet and then deploy it.
+
+// end::elser[]
 
 // tag::hugging-face[]
 
@@ -203,3 +223,32 @@ PUT _inference/text_embedding/amazon_bedrock_embeddings <1>
 <6> The model ID or ARN of the model to use.
 
 // end::amazon-bedrock[]
+
+// tag::alibabacloud-ai-search[]
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/alibabacloud_ai_search_embeddings <1>
+{
+    "service": "alibabacloud-ai-search",
+    "service_settings": {
+        "api_key": "<api_key>", <2>
+        "service_id": "<service_id>", <3>
+        "host": "<host>", <4>
+        "workspace": "<workspace>" <5>
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {infer} endpoint is `alibabacloud_ai_search_embeddings`.
+<2> The API key for accessing the AlibabaCloud AI Search API. You can find your API keys in
+your AlibabaCloud account under the
+https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[API keys section]. You need to provide
+your API key only once. The <<get-inference-api>> does not return your API
+key.
+<3> The AlibabaCloud AI Search embeddings model name, for example `ops-text-embedding-zh-001`.
+<4> The name our your AlibabaCloud AI Search host address.
+<5> The name our your AlibabaCloud AI Search workspace.
+
+// end::alibabacloud-ai-search[]
+
diff --git a/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc b/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc
index 2b3fcc1b6df9f..7eb27d5428956 100644
--- a/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc
@@ -4,31 +4,39 @@
 ++++
 <titleabbrev>Watermark errors</titleabbrev>
 ++++
-:keywords: {es}, high watermark, low watermark, full disk
+:keywords: {es}, high watermark, low watermark, full disk, flood stage watermark
 
 When a data node is critically low on disk space and has reached the
 <<cluster-routing-flood-stage,flood-stage disk usage watermark>>, the following
 error is logged: `Error: disk usage exceeded flood-stage watermark, index has read-only-allow-delete block`. 
 
-To prevent a full disk, when a node reaches this watermark, {es} blocks writes
+To prevent a full disk, when a node reaches this watermark, {es} <<index-block-settings,blocks writes>>
 to any index with a shard on the node. If the block affects related system
-indices, {kib} and other {stack} features may become unavailable.
+indices, {kib} and other {stack} features may become unavailable. For example, 
+this could induce {kib}'s `Kibana Server is not Ready yet` 
+{kibana-ref}/access.html#not-ready[error message]. 
 
 {es} will automatically remove the write block when the affected node's disk
-usage goes below the <<cluster-routing-watermark-high,high disk watermark>>. To
-achieve this, {es} automatically moves some of the affected node's shards to
-other nodes in the same data tier.
+usage falls below the <<cluster-routing-watermark-high,high disk watermark>>. 
+To achieve this, {es} attempts to rebalance some of the affected node's shards 
+to other nodes in the same data tier.
 
-To verify that shards are moving off the affected node, use the <<cat-shards,cat
-shards API>>.
+[[fix-watermark-errors-rebalance]]
+==== Monitor rebalancing
+
+To verify that shards are moving off the affected node until it falls below high 
+watermark., use the <<cat-shards,cat shards API>> and <<cat-recovery,cat recovery API>>: 
 
 [source,console]
 ----
 GET _cat/shards?v=true
+
+GET _cat/recovery?v=true&active_only=true
 ----
 
-If shards remain on the node, use the <<cluster-allocation-explain,cluster
-allocation explanation API>> to get an explanation for their allocation status.
+If shards remain on the node keeping it about high watermark, use the 
+<<cluster-allocation-explain,cluster allocation explanation API>> to get an 
+explanation for their allocation status.
 
 [source,console]
 ----
@@ -36,16 +44,18 @@ GET _cluster/allocation/explain
 {
   "index": "my-index",
   "shard": 0,
-  "primary": false,
-  "current_node": "my-node"
+  "primary": false
 }
 ----
 // TEST[s/^/PUT my-index\n/]
 // TEST[s/"primary": false,/"primary": false/]
-// TEST[s/"current_node": "my-node"//]
 
-To immediately restore write operations, you can temporarily increase the disk
-watermarks and remove the write block.
+[[fix-watermark-errors-temporary]]
+==== Temporary Relief
+
+To immediately restore write operations, you can temporarily increase the 
+<<disk-based-shard-allocation,disk watermarks>> and remove the 
+<<index-block-settings,write block>>.
 
 [source,console]
 ----
@@ -70,18 +80,7 @@ PUT */_settings?expand_wildcards=all
 ----
 // TEST[s/^/PUT my-index\n/]
 
-As a long-term solution, we recommend you add nodes to the affected data tiers
-or upgrade existing nodes to increase disk space. To free up additional disk
-space, you can delete unneeded indices using the <<indices-delete-index,delete
-index API>>.
-
-[source,console]
-----
-DELETE my-index
-----
-// TEST[s/^/PUT my-index\n/]
-
-When a long-term solution is in place, reset or reconfigure the disk watermarks.
+When a long-term solution is in place, to reset or reconfigure the disk watermarks:
 
 [source,console]
 ----
@@ -99,3 +98,21 @@ PUT _cluster/settings
   }
 }
 ----
+
+[[fix-watermark-errors-resolve]]
+==== Resolve
+
+As a long-term solution, we recommend you do one of the following best suited 
+to your use case: 
+
+* add nodes to the affected <<data-tiers,data tiers>>
+
+* upgrade existing nodes to increase disk space
++
+TIP: On {ess}, https://support.elastic.co[Elastic Support] intervention may 
+become necessary if <<cluster-health,cluster health>> reaches `status:red`. 
+
+* delete unneeded indices using the <<indices-delete-index,delete index API>>
+
+* update related <<index-lifecycle-management,ILM policy>> to push indices 
+through to later <<data-tiers,data tiers>>
diff --git a/docs/reference/troubleshooting/network-timeouts.asciidoc b/docs/reference/troubleshooting/network-timeouts.asciidoc
index ef942ac1d268d..ef666c09f87db 100644
--- a/docs/reference/troubleshooting/network-timeouts.asciidoc
+++ b/docs/reference/troubleshooting/network-timeouts.asciidoc
@@ -16,20 +16,22 @@ end::troubleshooting-network-timeouts-gc-vm[]
 
 tag::troubleshooting-network-timeouts-packet-capture-elections[]
 * Packet captures will reveal system-level and network-level faults, especially
-if you capture the network traffic simultaneously at all relevant nodes. You
-should be able to observe any retransmissions, packet loss, or other delays on
-the connections between the nodes.
+if you capture the network traffic simultaneously at all relevant nodes and
+analyse it alongside the {es} logs from those nodes. You should be able to
+observe any retransmissions, packet loss, or other delays on the connections
+between the nodes.
 end::troubleshooting-network-timeouts-packet-capture-elections[]
 
 tag::troubleshooting-network-timeouts-packet-capture-fault-detection[]
 * Packet captures will reveal system-level and network-level faults, especially
 if you capture the network traffic simultaneously at the elected master and the
-faulty node. The connection used for follower checks is not used for any other
-traffic so it can be easily identified from the flow pattern alone, even if TLS
-is in use: almost exactly every second there will be a few hundred bytes sent
-each way, first the request by the master and then the response by the
-follower. You should be able to observe any retransmissions, packet loss, or
-other delays on such a connection.
+faulty node and analyse it alongside the {es} logs from those nodes. The
+connection used for follower checks is not used for any other traffic so it can
+be easily identified from the flow pattern alone, even if TLS is in use: almost
+exactly every second there will be a few hundred bytes sent each way, first the
+request by the master and then the response by the follower. You should be able
+to observe any retransmissions, packet loss, or other delays on such a
+connection.
 end::troubleshooting-network-timeouts-packet-capture-fault-detection[]
 
 tag::troubleshooting-network-timeouts-threads[]
diff --git a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc
index 387ebcdcd43c0..cbb35f7731034 100644
--- a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc
+++ b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc
@@ -1,4 +1,316 @@
 [[troubleshooting-unstable-cluster]]
 == Troubleshooting an unstable cluster
 
-include::../modules/discovery/fault-detection.asciidoc[tag=troubleshooting,leveloffset=-2]
\ No newline at end of file
+Normally, a node will only leave a cluster if deliberately shut down. If a node
+leaves the cluster unexpectedly, it's important to address the cause. A cluster
+in which nodes leave unexpectedly is unstable and can create several issues.
+For instance:
+
+* The cluster health may be yellow or red.
+
+* Some shards will be initializing and other shards may be failing.
+
+* Search, indexing, and monitoring operations may fail and report exceptions in
+logs.
+
+* The `.security` index may be unavailable, blocking access to the cluster.
+
+* The master may appear busy due to frequent cluster state updates.
+
+To troubleshoot a cluster in this state, first ensure the cluster has a
+<<discovery-troubleshooting,stable master>>. Next, focus on the nodes
+unexpectedly leaving the cluster ahead of all other issues. It will not be
+possible to solve other issues until the cluster has a stable master node and
+stable node membership.
+
+Diagnostics and statistics are usually not useful in an unstable cluster. These
+tools only offer a view of the state of the cluster at a single point in time.
+Instead, look at the cluster logs to see the pattern of behaviour over time.
+Focus particularly on logs from the elected master. When a node leaves the
+cluster, logs for the elected master include a message like this (with line
+breaks added to make it easier to read):
+
+[source,text]
+----
+[2022-03-21T11:02:35,513][INFO ][o.e.c.c.NodeLeftExecutor] [instance-0000000000]
+    node-left: [{instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{aNlyORLASam1ammv2DzYXA}{172.27.47.21}{172.27.47.21:19054}{m}]
+    with reason [disconnected]
+----
+
+This message says that the `NodeLeftExecutor` on the elected master
+(`instance-0000000000`) processed a `node-left` task, identifying the node that
+was removed and the reason for its removal. When the node joins the cluster
+again, logs for the elected master will include a message like this (with line
+breaks added to make it easier to read):
+
+[source,text]
+----
+[2022-03-21T11:02:59,892][INFO ][o.e.c.c.NodeJoinExecutor] [instance-0000000000]
+    node-join: [{instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{UNw_RuazQCSBskWZV8ID_w}{172.27.47.21}{172.27.47.21:19054}{m}]
+    with reason [joining after restart, removed [24s] ago with reason [disconnected]]
+----
+
+This message says that the `NodeJoinExecutor` on the elected master
+(`instance-0000000000`) processed a `node-join` task, identifying the node that
+was added to the cluster and the reason for the task.
+
+Other nodes may log similar messages, but report fewer details:
+
+[source,text]
+----
+[2020-01-29T11:02:36,985][INFO ][o.e.c.s.ClusterApplierService]
+    [instance-0000000001] removed {
+        {instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{aNlyORLASam1ammv2DzYXA}{172.27.47.21}{172.27.47.21:19054}{m}
+        {tiebreaker-0000000003}{UNw_RuazQCSBskWZV8ID_w}{bltyVOQ-RNu20OQfTHSLtA}{172.27.161.154}{172.27.161.154:19251}{mv}
+    }, term: 14, version: 1653415, reason: Publication{term=14, version=1653415}
+----
+
+These messages are not especially useful for troubleshooting, so focus on the
+ones from the `NodeLeftExecutor` and `NodeJoinExecutor` which are only emitted
+on the elected master and which contain more details. If you don't see the
+messages from the `NodeLeftExecutor` and `NodeJoinExecutor`, check that:
+
+* You're looking at the logs for the elected master node.
+
+* The logs cover the correct time period.
+
+* Logging is enabled at `INFO` level.
+
+Nodes will also log a message containing `master node changed` whenever they
+start or stop following the elected master. You can use these messages to
+determine each node's view of the state of the master over time.
+
+If a node restarts, it will leave the cluster and then join the cluster again.
+When it rejoins, the `NodeJoinExecutor` will log that it processed a
+`node-join` task indicating that the node is `joining after restart`. If a node
+is unexpectedly restarting, look at the node's logs to see why it is shutting
+down.
+
+The <<health-api>> API on the affected node will also provide some useful
+information about the situation.
+
+If the node did not restart then you should look at the reason for its
+departure more closely. Each reason has different troubleshooting steps,
+described below. There are three possible reasons:
+
+* `disconnected`: The connection from the master node to the removed node was
+closed.
+
+* `lagging`: The master published a cluster state update, but the removed node
+did not apply it within the permitted timeout. By default, this timeout is 2
+minutes. Refer to <<modules-discovery-settings>> for information about the
+settings which control this mechanism.
+
+* `followers check retry count exceeded`: The master sent a number of
+consecutive health checks to the removed node. These checks were rejected or
+timed out. By default, each health check times out after 10 seconds and {es}
+removes the node removed after three consecutively failed health checks. Refer
+to <<modules-discovery-settings>> for information about the settings which
+control this mechanism.
+
+[discrete]
+[[troubleshooting-unstable-cluster-disconnected]]
+=== Diagnosing `disconnected` nodes
+
+Nodes typically leave the cluster with reason `disconnected` when they shut
+down, but if they rejoin the cluster without restarting then there is some
+other problem.
+
+{es} is designed to run on a fairly reliable network. It opens a number of TCP
+connections between nodes and expects these connections to remain open
+<<long-lived-connections,forever>>. If a connection is closed then {es} will
+try and reconnect, so the occasional blip may fail some in-flight operations
+but should otherwise have limited impact on the cluster. In contrast,
+repeatedly-dropped connections will severely affect its operation.
+
+The connections from the elected master node to every other node in the cluster
+are particularly important. The elected master never spontaneously closes its
+outbound connections to other nodes. Similarly, once an inbound connection is
+fully established, a node never spontaneously it unless the node is shutting
+down.
+
+If you see a node unexpectedly leave the cluster with the `disconnected`
+reason, something other than {es} likely caused the connection to close. A
+common cause is a misconfigured firewall with an improper timeout or another
+policy that's <<long-lived-connections,incompatible with {es}>>. It could also
+be caused by general connectivity issues, such as packet loss due to faulty
+hardware or network congestion. If you're an advanced user, configure the
+following loggers to get more detailed information about network exceptions:
+
+[source,yaml]
+----
+logger.org.elasticsearch.transport.TcpTransport: DEBUG
+logger.org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport: DEBUG
+----
+
+If these logs do not show enough information to diagnose the problem, obtain a
+packet capture simultaneously from the nodes at both ends of an unstable
+connection and analyse it alongside the {es} logs from those nodes to determine
+if traffic between the nodes is being disrupted by another device on the
+network.
+
+[discrete]
+[[troubleshooting-unstable-cluster-lagging]]
+=== Diagnosing `lagging` nodes
+
+{es} needs every node to process cluster state updates reasonably quickly. If a
+node takes too long to process a cluster state update, it can be harmful to the
+cluster. The master will remove these nodes with the `lagging` reason. Refer to
+<<modules-discovery-settings>> for information about the settings which control
+this mechanism.
+
+Lagging is typically caused by performance issues on the removed node. However,
+a node may also lag due to severe network delays. To rule out network delays,
+ensure that `net.ipv4.tcp_retries2` is <<system-config-tcpretries,configured
+properly>>. Log messages that contain `warn threshold` may provide more
+information about the root cause.
+
+If you're an advanced user, you can get more detailed information about what
+the node was doing when it was removed by configuring the following logger:
+
+[source,yaml]
+----
+logger.org.elasticsearch.cluster.coordination.LagDetector: DEBUG
+----
+
+When this logger is enabled, {es} will attempt to run the
+<<cluster-nodes-hot-threads>> API on the faulty node and report the results in
+the logs on the elected master. The results are compressed, encoded, and split
+into chunks to avoid truncation:
+
+[source,text]
+----
+[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 1]: H4sIAAAAAAAA/x...
+[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 2]: p7x3w1hmOQVtuV...
+[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 3]: v7uTboMGDbyOy+...
+[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 4]: 4tse0RnPnLeDNN...
+[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] (gzip compressed, base64-encoded, and split into 4 parts on preceding log lines)
+----
+
+To reconstruct the output, base64-decode the data and decompress it using
+`gzip`. For instance, on Unix-like systems:
+
+[source,sh]
+----
+cat lagdetector.log | sed -e 's/.*://' | base64 --decode | gzip --decompress
+----
+
+[discrete]
+[[troubleshooting-unstable-cluster-follower-check]]
+=== Diagnosing `follower check retry count exceeded` nodes
+
+Nodes sometimes leave the cluster with reason `follower check retry count
+exceeded` when they shut down, but if they rejoin the cluster without
+restarting then there is some other problem.
+
+{es} needs every node to respond to network messages successfully and
+reasonably quickly. If a node rejects requests or does not respond at all then
+it can be harmful to the cluster. If enough consecutive checks fail then the
+master will remove the node with reason `follower check retry count exceeded`
+and will indicate in the `node-left` message how many of the consecutive
+unsuccessful checks failed and how many of them timed out. Refer to
+<<modules-discovery-settings>> for information about the settings which control
+this mechanism.
+
+Timeouts and failures may be due to network delays or performance problems on
+the affected nodes. Ensure that `net.ipv4.tcp_retries2` is
+<<system-config-tcpretries,configured properly>> to eliminate network delays as
+a possible cause for this kind of instability. Log messages containing
+`warn threshold` may give further clues about the cause of the instability.
+
+If the last check failed with an exception then the exception is reported, and
+typically indicates the problem that needs to be addressed. If any of the
+checks timed out then narrow down the problem as follows.
+
+include::network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-gc-vm]
+
+include::network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-packet-capture-fault-detection]
+
+include::network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-threads]
+
+By default the follower checks will time out after 30s, so if node departures
+are unpredictable then capture stack dumps every 15s to be sure that at least
+one stack dump was taken at the right time.
+
+[discrete]
+[[troubleshooting-unstable-cluster-shardlockobtainfailedexception]]
+=== Diagnosing `ShardLockObtainFailedException` failures
+
+If a node leaves and rejoins the cluster then {es} will usually shut down and
+re-initialize its shards. If the shards do not shut down quickly enough then
+{es} may fail to re-initialize them due to a `ShardLockObtainFailedException`.
+
+To gather more information about the reason for shards shutting down slowly,
+configure the following logger:
+
+[source,yaml]
+----
+logger.org.elasticsearch.env.NodeEnvironment: DEBUG
+----
+
+When this logger is enabled, {es} will attempt to run the
+<<cluster-nodes-hot-threads>> API whenever it encounters a
+`ShardLockObtainFailedException`. The results are compressed, encoded, and
+split into chunks to avoid truncation:
+
+[source,text]
+----
+[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 1]: H4sIAAAAAAAA/x...
+[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 2]: p7x3w1hmOQVtuV...
+[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 3]: v7uTboMGDbyOy+...
+[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 4]: 4tse0RnPnLeDNN...
+[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] (gzip compressed, base64-encoded, and split into 4 parts on preceding log lines)
+----
+
+To reconstruct the output, base64-decode the data and decompress it using
+`gzip`. For instance, on Unix-like systems:
+
+[source,sh]
+----
+cat shardlock.log | sed -e 's/.*://' | base64 --decode | gzip --decompress
+----
+
+[discrete]
+[[troubleshooting-unstable-cluster-network]]
+=== Diagnosing other network disconnections
+
+{es} is designed to run on a fairly reliable network. It opens a number of TCP
+connections between nodes and expects these connections to remain open
+<<long-lived-connections,forever>>. If a connection is closed then {es} will
+try and reconnect, so the occasional blip may fail some in-flight operations
+but should otherwise have limited impact on the cluster. In contrast,
+repeatedly-dropped connections will severely affect its operation.
+
+{es} nodes will only actively close an outbound connection to another node if
+the other node leaves the cluster. See
+<<cluster-fault-detection-troubleshooting>> for further information about
+identifying and troubleshooting this situation. If an outbound connection
+closes for some other reason, nodes will log a message such as the following:
+
+[source,text]
+----
+[INFO ][o.e.t.ClusterConnectionManager] [node-1] transport connection to [{node-2}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] closed by remote
+----
+
+Similarly, once an inbound connection is fully established, a node never
+spontaneously closes it unless the node is shutting down.
+
+Therefore if you see a node report that a connection to another node closed
+unexpectedly, something other than {es} likely caused the connection to close.
+A common cause is a misconfigured firewall with an improper timeout or another
+policy that's <<long-lived-connections,incompatible with {es}>>. It could also
+be caused by general connectivity issues, such as packet loss due to faulty
+hardware or network congestion. If you're an advanced user, configure the
+following loggers to get more detailed information about network exceptions:
+
+[source,yaml]
+----
+logger.org.elasticsearch.transport.TcpTransport: DEBUG
+logger.org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport: DEBUG
+----
+
+If these logs do not show enough information to diagnose the problem, obtain a
+packet capture simultaneously from the nodes at both ends of an unstable
+connection and analyse it alongside the {es} logs from those nodes to determine
+if traffic between the nodes is being disrupted by another device on the
+network.
diff --git a/docs/reference/upgrade/disable-shard-alloc.asciidoc b/docs/reference/upgrade/disable-shard-alloc.asciidoc
index a93b6dfc6c60b..f69a673095257 100644
--- a/docs/reference/upgrade/disable-shard-alloc.asciidoc
+++ b/docs/reference/upgrade/disable-shard-alloc.asciidoc
@@ -17,3 +17,7 @@ PUT _cluster/settings
 }
 --------------------------------------------------
 // TEST[skip:indexes don't assign]
+
+You can also consider <<modules-gateway,gateway settings>> when restarting 
+large clusters to reduce initial strain while nodes are processing 
+<<modules-discovery,through discovery>>. 
\ No newline at end of file
diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml
index 792330fd3613b..12f7776add17b 100644
--- a/gradle/build.versions.toml
+++ b/gradle/build.versions.toml
@@ -44,6 +44,6 @@ snakeyaml = { group = "org.yaml", name = "snakeyaml", version = { strictly = "2.
 spock-core = { group = "org.spockframework", name="spock-core", version.ref="spock" }
 spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" }
 spock-platform = { group = "org.spockframework", name="spock-bom", version.ref="spock" }
-spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.22.0"
+spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.25.0"
 wiremock = "com.github.tomakehurst:wiremock-jre8-standalone:2.23.2"
 xmlunit-core = "org.xmlunit:xmlunit-core:2.8.2"
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index 5e26d96c4ca17..3a4f5ef9d240c 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -119,29 +119,44 @@
             <sha256 value="bc818ee3015f7cea73d5a603fc7a542ad82ebb5799e406d9abc81ac42caa90f0" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.azure" name="azure-core" version="1.34.0">
-         <artifact name="azure-core-1.34.0.jar">
-            <sha256 value="74f02bbac2dd5485b4e3b3fb7b8cb70232039af27014a7d2938d0c186e051e40" origin="Generated by Gradle"/>
+      <component group="com.azure" name="azure-core" version="1.51.0">
+         <artifact name="azure-core-1.51.0.jar">
+            <sha256 value="9f39244d3cc201a6fd69eb5800d9951624bcac548aee3b51efd642e4acbf457c" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.azure" name="azure-core-http-netty" version="1.12.7">
-         <artifact name="azure-core-http-netty-1.12.7.jar">
-            <sha256 value="24d15d722bef5be7065b835ff413ab045136bd5a3927a00647cfa7413bea3bce" origin="Generated by Gradle"/>
+      <component group="com.azure" name="azure-core-http-netty" version="1.15.3">
+         <artifact name="azure-core-http-netty-1.15.3.jar">
+            <sha256 value="18e9932c0aa3f42cee80fccee1907849fa9db5be40b8cffa21d1a7fc1d7457e5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.azure" name="azure-storage-blob" version="12.20.1">
-         <artifact name="azure-storage-blob-12.20.1.jar">
-            <sha256 value="9a568f03f555cd3dc7c46fed557ea65a0aa00b5eaff1e47e3e21462a863504ba" origin="Generated by Gradle"/>
+      <component group="com.azure" name="azure-identity" version="1.13.2">
+         <artifact name="azure-identity-1.13.2.jar">
+            <sha256 value="f00ebc38b1dcf75a35129b8b0ad434514c84fd310c7afaf12824bd04e590f243" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.azure" name="azure-storage-common" version="12.19.1">
-         <artifact name="azure-storage-common-12.19.1.jar">
-            <sha256 value="0e67a261c1088b9d688d7461e27da95f6830bdde713420b0d783ae1025c4d384" origin="Generated by Gradle"/>
+      <component group="com.azure" name="azure-json" version="1.2.0">
+         <artifact name="azure-json-1.2.0.jar">
+            <sha256 value="c7419ae04f668eb4dc3864cd64a6b77207abd93b9ee784f042dea7cb452840de" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.azure" name="azure-storage-internal-avro" version="12.5.3">
-         <artifact name="azure-storage-internal-avro-12.5.3.jar">
-            <sha256 value="7cbebf7710808f3689fb6acf1384e710e13ecd74bfedbe6fa8cb467eaa156d83" origin="Generated by Gradle"/>
+      <component group="com.azure" name="azure-storage-blob" version="12.27.1">
+         <artifact name="azure-storage-blob-12.27.1.jar">
+            <sha256 value="31915426834400cac854f48441c168d55aa6fc054527f28f1d242a7067affd14" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.azure" name="azure-storage-common" version="12.26.1">
+         <artifact name="azure-storage-common-12.26.1.jar">
+            <sha256 value="b0297ac1a9017ccd8a1e5cf41fb8d00ff0adbdd06849f6c5aafb3208708264dd" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.azure" name="azure-storage-internal-avro" version="12.12.1">
+         <artifact name="azure-storage-internal-avro-12.12.1.jar">
+            <sha256 value="b5405fb42fa93413dfb24bbdec57d1a9915810c5cda5fe27dd2fe5b27f8d8767" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.azure" name="azure-xml" version="1.1.0">
+         <artifact name="azure-xml-1.1.0.jar">
+            <sha256 value="1006208324a08dadd5463ad2edfa3057a1ec3adee9d79c3ca00b14ef2f66cbce" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.bettercloud" name="vault-java-driver" version="4.1.0">
@@ -189,19 +204,14 @@
             <sha256 value="095fd1dc77888c073f0be39a018156ee526722798b09de9e285ef2135e16eac4" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.diffplug.durian" name="durian-swt.os" version="4.1.1">
-         <artifact name="durian-swt.os-4.1.1.jar">
-            <sha256 value="9ff15ac4f14c4ef9e241108392c43347916a433658c5ad971e999df1191b3230" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
       <component group="com.diffplug.durian" name="durian-swt.os" version="4.2.0">
          <artifact name="durian-swt.os-4.2.0.jar">
             <sha256 value="f21e572bf9fbb549a998cb7e2f79b6b9a3ab9623371ac130b12509523f7b748f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.diffplug.durian" name="durian-swt.os" version="4.2.1">
-         <artifact name="durian-swt.os-4.2.1.jar">
-            <sha256 value="4075cef438707d669b8d468500289af99fe87cb3fa69b7f564d9cefa4c838b30" origin="Generated by Gradle"/>
+      <component group="com.diffplug.durian" name="durian-swt.os" version="4.2.2">
+         <artifact name="durian-swt.os-4.2.2.jar">
+            <sha256 value="6b531c6b4be581a8b32ead861ddc15c2c93b20c665fb4d33e0381483c24445f4" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.diffplug.spotless" name="spotless-eclipse-base" version="3.5.2">
@@ -214,36 +224,19 @@
             <sha256 value="e983d13b1051f8dc49820970aa8da80364857cecf46d78d1b509366e523186bd" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.diffplug.spotless" name="spotless-lib" version="2.38.0">
-         <artifact name="spotless-lib-2.38.0.jar">
-            <sha256 value="de867a4ad15042970c85bd8b822560be501e1d313f588bb8f5ddaab5759219f3" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.diffplug.spotless" name="spotless-lib" version="2.42.0">
-         <artifact name="spotless-lib-2.42.0.jar">
-            <sha256 value="aa3678c924f7a55475e3a923fe9a4b34fb7cf0ea1cb81039378ef8e04b3f00fe" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.diffplug.spotless" name="spotless-lib-extra" version="2.38.0">
-         <artifact name="spotless-lib-extra-2.38.0.jar">
-            <sha256 value="ea247cf4548748ad8e8c31101dbb287dbe523faaf57e3a553e834aa6d67118fc" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.diffplug.spotless" name="spotless-lib-extra" version="2.42.0">
-         <artifact name="spotless-lib-extra-2.42.0.jar">
-            <sha256 value="ed823e2a980c87d7df749a89c2356b11b36c15de2c32eab31ca990f7ad6fafb2" origin="Generated by Gradle"/>
+      <component group="com.diffplug.spotless" name="spotless-lib" version="2.45.0">
+         <artifact name="spotless-lib-2.45.0.jar">
+            <sha256 value="b25969972e1d980280ca2ae529197ed9b3160aae48b506cf1935f01bd1318667" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.diffplug.spotless" name="spotless-plugin-gradle" version="6.18.0">
-         <artifact name="spotless-plugin-gradle-6.18.0.jar">
-            <sha256 value="8f27c05d31a4389259ba54938c476427ae8a88cc1540d421623fdc6c9fc86f7b" origin="Generated by Gradle">
-               <also-trust value="daf622fc8d6dbfe53b7ec08f80ac64fdd2f9cb00f175e69eb91a8f0da1565970"/>
-            </sha256>
+      <component group="com.diffplug.spotless" name="spotless-lib-extra" version="2.45.0">
+         <artifact name="spotless-lib-extra-2.45.0.jar">
+            <sha256 value="602cbbcd3828ee9cfb2e30a7f9b30335c6927130771414d4cdd294d21fe5cb67" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.diffplug.spotless" name="spotless-plugin-gradle" version="6.22.0">
-         <artifact name="spotless-plugin-gradle-6.22.0.jar">
-            <sha256 value="32026f34b42b7b52ca9178d9c36802634f8caedd5a857764c3157ed8792277d1" origin="Generated by Gradle"/>
+      <component group="com.diffplug.spotless" name="spotless-plugin-gradle" version="6.25.0">
+         <artifact name="spotless-plugin-gradle-6.25.0.jar">
+            <sha256 value="f5eb908a4c5da46299e7543faada0402d2c4b77d58c7b432d4b6e59349b280a3" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.ethlo.time" name="itu" version="1.7.0">
@@ -271,11 +264,6 @@
             <sha256 value="91adcd3dcf5fd9a16499934e7536a23d456692a0093e3d4fd52f138c3936348c" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.core" name="jackson-annotations" version="2.15.4">
-         <artifact name="jackson-annotations-2.15.4.jar">
-            <sha256 value="f204ebbd552614a22b8531ffe350d47f8fd42c45bb60517c07974dc27a5a1dd3" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
       <component group="com.fasterxml.jackson.core" name="jackson-core" version="2.10.5">
          <artifact name="jackson-core-2.10.5.jar">
             <sha256 value="2656010d1e921ac69b76fc7e0c0f5a6b14aca62fa9603e78831e6148eb7c77ba" origin="Generated by Gradle"/>
@@ -296,9 +284,14 @@
             <sha256 value="5b483f68fa9dd6aa37da37d1f79dd5c4b9464238f4f0660a242cb6b5c724950c" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.core" name="jackson-core" version="2.15.4">
-         <artifact name="jackson-core-2.15.4.jar">
-            <sha256 value="8dc9210dd285db366f45f518dd1e6a9ccfeb0f1a8e184a899fe96d29edf1fd94" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.core" name="jackson-core" version="2.17.0">
+         <artifact name="jackson-core-2.17.0.jar">
+            <sha256 value="55be130f6a68038088a261856c4e383ce79957a0fc1a29ecb213a9efd6ef4389" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.core" name="jackson-core" version="2.17.2">
+         <artifact name="jackson-core-2.17.2.jar">
+            <sha256 value="721a189241dab0525d9e858e5cb604d3ecc0ede081e2de77d6f34fa5779a5b46" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.fasterxml.jackson.core" name="jackson-databind" version="2.10.5.1">
@@ -331,6 +324,16 @@
             <sha256 value="da59dba8212ea0886cf2ecf0e67365850db417b92a238e746a13381028cd2d75" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-cbor" version="2.17.0">
+         <artifact name="jackson-dataformat-cbor-2.17.0.jar">
+            <sha256 value="d7050cb5e622848d5be382848b1792e8244d529e24c84ba1cc27aabf2496fe3b" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-cbor" version="2.17.2">
+         <artifact name="jackson-dataformat-cbor-2.17.2.jar">
+            <sha256 value="1d7dc634ba3d7981d7017f4f44d43cf371e922c80028b4eb3e5374ce6546d5f6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-smile" version="2.14.2">
          <artifact name="jackson-dataformat-smile-2.14.2.jar">
             <sha256 value="9c279bb29770de09289c14cf8862dd195112687cd7fde815919f54a9157ce213" origin="Generated by Gradle"/>
@@ -341,14 +344,19 @@
             <sha256 value="bc9d5020ffce81eca245c9148054600134a209b3d279d7be38fa4f2c07f7a505" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-xml" version="2.13.4">
-         <artifact name="jackson-dataformat-xml-2.13.4.jar">
-            <sha256 value="76fbf0cedd51af6a13aba39c27c8c29a5a280dc24ee66577d559e4660d8709ce" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-smile" version="2.17.0">
+         <artifact name="jackson-dataformat-smile-2.17.0.jar">
+            <sha256 value="e6b2bf2a53d1f88ec02f1833da24c208aa299876e47d37f7920b1e83e214baab" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-xml" version="2.15.4">
-         <artifact name="jackson-dataformat-xml-2.15.4.jar">
-            <sha256 value="90d8109cda7b90c494a7bfde44e96e2fa25021191b67a5924dfa5cbd698025c3" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-smile" version="2.17.2">
+         <artifact name="jackson-dataformat-smile-2.17.2.jar">
+            <sha256 value="120b07ad7b7b9097d9a174ab6176e07b6409fb7af3f63fcf3d7147c053be0d29" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-xml" version="2.15.0">
+         <artifact name="jackson-dataformat-xml-2.15.0.jar">
+            <sha256 value="e3137c89b08d0c3d9f7dfeae11d5b941b310f5f7bfdea90063f29c0b3b1f6807" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-yaml" version="2.14.2">
@@ -361,14 +369,19 @@
             <sha256 value="691a0a605de15ea610180317331957fe6f7e91f45471ee45cbf9832d74291c31" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.datatype" name="jackson-datatype-jsr310" version="2.13.4">
-         <artifact name="jackson-datatype-jsr310-2.13.4.jar">
-            <sha256 value="5ad75d210bacc17271925da28e1f393aaf8c83f6c92fbe5b2ed61954b84decf7" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-yaml" version="2.17.0">
+         <artifact name="jackson-dataformat-yaml-2.17.0.jar">
+            <sha256 value="46b65ace036b01743710bcfc9e7f041eded2fc82ba0d3d83e19b32c818c18b4c" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.datatype" name="jackson-datatype-jsr310" version="2.15.4">
-         <artifact name="jackson-datatype-jsr310-2.15.4.jar">
-            <sha256 value="472498cbba2726012ff82f86fc8feef9593663bda1a695a17db804a63fa733ff" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-yaml" version="2.17.2">
+         <artifact name="jackson-dataformat-yaml-2.17.2.jar">
+            <sha256 value="941bcd8b1381bb3b0d726fab41624fa8ece0ee7b6cf2860ad95e8157ce673376" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.datatype" name="jackson-datatype-jsr310" version="2.15.0">
+         <artifact name="jackson-datatype-jsr310-2.15.0.jar">
+            <sha256 value="d5deb92d59668a8de0ac8da818d5d9c0fdfb4db6b2f1d66a779fe357f310d648" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.fasterxml.jackson.jaxrs" name="jackson-jaxrs-base" version="2.10.5">
@@ -386,14 +399,9 @@
             <sha256 value="994a0a510a35d55a869567807075736597da97e9d36ad1ebaff5e37def5a55d3" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.module" name="jackson-module-jaxb-annotations" version="2.13.4">
-         <artifact name="jackson-module-jaxb-annotations-2.13.4.jar">
-            <sha256 value="b23725fd92b783e3ddc149d23f565b9da5bdfc98361be81488ff4e45e6735ba1" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.fasterxml.jackson.module" name="jackson-module-jaxb-annotations" version="2.15.4">
-         <artifact name="jackson-module-jaxb-annotations-2.15.4.jar">
-            <sha256 value="20bf4d2ce22fa76c6feba48dc2e770bfeb313a36f984e00e3403af405e1c83b2" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.module" name="jackson-module-jaxb-annotations" version="2.15.0">
+         <artifact name="jackson-module-jaxb-annotations-2.15.0.jar">
+            <sha256 value="7f2083eb676dfbcfb380f4e634ad41b9ba6f25d6bb5a462465d8adc83faca0d5" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.fasterxml.woodstox" name="woodstox-core" version="5.3.0">
@@ -401,9 +409,9 @@
             <sha256 value="b2bd29c31fda49a9b28a22b9e5c2b26443bcfa99c1a28eab70ab9c7d349b5002" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.woodstox" name="woodstox-core" version="6.4.0">
-         <artifact name="woodstox-core-6.4.0.jar">
-            <sha256 value="d6eb9f2f40049a7a808baf11ffba0737648e62ff52fde9271d808e5d57a27279" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.woodstox" name="woodstox-core" version="6.7.0">
+         <artifact name="woodstox-core-6.7.0.jar">
+            <sha256 value="81cdeef50567735bda9f6b4aabe0cc0a3f6c04f15569192bc6505393d2612c25" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.github.breskeby" name="shadow" version="3b035f2">
@@ -551,6 +559,11 @@
             <sha256 value="0cdd163ce3598a20fc04eee71b140b24f6f2a3b35f0a499dbbdd9852e83fbfaf" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.code.gson" name="gson" version="2.11.0">
+         <artifact name="gson-2.11.0.jar">
+            <sha256 value="57928d6e5a6edeb2abd3770a8f95ba44dce45f3b23b7a9dc2b309c581552a78b" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.code.gson" name="gson" version="2.2.4">
          <artifact name="gson-2.2.4.jar">
             <sha256 value="c0328cd07ca9e363a5acd00c1cf4afe8cf554bd6d373834981ba05cebec687fb" origin="Generated by Gradle"/>
@@ -561,9 +574,9 @@
             <sha256 value="c8fb4839054d280b3033f800d1f5a97de2f028eb8ba2eb458ad287e536f3f25f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.errorprone" name="error_prone_annotations" version="2.1.3">
-         <artifact name="error_prone_annotations-2.1.3.jar">
-            <sha256 value="03d0329547c13da9e17c634d1049ea2ead093925e290567e1a364fd6b1fc7ff8" origin="Generated by Gradle"/>
+      <component group="com.google.crypto.tink" name="tink" version="1.14.0">
+         <artifact name="tink-1.14.0.jar">
+            <sha256 value="47b2248705e0c9771bc259f22465a79655c1296e2d47aaee852adb7cdacb6198" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.google.errorprone" name="error_prone_annotations" version="2.11.0">
@@ -581,6 +594,11 @@
             <sha256 value="6ebd22ca1b9d8ec06d41de8d64e0596981d9607b42035f9ed374f9de271a481a" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.errorprone" name="error_prone_annotations" version="2.21.1">
+         <artifact name="error_prone_annotations-2.21.1.jar">
+            <sha256 value="d1f3c66aa91ac52549e00ae3b208ba4b9af7d72d68f230643553beb38e6118ac" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.errorprone" name="error_prone_annotations" version="2.3.4">
          <artifact name="error_prone_annotations-2.3.4.jar">
             <sha256 value="baf7d6ea97ce606c53e11b6854ba5f2ce7ef5c24dddf0afa18d1260bd25b002c" origin="Generated by Gradle"/>
@@ -591,14 +609,9 @@
             <sha256 value="8d10cac2ea9878896077ba437d76fdb1b9a07f55a863c560bb8a024b04103f8b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.googlejavaformat" name="google-java-format" version="1.16.0">
-         <artifact name="google-java-format-1.16.0.jar">
-            <sha256 value="0cff5d0230ba20d538f3f70b2aa68bd33f9fdc69768cde07337c563c23eb7c43" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.google.googlejavaformat" name="google-java-format" version="1.17.0">
-         <artifact name="google-java-format-1.17.0.jar">
-            <sha256 value="631ba54c39f6c20df027dc1420736df2e5e43c581880efdd1e46ddb4ce050e3e" origin="Generated by Gradle"/>
+      <component group="com.google.googlejavaformat" name="google-java-format" version="1.19.2">
+         <artifact name="google-java-format-1.19.2.jar">
+            <sha256 value="bac84458eb12499585f2fabb1ac13bbe5b455c120bf3d19db21597814a27c863" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.google.guava" name="failureaccess" version="1.0">
@@ -616,11 +629,6 @@
             <sha256 value="a896857d07845d38c7dc5bbc0457b6d9b0f62ecffda010e5e9ec12d561f676d3" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.guava" name="guava" version="25.1-android">
-         <artifact name="guava-25.1-android.jar">
-            <sha256 value="f7b8f8fed176b9cf6831b98cb07320d7fbe91d99b29999f752c3821dfe45bdc8" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
       <component group="com.google.guava" name="guava" version="27.0-jre">
          <artifact name="guava-27.0-jre.jar">
             <sha256 value="63b09db6861011e7fb2481be7790c7fd4b03f0bb884b3de2ecba8823ad19bf3f" origin="Generated by Gradle"/>
@@ -646,6 +654,11 @@
             <sha256 value="91fbba37f1c8b251cf9ea9e7d3a369eb79eb1e6a5df1d4bbf483dd0380740281" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.guava" name="guava" version="32.1.3-jre">
+         <artifact name="guava-32.1.3-jre.jar">
+            <sha256 value="6d4e2b5a118aab62e6e5e29d185a0224eed82c85c40ac3d33cf04a270c3b3744" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.guava" name="listenablefuture" version="9999.0-empty-to-avoid-conflict-with-guava">
          <artifact name="listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar">
             <sha256 value="b372a037d4230aa57fbeffdef30fd6123f9c0c2db85d0aced00c91b974f33f99" origin="Generated by Gradle"/>
@@ -741,6 +754,11 @@
             <sha256 value="1b78b4a76a71512debfdff8f8fc5aef6bfd459f65758fecf7aff245e6e6301e4" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.protobuf" name="protobuf-java" version="4.27.0">
+         <artifact name="protobuf-java-4.27.0.jar">
+            <sha256 value="9072e60fe66cff5d6c0f11a1df21d8f3e4b29b5ee782b45c3fc75f59fbe2b839" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.protobuf" name="protobuf-java-util" version="3.21.9">
          <artifact name="protobuf-java-util-3.21.9.jar">
             <sha256 value="a78226d02551ca2cb34b24b4168a75b4a84041cc3333f3725bb5ad9806a80b77" origin="Generated by Gradle"/>
@@ -876,6 +894,16 @@
             <sha256 value="f1eae7a4e14944940e54605d7b512bdd3296efe6353af6bfb15c1b97ed06d8ec" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.microsoft.azure" name="msal4j" version="1.16.2">
+         <artifact name="msal4j-1.16.2.jar">
+            <sha256 value="23478ea37253045c58b8b80367a89b48cbf2bd3e3ef43c2ddd3e7d96b853ef43" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.microsoft.azure" name="msal4j-persistence-extension" version="1.3.0">
+         <artifact name="msal4j-persistence-extension-1.3.0.jar">
+            <sha256 value="dfc41c817fbfa76057af6ffe4379dbca6a5e16b8e87df8bdda23f371756c2d09" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.microsoft.sqlserver" name="mssql-jdbc" version="6.2.1.jre7">
          <artifact name="mssql-jdbc-6.2.1.jre7.jar">
             <sha256 value="9cfa259450ae3471d2e6e2c3d8aefcce236e3daef8b3734d21dc93c3a5bbe806" origin="Generated by Gradle"/>
@@ -911,19 +939,31 @@
             <sha256 value="260647c459cc7de269b2a8f576e5693e434df888fd4801e92365fe18b9992cbc" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.nimbusds" name="content-type" version="2.3">
+         <artifact name="content-type-2.3.jar">
+            <sha256 value="60349793e006fba96b532cb0c21e10e969fe0db8d87f91c3b9eaf82ba2998895" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.nimbusds" name="lang-tag" version="1.4.4">
          <artifact name="lang-tag-1.4.4.jar">
             <sha256 value="e49d2c694bb80c7036c177f2aabf53b7156061a68bd19dfd60e2bd370709e0c5" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.nimbusds" name="lang-tag" version="1.7">
+         <artifact name="lang-tag-1.7.jar">
+            <sha256 value="e8c1c594e2425bdbea2d860de55c69b69fc5d59454452449a0f0913c2a5b8a31" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.nimbusds" name="nimbus-jose-jwt" version="4.41.1">
          <artifact name="nimbus-jose-jwt-4.41.1.jar">
             <sha256 value="fbfd0d5f2b2f86758b821daa5e79b5d7c965edd9dc1b2cc80b515df1c6ddc22d" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.nimbusds" name="nimbus-jose-jwt" version="9.23">
-         <artifact name="nimbus-jose-jwt-9.23.jar">
-            <sha256 value="33ab8084fdae1d75be1b061b1489d4a12045bd7b50c2e24ff152911e4551ec07" origin="Generated by Gradle"/>
+      <component group="com.nimbusds" name="nimbus-jose-jwt" version="9.37.3">
+         <artifact name="nimbus-jose-jwt-9.37.3.jar">
+            <sha256 value="12ae4a3a260095d7aeba2adea7ae396e8b9570db8b7b409e09a824c219cc0444" origin="Generated by Gradle">
+               <also-trust value="afc63b689d881439b95f343b1dca750391edac63b87392be4d90d19c94ccafbe"/>
+            </sha256>
          </artifact>
       </component>
       <component group="com.nimbusds" name="nimbus-jose-jwt" version="9.8.1">
@@ -931,6 +971,16 @@
             <sha256 value="7664cf8c6f2adadf600287812b32878277beda54912eab9d4c2932cd50cb704a" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.nimbusds" name="oauth2-oidc-sdk" version="11.10.1">
+         <artifact name="oauth2-oidc-sdk-11.10.1.jar">
+            <sha256 value="9e51b2c17503cdd3eb97f41491c712aff7783bb3c67185d789f44ccf2a603b26" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.nimbusds" name="oauth2-oidc-sdk" version="11.9.1">
+         <artifact name="oauth2-oidc-sdk-11.9.1.jar">
+            <sha256 value="0820c9690966304d075347b88e81ae490213440fc4d2c84f3d370d41941b2b9c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.nimbusds" name="oauth2-oidc-sdk" version="9.37">
          <artifact name="oauth2-oidc-sdk-9.37.jar">
             <sha256 value="44a04bbed5ae3f6d198aa73ee6b545c476e528ec1a267ef3e9f7033f886dd6fe" origin="Generated by Gradle"/>
@@ -961,14 +1011,9 @@
             <sha256 value="88ac9fd1bb51f82bcc664cc1eb9c225c90dc4389d660231b4cc737bebfe7d0aa" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.squareup.okhttp3" name="okhttp" version="4.10.0">
-         <artifact name="okhttp-4.10.0.jar">
-            <sha256 value="7580f14fa1691206e37081ad3f92063b1603b328da0bb316f2fef02e0562e7ec" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.squareup.okhttp3" name="okhttp" version="4.11.0">
-         <artifact name="okhttp-4.11.0.jar">
-            <sha256 value="ee8f6bd6cd1257013d748330f4ca147638a9fbcb52fb388d5ac93cf53408745d" origin="Generated by Gradle"/>
+      <component group="com.squareup.okhttp3" name="okhttp" version="4.12.0">
+         <artifact name="okhttp-4.12.0.jar">
+            <sha256 value="b1050081b14bb7a3a7e55a4d3ef01b5dcfabc453b4573a4fc019767191d5f4e0" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.squareup.okio" name="okio" version="1.4.0">
@@ -981,14 +1026,9 @@
             <sha256 value="114bdc1f47338a68bcbc95abf2f5cdc72beeec91812f2fcd7b521c1937876266" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.squareup.okio" name="okio-jvm" version="3.0.0">
-         <artifact name="okio-jvm-3.0.0.jar">
-            <sha256 value="be64a0cc1f28ea9cd5c970dd7e7557af72c808d738c495b397bf897c9921e907" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.squareup.okio" name="okio-jvm" version="3.2.0">
-         <artifact name="okio-jvm-3.2.0.jar">
-            <sha256 value="b642baef4c570055de4cb3d1667b2b16dced901ff8066345a063691aa06025a4" origin="Generated by Gradle"/>
+      <component group="com.squareup.okio" name="okio-jvm" version="3.6.0">
+         <artifact name="okio-jvm-3.6.0.jar">
+            <sha256 value="67543f0736fc422ae927ed0e504b98bc5e269fda0d3500579337cb713da28412" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.sun.activation" name="jakarta.activation" version="1.2.1">
@@ -1261,14 +1301,14 @@
             <sha256 value="7b478e69fb613a05c2e1cb0ecfd257618cfb477333660914440a30b5afc712ff" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="dev.equo.ide" name="solstice" version="1.0.3">
-         <artifact name="solstice-1.0.3.jar">
-            <sha256 value="0bfbd724f4ebdbabfc384ba90498702a45b449c92817d2844444713229d2cbf3" origin="Generated by Gradle"/>
+      <component group="dev.equo.ide" name="solstice" version="1.7.4">
+         <artifact name="solstice-1.7.4.jar">
+            <sha256 value="1288c1010299cb8434218e370f2291f2d6ea49b3ccb34b5082080f7c270a254c" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="dev.equo.ide" name="solstice" version="1.7.3">
-         <artifact name="solstice-1.7.3.jar">
-            <sha256 value="a8cfefb2c28b906c920ddb13024f557f57c68668ef8c748662e5b86cde70f55b" origin="Generated by Gradle"/>
+      <component group="dev.equo.ide" name="solstice" version="1.7.5">
+         <artifact name="solstice-1.7.5.jar">
+            <sha256 value="06e14bc43acc331dab6b5ea201fc67364ed123f982c85fa5131f0817ed65ae4f" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="dnsjava" name="dnsjava" version="2.1.7">
@@ -1471,19 +1511,19 @@
             <sha256 value="3cabdae2244c999addebb8c31ae452fbdc874b4f26a163539954b8eeb5d6acc6" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.projectreactor" name="reactor-core" version="3.4.34">
-         <artifact name="reactor-core-3.4.34.jar">
-            <sha256 value="b324c0921f97bd73a9aebffbf59ac45eeb48d9eab9bce67c23ad0bc83b513d87" origin="Generated by Gradle"/>
+      <component group="io.projectreactor" name="reactor-core" version="3.4.38">
+         <artifact name="reactor-core-3.4.38.jar">
+            <sha256 value="a9255f60d92e8fc58c3c87320cc439936e08227db65bd88eb97e844af853e608" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.projectreactor.netty" name="reactor-netty-core" version="1.0.39">
-         <artifact name="reactor-netty-core-1.0.39.jar">
-            <sha256 value="f7c2330852a7c2a96de4847eb1dc906a458c9573a2cdde3cefc69300c0c5c68c" origin="Generated by Gradle"/>
+      <component group="io.projectreactor.netty" name="reactor-netty-core" version="1.0.45">
+         <artifact name="reactor-netty-core-1.0.45.jar">
+            <sha256 value="b38fdbc0618bc193359bb32807bc17b691fed42259dd27aa02b18ddbcf2b0844" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.projectreactor.netty" name="reactor-netty-http" version="1.0.39">
-         <artifact name="reactor-netty-http-1.0.39.jar">
-            <sha256 value="399eeee2917e87044b0c64a47456c69bb85edec284a5643e4f6c67affecbbdcc" origin="Generated by Gradle"/>
+      <component group="io.projectreactor.netty" name="reactor-netty-http" version="1.0.45">
+         <artifact name="reactor-netty-http-1.0.45.jar">
+            <sha256 value="f657ebfcf8115008fa8c55e75200da886a45f22c488d82ddaaf013ffccb65205" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.sgr" name="s2-geometry-library-java" version="1.0.1">
@@ -1501,6 +1541,11 @@
             <sha256 value="69156304079bdeed9fc0ae3b39389f19b3cc4ba4443bc80508995394ead742ea" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="jakarta.xml.bind" name="jakarta.xml.bind-api" version="2.3.3">
+         <artifact name="jakarta.xml.bind-api-2.3.3.jar">
+            <sha256 value="c04539f472e9a6dd0c7685ea82d677282269ab8e7baca2e14500e381e0c6cec5" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="javax.activation" name="activation" version="1.1">
          <artifact name="activation-1.1.jar">
             <sha256 value="2881c79c9d6ef01c58e62beea13e9d1ac8b8baa16f2fc198ad6e6776defdcdd3" origin="Generated by Gradle"/>
@@ -1649,6 +1694,11 @@
             <sha256 value="0271ae7fc162a5e69c337f36d86fdb94a8a232c5c42a80d8a7424071addd1fdc" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="net.java.dev.jna" name="jna-platform" version="5.12.1">
+         <artifact name="jna-platform-5.12.1.jar">
+            <sha256 value="8ce969116cac95bd61b07a8d5e07174b352e63301473caac72c395e3c08488d2" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="net.java.dev.jna" name="jna-platform" version="5.7.0">
          <artifact name="jna-platform-5.7.0.jar">
             <sha256 value="42e020705692eddbd285e2b72ef0ff468f51a926382569c45f4e9cea4602ad1e" origin="Generated by Gradle"/>
@@ -1674,6 +1724,11 @@
             <sha256 value="0972bbc99437c4163acd09b630e6c77eab4cfab8a9594621c95466c0c6645396" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="net.minidev" name="accessors-smart" version="2.5.0">
+         <artifact name="accessors-smart-2.5.0.jar">
+            <sha256 value="12314fc6881d66a413fd66370787adba16e504fbf7e138690b0f3952e3fbd321" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="net.minidev" name="json-smart" version="2.3">
          <artifact name="json-smart-2.3.jar">
             <sha256 value="903f48c8aa4c3f6426440b8d32de89fa1dc23b1169abde25e4e1d068aa67708b" origin="Generated by Gradle"/>
@@ -1689,6 +1744,16 @@
             <sha256 value="64072f56d9dff5040b2acec477c5d5e6bcebfc88c508f12acb26072d07942146" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="net.minidev" name="json-smart" version="2.5.0">
+         <artifact name="json-smart-2.5.0.jar">
+            <sha256 value="432b9e545848c4141b80717b26e367f83bf33f19250a228ce75da6e967da2bc7" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="net.minidev" name="json-smart" version="2.5.1">
+         <artifact name="json-smart-2.5.1.jar">
+            <sha256 value="86c0c189581b79b57b0719f443a724e9f628ffbb9eef645cf79194f5973a1001" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="net.nextencia" name="rrdiagram" version="0.9.4">
          <artifact name="rrdiagram-0.9.4.jar">
             <sha256 value="8f0855addca5320cfadedbf7d3d46b681f3f308b6e87d5d82f32637ba72256b6" origin="Generated by Gradle"/>
@@ -3182,11 +3247,6 @@
             <sha256 value="ac97f7b4b1d8e9337edfa0e34044f8d0efe7223f6ad8f3a85d54cc1018ea2e04" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.checkerframework" name="checker-compat-qual" version="2.0.0">
-         <artifact name="checker-compat-qual-2.0.0.jar">
-            <sha256 value="a40b2ce6d8551e5b90b1bf637064303f32944d61b52ab2014e38699df573941b" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
       <component group="org.checkerframework" name="checker-qual" version="2.10.0">
          <artifact name="checker-qual-2.10.0.jar">
             <sha256 value="d261fde25d590f6b69db7721d469ac1b0a19a17ccaaaa751c31f0d8b8260b894" origin="Generated by Gradle"/>
@@ -3207,16 +3267,16 @@
             <sha256 value="e316255bbfcd9fe50d165314b85abb2b33cb2a66a93c491db648e498a82c2de1" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.checkerframework" name="checker-qual" version="3.37.0">
+         <artifact name="checker-qual-3.37.0.jar">
+            <sha256 value="e4ce1376cc2735e1dde220b62ad0913f51297704daad155a33f386bc5db0d9f7" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.checkerframework" name="checker-qual" version="3.42.0">
          <artifact name="checker-qual-3.42.0.jar">
             <sha256 value="ccaedd33af0b7894d9f2f3b644f4d19e43928e32902e61ac4d10777830f5aac7" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.checkerframework" name="checker-qual" version="3.5.0">
-         <artifact name="checker-qual-3.5.0.jar">
-            <sha256 value="729990b3f18a95606fc2573836b6958bcdb44cb52bfbd1b7aa9c339cff35a5a4" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
       <component group="org.codehaus.groovy" name="groovy" version="3.0.9">
          <artifact name="groovy-3.0.9.jar">
             <sha256 value="77bf86897f295f8cae2e1f46b1eca109f487ba81b66ef24a2b6dcba1eb7d6ce7" origin="Generated by Gradle"/>
@@ -3312,6 +3372,11 @@
             <sha256 value="678567e48b51a42c65c699f266539ad3d676d4b1a5b0ad7d89ece8b9d5772579" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.codehaus.woodstox" name="stax2-api" version="4.2.2">
+         <artifact name="stax2-api-4.2.2.jar">
+            <sha256 value="a61c48d553efad78bc01fffc4ac528bebbae64cbaec170b2a5e39cf61eb51abe" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.cryptacular" name="cryptacular" version="1.2.4">
          <artifact name="cryptacular-1.2.4.jar">
             <sha256 value="97feff80494a54f1b5001f6f4bbdbd45cb64ccbb2dffeb679da9da9be0434b07" origin="Generated by Gradle"/>
@@ -3322,14 +3387,19 @@
             <sha256 value="c600d1ae61b5b0ff1391e00eb6fb390201e4612c3aaf2dc1b94050c8784840be" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.jdt" name="ecj" version="3.33.0">
-         <artifact name="ecj-3.33.0.jar">
-            <sha256 value="f7686c4960cf70c2ebc5c500a73a8cfc04541b730c18f1c5c21329889b137f45" origin="Generated by Gradle"/>
+      <component group="org.cryptomator" name="siv-mode" version="1.5.2">
+         <artifact name="siv-mode-1.5.2.jar">
+            <sha256 value="81a33da58e5e878ffe8972ef93cd0bfc3a9daa0f5e3af4dd151be2b1d73018b4" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.jdt" name="org.eclipse.jdt.core" version="3.33.0">
-         <artifact name="org.eclipse.jdt.core-3.33.0.jar">
-            <sha256 value="c03cbbf5ab447d954efb8268a58dd9f2fe70f9e5d9be0a8371f79d785357eba3" origin="Generated by Gradle"/>
+      <component group="org.eclipse.jdt" name="ecj" version="3.35.0">
+         <artifact name="ecj-3.35.0.jar">
+            <sha256 value="b89df382369f2d16b19b67085426dc0fb7401fe1ad9fba1806af4e9729f5d1c0" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.eclipse.jdt" name="org.eclipse.jdt.core" version="3.35.0">
+         <artifact name="org.eclipse.jdt.core-3.35.0.jar">
+            <sha256 value="8f5bcb00355b271638b9d82a8cccd0e733225cb74c4a3f0f55a7b75d43ee442a" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.eclipse.jetty" name="jetty-annotations" version="9.4.40.v20210413">
@@ -3447,69 +3517,49 @@
             <sha256 value="b564477d092241aaab50c84ac5dd1ac375c0182044d7e5659d3a60a90e2e801a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.core.commands" version="3.10.100">
-         <artifact name="org.eclipse.core.commands-3.10.100.jar">
-            <sha256 value="177c605efd78681e28765b869a2ff5284a79b02d133007a6169a64317cee8633" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="org.eclipse.platform" name="org.eclipse.core.commands" version="3.10.300">
-         <artifact name="org.eclipse.core.commands-3.10.300.jar">
-            <sha256 value="e589b4036723278bcd25e7aa190d1632a58569437a202d5c2b97c7ac9847b874" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="org.eclipse.platform" name="org.eclipse.core.commands" version="3.10.400">
-         <artifact name="org.eclipse.core.commands-3.10.400.jar">
-            <sha256 value="b467c30a1b56029b7338027f8beedd1956f1e19fb0c69b15f56db6968ef3fae6" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="org.eclipse.platform" name="org.eclipse.core.contenttype" version="3.8.0">
-         <artifact name="org.eclipse.core.contenttype-3.8.0.jar">
-            <sha256 value="f90fa8dd89b9da7d5e817281f963b46ad1414d33376354e541464434f0b23c90" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.core.commands" version="3.11.100">
+         <artifact name="org.eclipse.core.commands-3.11.100.jar">
+            <sha256 value="97a503e4c7711e93e0724cb5181eab8a3cb926fbe0a00ab84f13e69e1d156378" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.core.contenttype" version="3.8.200">
-         <artifact name="org.eclipse.core.contenttype-3.8.200.jar">
-            <sha256 value="d32716bfb03e7f3244a5df0f8816a8f0b6644a8c782f54062af8a3b00e1b4e45" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.core.contenttype" version="3.9.100">
+         <artifact name="org.eclipse.core.contenttype-3.9.100.jar">
+            <sha256 value="707e81707e8f2843be90f621aa3b7ff318dd3244e6f4a2982af55f66faf55987" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.core.expressions" version="3.8.200">
-         <artifact name="org.eclipse.core.expressions-3.8.200.jar">
-            <sha256 value="13dbbe3256f1199c69a79c3ba4c6ee79e6c1a0e73f67d49f5f3365b7ad387818" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.core.expressions" version="3.9.100">
+         <artifact name="org.eclipse.core.expressions-3.9.100.jar">
+            <sha256 value="72736638d17cef9183659590d99fd7262a601ed2e3723295737a638db24e23fd" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.core.filesystem" version="1.9.100">
-         <artifact name="org.eclipse.core.filesystem-1.9.100.jar">
-            <sha256 value="36e802c4d9a2c864e7d6b22b8d06f59927d113475ea04b3fd9bf6312d5a97ff6" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.core.filesystem" version="1.10.100">
+         <artifact name="org.eclipse.core.filesystem-1.10.100.jar">
+            <sha256 value="fb9c03bde38e2cfe2d5e32c4ed858e824882cec07e911ae5de2322bd33e6a0a2" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.core.filesystem" version="1.9.500">
-         <artifact name="org.eclipse.core.filesystem-1.9.500.jar">
-            <sha256 value="0c4810a1fa6e871c0bb23b29090a72c83b760af8171495c1325b4711b919072b" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.core.jobs" version="3.15.0">
+         <artifact name="org.eclipse.core.jobs-3.15.0.jar">
+            <sha256 value="857df690c6424a6558b5c96732958f03b9c209258562caf93d9fcbc13242fa11" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.core.jobs" version="3.13.300">
-         <artifact name="org.eclipse.core.jobs-3.13.300.jar">
-            <sha256 value="eb59627a7fcaf8c57dc34c75e24a8a54963ebfe1b7630d6fc72be882ea91eed4" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.core.resources" version="3.19.100">
+         <artifact name="org.eclipse.core.resources-3.19.100.jar">
+            <sha256 value="537af88de2c0e602446ab0fa9dcfec29c617f0891b96fc4cc3fba1655edcc91b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.core.resources" version="3.18.200">
-         <artifact name="org.eclipse.core.resources-3.18.200.jar">
-            <sha256 value="df6e64965c436c96c3d1de0481407a3c5523e320145dac01ca3d2edf4aa66738" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.core.runtime" version="3.29.0">
+         <artifact name="org.eclipse.core.runtime-3.29.0.jar">
+            <sha256 value="096373dff05c425cb37e4924b25b3532a8f8c2992f36d4b6266b0e05a19af814" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.core.runtime" version="3.26.100">
-         <artifact name="org.eclipse.core.runtime-3.26.100.jar">
-            <sha256 value="f90ebd2f052d602281840b2b8cd1f5a7b6d1cf2d5a85b7b0415c70456855d173" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.equinox.app" version="1.6.300">
+         <artifact name="org.eclipse.equinox.app-1.6.300.jar">
+            <sha256 value="eaeb288c9bc78916027d3933d7d1f78c1b8675d54a55a7abdea161b001b7adf1" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.equinox.app" version="1.6.200">
-         <artifact name="org.eclipse.equinox.app-1.6.200.jar">
-            <sha256 value="52840b5fe48ed8f50a26d792d21b2db73b622a0a10d64c18883da6fb85dcbc24" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="org.eclipse.platform" name="org.eclipse.equinox.common" version="3.17.100">
-         <artifact name="org.eclipse.equinox.common-3.17.100.jar">
-            <sha256 value="a3464c02a8a2b0c0dd73e00656064b6ab5cb59387c80bef71244ef18caa9d24d" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.equinox.common" version="3.18.100">
+         <artifact name="org.eclipse.equinox.common-3.18.100.jar">
+            <sha256 value="7c15db46dda5f6f1ff9ecac10eb6ffa9ed1cbd0ed5e6996cb95f0e390902700e" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.eclipse.platform" name="org.eclipse.equinox.event" version="1.6.200">
@@ -3517,19 +3567,19 @@
             <sha256 value="b83c7f7023a3c30daa75e112f3ad2c6d8c30108e19bbe84ec2d25b8bad0efe7a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.equinox.preferences" version="3.10.200">
-         <artifact name="org.eclipse.equinox.preferences-3.10.200.jar">
-            <sha256 value="3312f21ef759b30a6407ecb1fe8f6b552f2e3661f66bdec55831d05a36aa8ff5" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.equinox.preferences" version="3.10.300">
+         <artifact name="org.eclipse.equinox.preferences-3.10.300.jar">
+            <sha256 value="0e231ed4443b1bb21e16ec13a83e975bc04b0503eac2ce6f4b4a7d7b2f53956d" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.equinox.registry" version="3.11.200">
-         <artifact name="org.eclipse.equinox.registry-3.11.200.jar">
-            <sha256 value="7a3668ca406930213e3edb8024ac72c51fbc3f289de63a33254c8070aa8d0a3f" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.equinox.registry" version="3.11.300">
+         <artifact name="org.eclipse.equinox.registry-3.11.300.jar">
+            <sha256 value="3acbb9c857bae9093f3308aff5abe72be5eabcb23e2791139d3571cade659da0" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.equinox.supplement" version="1.10.600">
-         <artifact name="org.eclipse.equinox.supplement-1.10.600.jar">
-            <sha256 value="d36dd1d1b0db36a3dad7a7b7f2f93ca48140fc3e663b47b0d56aed5c5e6fd655" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.equinox.supplement" version="1.10.700">
+         <artifact name="org.eclipse.equinox.supplement-1.10.700.jar">
+            <sha256 value="5df4309856c79492807c17f116ca112ea83c7dd04c2c01049f5d3b4811135068" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.eclipse.platform" name="org.eclipse.osgi" version="3.18.300">
@@ -3537,9 +3587,14 @@
             <sha256 value="bab943e58edd17309238672dba7a45aec9e2dacbdddb818a8cf17723ea13fa22" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.platform" name="org.eclipse.text" version="3.12.300">
-         <artifact name="org.eclipse.text-3.12.300.jar">
-            <sha256 value="6c4f4f01397ae566cb7007ca885ffc5d1813c913d6602d78a8f81801a39060e8" origin="Generated by Gradle"/>
+      <component group="org.eclipse.platform" name="org.eclipse.osgi" version="3.18.500">
+         <artifact name="org.eclipse.osgi-3.18.500.jar">
+            <sha256 value="80b275d58379723b291ea650249cc3809c8f10b34f78aaf988132cd6d434ab4e" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.eclipse.platform" name="org.eclipse.text" version="3.13.100">
+         <artifact name="org.eclipse.text-3.13.100.jar">
+            <sha256 value="89afa204e8e686eaa5f1464a6ceb65ac1b341ad1d0326cab6716c6733ecaf5ac" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.eclipse.sisu" name="org.eclipse.sisu.inject" version="0.3.3">
@@ -3627,54 +3682,24 @@
             <sha256 value="edc8e3ec9796a5f41c1ae44b2d318507ee6ac1212f121d93d33699b3d0aff638" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib" version="1.4.21">
-         <artifact name="kotlin-stdlib-1.4.21.jar">
-            <sha256 value="f78c5d8c09db985912ab83a1de3c3b53ddf208d7b151f06a72358ea3e137d01b" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib" version="1.6.20">
-         <artifact name="kotlin-stdlib-1.6.20.jar">
-            <sha256 value="eeb51c2b67b26233fd81d0bc4f8044ec849718890905763ceffd84a31e2cb799" origin="Generated by Gradle"/>
+      <component group="org.jetbrains.kotlin" name="kotlin-stdlib" version="1.9.10">
+         <artifact name="kotlin-stdlib-1.9.10.jar">
+            <sha256 value="55e989c512b80907799f854309f3bc7782c5b3d13932442d0379d5c472711504" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-common" version="1.4.21">
-         <artifact name="kotlin-stdlib-common-1.4.21.jar">
-            <sha256 value="812cf197d9c4c67e1f47f95e2d72a9b600f0d1124560617bfe9850773eccbcff" origin="Generated by Gradle"/>
+      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-common" version="1.9.10">
+         <artifact name="kotlin-stdlib-common-1.9.10.jar">
+            <sha256 value="cde3341ba18a2ba262b0b7cf6c55b20c90e8d434e42c9a13e6a3f770db965a88" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-common" version="1.6.20">
-         <artifact name="kotlin-stdlib-common-1.6.20.jar">
-            <sha256 value="8da40a2520d30dcb1012176fe93d24e82d08a3e346c37e0343b0fb6f64f6be01" origin="Generated by Gradle"/>
+      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk7" version="1.9.10">
+         <artifact name="kotlin-stdlib-jdk7-1.9.10.jar">
+            <sha256 value="ac6361bf9ad1ed382c2103d9712c47cdec166232b4903ed596e8876b0681c9b7" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk7" version="1.4.21">
-         <artifact name="kotlin-stdlib-jdk7-1.4.21.jar">
-            <sha256 value="50de5f7dad6235064ac6c0ff577f095a91b3306c2547d8bc372291587495024a" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk7" version="1.5.31">
-         <artifact name="kotlin-stdlib-jdk7-1.5.31.jar">
-            <sha256 value="a25bf47353ce899d843cbddee516d621a73473e7fba97f8d0301e7b4aed7c15f" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk7" version="1.6.20">
-         <artifact name="kotlin-stdlib-jdk7-1.6.20.jar">
-            <sha256 value="aa2fa2e81355c4d98dd97da2169bf401f842261378f5b1cbea1aa11855d67620" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk8" version="1.4.21">
-         <artifact name="kotlin-stdlib-jdk8-1.4.21.jar">
-            <sha256 value="8ec3db1516948b2d3524e3afbe75cb5ac59e02d98cb6ef586ef57ba63ca8d11f" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk8" version="1.5.31">
-         <artifact name="kotlin-stdlib-jdk8-1.5.31.jar">
-            <sha256 value="b548f7767aacf029d2417e47440742bd6d3ebede19b60386e23554ce5c4c5fdc" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk8" version="1.6.20">
-         <artifact name="kotlin-stdlib-jdk8-1.6.20.jar">
-            <sha256 value="fdab1bf120e2b5e7ab6d7888e9ebc024ec6b8ca729361296395dab634b213695" origin="Generated by Gradle"/>
+      <component group="org.jetbrains.kotlin" name="kotlin-stdlib-jdk8" version="1.9.10">
+         <artifact name="kotlin-stdlib-jdk8-1.9.10.jar">
+            <sha256 value="a4c74d94d64ce1abe53760fe0389dd941f6fc558d0dab35e47c085a11ec80f28" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.jline" name="jline" version="3.9.0">
@@ -4117,6 +4142,11 @@
             <sha256 value="878fbe521731c072d14d2d65b983b1beae6ad06fda0007b6a8bae81f73f433c4" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.ow2.asm" name="asm-analysis" version="9.7">
+         <artifact name="asm-analysis-9.7.jar">
+            <sha256 value="7bc6bcbc21379948a0c8c467fb0f864206e5b818f6bc0b546872f5c9f941556f" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.ow2.asm" name="asm-commons" version="7.2">
          <artifact name="asm-commons-7.2.jar">
             <sha256 value="0e86b8b179c5fb223d1a880a0ff4960b6978223984b94e62e71135f2d8ea3558" origin="Generated by Gradle"/>
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index 2c3521197d7c4..a4b76b9530d66 100644
Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index efe2ff3449216..9036682bf0f0c 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip
+distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java b/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java
new file mode 100644
index 0000000000000..0fe816bd3721d
--- /dev/null
+++ b/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.core;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to identify a block of code (a whole class, a method, or a field) that needs to be reviewed (for cleanup, remove or change)
+ * before releasing 10.0
+ */
+@Retention(RetentionPolicy.SOURCE)
+@Target({ ElementType.LOCAL_VARIABLE, ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE })
+public @interface UpdateForV10 {
+}
diff --git a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java
index f3f53f1b3c5ea..3c01e490369de 100644
--- a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java
+++ b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java
@@ -203,7 +203,7 @@ public Map<String, String> parse(String inputString) {
             DissectKey key = dissectPair.key();
             byte[] delimiter = dissectPair.delimiter().getBytes(StandardCharsets.UTF_8);
             // start dissection after the first delimiter
-            int i = leadingDelimiter.length();
+            int i = leadingDelimiter.getBytes(StandardCharsets.UTF_8).length;
             int valueStart = i;
             int lookAheadMatches;
             // start walking the input string byte by byte, look ahead for matches where needed
diff --git a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java
index 431b26fc1155d..2893e419a84a3 100644
--- a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java
+++ b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java
@@ -211,6 +211,18 @@ public void testMatchUnicode() {
         assertMatch("%{a->}࿏%{b}", "⟳༒࿏࿏࿏࿏࿏༒⟲", Arrays.asList("a", "b"), Arrays.asList("⟳༒", "༒⟲"));
         assertMatch("%{*a}࿏%{&a}", "⟳༒࿏༒⟲", Arrays.asList("⟳༒"), Arrays.asList("༒⟲"));
         assertMatch("%{}࿏%{a}", "⟳༒࿏༒⟲", Arrays.asList("a"), Arrays.asList("༒⟲"));
+        assertMatch(
+            "Zürich, the %{adjective} city in Switzerland",
+            "Zürich, the largest city in Switzerland",
+            Arrays.asList("adjective"),
+            Arrays.asList("largest")
+        );
+        assertMatch(
+            "Zürich, the %{one} city in Switzerland; Zürich, the %{two} city in Switzerland",
+            "Zürich, the largest city in Switzerland; Zürich, the LARGEST city in Switzerland",
+            Arrays.asList("one", "two"),
+            Arrays.asList("largest", "LARGEST")
+        );
     }
 
     public void testMatchRemainder() {
diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java
index d233dcc81a3fc..1e7ac3f8097e9 100644
--- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java
+++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java
@@ -43,6 +43,7 @@ public class WellKnownText {
     public static final String RPAREN = ")";
     public static final String COMMA = ",";
     public static final String NAN = "NaN";
+    public static final int MAX_NESTED_DEPTH = 1000;
 
     private static final String NUMBER = "<NUMBER>";
     private static final String EOF = "END-OF-STREAM";
@@ -425,7 +426,7 @@ public static Geometry fromWKT(GeometryValidator validator, boolean coerce, Stri
             tokenizer.whitespaceChars('\r', '\r');
             tokenizer.whitespaceChars('\n', '\n');
             tokenizer.commentChar('#');
-            Geometry geometry = parseGeometry(tokenizer, coerce);
+            Geometry geometry = parseGeometry(tokenizer, coerce, 0);
             validator.validate(geometry);
             return geometry;
         } finally {
@@ -436,40 +437,35 @@ public static Geometry fromWKT(GeometryValidator validator, boolean coerce, Stri
     /**
      * parse geometry from the stream tokenizer
      */
-    private static Geometry parseGeometry(StreamTokenizer stream, boolean coerce) throws IOException, ParseException {
+    private static Geometry parseGeometry(StreamTokenizer stream, boolean coerce, int depth) throws IOException, ParseException {
         final String type = nextWord(stream).toLowerCase(Locale.ROOT);
-        switch (type) {
-            case "point":
-                return parsePoint(stream);
-            case "multipoint":
-                return parseMultiPoint(stream);
-            case "linestring":
-                return parseLine(stream);
-            case "multilinestring":
-                return parseMultiLine(stream);
-            case "polygon":
-                return parsePolygon(stream, coerce);
-            case "multipolygon":
-                return parseMultiPolygon(stream, coerce);
-            case "bbox":
-                return parseBBox(stream);
-            case "geometrycollection":
-                return parseGeometryCollection(stream, coerce);
-            case "circle": // Not part of the standard, but we need it for internal serialization
-                return parseCircle(stream);
-        }
-        throw new IllegalArgumentException("Unknown geometry type: " + type);
-    }
-
-    private static GeometryCollection<Geometry> parseGeometryCollection(StreamTokenizer stream, boolean coerce) throws IOException,
-        ParseException {
+        return switch (type) {
+            case "point" -> parsePoint(stream);
+            case "multipoint" -> parseMultiPoint(stream);
+            case "linestring" -> parseLine(stream);
+            case "multilinestring" -> parseMultiLine(stream);
+            case "polygon" -> parsePolygon(stream, coerce);
+            case "multipolygon" -> parseMultiPolygon(stream, coerce);
+            case "bbox" -> parseBBox(stream);
+            case "geometrycollection" -> parseGeometryCollection(stream, coerce, depth + 1);
+            case "circle" -> // Not part of the standard, but we need it for internal serialization
+                parseCircle(stream);
+            default -> throw new IllegalArgumentException("Unknown geometry type: " + type);
+        };
+    }
+
+    private static GeometryCollection<Geometry> parseGeometryCollection(StreamTokenizer stream, boolean coerce, int depth)
+        throws IOException, ParseException {
         if (nextEmptyOrOpen(stream).equals(EMPTY)) {
             return GeometryCollection.EMPTY;
         }
+        if (depth > MAX_NESTED_DEPTH) {
+            throw new ParseException("maximum nested depth of " + MAX_NESTED_DEPTH + " exceeded", stream.lineno());
+        }
         List<Geometry> shapes = new ArrayList<>();
-        shapes.add(parseGeometry(stream, coerce));
+        shapes.add(parseGeometry(stream, coerce, depth));
         while (nextCloserOrComma(stream).equals(COMMA)) {
-            shapes.add(parseGeometry(stream, coerce));
+            shapes.add(parseGeometry(stream, coerce, depth));
         }
         return new GeometryCollection<>(shapes);
     }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java
index 6a7bda7f9e0bb..b3f7aa610153b 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java
@@ -19,6 +19,8 @@
 import java.util.Arrays;
 import java.util.Collections;
 
+import static org.hamcrest.Matchers.containsString;
+
 public class GeometryCollectionTests extends BaseGeometryTestCase<GeometryCollection<Geometry>> {
     @Override
     protected GeometryCollection<Geometry> createTestInstance(boolean hasAlt) {
@@ -65,6 +67,31 @@ public void testInitValidation() {
         StandardValidator.instance(true).validate(new GeometryCollection<Geometry>(Collections.singletonList(new Point(20, 10, 30))));
     }
 
+    public void testDeeplyNestedCollection() throws IOException, ParseException {
+        String wkt = makeDeeplyNestedGeometryCollectionWKT(WellKnownText.MAX_NESTED_DEPTH);
+        Geometry parsed = WellKnownText.fromWKT(GeographyValidator.instance(true), true, wkt);
+        assertEquals(WellKnownText.MAX_NESTED_DEPTH, countNestedGeometryCollections((GeometryCollection<?>) parsed));
+    }
+
+    public void testTooDeeplyNestedCollection() {
+        String wkt = makeDeeplyNestedGeometryCollectionWKT(WellKnownText.MAX_NESTED_DEPTH + 1);
+        ParseException ex = expectThrows(ParseException.class, () -> WellKnownText.fromWKT(GeographyValidator.instance(true), true, wkt));
+        assertThat(ex.getMessage(), containsString("maximum nested depth of " + WellKnownText.MAX_NESTED_DEPTH));
+    }
+
+    private String makeDeeplyNestedGeometryCollectionWKT(int depth) {
+        return "GEOMETRYCOLLECTION (".repeat(depth) + "POINT (20.0 10.0)" + ")".repeat(depth);
+    }
+
+    private int countNestedGeometryCollections(GeometryCollection<?> geometry) {
+        int count = 1;
+        while (geometry.get(0) instanceof GeometryCollection<?> g) {
+            count += 1;
+            geometry = g;
+        }
+        return count;
+    }
+
     @Override
     protected GeometryCollection<Geometry> mutateInstance(GeometryCollection<Geometry> instance) {
         return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
diff --git a/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java b/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java
index bcf9253866931..3b10d58815169 100644
--- a/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java
+++ b/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java
@@ -8,12 +8,17 @@
 
 package org.elasticsearch.grok;
 
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Deque;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 
 public class PatternBank {
 
@@ -57,52 +62,102 @@ public PatternBank extendWith(Map<String, String> extraPatterns) {
     }
 
     /**
-     * Checks whether patterns reference each other in a circular manner and if so fail with an exception.
+     * Checks whether patterns reference each other in a circular manner and if so fail with an IllegalArgumentException. It will also
+     * fail if any pattern value contains a pattern name that does not exist in the bank.
      * <p>
      * In a pattern, anything between <code>%{</code> and <code>}</code> or <code>:</code> is considered
      * a reference to another named pattern. This method will navigate to all these named patterns and
      * check for a circular reference.
      */
     static void forbidCircularReferences(Map<String, String> bank) {
-        // first ensure that the pattern bank contains no simple circular references (i.e., any pattern
-        // containing an immediate reference to itself) as those can cause the remainder of this algorithm
-        // to recurse infinitely
-        for (Map.Entry<String, String> entry : bank.entrySet()) {
-            if (patternReferencesItself(entry.getValue(), entry.getKey())) {
-                throw new IllegalArgumentException("circular reference in pattern [" + entry.getKey() + "][" + entry.getValue() + "]");
+        Set<String> allVisitedNodes = new HashSet<>();
+        Set<String> nodesVisitedMoreThanOnceInAPath = new HashSet<>();
+        // Walk the full path starting at each node in the graph:
+        for (String traversalStartNode : bank.keySet()) {
+            if (nodesVisitedMoreThanOnceInAPath.contains(traversalStartNode) == false && allVisitedNodes.contains(traversalStartNode)) {
+                // If we have seen this node before in a path, and it only appeared once in that path, there is no need to check it again
+                continue;
             }
-        }
-
-        // next, recursively check any other pattern names referenced in each pattern
-        for (Map.Entry<String, String> entry : bank.entrySet()) {
-            String name = entry.getKey();
-            String pattern = entry.getValue();
-            innerForbidCircularReferences(bank, name, new ArrayList<>(), pattern);
+            Set<String> visitedFromThisStartNode = new LinkedHashSet<>();
+            /*
+             * This stack records where we are in the graph. Each String[] in the stack represents a collection of neighbors to the first
+             * non-null node in the layer below it. Null means that the path from that location has been fully traversed. Once all nodes
+             * at a layer have been set to null, the layer is popped. So for example say we have the graph
+             * ( 1 -> (2 -> (4, 5, 8), 3 -> (6, 7))) then when we are at 6 via 1 -> 3 -> 6, the stack looks like this:
+             * [6, 7]
+             * [null, 3]
+             * [1]
+             */
+            Deque<String[]> stack = new ArrayDeque<>();
+            stack.push(new String[] { traversalStartNode });
+            // This is used so that we know that we're unwinding the stack and know not to get the current node's neighbors again.
+            boolean unwinding = false;
+            while (stack.isEmpty() == false) {
+                String[] currentLevel = stack.peek();
+                int firstNonNullIndex = findFirstNonNull(currentLevel);
+                String node = currentLevel[firstNonNullIndex];
+                boolean endOfThisPath = false;
+                if (unwinding) {
+                    // We have completed all of this node's neighbors and have popped back to the node
+                    endOfThisPath = true;
+                } else if (traversalStartNode.equals(node) && stack.size() > 1) {
+                    Deque<String> reversedPath = new ArrayDeque<>();
+                    for (String[] level : stack) {
+                        reversedPath.push(level[findFirstNonNull(level)]);
+                    }
+                    throw new IllegalArgumentException("circular reference detected: " + String.join("->", reversedPath));
+                } else if (visitedFromThisStartNode.contains(node)) {
+                    /*
+                     * We are only looking for a cycle starting and ending at traversalStartNode right now. But this node has been
+                     * visited more than once in the path rooted at traversalStartNode. This could be because it is a cycle, or could be
+                     * because two nodes in the path both point to it. We add it to nodesVisitedMoreThanOnceInAPath so that we make sure
+                     * to check the path rooted at this node later.
+                     */
+                    nodesVisitedMoreThanOnceInAPath.add(node);
+                    endOfThisPath = true;
+                } else {
+                    visitedFromThisStartNode.add(node);
+                    String[] neighbors = getPatternNamesForPattern(bank, node);
+                    if (neighbors.length == 0) {
+                        endOfThisPath = true;
+                    } else {
+                        stack.push(neighbors);
+                    }
+                }
+                if (endOfThisPath) {
+                    if (firstNonNullIndex == currentLevel.length - 1) {
+                        // We have handled all the neighbors at this level -- there are no more non-null ones
+                        stack.pop();
+                        unwinding = true;
+                    } else {
+                        currentLevel[firstNonNullIndex] = null;
+                        unwinding = false;
+                    }
+                } else {
+                    unwinding = false;
+                }
+            }
+            allVisitedNodes.addAll(visitedFromThisStartNode);
         }
     }
 
-    private static void innerForbidCircularReferences(Map<String, String> bank, String patternName, List<String> path, String pattern) {
-        if (patternReferencesItself(pattern, patternName)) {
-            String message;
-            if (path.isEmpty()) {
-                message = "circular reference in pattern [" + patternName + "][" + pattern + "]";
-            } else {
-                message = "circular reference in pattern ["
-                    + path.remove(path.size() - 1)
-                    + "]["
-                    + pattern
-                    + "] back to pattern ["
-                    + patternName
-                    + "]";
-                // add rest of the path:
-                if (path.isEmpty() == false) {
-                    message += " via patterns [" + String.join("=>", path) + "]";
-                }
+    private static int findFirstNonNull(String[] level) {
+        for (int i = 0; i < level.length; i++) {
+            if (level[i] != null) {
+                return i;
             }
-            throw new IllegalArgumentException(message);
         }
+        return -1;
+    }
 
-        // next check any other pattern names found in the pattern
+    /**
+     * This method returns the array of pattern names (if any) found in the bank for the pattern named patternName. If no pattern names
+     * are found, an empty array is returned. If any of the list of pattern names to be returned does not exist in the bank, an exception
+     * is thrown.
+     */
+    private static String[] getPatternNamesForPattern(Map<String, String> bank, String patternName) {
+        String pattern = bank.get(patternName);
+        List<String> patternReferences = new ArrayList<>();
         for (int i = pattern.indexOf("%{"); i != -1; i = pattern.indexOf("%{", i + 1)) {
             int begin = i + 2;
             int bracketIndex = pattern.indexOf('}', begin);
@@ -112,25 +167,22 @@ private static void innerForbidCircularReferences(Map<String, String> bank, Stri
                 end = bracketIndex;
             } else if (columnIndex != -1 && bracketIndex == -1) {
                 end = columnIndex;
-            } else if (bracketIndex != -1 && columnIndex != -1) {
+            } else if (bracketIndex != -1) {
                 end = Math.min(bracketIndex, columnIndex);
             } else {
                 throw new IllegalArgumentException("pattern [" + pattern + "] has an invalid syntax");
             }
             String otherPatternName = pattern.substring(begin, end);
-            path.add(otherPatternName);
-            String otherPattern = bank.get(otherPatternName);
-            if (otherPattern == null) {
-                throw new IllegalArgumentException(
-                    "pattern [" + patternName + "] is referencing a non-existent pattern [" + otherPatternName + "]"
-                );
+            if (patternReferences.contains(otherPatternName) == false) {
+                patternReferences.add(otherPatternName);
+                String otherPattern = bank.get(otherPatternName);
+                if (otherPattern == null) {
+                    throw new IllegalArgumentException(
+                        "pattern [" + patternName + "] is referencing a non-existent pattern [" + otherPatternName + "]"
+                    );
+                }
             }
-
-            innerForbidCircularReferences(bank, patternName, path, otherPattern);
         }
-    }
-
-    private static boolean patternReferencesItself(String pattern, String patternName) {
-        return pattern.contains("%{" + patternName + "}") || pattern.contains("%{" + patternName + ":");
+        return patternReferences.toArray(new String[0]);
     }
 }
diff --git a/libs/grok/src/test/java/org/elasticsearch/grok/MatcherWatchdogTests.java b/libs/grok/src/test/java/org/elasticsearch/grok/MatcherWatchdogTests.java
index b66778743aec0..5ed1a7d13b80a 100644
--- a/libs/grok/src/test/java/org/elasticsearch/grok/MatcherWatchdogTests.java
+++ b/libs/grok/src/test/java/org/elasticsearch/grok/MatcherWatchdogTests.java
@@ -7,12 +7,12 @@
  */
 package org.elasticsearch.grok;
 
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.test.ESTestCase;
 import org.joni.Matcher;
 import org.mockito.Mockito;
 
 import java.util.Map;
-import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -77,16 +77,17 @@ public void testIdleIfNothingRegistered() throws Exception {
         );
         // Periodic action is not scheduled because no thread is registered
         verifyNoMoreInteractions(threadPool);
-        CompletableFuture<Runnable> commandFuture = new CompletableFuture<>();
+
+        PlainActionFuture<Runnable> commandFuture = new PlainActionFuture<>();
         // Periodic action is scheduled because a thread is registered
         doAnswer(invocationOnMock -> {
-            commandFuture.complete((Runnable) invocationOnMock.getArguments()[0]);
+            commandFuture.onResponse(invocationOnMock.getArgument(0));
             return null;
         }).when(threadPool).schedule(any(Runnable.class), eq(interval), eq(TimeUnit.MILLISECONDS));
         Matcher matcher = mock(Matcher.class);
         watchdog.register(matcher);
         // Registering the first thread should have caused the command to get scheduled again
-        Runnable command = commandFuture.get(1L, TimeUnit.MILLISECONDS);
+        Runnable command = safeGet(commandFuture);
         Mockito.reset(threadPool);
         watchdog.unregister(matcher);
         command.run();
diff --git a/libs/grok/src/test/java/org/elasticsearch/grok/PatternBankTests.java b/libs/grok/src/test/java/org/elasticsearch/grok/PatternBankTests.java
index dcc7ab431611a..08a4965cdb371 100644
--- a/libs/grok/src/test/java/org/elasticsearch/grok/PatternBankTests.java
+++ b/libs/grok/src/test/java/org/elasticsearch/grok/PatternBankTests.java
@@ -11,8 +11,13 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.Map;
-import java.util.TreeMap;
+import java.util.Set;
+
+import static org.elasticsearch.test.ESTestCase.randomBoolean;
+import static org.hamcrest.Matchers.containsString;
 
 public class PatternBankTests extends ESTestCase {
 
@@ -32,7 +37,7 @@ public void testBankCannotBeNull() {
 
     public void testConstructorValidatesCircularReferences() {
         var e = expectThrows(IllegalArgumentException.class, () -> new PatternBank(Map.of("NAME", "!!!%{NAME}!!!")));
-        assertEquals("circular reference in pattern [NAME][!!!%{NAME}!!!]", e.getMessage());
+        assertEquals("circular reference detected: NAME->NAME", e.getMessage());
     }
 
     public void testExtendWith() {
@@ -48,36 +53,36 @@ public void testExtendWith() {
 
     public void testCircularReference() {
         var e = expectThrows(IllegalArgumentException.class, () -> PatternBank.forbidCircularReferences(Map.of("NAME", "!!!%{NAME}!!!")));
-        assertEquals("circular reference in pattern [NAME][!!!%{NAME}!!!]", e.getMessage());
+        assertEquals("circular reference detected: NAME->NAME", e.getMessage());
 
         e = expectThrows(IllegalArgumentException.class, () -> PatternBank.forbidCircularReferences(Map.of("NAME", "!!!%{NAME:name}!!!")));
-        assertEquals("circular reference in pattern [NAME][!!!%{NAME:name}!!!]", e.getMessage());
+        assertEquals("circular reference detected: NAME->NAME", e.getMessage());
 
         e = expectThrows(
             IllegalArgumentException.class,
             () -> { PatternBank.forbidCircularReferences(Map.of("NAME", "!!!%{NAME:name:int}!!!")); }
         );
-        assertEquals("circular reference in pattern [NAME][!!!%{NAME:name:int}!!!]", e.getMessage());
+        assertEquals("circular reference detected: NAME->NAME", e.getMessage());
 
         e = expectThrows(IllegalArgumentException.class, () -> {
-            Map<String, String> bank = new TreeMap<>();
+            Map<String, String> bank = new LinkedHashMap<>();
             bank.put("NAME1", "!!!%{NAME2}!!!");
             bank.put("NAME2", "!!!%{NAME1}!!!");
             PatternBank.forbidCircularReferences(bank);
         });
-        assertEquals("circular reference in pattern [NAME2][!!!%{NAME1}!!!] back to pattern [NAME1]", e.getMessage());
+        assertEquals("circular reference detected: NAME1->NAME2->NAME1", e.getMessage());
 
         e = expectThrows(IllegalArgumentException.class, () -> {
-            Map<String, String> bank = new TreeMap<>();
+            Map<String, String> bank = new LinkedHashMap<>();
             bank.put("NAME1", "!!!%{NAME2}!!!");
             bank.put("NAME2", "!!!%{NAME3}!!!");
             bank.put("NAME3", "!!!%{NAME1}!!!");
             PatternBank.forbidCircularReferences(bank);
         });
-        assertEquals("circular reference in pattern [NAME3][!!!%{NAME1}!!!] back to pattern [NAME1] via patterns [NAME2]", e.getMessage());
+        assertEquals("circular reference detected: NAME1->NAME2->NAME3->NAME1", e.getMessage());
 
         e = expectThrows(IllegalArgumentException.class, () -> {
-            Map<String, String> bank = new TreeMap<>();
+            Map<String, String> bank = new LinkedHashMap<>();
             bank.put("NAME1", "!!!%{NAME2}!!!");
             bank.put("NAME2", "!!!%{NAME3}!!!");
             bank.put("NAME3", "!!!%{NAME4}!!!");
@@ -85,10 +90,78 @@ public void testCircularReference() {
             bank.put("NAME5", "!!!%{NAME1}!!!");
             PatternBank.forbidCircularReferences(bank);
         });
-        assertEquals(
-            "circular reference in pattern [NAME5][!!!%{NAME1}!!!] back to pattern [NAME1] via patterns [NAME2=>NAME3=>NAME4]",
-            e.getMessage()
-        );
+        assertEquals("circular reference detected: NAME1->NAME2->NAME3->NAME4->NAME5->NAME1", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> {
+            Map<String, String> bank = new LinkedHashMap<>();
+            bank.put("NAME1", "!!!%{NAME2}!!!");
+            bank.put("NAME2", "!!!%{NAME3}!!!");
+            bank.put("NAME3", "!!!%{NAME2}!!!");
+            PatternBank.forbidCircularReferences(bank);
+        });
+        assertEquals("circular reference detected: NAME2->NAME3->NAME2", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> {
+            Map<String, String> bank = new LinkedHashMap<>();
+            bank.put("NAME1", "!!!%{NAME2}!!!");
+            bank.put("NAME2", "!!!%{NAME2}!!%{NAME3}!");
+            bank.put("NAME3", "!!!%{NAME1}!!!");
+            PatternBank.forbidCircularReferences(bank);
+        });
+        assertEquals("circular reference detected: NAME1->NAME2->NAME3->NAME1", e.getMessage());
+
+        {
+            Map<String, String> bank = new HashMap<>();
+            bank.put("NAME1", "!!!%{NAME2}!!!%{NAME3}%{NAME4}");
+            bank.put("NAME2", "!!!%{NAME3}!!!");
+            bank.put("NAME3", "!!!!!!");
+            bank.put("NAME4", "!!!%{NAME5}!!!");
+            bank.put("NAME5", "!!!!!!");
+            PatternBank.forbidCircularReferences(bank);
+        }
+
+        e = expectThrows(IllegalArgumentException.class, () -> {
+            Map<String, String> bank = new LinkedHashMap<>();
+            bank.put("NAME1", "!!!%{NAME2}!!!%{NAME3}%{NAME4}");
+            bank.put("NAME2", "!!!%{NAME3}!!!");
+            bank.put("NAME3", "!!!!!!");
+            bank.put("NAME4", "!!!%{NAME5}!!!");
+            bank.put("NAME5", "!!!%{NAME1}!!!");
+            PatternBank.forbidCircularReferences(bank);
+        });
+        assertEquals("circular reference detected: NAME1->NAME4->NAME5->NAME1", e.getMessage());
+
+        {
+            Map<String, String> bank = new HashMap<>();
+            bank.put("NAME1", "!!!%{NAME2}!!!");
+            bank.put("NAME2", "!!!%{NAME3}!!!");
+            bank.put("NAME3", "!!!!!!");
+            bank.put("NAME4", "!!!%{NAME5}!!!");
+            bank.put("NAME5", "!!!%{NAME1}!!!");
+            PatternBank.forbidCircularReferences(bank);
+        }
+
+        e = expectThrows(IllegalArgumentException.class, () -> {
+            Map<String, String> bank = new LinkedHashMap<>();
+            bank.put("NAME1", "!!!%{NAME2} %{NAME3}!!!");
+            bank.put("NAME2", "!!!%{NAME4} %{NAME5}!!!");
+            bank.put("NAME3", "!!!!!!");
+            bank.put("NAME4", "!!!!!!");
+            bank.put("NAME5", "!!!%{NAME1}!!!");
+            PatternBank.forbidCircularReferences(bank);
+        });
+        assertEquals("circular reference detected: NAME1->NAME2->NAME5->NAME1", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> {
+            Map<String, String> bank = new LinkedHashMap<>();
+            bank.put("NAME1", "!!!%{NAME2} %{NAME3}!!!");
+            bank.put("NAME2", "!!!%{NAME4} %{NAME5}!!!");
+            bank.put("NAME3", "!!!%{NAME1}!!!");
+            bank.put("NAME4", "!!!!!!");
+            bank.put("NAME5", "!!!!!!");
+            PatternBank.forbidCircularReferences(bank);
+        });
+        assertEquals("circular reference detected: NAME1->NAME3->NAME1", e.getMessage());
     }
 
     public void testCircularSelfReference() {
@@ -96,7 +169,7 @@ public void testCircularSelfReference() {
             IllegalArgumentException.class,
             () -> PatternBank.forbidCircularReferences(Map.of("ANOTHER", "%{INT}", "INT", "%{INT}"))
         );
-        assertEquals("circular reference in pattern [INT][%{INT}]", e.getMessage());
+        assertEquals("circular reference detected: INT->INT", e.getMessage());
     }
 
     public void testInvalidPatternReferences() {
@@ -112,4 +185,80 @@ public void testInvalidPatternReferences() {
         );
         assertEquals("pattern [%{VALID] has an invalid syntax", e.getMessage());
     }
+
+    public void testDeepGraphOfPatterns() {
+        Map<String, String> patternBankMap = randomBoolean() ? new HashMap<>() : new LinkedHashMap<>();
+        final int nodeCount = 20_000;
+        for (int i = 0; i < nodeCount - 1; i++) {
+            patternBankMap.put("FOO" + i, "%{FOO" + (i + 1) + "}");
+        }
+        patternBankMap.put("FOO" + (nodeCount - 1), "foo");
+        new PatternBank(patternBankMap);
+    }
+
+    public void testRandomBanksWithoutCycles() {
+        /*
+         * This creates a large number of pattens, each of which refers to a large number of patterns. But there are no cycles in any of
+         * these since each pattern only references patterns with a higher ID. We don't expect any exceptions here.
+         */
+        Map<String, String> patternBankMap = randomBoolean() ? new HashMap<>() : new LinkedHashMap<>();
+        final int nodeCount = 500;
+        for (int i = 0; i < nodeCount - 1; i++) {
+            StringBuilder patternBuilder = new StringBuilder();
+            for (int j = 0; j < randomIntBetween(0, 20); j++) {
+                patternBuilder.append("%{FOO-" + randomIntBetween(i + 1, nodeCount - 1) + "}");
+            }
+            patternBankMap.put("FOO-" + i, patternBuilder.toString());
+        }
+        patternBankMap.put("FOO-" + (nodeCount - 1), "foo");
+        new PatternBank(patternBankMap);
+    }
+
+    public void testRandomBanksWithCycles() {
+        /*
+         * This creates a large number of pattens, each of which refers to a large number of patterns. We have at least one cycle because
+         * we pick a node at random, and make sure that a node that it links (or one of its descendants) to links back. If no descendant
+         * links back to it, we create an artificial cycle at the end.
+         */
+        Map<String, String> patternBankMap = new LinkedHashMap<>();
+        final int nodeCount = 500;
+        int nodeToHaveCycle = randomIntBetween(0, nodeCount);
+        int nodeToPotentiallyCreateCycle = -1;
+        boolean haveCreatedCycle = false;
+        for (int i = 0; i < nodeCount - 1; i++) {
+            StringBuilder patternBuilder = new StringBuilder();
+            int numberOfLinkedPatterns = randomIntBetween(1, 20);
+            int nodeToLinkBackIndex = randomIntBetween(0, numberOfLinkedPatterns);
+            Set<Integer> childNodes = new HashSet<>();
+            for (int j = 0; j < numberOfLinkedPatterns; j++) {
+                int childNode = randomIntBetween(i + 1, nodeCount - 1);
+                childNodes.add(childNode);
+                patternBuilder.append("%{FOO-" + childNode + "}");
+                if (i == nodeToHaveCycle) {
+                    if (nodeToLinkBackIndex == j) {
+                        nodeToPotentiallyCreateCycle = childNode;
+                    }
+                }
+            }
+            if (i == nodeToPotentiallyCreateCycle) {
+                // We either create the cycle here, or randomly pick a child node to maybe create the cycle
+                if (randomBoolean()) {
+                    patternBuilder.append("%{FOO-" + nodeToHaveCycle + "}");
+                    haveCreatedCycle = true;
+                } else {
+                    nodeToPotentiallyCreateCycle = randomFrom(childNodes);
+                }
+            }
+            patternBankMap.put("FOO-" + i, patternBuilder.toString());
+        }
+        if (haveCreatedCycle) {
+            patternBankMap.put("FOO-" + (nodeCount - 1), "foo");
+        } else {
+            // We didn't randomly create a cycle, so just force one in this last pattern
+            nodeToHaveCycle = nodeCount - 1;
+            patternBankMap.put("FOO-" + nodeToHaveCycle, "%{FOO-" + nodeToHaveCycle + "}");
+        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new PatternBank(patternBankMap));
+        assertThat(e.getMessage(), containsString("FOO-" + nodeToHaveCycle));
+    }
 }
diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java
index 13218a9b206a5..30801b4f0b078 100644
--- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java
+++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.logstashbridge.StableBridgeAPI;
 import org.elasticsearch.logstashbridge.common.SettingsBridge;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.concurrent.TimeUnit;
@@ -17,7 +18,7 @@
 public class ThreadPoolBridge extends StableBridgeAPI.Proxy<ThreadPool> {
 
     public ThreadPoolBridge(final SettingsBridge settingsBridge) {
-        this(new ThreadPool(settingsBridge.unwrap(), MeterRegistry.NOOP));
+        this(new ThreadPool(settingsBridge.unwrap(), MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders()));
     }
 
     public ThreadPoolBridge(final ThreadPool delegate) {
diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java
index 454581ae70b51..e0233187425ea 100644
--- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java
+++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java
@@ -8,14 +8,15 @@
 
 package org.elasticsearch.nativeaccess.jna;
 
+import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.nativeaccess.lib.JavaLibrary;
 import org.elasticsearch.nativeaccess.lib.Kernel32Library;
 import org.elasticsearch.nativeaccess.lib.LinuxCLibrary;
+import org.elasticsearch.nativeaccess.lib.LoaderHelper;
 import org.elasticsearch.nativeaccess.lib.MacCLibrary;
 import org.elasticsearch.nativeaccess.lib.NativeLibrary;
 import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider;
 import org.elasticsearch.nativeaccess.lib.PosixCLibrary;
-import org.elasticsearch.nativeaccess.lib.SystemdLibrary;
 import org.elasticsearch.nativeaccess.lib.VectorLibrary;
 import org.elasticsearch.nativeaccess.lib.ZstdLibrary;
 
@@ -24,6 +25,10 @@
 
 public class JnaNativeLibraryProvider extends NativeLibraryProvider {
 
+    static {
+        setJnaLibraryPath();
+    }
+
     public JnaNativeLibraryProvider() {
         super(
             "jna",
@@ -38,8 +43,6 @@ public JnaNativeLibraryProvider() {
                 JnaMacCLibrary::new,
                 Kernel32Library.class,
                 JnaKernel32Library::new,
-                SystemdLibrary.class,
-                JnaSystemdLibrary::new,
                 ZstdLibrary.class,
                 JnaZstdLibrary::new,
                 VectorLibrary.class,
@@ -48,6 +51,11 @@ public JnaNativeLibraryProvider() {
         );
     }
 
+    @SuppressForbidden(reason = "jna library path must be set for load library to work with our own libs")
+    private static void setJnaLibraryPath() {
+        System.setProperty("jna.library.path", LoaderHelper.platformLibDir.toString());
+    }
+
     private static Supplier<NativeLibrary> notImplemented() {
         return () -> { throw new AssertionError(); };
     }
diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java
index d984d239e0b39..82a69e4864d94 100644
--- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java
+++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaPosixCLibrary.java
@@ -16,6 +16,7 @@
 import com.sun.jna.Pointer;
 import com.sun.jna.Structure;
 
+import org.elasticsearch.nativeaccess.CloseableByteBuffer;
 import org.elasticsearch.nativeaccess.lib.PosixCLibrary;
 
 import java.util.Arrays;
@@ -109,6 +110,16 @@ public long bytesalloc() {
         }
     }
 
+    public static class JnaSockAddr implements SockAddr {
+        final Memory memory;
+
+        JnaSockAddr(String path) {
+            this.memory = new Memory(110);
+            memory.setShort(0, AF_UNIX);
+            memory.setString(2, path, "UTF-8");
+        }
+    }
+
     private interface NativeFunctions extends Library {
         int geteuid();
 
@@ -126,6 +137,12 @@ private interface NativeFunctions extends Library {
 
         int close(int fd);
 
+        int socket(int domain, int type, int protocol);
+
+        int connect(int sockfd, Pointer addr, int addrlen);
+
+        long send(int sockfd, Pointer buf, long buflen, int flags);
+
         String strerror(int errno);
     }
 
@@ -235,6 +252,30 @@ public int fstat64(int fd, Stat64 stats) {
         return fstat64.fstat64(fd, jnaStats.memory);
     }
 
+    @Override
+    public int socket(int domain, int type, int protocol) {
+        return functions.socket(domain, type, protocol);
+    }
+
+    @Override
+    public SockAddr newUnixSockAddr(String path) {
+        return new JnaSockAddr(path);
+    }
+
+    @Override
+    public int connect(int sockfd, SockAddr addr) {
+        assert addr instanceof JnaSockAddr;
+        var jnaAddr = (JnaSockAddr) addr;
+        return functions.connect(sockfd, jnaAddr.memory, (int) jnaAddr.memory.size());
+    }
+
+    @Override
+    public long send(int sockfd, CloseableByteBuffer buffer, int flags) {
+        assert buffer instanceof JnaCloseableByteBuffer;
+        var nativeBuffer = (JnaCloseableByteBuffer) buffer;
+        return functions.send(sockfd, nativeBuffer.memory, nativeBuffer.buffer().remaining(), flags);
+    }
+
     @Override
     public String strerror(int errno) {
         return functions.strerror(errno);
diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaSystemdLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaSystemdLibrary.java
deleted file mode 100644
index f06361e8807c5..0000000000000
--- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaSystemdLibrary.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.nativeaccess.jna;
-
-import com.sun.jna.Library;
-import com.sun.jna.Native;
-
-import org.elasticsearch.nativeaccess.lib.SystemdLibrary;
-
-class JnaSystemdLibrary implements SystemdLibrary {
-    private interface NativeFunctions extends Library {
-        int sd_notify(int unset_environment, String state);
-    }
-
-    private final NativeFunctions functions;
-
-    JnaSystemdLibrary() {
-        this.functions = Native.load("libsystemd.so.0", NativeFunctions.class);
-    }
-
-    @Override
-    public int sd_notify(int unset_environment, String state) {
-        return functions.sd_notify(unset_environment, state);
-    }
-}
diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java
index f6e6035a8aba6..e1ea28e8786f5 100644
--- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java
+++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/LinuxNativeAccess.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.nativeaccess.lib.LinuxCLibrary.SockFProg;
 import org.elasticsearch.nativeaccess.lib.LinuxCLibrary.SockFilter;
 import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider;
-import org.elasticsearch.nativeaccess.lib.SystemdLibrary;
+import org.elasticsearch.nativeaccess.lib.PosixCLibrary;
 
 import java.util.Map;
 
@@ -92,7 +92,14 @@ record Arch(
     LinuxNativeAccess(NativeLibraryProvider libraryProvider) {
         super("Linux", libraryProvider, new PosixConstants(-1L, 9, 1, 8, 64, 144, 48, 64));
         this.linuxLibc = libraryProvider.getLibrary(LinuxCLibrary.class);
-        this.systemd = new Systemd(libraryProvider.getLibrary(SystemdLibrary.class));
+        String socketPath = System.getenv("NOTIFY_SOCKET");
+        if (socketPath == null) {
+            this.systemd = null; // not running under systemd
+        } else {
+            logger.debug("Systemd socket path: {}", socketPath);
+            var buffer = newBuffer(64);
+            this.systemd = new Systemd(libraryProvider.getLibrary(PosixCLibrary.class), socketPath, buffer);
+        }
     }
 
     @Override
diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java
index 2ce09e567c284..1cf08fce0c2af 100644
--- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java
+++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/PosixNativeAccess.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.nativeaccess;
 
+import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider;
 import org.elasticsearch.nativeaccess.lib.PosixCLibrary;
 import org.elasticsearch.nativeaccess.lib.VectorLibrary;
@@ -147,10 +148,22 @@ public OptionalLong allocatedSizeInBytes(Path path) {
         return OptionalLong.of(stats.st_blocks() * 512);
     }
 
+    @SuppressForbidden(reason = "Using mkdirs")
     @Override
     public void tryPreallocate(Path file, long newSize) {
+        var absolutePath = file.toAbsolutePath();
+        var directory = absolutePath.getParent();
+        directory.toFile().mkdirs();
         // get fd and current size, then pass to OS variant
-        int fd = libc.open(file.toAbsolutePath().toString(), O_WRONLY, constants.O_CREAT());
+        // We pass down O_CREAT, so open will create the file if it does not exist.
+        // From the open man page (https://www.man7.org/linux/man-pages/man2/open.2.html):
+        // - The mode parameter is needed when specifying O_CREAT
+        // - The effective mode is modified by the process's umask: in the absence of a default ACL, the mode of the created file is
+        // (mode & ~umask).
+        // We choose to pass down 0666 (r/w permission for user/group/others) to mimic what the JDK does for its open operations;
+        // see for example the fileOpen implementation in libjava:
+        // https://github.com/openjdk/jdk/blob/98562166e4a4c8921709014423c6cbc993aa0d97/src/java.base/unix/native/libjava/io_util_md.c#L105
+        int fd = libc.open(absolutePath.toString(), O_WRONLY | constants.O_CREAT(), 0666);
         if (fd == -1) {
             logger.warn("Could not open file [" + file + "] to preallocate size: " + libc.strerror(libc.errno()));
             return;
diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/Systemd.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/Systemd.java
index 4deade118b788..058cfe77b1ff3 100644
--- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/Systemd.java
+++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/Systemd.java
@@ -10,17 +10,28 @@
 
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
-import org.elasticsearch.nativeaccess.lib.SystemdLibrary;
+import org.elasticsearch.nativeaccess.lib.PosixCLibrary;
 
-import java.util.Locale;
+import java.nio.charset.StandardCharsets;
 
+/**
+ * Wraps access to notifications to systemd.
+ * <p>
+ * Systemd notifications are done through a Unix socket. Although Java does support
+ * opening unix sockets, it unfortunately does not support datagram sockets. This class
+ * instead opens and communicates with the socket using native methods.
+ */
 public class Systemd {
     private static final Logger logger = LogManager.getLogger(Systemd.class);
 
-    private final SystemdLibrary lib;
+    private final PosixCLibrary libc;
+    private final String socketPath;
+    private final CloseableByteBuffer buffer;
 
-    Systemd(SystemdLibrary lib) {
-        this.lib = lib;
+    Systemd(PosixCLibrary libc, String socketPath, CloseableByteBuffer buffer) {
+        this.libc = libc;
+        this.socketPath = socketPath;
+        this.buffer = buffer;
     }
 
     /**
@@ -41,15 +52,61 @@ public void notify_stopping() {
     }
 
     private void notify(String state, boolean warnOnError) {
-        int rc = lib.sd_notify(0, state);
-        logger.trace("sd_notify({}, {}) returned [{}]", 0, state, rc);
-        if (rc < 0) {
-            String message = String.format(Locale.ROOT, "sd_notify(%d, %s) returned error [%d]", 0, state, rc);
-            if (warnOnError) {
-                logger.warn(message);
+        int sockfd = libc.socket(PosixCLibrary.AF_UNIX, PosixCLibrary.SOCK_DGRAM, 0);
+        if (sockfd < 0) {
+            throwOrLog("Could not open systemd socket: " + libc.strerror(libc.errno()), warnOnError);
+            return;
+        }
+        RuntimeException error = null;
+        try {
+            var sockAddr = libc.newUnixSockAddr(socketPath);
+            if (libc.connect(sockfd, sockAddr) != 0) {
+                throwOrLog("Could not connect to systemd socket: " + libc.strerror(libc.errno()), warnOnError);
+                return;
+            }
+
+            byte[] bytes = state.getBytes(StandardCharsets.US_ASCII);
+            final long bytesSent;
+            synchronized (buffer) {
+                buffer.buffer().clear();
+                buffer.buffer().put(0, bytes);
+                buffer.buffer().limit(bytes.length);
+                bytesSent = libc.send(sockfd, buffer, 0);
+            }
+
+            if (bytesSent == -1) {
+                throwOrLog("Failed to send message (" + state + ") to systemd socket: " + libc.strerror(libc.errno()), warnOnError);
+            } else if (bytesSent != bytes.length) {
+                throwOrLog("Not all bytes of message (" + state + ") sent to systemd socket (sent " + bytesSent + ")", warnOnError);
             } else {
-                throw new RuntimeException(message);
+                logger.trace("Message (" + state + ") sent to systemd");
+            }
+        } catch (RuntimeException e) {
+            error = e;
+        } finally {
+            if (libc.close(sockfd) != 0) {
+                try {
+                    throwOrLog("Could not close systemd socket: " + libc.strerror(libc.errno()), warnOnError);
+                } catch (RuntimeException e) {
+                    if (error != null) {
+                        error.addSuppressed(e);
+                        throw error;
+                    } else {
+                        throw e;
+                    }
+                }
+            } else if (error != null) {
+                throw error;
             }
         }
     }
+
+    private void throwOrLog(String message, boolean warnOnError) {
+        if (warnOnError) {
+            logger.warn(message);
+        } else {
+            logger.error(message);
+            throw new RuntimeException(message);
+        }
+    }
 }
diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LoaderHelper.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LoaderHelper.java
new file mode 100644
index 0000000000000..42ca60b81a027
--- /dev/null
+++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LoaderHelper.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.nativeaccess.lib;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * A utility for loading libraries from Elasticsearch's platform specific lib dir.
+ */
+public class LoaderHelper {
+    public static final Path platformLibDir = findPlatformLibDir();
+
+    private static Path findPlatformLibDir() {
+        // tests don't have an ES install, so the platform dir must be passed in explicitly
+        String path = System.getProperty("es.nativelibs.path");
+        if (path != null) {
+            return Paths.get(path);
+        }
+
+        Path platformDir = Paths.get("lib", "platform");
+
+        String osname = System.getProperty("os.name");
+        String os;
+        if (osname.startsWith("Windows")) {
+            os = "windows";
+        } else if (osname.startsWith("Linux")) {
+            os = "linux";
+        } else if (osname.startsWith("Mac OS")) {
+            os = "darwin";
+        } else {
+            os = "unsupported_os[" + osname + "]";
+        }
+        String archname = System.getProperty("os.arch");
+        String arch;
+        if (archname.equals("amd64") || archname.equals("x86_64")) {
+            arch = "x64";
+        } else if (archname.equals("aarch64")) {
+            arch = archname;
+        } else {
+            arch = "unsupported_arch[" + archname + "]";
+        }
+        return platformDir.resolve(os + "-" + arch);
+    }
+
+    public static void loadLibrary(String libname) {
+        Path libpath = platformLibDir.resolve(System.mapLibraryName(libname));
+        if (Files.exists(libpath) == false) {
+            throw new UnsatisfiedLinkError("Native library [" + libpath + "] does not exist");
+        }
+        System.load(libpath.toAbsolutePath().toString());
+    }
+
+    private LoaderHelper() {} // no construction
+}
diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java
index faa0e861dc63f..cdd0a56c52a90 100644
--- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java
+++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java
@@ -9,5 +9,5 @@
 package org.elasticsearch.nativeaccess.lib;
 
 /** A marker interface for libraries that can be loaded by {@link org.elasticsearch.nativeaccess.lib.NativeLibraryProvider} */
-public sealed interface NativeLibrary permits JavaLibrary, PosixCLibrary, LinuxCLibrary, MacCLibrary, Kernel32Library, SystemdLibrary,
-    VectorLibrary, ZstdLibrary {}
+public sealed interface NativeLibrary permits JavaLibrary, PosixCLibrary, LinuxCLibrary, MacCLibrary, Kernel32Library, VectorLibrary,
+    ZstdLibrary {}
diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java
index 0e7d07d0ad623..ac34fcb23b3eb 100644
--- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java
+++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/PosixCLibrary.java
@@ -8,11 +8,19 @@
 
 package org.elasticsearch.nativeaccess.lib;
 
+import org.elasticsearch.nativeaccess.CloseableByteBuffer;
+
 /**
  * Provides access to methods in libc.so available on POSIX systems.
  */
 public non-sealed interface PosixCLibrary extends NativeLibrary {
 
+    /** socket domain indicating unix file socket */
+    short AF_UNIX = 1;
+
+    /** socket type indicating a datagram-oriented socket */
+    int SOCK_DGRAM = 2;
+
     /**
      * Gets the effective userid of the current process.
      *
@@ -68,8 +76,6 @@ interface Stat64 {
 
     int open(String pathname, int flags);
 
-    int close(int fd);
-
     int fstat64(int fd, Stat64 stats);
 
     int ftruncate(int fd, long length);
@@ -90,6 +96,55 @@ interface FStore {
 
     int fcntl(int fd, int cmd, FStore fst);
 
+    /**
+     * Open a file descriptor to connect to a socket.
+     *
+     * @param domain The socket protocol family, eg AF_UNIX
+     * @param type The socket type, eg SOCK_DGRAM
+     * @param protocol The protocol for the given protocl family, normally 0
+     * @return an open file descriptor, or -1 on failure with errno set
+     * @see <a href="https://man7.org/linux/man-pages/man2/socket.2.html">socket manpage</a>
+     */
+    int socket(int domain, int type, int protocol);
+
+    /**
+     * Marker interface for sockaddr struct implementations.
+     */
+    interface SockAddr {}
+
+    /**
+     * Create a sockaddr for the AF_UNIX family.
+     */
+    SockAddr newUnixSockAddr(String path);
+
+    /**
+     * Connect a socket to an address.
+     *
+     * @param sockfd An open socket file descriptor
+     * @param addr The address to connect to
+     * @return 0 on success, -1 on failure with errno set
+     */
+    int connect(int sockfd, SockAddr addr);
+
+    /**
+     * Send a message to a socket.
+     *
+     * @param sockfd The open socket file descriptor
+     * @param buffer The message bytes to send
+     * @param flags Flags that may adjust how the message is sent
+     * @return The number of bytes sent, or -1 on failure with errno set
+     * @see <a href="https://man7.org/linux/man-pages/man2/sendto.2.html">send manpage</a>
+     */
+    long send(int sockfd, CloseableByteBuffer buffer, int flags);
+
+    /**
+     * Close a file descriptor
+     * @param fd The file descriptor to close
+     * @return 0 on success, -1 on failure with errno set
+     * @see <a href="https://man7.org/linux/man-pages/man2/close.2.html">close manpage</a>
+     */
+    int close(int fd);
+
     /**
      * Return a string description for an error.
      *
diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/SystemdLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/SystemdLibrary.java
deleted file mode 100644
index 3c4ffefb6e41f..0000000000000
--- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/SystemdLibrary.java
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.nativeaccess.lib;
-
-public non-sealed interface SystemdLibrary extends NativeLibrary {
-    int sd_notify(int unset_environment, String state);
-}
diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java
index a3ddc0d59890d..0294b721aa6a8 100644
--- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java
+++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkKernel32Library.java
@@ -56,7 +56,7 @@ class JdkKernel32Library implements Kernel32Library {
     );
     private static final MethodHandle SetProcessWorkingSetSize$mh = downcallHandleWithError(
         "SetProcessWorkingSetSize",
-        FunctionDescriptor.of(ADDRESS, JAVA_LONG, JAVA_LONG)
+        FunctionDescriptor.of(JAVA_BOOLEAN, ADDRESS, JAVA_LONG, JAVA_LONG)
     );
     private static final MethodHandle GetCompressedFileSizeW$mh = downcallHandleWithError(
         "GetCompressedFileSizeW",
@@ -115,7 +115,7 @@ static class JdkAddress implements Address {
 
         @Override
         public Address add(long offset) {
-            return new JdkAddress(MemorySegment.ofAddress(address.address()));
+            return new JdkAddress(MemorySegment.ofAddress(address.address() + offset));
         }
     }
 
diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java
index cbd43a394379b..1ac7d6c6f897d 100644
--- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java
+++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.nativeaccess.lib.MacCLibrary;
 import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider;
 import org.elasticsearch.nativeaccess.lib.PosixCLibrary;
-import org.elasticsearch.nativeaccess.lib.SystemdLibrary;
 import org.elasticsearch.nativeaccess.lib.VectorLibrary;
 import org.elasticsearch.nativeaccess.lib.ZstdLibrary;
 
@@ -36,8 +35,6 @@ public JdkNativeLibraryProvider() {
                 JdkMacCLibrary::new,
                 Kernel32Library.class,
                 JdkKernel32Library::new,
-                SystemdLibrary.class,
-                JdkSystemdLibrary::new,
                 ZstdLibrary.class,
                 JdkZstdLibrary::new,
                 VectorLibrary.class,
diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java
index 7affd0614461d..f5e3132b76b56 100644
--- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java
+++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkPosixCLibrary.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
+import org.elasticsearch.nativeaccess.CloseableByteBuffer;
 import org.elasticsearch.nativeaccess.lib.PosixCLibrary;
 
 import java.lang.foreign.Arena;
@@ -24,8 +25,10 @@
 
 import static java.lang.foreign.MemoryLayout.PathElement.groupElement;
 import static java.lang.foreign.ValueLayout.ADDRESS;
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
 import static java.lang.foreign.ValueLayout.JAVA_INT;
 import static java.lang.foreign.ValueLayout.JAVA_LONG;
+import static java.lang.foreign.ValueLayout.JAVA_SHORT;
 import static org.elasticsearch.nativeaccess.jdk.LinkerHelper.downcallHandle;
 import static org.elasticsearch.nativeaccess.jdk.MemorySegmentUtil.varHandleWithoutOffset;
 
@@ -89,6 +92,18 @@ class JdkPosixCLibrary implements PosixCLibrary {
         }
         fstat$mh = fstat;
     }
+    private static final MethodHandle socket$mh = downcallHandleWithErrno(
+        "socket",
+        FunctionDescriptor.of(JAVA_INT, JAVA_INT, JAVA_INT, JAVA_INT)
+    );
+    private static final MethodHandle connect$mh = downcallHandleWithErrno(
+        "connect",
+        FunctionDescriptor.of(JAVA_INT, JAVA_INT, ADDRESS, JAVA_INT)
+    );
+    private static final MethodHandle send$mh = downcallHandleWithErrno(
+        "send",
+        FunctionDescriptor.of(JAVA_LONG, JAVA_INT, ADDRESS, JAVA_LONG, JAVA_INT)
+    );
 
     static final MemorySegment errnoState = Arena.ofAuto().allocate(CAPTURE_ERRNO_LAYOUT);
 
@@ -226,6 +241,44 @@ public int fstat64(int fd, Stat64 stat64) {
         }
     }
 
+    @Override
+    public int socket(int domain, int type, int protocol) {
+        try {
+            return (int) socket$mh.invokeExact(errnoState, domain, type, protocol);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
+    @Override
+    public SockAddr newUnixSockAddr(String path) {
+        return new JdkSockAddr(path);
+    }
+
+    @Override
+    public int connect(int sockfd, SockAddr addr) {
+        assert addr instanceof JdkSockAddr;
+        var jdkAddr = (JdkSockAddr) addr;
+        try {
+            return (int) connect$mh.invokeExact(errnoState, sockfd, jdkAddr.segment, (int) jdkAddr.segment.byteSize());
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
+    @Override
+    public long send(int sockfd, CloseableByteBuffer buffer, int flags) {
+        assert buffer instanceof JdkCloseableByteBuffer;
+        var nativeBuffer = (JdkCloseableByteBuffer) buffer;
+        var segment = nativeBuffer.segment;
+        try {
+            logger.info("Sending {} bytes to socket", buffer.buffer().remaining());
+            return (long) send$mh.invokeExact(errnoState, sockfd, segment, (long) buffer.buffer().remaining(), flags);
+        } catch (Throwable t) {
+            throw new AssertionError(t);
+        }
+    }
+
     static class JdkRLimit implements RLimit {
         private static final MemoryLayout layout = MemoryLayout.structLayout(JAVA_LONG, JAVA_LONG);
         private static final VarHandle rlim_cur$vh = varHandleWithoutOffset(layout, groupElement(0));
@@ -326,4 +379,15 @@ public long bytesalloc() {
             return (long) st_bytesalloc$vh.get(segment);
         }
     }
+
+    private static class JdkSockAddr implements SockAddr {
+        private static final MemoryLayout layout = MemoryLayout.structLayout(JAVA_SHORT, MemoryLayout.sequenceLayout(108, JAVA_BYTE));
+        final MemorySegment segment;
+
+        JdkSockAddr(String path) {
+            segment = Arena.ofAuto().allocate(layout);
+            segment.set(JAVA_SHORT, 0, AF_UNIX);
+            MemorySegmentUtil.setString(segment, 2, path);
+        }
+    }
 }
diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java
deleted file mode 100644
index c34c8c070edc5..0000000000000
--- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.nativeaccess.jdk;
-
-import org.elasticsearch.nativeaccess.lib.SystemdLibrary;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.lang.foreign.Arena;
-import java.lang.foreign.FunctionDescriptor;
-import java.lang.foreign.MemorySegment;
-import java.lang.invoke.MethodHandle;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static java.lang.foreign.ValueLayout.ADDRESS;
-import static java.lang.foreign.ValueLayout.JAVA_INT;
-import static org.elasticsearch.nativeaccess.jdk.LinkerHelper.downcallHandle;
-
-class JdkSystemdLibrary implements SystemdLibrary {
-
-    static {
-        // Find and load libsystemd. We attempt all instances of
-        // libsystemd in case of multiarch systems, and stop when
-        // one is successfully loaded. If none can be loaded,
-        // UnsatisfiedLinkError will be thrown.
-        List<String> paths = findLibSystemd();
-        if (paths.isEmpty()) {
-            String libpath = System.getProperty("java.library.path");
-            throw new UnsatisfiedLinkError("Could not find libsystemd in java.library.path: " + libpath);
-        }
-        UnsatisfiedLinkError last = null;
-        for (String path : paths) {
-            try {
-                System.load(path);
-                last = null;
-                break;
-            } catch (UnsatisfiedLinkError e) {
-                last = e;
-            }
-        }
-        if (last != null) {
-            throw last;
-        }
-    }
-
-    // findLibSystemd returns a list of paths to instances of libsystemd
-    // found within java.library.path.
-    static List<String> findLibSystemd() {
-        // Note: on some systems libsystemd does not have a non-versioned symlink.
-        // System.loadLibrary only knows how to find non-versioned library files,
-        // so we must manually check the library path to find what we need.
-        final Path libsystemd = Paths.get("libsystemd.so.0");
-        final String libpath = System.getProperty("java.library.path");
-        final List<String> foundPaths = new ArrayList<>();
-        Arrays.stream(libpath.split(":")).map(Paths::get).filter(Files::exists).forEach(rootPath -> {
-            try {
-                Files.walkFileTree(rootPath, new SimpleFileVisitor<>() {
-                    @Override
-                    public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
-                        if (Files.isReadable(dir)) {
-                            return FileVisitResult.CONTINUE;
-                        }
-                        return FileVisitResult.SKIP_SUBTREE;
-                    }
-
-                    @Override
-                    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
-                        if (file.getFileName().equals(libsystemd)) {
-                            foundPaths.add(file.toAbsolutePath().toString());
-                        }
-                        return FileVisitResult.CONTINUE;
-                    }
-
-                    @Override
-                    public FileVisitResult visitFileFailed(Path file, IOException exc) {
-                        return FileVisitResult.CONTINUE;
-                    }
-                });
-            } catch (IOException e) {
-                throw new UncheckedIOException(e);
-            }
-        });
-        return foundPaths;
-    }
-
-    private static final MethodHandle sd_notify$mh = downcallHandle("sd_notify", FunctionDescriptor.of(JAVA_INT, JAVA_INT, ADDRESS));
-
-    @Override
-    public int sd_notify(int unset_environment, String state) {
-        try (Arena arena = Arena.ofConfined()) {
-            MemorySegment nativeState = MemorySegmentUtil.allocateString(arena, state);
-            return (int) sd_notify$mh.invokeExact(unset_environment, nativeState);
-        } catch (Throwable t) {
-            throw new AssertionError(t);
-        }
-    }
-}
diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java
index c92ad654c9b9a..a1032f1381d94 100644
--- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java
+++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.nativeaccess.jdk;
 
 import org.elasticsearch.nativeaccess.VectorSimilarityFunctions;
+import org.elasticsearch.nativeaccess.lib.LoaderHelper;
 import org.elasticsearch.nativeaccess.lib.VectorLibrary;
 
 import java.lang.foreign.FunctionDescriptor;
@@ -29,7 +30,7 @@ public final class JdkVectorLibrary implements VectorLibrary {
     static final VectorSimilarityFunctions INSTANCE;
 
     static {
-        System.loadLibrary("vec");
+        LoaderHelper.loadLibrary("vec");
         final MethodHandle vecCaps$mh = downcallHandle("vec_caps", FunctionDescriptor.of(JAVA_INT));
 
         try {
diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java
index e3e972bc19d72..284ac134d2036 100644
--- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java
+++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.nativeaccess.jdk;
 
 import org.elasticsearch.nativeaccess.CloseableByteBuffer;
+import org.elasticsearch.nativeaccess.lib.LoaderHelper;
 import org.elasticsearch.nativeaccess.lib.ZstdLibrary;
 
 import java.lang.foreign.FunctionDescriptor;
@@ -24,7 +25,7 @@
 class JdkZstdLibrary implements ZstdLibrary {
 
     static {
-        System.loadLibrary("zstd");
+        LoaderHelper.loadLibrary("zstd");
     }
 
     private static final MethodHandle compressBound$mh = downcallHandle("ZSTD_compressBound", FunctionDescriptor.of(JAVA_LONG, JAVA_INT));
diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java
index c65711af0f63f..6c4c9bd0111c0 100644
--- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java
+++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java
@@ -22,6 +22,10 @@ static String getString(MemorySegment segment, long offset) {
         return segment.getUtf8String(offset);
     }
 
+    static void setString(MemorySegment segment, long offset, String value) {
+        segment.setUtf8String(offset, value);
+    }
+
     static MemorySegment allocateString(Arena arena, String s) {
         return arena.allocateUtf8String(s);
     }
diff --git a/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java b/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java
index 25c449337e294..23d9919603ab4 100644
--- a/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java
+++ b/libs/native/src/main22/java/org/elasticsearch/nativeaccess/jdk/MemorySegmentUtil.java
@@ -20,6 +20,10 @@ static String getString(MemorySegment segment, long offset) {
         return segment.getString(offset);
     }
 
+    static void setString(MemorySegment segment, long offset, String value) {
+        segment.setString(offset, value);
+    }
+
     static MemorySegment allocateString(Arena arena, String s) {
         return arena.allocateFrom(s);
     }
diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/PreallocateTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/PreallocateTests.java
index c5d427c3aa47b..a95f4df6aef6f 100644
--- a/libs/native/src/test/java/org/elasticsearch/nativeaccess/PreallocateTests.java
+++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/PreallocateTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.nativeaccess;
 
 import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -17,8 +18,14 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class PreallocateTests extends ESTestCase {
-    public void testPreallocate() throws IOException {
+
+    @Before
+    public void setup() {
         assumeFalse("no preallocate on windows", System.getProperty("os.name").startsWith("Windows"));
+        assumeFalse("preallocate not supported on encrypted block devices", "encryption-at-rest".equals(System.getenv("BUILDKITE_LABEL")));
+    }
+
+    public void testPreallocate() throws IOException {
         Path file = createTempFile();
         long size = 1024 * 1024; // 1 MB
         var nativeAccess = NativeAccess.instance();
@@ -27,4 +34,24 @@ public void testPreallocate() throws IOException {
         assertTrue(foundSize.isPresent());
         assertThat(foundSize.getAsLong(), equalTo(size));
     }
+
+    public void testPreallocateNonExistingFile() {
+        Path file = createTempDir().resolve("intermediate-dir").resolve("test-preallocate");
+        long size = 1024 * 1024; // 1 MB
+        var nativeAccess = NativeAccess.instance();
+        nativeAccess.tryPreallocate(file, size);
+        OptionalLong foundSize = nativeAccess.allocatedSizeInBytes(file);
+        assertTrue(foundSize.isPresent());
+        assertThat(foundSize.getAsLong(), equalTo(size));
+    }
+
+    public void testPreallocateNonExistingDirectory() {
+        Path file = createTempDir().resolve("intermediate-dir").resolve("test-preallocate");
+        long size = 1024 * 1024; // 1 MB
+        var nativeAccess = NativeAccess.instance();
+        nativeAccess.tryPreallocate(file, size);
+        OptionalLong foundSize = nativeAccess.allocatedSizeInBytes(file);
+        assertTrue(foundSize.isPresent());
+        assertThat(foundSize.getAsLong(), equalTo(size));
+    }
 }
diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java
index 9875878d8658a..cda4fc8c55444 100644
--- a/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java
+++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java
@@ -49,7 +49,7 @@ public void testSystemPropertyDisabled() throws Exception {
             "-Xms4m",
             "-cp",
             jarPath + File.pathSeparator + System.getProperty("java.class.path"),
-            "-Djava.library.path=" + System.getProperty("java.library.path"),
+            "-Des.nativelibs.path=" + System.getProperty("es.nativelibs.path"),
             "p.Test"
         ).start();
         String output = new String(process.getInputStream().readAllBytes(), UTF_8);
diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/KeyStoreUtil.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/KeyStoreUtil.java
index aebee89297a88..7f5b005e28470 100644
--- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/KeyStoreUtil.java
+++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/KeyStoreUtil.java
@@ -106,8 +106,12 @@ public static KeyStore filter(KeyStore store, Predicate<KeyStoreEntry> filter) {
      * @param certificates The root certificates to trust
      */
     public static KeyStore buildTrustStore(Iterable<Certificate> certificates) throws GeneralSecurityException {
+        return buildTrustStore(certificates, KeyStore.getDefaultType());
+    }
+
+    public static KeyStore buildTrustStore(Iterable<Certificate> certificates, String type) throws GeneralSecurityException {
         assert certificates != null : "Cannot create keystore with null certificates";
-        KeyStore store = buildNewKeyStore();
+        KeyStore store = buildNewKeyStore(type);
         int counter = 0;
         for (Certificate certificate : certificates) {
             store.setCertificateEntry("cert-" + counter, certificate);
@@ -117,7 +121,11 @@ public static KeyStore buildTrustStore(Iterable<Certificate> certificates) throw
     }
 
     private static KeyStore buildNewKeyStore() throws GeneralSecurityException {
-        KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
+        return buildNewKeyStore(KeyStore.getDefaultType());
+    }
+
+    private static KeyStore buildNewKeyStore(String type) throws GeneralSecurityException {
+        KeyStore keyStore = KeyStore.getInstance(type);
         try {
             keyStore.load(null, null);
         } catch (IOException e) {
diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java
index 0be2b68d76a21..fc22bda52e104 100644
--- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java
+++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java
@@ -302,9 +302,13 @@ private void merge(
                 addThis = projectedW <= wLimit;
             }
             if (i == 1 || i == incomingCount - 1) {
-                // force last centroid to never merge
+                // force first and last centroid to never merge
                 addThis = false;
             }
+            if (lastUsedCell == mean.length - 1) {
+                // use the last centroid, there's no more
+                addThis = true;
+            }
 
             if (addThis) {
                 // next point will fit
diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java
index 16a81bad50756..9fadf2218f203 100644
--- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java
+++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java
@@ -151,4 +151,14 @@ public void testFill() {
             i++;
         }
     }
+
+    public void testLargeInputSmallCompression() {
+        MergingDigest td = new MergingDigest(10);
+        for (int i = 0; i < 10_000_000; i++) {
+            td.add(between(0, 3_600_000));
+        }
+        assertTrue(td.centroidCount() < 100);
+        assertTrue(td.quantile(0.00001) < 100_000);
+        assertTrue(td.quantile(0.99999) > 3_000_000);
+    }
 }
diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java
index 72b460da19da2..815346100532c 100644
--- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java
+++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java
@@ -152,7 +152,7 @@ public void testQuantile() {
         hist2.compress();
         double x1 = hist1.quantile(0.5);
         double x2 = hist2.quantile(0.5);
-        assertEquals(Dist.quantile(0.5, data), x1, 0.2);
+        assertEquals(Dist.quantile(0.5, data), x1, 0.25);
         assertEquals(x1, x2, 0.01);
     }
 
diff --git a/libs/x-content/impl/build.gradle b/libs/x-content/impl/build.gradle
index 41b65044735ca..6cf278e826d4c 100644
--- a/libs/x-content/impl/build.gradle
+++ b/libs/x-content/impl/build.gradle
@@ -12,7 +12,7 @@ base {
   archivesName = "x-content-impl"
 }
 
-String jacksonVersion = "2.15.0"
+String jacksonVersion = "2.17.2"
 
 dependencies {
   compileOnly project(':libs:elasticsearch-core')
diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java
index 2a8e7a4dfa12c..3aa8323eb5495 100644
--- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java
+++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/cbor/CborXContentImpl.java
@@ -63,8 +63,8 @@ public XContentType type() {
     }
 
     @Override
-    public byte streamSeparator() {
-        throw new XContentParseException("cbor does not support stream parsing...");
+    public byte bulkSeparator() {
+        throw new XContentParseException("cbor does not support bulk parsing...");
     }
 
     @Override
diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java
index 2e4925b4a853e..4e04230a7486e 100644
--- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java
+++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java
@@ -54,6 +54,8 @@ public static final XContent jsonXContent() {
         jsonFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
         jsonFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true);
         jsonFactory.configure(JsonParser.Feature.USE_FAST_DOUBLE_PARSER, true);
+        // keeping existing behavior of including source, for now
+        jsonFactory.configure(JsonParser.Feature.INCLUDE_SOURCE_IN_LOCATION, true);
         jsonXContent = new JsonXContentImpl();
     }
 
@@ -65,7 +67,7 @@ public XContentType type() {
     }
 
     @Override
-    public byte streamSeparator() {
+    public byte bulkSeparator() {
         return '\n';
     }
 
diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java
index 3c774c582c638..83528980c2b52 100644
--- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java
+++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/smile/SmileXContentImpl.java
@@ -65,7 +65,7 @@ public XContentType type() {
     }
 
     @Override
-    public byte streamSeparator() {
+    public byte bulkSeparator() {
         return (byte) 0xFF;
     }
 
diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java
index 6a22508ba51c6..6e1496bfffd7b 100644
--- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java
+++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/yaml/YamlXContentImpl.java
@@ -61,8 +61,8 @@ public XContentType type() {
     }
 
     @Override
-    public byte streamSeparator() {
-        throw new UnsupportedOperationException("yaml does not support stream parsing...");
+    public byte bulkSeparator() {
+        throw new UnsupportedOperationException("yaml does not support bulk parsing...");
     }
 
     @Override
diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java
index 146f90b8e2510..56eb308eaebae 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContent.java
@@ -24,7 +24,7 @@ public interface XContent {
      */
     XContentType type();
 
-    byte streamSeparator();
+    byte bulkSeparator();
 
     @Deprecated
     boolean detectContent(byte[] bytes, int offset, int length);
diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerator.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerator.java
index 5037ed0b40664..add5a913faf8a 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerator.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentGenerator.java
@@ -148,6 +148,12 @@ default void copyCurrentEvent(XContentParser parser) throws IOException {
                     case LONG -> writeNumber(parser.longValue());
                     case FLOAT -> writeNumber(parser.floatValue());
                     case DOUBLE -> writeNumber(parser.doubleValue());
+                    case BIG_INTEGER -> writeNumber((BigInteger) parser.numberValue());
+                    // note: BIG_DECIMAL is not supported, ES only supports up to double.
+                    // BIG_INTEGER above is only for representing unsigned long
+                    default -> {
+                        assert false : "missing xcontent number handling for type [" + parser.numberType() + "]";
+                    }
                 }
                 break;
             case VALUE_BOOLEAN:
@@ -158,6 +164,9 @@ default void copyCurrentEvent(XContentParser parser) throws IOException {
                 break;
             case VALUE_EMBEDDED_OBJECT:
                 writeBinary(parser.binaryValue());
+                break;
+            default:
+                assert false : "missing xcontent token handling for token [" + parser.text() + "]";
         }
     }
 
diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java
index be100e1a6d120..9672c73ef56df 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/AbstractXContentParser.java
@@ -151,11 +151,8 @@ public int intValue(boolean coerce) throws IOException {
 
     protected abstract int doIntValue() throws IOException;
 
-    private static BigInteger LONG_MAX_VALUE_AS_BIGINTEGER = BigInteger.valueOf(Long.MAX_VALUE);
-    private static BigInteger LONG_MIN_VALUE_AS_BIGINTEGER = BigInteger.valueOf(Long.MIN_VALUE);
-    // weak bounds on the BigDecimal representation to allow for coercion
-    private static BigDecimal BIGDECIMAL_GREATER_THAN_LONG_MAX_VALUE = BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE);
-    private static BigDecimal BIGDECIMAL_LESS_THAN_LONG_MIN_VALUE = BigDecimal.valueOf(Long.MIN_VALUE).subtract(BigDecimal.ONE);
+    private static final BigInteger LONG_MAX_VALUE_AS_BIGINTEGER = BigInteger.valueOf(Long.MAX_VALUE);
+    private static final BigInteger LONG_MIN_VALUE_AS_BIGINTEGER = BigInteger.valueOf(Long.MIN_VALUE);
 
     /** Return the long that {@code stringValue} stores or throws an exception if the
      *  stored value cannot be converted to a long that stores the exact same
@@ -170,11 +167,21 @@ private static long toLong(String stringValue, boolean coerce) {
         final BigInteger bigIntegerValue;
         try {
             final BigDecimal bigDecimalValue = new BigDecimal(stringValue);
-            if (bigDecimalValue.compareTo(BIGDECIMAL_GREATER_THAN_LONG_MAX_VALUE) >= 0
-                || bigDecimalValue.compareTo(BIGDECIMAL_LESS_THAN_LONG_MIN_VALUE) <= 0) {
+            // long can have a maximum of 19 digits - any more than that cannot be a long
+            // the scale is stored as the negation, so negative scale -> big number
+            if (bigDecimalValue.scale() < -19) {
                 throw new IllegalArgumentException("Value [" + stringValue + "] is out of range for a long");
             }
-            bigIntegerValue = coerce ? bigDecimalValue.toBigInteger() : bigDecimalValue.toBigIntegerExact();
+            // large scale -> very small number
+            if (bigDecimalValue.scale() > 19) {
+                if (coerce) {
+                    bigIntegerValue = BigInteger.ZERO;
+                } else {
+                    throw new ArithmeticException("Number has a decimal part");
+                }
+            } else {
+                bigIntegerValue = coerce ? bigDecimalValue.toBigInteger() : bigDecimalValue.toBigIntegerExact();
+            }
         } catch (ArithmeticException e) {
             throw new IllegalArgumentException("Value [" + stringValue + "] has a decimal part");
         } catch (NumberFormatException e) {
diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentGeneratorTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentGeneratorTests.java
new file mode 100644
index 0000000000000..ab141f9af484c
--- /dev/null
+++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentGeneratorTests.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.xcontent;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class XContentGeneratorTests extends ESTestCase {
+
+    public void testCopyCurrentEventRoundtrip() throws Exception {
+        assertTypeCopy("null", "null");
+        assertTypeCopy("string", "\"hi\"");
+        assertTypeCopy("integer", "1");
+        assertTypeCopy("float", "1.0");
+        assertTypeCopy("long", "5000000000");
+        assertTypeCopy("double", "1.123456789");
+        assertTypeCopy("biginteger", "18446744073709551615");
+    }
+
+    private void assertTypeCopy(String typename, String value) throws Exception {
+        var input = String.format(Locale.ROOT, "{\"%s\":%s,\"%s_in_array\":[%s]}", typename, value, typename, value);
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        try (
+            var generator = JsonXContent.jsonXContent.createGenerator(outputStream);
+            var parser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, input)
+        ) {
+            XContentParser.Token token;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                generator.copyCurrentEvent(parser);
+            }
+            generator.copyCurrentEvent(parser); // copy end object too
+        }
+        assertThat(outputStream.toString(StandardCharsets.UTF_8), equalTo(input));
+    }
+}
diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java
index c8df9929d007b..58cb0af79e103 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/XContentParserTests.java
@@ -31,6 +31,7 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.in;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.junit.internal.matchers.ThrowableMessageMatcher.hasMessage;
 
@@ -74,6 +75,73 @@ public void testFloat() throws IOException {
         }
     }
 
+    public void testLongCoercion() throws IOException {
+        XContentType xContentType = randomFrom(XContentType.values());
+
+        try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
+            builder.startObject();
+
+            builder.field("five", "5.5");
+            builder.field("minusFive", "-5.5");
+
+            builder.field("minNegative", "-9.2233720368547758089999e18");
+            builder.field("tooNegative", "-9.223372036854775809e18");
+            builder.field("maxPositive", "9.2233720368547758079999e18");
+            builder.field("tooPositive", "9.223372036854775808e18");
+
+            builder.field("expTooBig", "2e100");
+            builder.field("minusExpTooBig", "-2e100");
+            builder.field("maxPositiveExp", "1e2147483647");
+            builder.field("tooPositiveExp", "1e2147483648");
+
+            builder.field("expTooSmall", "2e-100");
+            builder.field("minusExpTooSmall", "-2e-100");
+            builder.field("maxNegativeExp", "1e-2147483647");
+
+            builder.field("tooNegativeExp", "1e-2147483648");
+
+            builder.endObject();
+
+            try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) {
+                assertThat(parser.nextToken(), is(XContentParser.Token.START_OBJECT));
+
+                assertFieldWithValue("five", 5L, parser);
+                assertFieldWithValue("minusFive", -5L, parser); // Rounds toward zero
+
+                assertFieldWithValue("minNegative", Long.MIN_VALUE, parser);
+                assertFieldWithInvalidLongValue("tooNegative", parser);
+                assertFieldWithValue("maxPositive", Long.MAX_VALUE, parser);
+                assertFieldWithInvalidLongValue("tooPositive", parser);
+
+                assertFieldWithInvalidLongValue("expTooBig", parser);
+                assertFieldWithInvalidLongValue("minusExpTooBig", parser);
+                assertFieldWithInvalidLongValue("maxPositiveExp", parser);
+                assertFieldWithInvalidLongValue("tooPositiveExp", parser);
+
+                // too small goes to zero
+                assertFieldWithValue("expTooSmall", 0L, parser);
+                assertFieldWithValue("minusExpTooSmall", 0L, parser);
+                assertFieldWithValue("maxNegativeExp", 0L, parser);
+
+                assertFieldWithInvalidLongValue("tooNegativeExp", parser);
+            }
+        }
+    }
+
+    private static void assertFieldWithValue(String fieldName, long fieldValue, XContentParser parser) throws IOException {
+        assertThat(parser.nextToken(), is(XContentParser.Token.FIELD_NAME));
+        assertThat(parser.currentName(), is(fieldName));
+        assertThat(parser.nextToken(), is(XContentParser.Token.VALUE_STRING));
+        assertThat(parser.longValue(), equalTo(fieldValue));
+    }
+
+    private static void assertFieldWithInvalidLongValue(String fieldName, XContentParser parser) throws IOException {
+        assertThat(parser.nextToken(), is(XContentParser.Token.FIELD_NAME));
+        assertThat(parser.currentName(), is(fieldName));
+        assertThat(parser.nextToken(), is(XContentParser.Token.VALUE_STRING));
+        expectThrows(IllegalArgumentException.class, parser::longValue);
+    }
+
     public void testReadList() throws IOException {
         assertThat(readList("{\"foo\": [\"bar\"]}"), contains("bar"));
         assertThat(readList("{\"foo\": [\"bar\",\"baz\"]}"), contains("bar", "baz"));
diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java
index 18b24123e6cf0..63f58bbb75713 100644
--- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java
+++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.aggregations.AggregationIntegTestCase;
 import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries;
 import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
@@ -102,15 +101,11 @@ private CreateIndexResponse prepareTimeSeriesIndex(
         final String[] routingDimensions
     ) {
         return prepareCreate("index").setSettings(
-            Settings.builder()
-                .put("mode", "time_series")
+            indexSettings(randomIntBetween(1, 3), randomIntBetween(1, 3)).put("mode", "time_series")
                 .put("routing_path", String.join(",", routingDimensions))
-                .put("index.number_of_shards", randomIntBetween(1, 3))
-                .put("index.number_of_replicas", randomIntBetween(1, 3))
                 .put("time_series.start_time", startMillis)
                 .put("time_series.end_time", endMillis)
                 .put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), 4192)
-                .build()
         ).setMapping(mapping).get();
     }
 
diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java
index 3287f50ab1739..2967e6f5e322f 100644
--- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java
+++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.aggregations.AggregationIntegTestCase;
 import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries;
 import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
@@ -103,15 +102,11 @@ private CreateIndexResponse prepareTimeSeriesIndex(
         final String[] routingDimensions
     ) {
         return prepareCreate("index").setSettings(
-            Settings.builder()
-                .put("mode", "time_series")
+            indexSettings(randomIntBetween(1, 3), randomIntBetween(1, 3)).put("mode", "time_series")
                 .put("routing_path", String.join(",", routingDimensions))
-                .put("index.number_of_shards", randomIntBetween(1, 3))
-                .put("index.number_of_replicas", randomIntBetween(1, 3))
                 .put("time_series.start_time", startMillis)
                 .put("time_series.end_time", endMillis)
                 .put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), 4192)
-                .build()
         ).setMapping(mapping).get();
     }
 
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java
index 1e3042f8cf1e4..b813c9ec50c83 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java
@@ -213,7 +213,7 @@ public InternalAutoDateHistogram(StreamInput in) throws IOException {
             bucketInnerInterval = 1; // Calculated on merge.
         }
         // we changed the order format in 8.13 for partial reduce, therefore we need to order them to perform merge sort
-        if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.HISTOGRAM_AGGS_KEY_SORTED)) {
+        if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_14_0)) {
             // list is mutable by #readCollectionAsList contract
             buckets.sort(Comparator.comparingLong(b -> b.key));
         }
diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle
index 77fd095806d10..1fc42a1b294fe 100644
--- a/modules/analysis-common/build.gradle
+++ b/modules/analysis-common/build.gradle
@@ -36,3 +36,6 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task ->
   task.skipTest("search.query/50_queries_with_synonyms/Test common terms query with stacked tokens", "#42654 -  `common` query throws an exception")
 }
 
+artifacts {
+  restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
+}
diff --git a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java
index e8164bfbb8f36..4be28483e0d30 100644
--- a/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java
+++ b/modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java
@@ -13,6 +13,9 @@
 import org.elasticsearch.action.admin.indices.analyze.ReloadAnalyzersRequest;
 import org.elasticsearch.action.admin.indices.analyze.ReloadAnalyzersResponse;
 import org.elasticsearch.action.admin.indices.analyze.TransportReloadAnalyzersAction;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
@@ -24,13 +27,14 @@
 import java.io.PrintWriter;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.nio.file.OpenOption;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -55,9 +59,8 @@ private void testSynonymsUpdate(boolean preview) throws FileNotFoundException, I
         Path config = internalCluster().getInstance(Environment.class).configFile();
         String synonymsFileName = "synonyms.txt";
         Path synonymsFile = config.resolve(synonymsFileName);
-        try (PrintWriter out = new PrintWriter(new OutputStreamWriter(Files.newOutputStream(synonymsFile), StandardCharsets.UTF_8))) {
-            out.println("foo, baz");
-        }
+        writeFile(synonymsFile, "foo, baz");
+
         assertAcked(
             indicesAdmin().prepareCreate("test")
                 .setSettings(
@@ -75,48 +78,114 @@ private void testSynonymsUpdate(boolean preview) throws FileNotFoundException, I
 
         assertHitCount(prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")), 1L);
         assertHitCount(prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")), 0L);
-        Response analyzeResponse = indicesAdmin().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get();
-        assertEquals(2, analyzeResponse.getTokens().size());
-        assertEquals("foo", analyzeResponse.getTokens().get(0).getTerm());
-        assertEquals("baz", analyzeResponse.getTokens().get(1).getTerm());
+        assertAnalysis("test", "my_synonym_analyzer", "foo", Set.of("foo", "baz"));
 
         // now update synonyms file several times and trigger reloading
         for (int i = 0; i < 10; i++) {
             String testTerm = randomAlphaOfLength(10);
-            try (
-                PrintWriter out = new PrintWriter(
-                    new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8)
-                )
-            ) {
-                out.println("foo, baz, " + testTerm);
-            }
+            writeFile(synonymsFile, "foo, baz, " + testTerm, StandardOpenOption.WRITE);
+
             ReloadAnalyzersResponse reloadResponse = client().execute(
                 TransportReloadAnalyzersAction.TYPE,
                 new ReloadAnalyzersRequest(null, preview, "test")
             ).actionGet();
-            assertNoFailures(reloadResponse);
-            assertEquals(cluster().numDataNodes(), reloadResponse.getSuccessfulShards());
-            assertTrue(reloadResponse.getReloadDetails().containsKey("test"));
-            assertEquals("test", reloadResponse.getReloadDetails().get("test").getIndexName());
-            assertEquals(
-                Collections.singleton("my_synonym_analyzer"),
-                reloadResponse.getReloadDetails().get("test").getReloadedAnalyzers()
-            );
-
-            analyzeResponse = indicesAdmin().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get();
-            int expectedTokens = preview ? 2 : 3;
-            assertEquals(expectedTokens, analyzeResponse.getTokens().size());
-            Set<String> tokens = new HashSet<>();
-            analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
-            assertTrue(tokens.contains("foo"));
-            assertTrue(tokens.contains("baz"));
-            if (preview == false) {
-                assertTrue(tokens.contains(testTerm));
-            }
+            assertReloadAnalyzers(reloadResponse, cluster().numDataNodes(), Map.of("test", Set.of("my_synonym_analyzer")));
+
+            Set<String> expectedTokens = preview ? Set.of("foo", "baz") : Set.of("foo", "baz", testTerm);
+            assertAnalysis("test", "my_synonym_analyzer", "foo", expectedTokens);
 
             assertHitCount(prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")), 1L);
             long expectedHitCount = preview ? 0L : 1L;
             assertHitCount(prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", testTerm)), expectedHitCount);
         }
     }
+
+    public void testSynonymsUpdateInvalid() throws IOException {
+        final String indexName = "test-invalid";
+        final String synonymsFileName = "synonyms.txt";
+        final String fieldName = "field";
+
+        Path config = internalCluster().getInstance(Environment.class).configFile();
+        Path synonymsFile = config.resolve(synonymsFileName);
+        writeFile(synonymsFile, "foo, baz");
+
+        assertAcked(
+            indicesAdmin().prepareCreate(indexName)
+                .setSettings(
+                    indexSettings(cluster().numDataNodes(), 0).put("analysis.analyzer.my_synonym_analyzer.tokenizer", "standard")
+                        .putList("analysis.analyzer.my_synonym_analyzer.filter", "my_stop_filter", "my_synonym_filter")
+                        .put("analysis.filter.my_stop_filter.type", "stop")
+                        .put("analysis.filter.my_stop_filter.stopwords", "bar")
+                        .put("analysis.filter.my_synonym_filter.type", "synonym")
+                        .put("analysis.filter.my_synonym_filter.updateable", "true")
+                        .put("analysis.filter.my_synonym_filter.synonyms_path", synonymsFileName)
+                )
+                .setMapping(fieldName, "type=text,analyzer=standard,search_analyzer=my_synonym_analyzer")
+        );
+        ensureGreen(indexName);
+
+        prepareIndex(indexName).setId("1").setSource(fieldName, "foo").get();
+        assertNoFailures(indicesAdmin().prepareRefresh(indexName).get());
+
+        assertHitCount(prepareSearch(indexName).setQuery(QueryBuilders.matchQuery(fieldName, "baz")), 1);
+        assertHitCount(prepareSearch(indexName).setQuery(QueryBuilders.matchQuery(fieldName, "buzz")), 0);
+        assertAnalysis(indexName, "my_synonym_analyzer", "foo", Set.of("foo", "baz"));
+
+        // Add an invalid synonym to the file and reload analyzers
+        writeFile(synonymsFile, "foo, baz, bar, buzz");
+        ReloadAnalyzersResponse reloadResponse = client().execute(
+            TransportReloadAnalyzersAction.TYPE,
+            new ReloadAnalyzersRequest(null, false, indexName)
+        ).actionGet();
+        assertReloadAnalyzers(reloadResponse, cluster().numDataNodes(), Map.of(indexName, Set.of("my_synonym_analyzer")));
+
+        ensureGreen(indexName);
+        assertHitCount(prepareSearch(indexName).setQuery(QueryBuilders.matchQuery(fieldName, "baz")), 1);
+        assertHitCount(prepareSearch(indexName).setQuery(QueryBuilders.matchQuery(fieldName, "buzz")), 1);
+        assertAnalysis(indexName, "my_synonym_analyzer", "foo", Set.of("foo", "baz", "buzz"));
+
+        // Reload the index
+        reloadIndex(indexName);
+        ensureGreen(indexName);
+        assertHitCount(prepareSearch(indexName).setQuery(QueryBuilders.matchQuery(fieldName, "baz")), 1);
+        assertHitCount(prepareSearch(indexName).setQuery(QueryBuilders.matchQuery(fieldName, "buzz")), 1);
+        assertAnalysis(indexName, "my_synonym_analyzer", "foo", Set.of("foo", "baz", "buzz"));
+    }
+
+    private static void writeFile(Path path, String contents, OpenOption... options) throws IOException {
+        try (PrintWriter out = new PrintWriter(new OutputStreamWriter(Files.newOutputStream(path, options), StandardCharsets.UTF_8))) {
+            out.println(contents);
+        }
+    }
+
+    private static void assertAnalysis(String indexName, String analyzerName, String input, Set<String> expectedTokens) {
+        Response response = indicesAdmin().prepareAnalyze(indexName, input).setAnalyzer(analyzerName).get();
+        Set<String> actualTokens = response.getTokens().stream().map(AnalyzeToken::getTerm).collect(Collectors.toSet());
+        assertEquals(expectedTokens, actualTokens);
+    }
+
+    private static void assertReloadAnalyzers(
+        ReloadAnalyzersResponse response,
+        int expectedSuccessfulShards,
+        Map<String, Set<String>> expectedIndicesAndAnalyzers
+    ) {
+        assertNoFailures(response);
+        assertEquals(expectedSuccessfulShards, response.getSuccessfulShards());
+
+        assertEquals(expectedIndicesAndAnalyzers.size(), response.getReloadDetails().size());
+        for (var expectedIndexAndAnalyzers : expectedIndicesAndAnalyzers.entrySet()) {
+            String expectedIndexName = expectedIndexAndAnalyzers.getKey();
+            Set<String> expectedAnalyzers = expectedIndexAndAnalyzers.getValue();
+
+            ReloadAnalyzersResponse.ReloadDetails reloadDetails = response.getReloadDetails().get(expectedIndexName);
+            assertNotNull(reloadDetails);
+            assertEquals(expectedAnalyzers, reloadDetails.getReloadedAnalyzers());
+        }
+    }
+
+    private static void reloadIndex(String indexName) {
+        final TimeValue timeout = TimeValue.timeValueSeconds(30);
+        assertAcked(indicesAdmin().close(new CloseIndexRequest(indexName)).actionGet(timeout));
+        assertAcked(indicesAdmin().open(new OpenIndexRequest(indexName)).actionGet(timeout));
+    }
 }
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java
index 9626e44a74bd3..7691253313d50 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymGraphTokenFilterFactory.java
@@ -51,8 +51,8 @@ public TokenFilterFactory getChainAwareTokenFilterFactory(
         Function<String, TokenFilterFactory> allFilters
     ) {
         final Analyzer analyzer = buildSynonymAnalyzer(tokenizer, charFilters, previousTokenFilters);
-        ReaderWithOrigin rulesFromSettings = getRulesFromSettings(environment, context);
-        final SynonymMap synonyms = buildSynonyms(analyzer, rulesFromSettings);
+        ReaderWithOrigin rulesReader = synonymsSource.getRulesReader(this, context);
+        final SynonymMap synonyms = buildSynonyms(analyzer, rulesReader);
         final String name = name();
         return new TokenFilterFactory() {
             @Override
@@ -72,7 +72,7 @@ public AnalysisMode getAnalysisMode() {
 
             @Override
             public String getResourceName() {
-                return rulesFromSettings.resource();
+                return rulesReader.resource();
             }
         };
     }
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java
index 0462511f2ef87..762de01b93d51 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexService.IndexCreationContext;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
 import org.elasticsearch.index.analysis.Analysis;
 import org.elasticsearch.index.analysis.AnalysisMode;
@@ -34,6 +35,101 @@
 
 public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
 
+    protected enum SynonymsSource {
+        INLINE("synonyms") {
+            @Override
+            public ReaderWithOrigin getRulesReader(SynonymTokenFilterFactory factory, IndexCreationContext context) {
+                List<String> rulesList = Analysis.getWordList(
+                    factory.environment,
+                    factory.settings,
+                    SynonymsSource.INLINE.getSettingName()
+                );
+                StringBuilder sb = new StringBuilder();
+                for (String line : rulesList) {
+                    sb.append(line).append(System.lineSeparator());
+                }
+                return new ReaderWithOrigin(new StringReader(sb.toString()), "'" + factory.name() + "' analyzer settings");
+            }
+        },
+        INDEX("synonyms_set") {
+            @Override
+            public ReaderWithOrigin getRulesReader(SynonymTokenFilterFactory factory, IndexCreationContext context) {
+                if (factory.analysisMode != AnalysisMode.SEARCH_TIME) {
+                    throw new IllegalArgumentException(
+                        "Can't apply ["
+                            + SynonymsSource.INDEX.getSettingName()
+                            + "]! Loading synonyms from index is supported only for search time synonyms!"
+                    );
+                }
+                String synonymsSet = factory.settings.get(SynonymsSource.INDEX.getSettingName(), null);
+                // provide empty synonyms on index creation and index metadata checks to ensure that we
+                // don't block a master thread
+                ReaderWithOrigin reader;
+                if (context != IndexCreationContext.RELOAD_ANALYZERS) {
+                    reader = new ReaderWithOrigin(
+                        new StringReader(""),
+                        "fake empty [" + synonymsSet + "] synonyms_set in .synonyms index",
+                        synonymsSet
+                    );
+                } else {
+                    reader = new ReaderWithOrigin(
+                        Analysis.getReaderFromIndex(synonymsSet, factory.synonymsManagementAPIService),
+                        "[" + synonymsSet + "] synonyms_set in .synonyms index",
+                        synonymsSet
+                    );
+                }
+
+                return reader;
+            }
+        },
+        LOCAL_FILE("synonyms_path") {
+            @Override
+            public ReaderWithOrigin getRulesReader(SynonymTokenFilterFactory factory, IndexCreationContext context) {
+                String synonymsPath = factory.settings.get(SynonymsSource.LOCAL_FILE.getSettingName(), null);
+                return new ReaderWithOrigin(
+                    // Pass the inline setting name because "_path" is appended by getReaderFromFile
+                    Analysis.getReaderFromFile(factory.environment, synonymsPath, SynonymsSource.INLINE.getSettingName()),
+                    synonymsPath
+                );
+            }
+        };
+
+        private final String settingName;
+
+        SynonymsSource(String settingName) {
+            this.settingName = settingName;
+        }
+
+        public abstract ReaderWithOrigin getRulesReader(SynonymTokenFilterFactory factory, IndexCreationContext context);
+
+        public String getSettingName() {
+            return settingName;
+        }
+
+        public static SynonymsSource fromSettings(Settings settings) {
+            SynonymsSource synonymsSource;
+            if (settings.hasValue(SynonymsSource.INLINE.getSettingName())) {
+                synonymsSource = SynonymsSource.INLINE;
+            } else if (settings.hasValue(SynonymsSource.INDEX.getSettingName())) {
+                synonymsSource = SynonymsSource.INDEX;
+            } else if (settings.hasValue(SynonymsSource.LOCAL_FILE.getSettingName())) {
+                synonymsSource = SynonymsSource.LOCAL_FILE;
+            } else {
+                throw new IllegalArgumentException(
+                    "synonym requires either `"
+                        + SynonymsSource.INLINE.getSettingName()
+                        + "`, `"
+                        + SynonymsSource.INDEX.getSettingName()
+                        + "` or `"
+                        + SynonymsSource.LOCAL_FILE.getSettingName()
+                        + "` to be configured"
+                );
+            }
+
+            return synonymsSource;
+        }
+    }
+
     private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(SynonymTokenFilterFactory.class);
 
     private final String format;
@@ -43,6 +139,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
     protected final Environment environment;
     protected final AnalysisMode analysisMode;
     private final SynonymsManagementAPIService synonymsManagementAPIService;
+    protected final SynonymsSource synonymsSource;
 
     SynonymTokenFilterFactory(
         IndexSettings indexSettings,
@@ -62,10 +159,15 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
                     + "Instead, insert a lowercase filter in the filter chain before the synonym_graph filter."
             );
         }
+
+        this.synonymsSource = SynonymsSource.fromSettings(settings);
         this.expand = settings.getAsBoolean("expand", true);
-        this.lenient = settings.getAsBoolean("lenient", false);
         this.format = settings.get("format", "");
         boolean updateable = settings.getAsBoolean("updateable", false);
+        this.lenient = settings.getAsBoolean(
+            "lenient",
+            indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.LENIENT_UPDATEABLE_SYNONYMS) && updateable
+        );
         this.analysisMode = updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL;
         this.environment = env;
         this.synonymsManagementAPIService = synonymsManagementAPIService;
@@ -90,8 +192,8 @@ public TokenFilterFactory getChainAwareTokenFilterFactory(
         Function<String, TokenFilterFactory> allFilters
     ) {
         final Analyzer analyzer = buildSynonymAnalyzer(tokenizer, charFilters, previousTokenFilters);
-        ReaderWithOrigin rulesFromSettings = getRulesFromSettings(environment, context);
-        final SynonymMap synonyms = buildSynonyms(analyzer, rulesFromSettings);
+        ReaderWithOrigin rulesReader = synonymsSource.getRulesReader(this, context);
+        final SynonymMap synonyms = buildSynonyms(analyzer, rulesReader);
         final String name = name();
         return new TokenFilterFactory() {
             @Override
@@ -119,7 +221,7 @@ public AnalysisMode getAnalysisMode() {
 
             @Override
             public String getResourceName() {
-                return rulesFromSettings.resource();
+                return rulesReader.resource();
             }
         };
     }
@@ -152,43 +254,6 @@ SynonymMap buildSynonyms(Analyzer analyzer, ReaderWithOrigin rules) {
         }
     }
 
-    protected ReaderWithOrigin getRulesFromSettings(Environment env, IndexCreationContext context) {
-        if (settings.getAsList("synonyms", null) != null) {
-            List<String> rulesList = Analysis.getWordList(env, settings, "synonyms");
-            StringBuilder sb = new StringBuilder();
-            for (String line : rulesList) {
-                sb.append(line).append(System.lineSeparator());
-            }
-            return new ReaderWithOrigin(new StringReader(sb.toString()), "'" + name() + "' analyzer settings");
-        } else if (settings.get("synonyms_set") != null) {
-            if (analysisMode != AnalysisMode.SEARCH_TIME) {
-                throw new IllegalArgumentException(
-                    "Can't apply [synonyms_set]! " + "Loading synonyms from index is supported only for search time synonyms!"
-                );
-            }
-            String synonymsSet = settings.get("synonyms_set", null);
-            // provide fake synonyms on index creation and index metadata checks to ensure that we
-            // don't block a master thread
-            if (context != IndexCreationContext.RELOAD_ANALYZERS) {
-                return new ReaderWithOrigin(
-                    new StringReader("fake rule => fake"),
-                    "fake [" + synonymsSet + "] synonyms_set in .synonyms index",
-                    synonymsSet
-                );
-            }
-            return new ReaderWithOrigin(
-                Analysis.getReaderFromIndex(synonymsSet, synonymsManagementAPIService),
-                "[" + synonymsSet + "] synonyms_set in .synonyms index",
-                synonymsSet
-            );
-        } else if (settings.get("synonyms_path") != null) {
-            String synonyms_path = settings.get("synonyms_path", null);
-            return new ReaderWithOrigin(Analysis.getReaderFromFile(env, synonyms_path, "synonyms_path"), synonyms_path);
-        } else {
-            throw new IllegalArgumentException("synonym requires either `synonyms`, `synonyms_set` or `synonyms_path` to be configured");
-        }
-    }
-
     record ReaderWithOrigin(Reader reader, String origin, String resource) {
         ReaderWithOrigin(Reader reader, String origin) {
             this(reader, origin, null);
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java
index 8758c2478e873..7a2bd2a822988 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymsAnalysisTests.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -36,6 +37,7 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.function.BiConsumer;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -78,7 +80,7 @@ public void testSynonymsAnalysis() throws IOException {
     }
 
     public void testSynonymWordDeleteByAnalyzer() throws IOException {
-        Settings settings = Settings.builder()
+        Settings.Builder settingsBuilder = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
             .put("path.home", createTempDir().toString())
             .put("index.analysis.filter.my_synonym.type", "synonym")
@@ -86,17 +88,49 @@ public void testSynonymWordDeleteByAnalyzer() throws IOException {
             .put("index.analysis.filter.stop_within_synonym.type", "stop")
             .putList("index.analysis.filter.stop_within_synonym.stopwords", "kimchy", "elasticsearch")
             .put("index.analysis.analyzer.synonymAnalyzerWithStopSynonymBeforeSynonym.tokenizer", "whitespace")
-            .putList("index.analysis.analyzer.synonymAnalyzerWithStopSynonymBeforeSynonym.filter", "stop_within_synonym", "my_synonym")
-            .build();
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        try {
+            .putList("index.analysis.analyzer.synonymAnalyzerWithStopSynonymBeforeSynonym.filter", "stop_within_synonym", "my_synonym");
+
+        CheckedBiConsumer<IndexVersion, Boolean, IOException> assertIsLenient = (iv, updateable) -> {
+            Settings settings = settingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, iv)
+                .put("index.analysis.filter.my_synonym.updateable", updateable)
+                .build();
+            IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
             indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
-            fail("fail! due to synonym word deleted by analyzer");
-        } catch (Exception e) {
-            assertThat(e, instanceOf(IllegalArgumentException.class));
-            assertThat(e.getMessage(), startsWith("failed to build synonyms"));
-            assertThat(e.getMessage(), containsString("['my_synonym' analyzer settings]"));
-        }
+            match("synonymAnalyzerWithStopSynonymBeforeSynonym", "kimchy is the dude abides", "is the dude man!");
+        };
+
+        BiConsumer<IndexVersion, Boolean> assertIsNotLenient = (iv, updateable) -> {
+            Settings settings = settingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, iv)
+                .put("index.analysis.filter.my_synonym.updateable", updateable)
+                .build();
+            try {
+                IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
+                indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
+                fail("fail! due to synonym word deleted by analyzer");
+            } catch (Exception e) {
+                assertThat(e, instanceOf(IllegalArgumentException.class));
+                assertThat(e.getMessage(), startsWith("failed to build synonyms"));
+                assertThat(e.getMessage(), containsString("['my_synonym' analyzer settings]"));
+            }
+        };
+
+        // Test with an index version where lenient should always be false by default
+        IndexVersion randomNonLenientIndexVersion = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersions.MINIMUM_COMPATIBLE,
+            IndexVersions.INDEX_SORTING_ON_NESTED
+        );
+        assertIsNotLenient.accept(randomNonLenientIndexVersion, false);
+        assertIsNotLenient.accept(randomNonLenientIndexVersion, true);
+
+        // Test with an index version where the default lenient value is based on updateable
+        IndexVersion randomLenientIndexVersion = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersions.LENIENT_UPDATEABLE_SYNONYMS,
+            IndexVersion.current()
+        );
+        assertIsNotLenient.accept(randomLenientIndexVersion, false);
+        assertIsLenient.accept(randomLenientIndexVersion, true);
     }
 
     public void testSynonymWordDeleteByAnalyzerFromFile() throws IOException {
@@ -106,7 +140,7 @@ public void testSynonymWordDeleteByAnalyzerFromFile() throws IOException {
         Files.createDirectory(config);
         Files.copy(synonyms, config.resolve("synonyms.txt"));
 
-        Settings settings = Settings.builder()
+        Settings.Builder settingsBuilder = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
             .put("path.home", home)
             .put("index.analysis.filter.my_synonym.type", "synonym")
@@ -114,20 +148,52 @@ public void testSynonymWordDeleteByAnalyzerFromFile() throws IOException {
             .put("index.analysis.filter.stop_within_synonym.type", "stop")
             .putList("index.analysis.filter.stop_within_synonym.stopwords", "kimchy", "elasticsearch")
             .put("index.analysis.analyzer.synonymAnalyzerWithStopSynonymBeforeSynonym.tokenizer", "whitespace")
-            .putList("index.analysis.analyzer.synonymAnalyzerWithStopSynonymBeforeSynonym.filter", "stop_within_synonym", "my_synonym")
-            .build();
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        try {
+            .putList("index.analysis.analyzer.synonymAnalyzerWithStopSynonymBeforeSynonym.filter", "stop_within_synonym", "my_synonym");
+
+        CheckedBiConsumer<IndexVersion, Boolean, IOException> assertIsLenient = (iv, updateable) -> {
+            Settings settings = settingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, iv)
+                .put("index.analysis.filter.my_synonym.updateable", updateable)
+                .build();
+            IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
             indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
-            fail("fail! due to synonym word deleted by analyzer");
-        } catch (Exception e) {
-            assertThat(e, instanceOf(IllegalArgumentException.class));
-            assertThat(e.getMessage(), equalTo("failed to build synonyms from [synonyms.txt]"));
-        }
+            match("synonymAnalyzerWithStopSynonymBeforeSynonym", "kimchy is the dude abides", "is the dude man!");
+        };
+
+        BiConsumer<IndexVersion, Boolean> assertIsNotLenient = (iv, updateable) -> {
+            Settings settings = settingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, iv)
+                .put("index.analysis.filter.my_synonym.updateable", updateable)
+                .build();
+            try {
+                IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
+                indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
+                fail("fail! due to synonym word deleted by analyzer");
+            } catch (Exception e) {
+                assertThat(e, instanceOf(IllegalArgumentException.class));
+                assertThat(e.getMessage(), equalTo("failed to build synonyms from [synonyms.txt]"));
+            }
+        };
+
+        // Test with an index version where lenient should always be false by default
+        IndexVersion randomNonLenientIndexVersion = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersions.MINIMUM_COMPATIBLE,
+            IndexVersions.INDEX_SORTING_ON_NESTED
+        );
+        assertIsNotLenient.accept(randomNonLenientIndexVersion, false);
+        assertIsNotLenient.accept(randomNonLenientIndexVersion, true);
+
+        // Test with an index version where the default lenient value is based on updateable
+        IndexVersion randomLenientIndexVersion = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersions.LENIENT_UPDATEABLE_SYNONYMS,
+            IndexVersion.current()
+        );
+        assertIsNotLenient.accept(randomLenientIndexVersion, false);
+        assertIsLenient.accept(randomLenientIndexVersion, true);
     }
 
     public void testExpandSynonymWordDeleteByAnalyzer() throws IOException {
-        Settings settings = Settings.builder()
+        Settings.Builder settingsBuilder = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
             .put("path.home", createTempDir().toString())
             .put("index.analysis.filter.synonym_expand.type", "synonym")
@@ -135,17 +201,49 @@ public void testExpandSynonymWordDeleteByAnalyzer() throws IOException {
             .put("index.analysis.filter.stop_within_synonym.type", "stop")
             .putList("index.analysis.filter.stop_within_synonym.stopwords", "kimchy", "elasticsearch")
             .put("index.analysis.analyzer.synonymAnalyzerExpandWithStopBeforeSynonym.tokenizer", "whitespace")
-            .putList("index.analysis.analyzer.synonymAnalyzerExpandWithStopBeforeSynonym.filter", "stop_within_synonym", "synonym_expand")
-            .build();
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        try {
+            .putList("index.analysis.analyzer.synonymAnalyzerExpandWithStopBeforeSynonym.filter", "stop_within_synonym", "synonym_expand");
+
+        CheckedBiConsumer<IndexVersion, Boolean, IOException> assertIsLenient = (iv, updateable) -> {
+            Settings settings = settingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, iv)
+                .put("index.analysis.filter.synonym_expand.updateable", updateable)
+                .build();
+            IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
             indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
-            fail("fail! due to synonym word deleted by analyzer");
-        } catch (Exception e) {
-            assertThat(e, instanceOf(IllegalArgumentException.class));
-            assertThat(e.getMessage(), startsWith("failed to build synonyms"));
-            assertThat(e.getMessage(), containsString("['synonym_expand' analyzer settings]"));
-        }
+            match("synonymAnalyzerExpandWithStopBeforeSynonym", "kimchy is the dude abides", "is the dude abides man!");
+        };
+
+        BiConsumer<IndexVersion, Boolean> assertIsNotLenient = (iv, updateable) -> {
+            Settings settings = settingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, iv)
+                .put("index.analysis.filter.synonym_expand.updateable", updateable)
+                .build();
+            try {
+                IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
+                indexAnalyzers = createTestAnalysis(idxSettings, settings, new CommonAnalysisPlugin()).indexAnalyzers;
+                fail("fail! due to synonym word deleted by analyzer");
+            } catch (Exception e) {
+                assertThat(e, instanceOf(IllegalArgumentException.class));
+                assertThat(e.getMessage(), startsWith("failed to build synonyms"));
+                assertThat(e.getMessage(), containsString("['synonym_expand' analyzer settings]"));
+            }
+        };
+
+        // Test with an index version where lenient should always be false by default
+        IndexVersion randomNonLenientIndexVersion = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersions.MINIMUM_COMPATIBLE,
+            IndexVersions.INDEX_SORTING_ON_NESTED
+        );
+        assertIsNotLenient.accept(randomNonLenientIndexVersion, false);
+        assertIsNotLenient.accept(randomNonLenientIndexVersion, true);
+
+        // Test with an index version where the default lenient value is based on updateable
+        IndexVersion randomLenientIndexVersion = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersions.LENIENT_UPDATEABLE_SYNONYMS,
+            IndexVersion.current()
+        );
+        assertIsNotLenient.accept(randomLenientIndexVersion, false);
+        assertIsLenient.accept(randomLenientIndexVersion, true);
     }
 
     public void testSynonymsWrappedByMultiplexer() throws IOException {
@@ -272,7 +370,6 @@ public void testTokenFiltersBypassSynonymAnalysis() throws IOException {
         for (String factory : bypassingFactories) {
             TokenFilterFactory tff = plugin.getTokenFilters().get(factory).get(idxSettings, null, factory, settings);
             TokenizerFactory tok = new KeywordTokenizerFactory(idxSettings, null, "keyword", settings);
-            SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, null, "synonym", settings, null);
             Analyzer analyzer = SynonymTokenFilterFactory.buildSynonymAnalyzer(
                 tok,
                 Collections.emptyList(),
@@ -348,7 +445,6 @@ public void testDisallowedTokenFilters() throws IOException {
         for (String factory : disallowedFactories) {
             TokenFilterFactory tff = plugin.getTokenFilters().get(factory).get(idxSettings, null, factory, settings);
             TokenizerFactory tok = new KeywordTokenizerFactory(idxSettings, null, "keyword", settings);
-            SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, null, "synonym", settings, null);
 
             IllegalArgumentException e = expectThrows(
                 IllegalArgumentException.class,
@@ -372,7 +468,7 @@ private void match(String analyzerName, String source, String target) throws IOE
             sb.append(termAtt.toString()).append(" ");
         }
 
-        MatcherAssert.assertThat(target, equalTo(sb.toString().trim()));
+        MatcherAssert.assertThat(sb.toString().trim(), equalTo(target));
     }
 
 }
diff --git a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices.analyze/10_analyze.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices.analyze/15_analyze.yml
similarity index 100%
rename from modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices.analyze/10_analyze.yml
rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices.analyze/15_analyze.yml
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APMMeterRegistry.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APMMeterRegistry.java
index 382fc9417eac0..831e2f19e0126 100644
--- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APMMeterRegistry.java
+++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APMMeterRegistry.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.locks.ReentrantLock;
@@ -96,11 +97,11 @@ public DoubleCounter getDoubleCounter(String name) {
     }
 
     @Override
-    public DoubleAsyncCounter registerDoubleAsyncCounter(
+    public DoubleAsyncCounter registerDoublesAsyncCounter(
         String name,
         String description,
         String unit,
-        Supplier<DoubleWithAttributes> observer
+        Supplier<Collection<DoubleWithAttributes>> observer
     ) {
         try (ReleasableLock lock = registerLock.acquire()) {
             return register(doubleAsynchronousCounters, new DoubleAsyncCounterAdapter(meter, name, description, unit, observer));
@@ -125,7 +126,12 @@ public DoubleUpDownCounter getDoubleUpDownCounter(String name) {
     }
 
     @Override
-    public DoubleGauge registerDoubleGauge(String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+    public DoubleGauge registerDoublesGauge(
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<DoubleWithAttributes>> observer
+    ) {
         try (ReleasableLock lock = registerLock.acquire()) {
             return register(doubleGauges, new DoubleGaugeAdapter(meter, name, description, unit, observer));
         }
@@ -156,7 +162,12 @@ public LongCounter registerLongCounter(String name, String description, String u
     }
 
     @Override
-    public LongAsyncCounter registerLongAsyncCounter(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+    public LongAsyncCounter registerLongsAsyncCounter(
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<LongWithAttributes>> observer
+    ) {
         try (ReleasableLock lock = registerLock.acquire()) {
             return register(longAsynchronousCounters, new LongAsyncCounterAdapter(meter, name, description, unit, observer));
         }
@@ -185,7 +196,7 @@ public LongUpDownCounter getLongUpDownCounter(String name) {
     }
 
     @Override
-    public LongGauge registerLongGauge(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+    public LongGauge registerLongsGauge(String name, String description, String unit, Supplier<Collection<LongWithAttributes>> observer) {
         try (ReleasableLock lock = registerLock.acquire()) {
             return register(longGauges, new LongGaugeAdapter(meter, name, description, unit, observer));
         }
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleAsyncCounterAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleAsyncCounterAdapter.java
index 6b17a83619ef7..ab735c41ca890 100644
--- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleAsyncCounterAdapter.java
+++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleAsyncCounterAdapter.java
@@ -15,12 +15,19 @@
 import org.elasticsearch.telemetry.metric.DoubleAsyncCounter;
 import org.elasticsearch.telemetry.metric.DoubleWithAttributes;
 
+import java.util.Collection;
 import java.util.Objects;
 import java.util.function.Supplier;
 
 public class DoubleAsyncCounterAdapter extends AbstractInstrument<ObservableDoubleCounter> implements DoubleAsyncCounter {
 
-    public DoubleAsyncCounterAdapter(Meter meter, String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+    public DoubleAsyncCounterAdapter(
+        Meter meter,
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<DoubleWithAttributes>> observer
+    ) {
         super(meter, new Builder(name, description, unit, observer));
     }
 
@@ -30,9 +37,9 @@ public void close() throws Exception {
     }
 
     private static class Builder extends AbstractInstrument.Builder<ObservableDoubleCounter> {
-        private final Supplier<DoubleWithAttributes> observer;
+        private final Supplier<Collection<DoubleWithAttributes>> observer;
 
-        private Builder(String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+        private Builder(String name, String description, String unit, Supplier<Collection<DoubleWithAttributes>> observer) {
             super(name, description, unit);
             this.observer = Objects.requireNonNull(observer);
         }
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java
index ed6ecee66d696..2a9c2d45981ed 100644
--- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java
+++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.telemetry.apm.AbstractInstrument;
 import org.elasticsearch.telemetry.metric.DoubleWithAttributes;
 
+import java.util.Collection;
 import java.util.Objects;
 import java.util.function.Supplier;
 
@@ -24,7 +25,13 @@ public class DoubleGaugeAdapter extends AbstractInstrument<ObservableDoubleGauge
     implements
         org.elasticsearch.telemetry.metric.DoubleGauge {
 
-    public DoubleGaugeAdapter(Meter meter, String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+    public DoubleGaugeAdapter(
+        Meter meter,
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<DoubleWithAttributes>> observer
+    ) {
         super(meter, new Builder(name, description, unit, observer));
     }
 
@@ -34,9 +41,9 @@ public void close() throws Exception {
     }
 
     private static class Builder extends AbstractInstrument.Builder<ObservableDoubleGauge> {
-        private final Supplier<DoubleWithAttributes> observer;
+        private final Supplier<Collection<DoubleWithAttributes>> observer;
 
-        private Builder(String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+        private Builder(String name, String description, String unit, Supplier<Collection<DoubleWithAttributes>> observer) {
             super(name, description, unit);
             this.observer = Objects.requireNonNull(observer);
         }
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongAsyncCounterAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongAsyncCounterAdapter.java
index 14c58139d03e1..1bc21ef2c831c 100644
--- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongAsyncCounterAdapter.java
+++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongAsyncCounterAdapter.java
@@ -15,12 +15,19 @@
 import org.elasticsearch.telemetry.metric.LongAsyncCounter;
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
 
+import java.util.Collection;
 import java.util.Objects;
 import java.util.function.Supplier;
 
 public class LongAsyncCounterAdapter extends AbstractInstrument<ObservableLongCounter> implements LongAsyncCounter {
 
-    public LongAsyncCounterAdapter(Meter meter, String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+    public LongAsyncCounterAdapter(
+        Meter meter,
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<LongWithAttributes>> observer
+    ) {
         super(meter, new Builder(name, description, unit, observer));
     }
 
@@ -30,9 +37,9 @@ public void close() throws Exception {
     }
 
     private static class Builder extends AbstractInstrument.Builder<ObservableLongCounter> {
-        private final Supplier<LongWithAttributes> observer;
+        private final Supplier<Collection<LongWithAttributes>> observer;
 
-        private Builder(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+        private Builder(String name, String description, String unit, Supplier<Collection<LongWithAttributes>> observer) {
             super(name, description, unit);
             this.observer = Objects.requireNonNull(observer);
         }
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java
index 52c19c80c284f..eab9ed2eb5278 100644
--- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java
+++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.telemetry.apm.AbstractInstrument;
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
 
+import java.util.Collection;
 import java.util.Objects;
 import java.util.function.Supplier;
 
@@ -21,7 +22,7 @@
  * LongGaugeAdapter wraps an otel ObservableLongGauge
  */
 public class LongGaugeAdapter extends AbstractInstrument<ObservableLongGauge> implements org.elasticsearch.telemetry.metric.LongGauge {
-    public LongGaugeAdapter(Meter meter, String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+    public LongGaugeAdapter(Meter meter, String name, String description, String unit, Supplier<Collection<LongWithAttributes>> observer) {
         super(meter, new Builder(name, description, unit, observer));
     }
 
@@ -31,11 +32,11 @@ public void close() throws Exception {
     }
 
     private static class Builder extends AbstractInstrument.Builder<ObservableLongGauge> {
-        private final Supplier<LongWithAttributes> observer;
+        private final Supplier<Collection<LongWithAttributes>> observer;
 
-        private Builder(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+        private Builder(String name, String description, String unit, Supplier<Collection<LongWithAttributes>> observer) {
             super(name, description, unit);
-            this.observer = Objects.requireNonNull(observer);
+            this.observer = observer;
         }
 
         @Override
diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/OtelHelper.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/OtelHelper.java
index 3e8ab415bd25e..1d760c8c12791 100644
--- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/OtelHelper.java
+++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/OtelHelper.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.telemetry.metric.DoubleWithAttributes;
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
 
+import java.util.Collection;
 import java.util.Map;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
@@ -53,37 +54,45 @@ static Attributes fromMap(Map<String, Object> attributes) {
         return builder.build();
     }
 
-    static Consumer<ObservableDoubleMeasurement> doubleMeasurementCallback(Supplier<DoubleWithAttributes> observer) {
+    static Consumer<ObservableDoubleMeasurement> doubleMeasurementCallback(Supplier<Collection<DoubleWithAttributes>> observer) {
         return measurement -> {
-            DoubleWithAttributes observation;
+            Collection<DoubleWithAttributes> observations;
             try {
-                observation = observer.get();
+                observations = observer.get();
             } catch (RuntimeException err) {
                 assert false : "observer must not throw [" + err.getMessage() + "]";
                 logger.error("doubleMeasurementCallback observer unexpected error", err);
                 return;
             }
-            if (observation == null) {
+            if (observations == null) {
                 return;
             }
-            measurement.record(observation.value(), OtelHelper.fromMap(observation.attributes()));
+            for (DoubleWithAttributes observation : observations) {
+                if (observation != null) {
+                    measurement.record(observation.value(), OtelHelper.fromMap(observation.attributes()));
+                }
+            }
         };
     }
 
-    static Consumer<ObservableLongMeasurement> longMeasurementCallback(Supplier<LongWithAttributes> observer) {
+    static Consumer<ObservableLongMeasurement> longMeasurementCallback(Supplier<Collection<LongWithAttributes>> observer) {
         return measurement -> {
-            LongWithAttributes observation;
+            Collection<LongWithAttributes> observations;
             try {
-                observation = observer.get();
+                observations = observer.get();
             } catch (RuntimeException err) {
                 assert false : "observer must not throw [" + err.getMessage() + "]";
                 logger.error("longMeasurementCallback observer unexpected error", err);
                 return;
             }
-            if (observation == null) {
+            if (observations == null) {
                 return;
             }
-            measurement.record(observation.value(), OtelHelper.fromMap(observation.attributes()));
+            for (LongWithAttributes observation : observations) {
+                if (observation != null) {
+                    measurement.record(observation.value(), OtelHelper.fromMap(observation.attributes()));
+                }
+            }
         };
     }
 }
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java
index a4c9a9d3e1c67..ed09bd2462bb3 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java
@@ -112,12 +112,8 @@ public void resetClusterSetting() {
     public void testRolloverOnAutoShardCondition() throws Exception {
         final String dataStreamName = "logs-es";
 
-        putComposableIndexTemplate(
-            "my-template",
-            List.of("logs-*"),
-            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
-        );
-        final var createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        putComposableIndexTemplate("my-template", List.of("logs-*"), indexSettings(3, 0).build());
+        final var createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName);
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet());
 
         indexDocs(dataStreamName, randomIntBetween(100, 200));
@@ -277,12 +273,8 @@ public void testReduceShardsOnRollover() throws IOException {
         final String dataStreamName = "logs-es";
 
         // start with 3 shards
-        putComposableIndexTemplate(
-            "my-template",
-            List.of("logs-*"),
-            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
-        );
-        final var createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        putComposableIndexTemplate("my-template", List.of("logs-*"), indexSettings(3, 0).build());
+        final var createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName);
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet());
 
         indexDocs(dataStreamName, randomIntBetween(100, 200));
@@ -391,12 +383,8 @@ public void testReduceShardsOnRollover() throws IOException {
     public void testLazyRolloverKeepsPreviousAutoshardingDecision() throws IOException {
         final String dataStreamName = "logs-es";
 
-        putComposableIndexTemplate(
-            "my-template",
-            List.of("logs-*"),
-            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
-        );
-        final var createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        putComposableIndexTemplate("my-template", List.of("logs-*"), indexSettings(3, 0).build());
+        final var createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName);
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet());
 
         indexDocs(dataStreamName, randomIntBetween(100, 200));
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
index f79eea8676b3e..e3da69b7b2f0b 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.RequestBuilder;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
+import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@@ -151,14 +153,18 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     public void testBasicScenario() throws Exception {
         List<String> backingIndices = new ArrayList<>(4);
         putComposableIndexTemplate("id1", List.of("metrics-foo*"));
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "metrics-foo"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         putComposableIndexTemplate("id2", List.of("metrics-bar*"));
-        createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar");
+        createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "metrics-bar");
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         getDataStreamResponse.getDataStreams().sort(Comparator.comparing(dataStreamInfo -> dataStreamInfo.getDataStream().getName()));
@@ -228,7 +234,7 @@ public void testBasicScenario() throws Exception {
         verifyDocs("metrics-bar", numDocsBar + numDocsBar2, 1, 2);
         verifyDocs("metrics-foo", numDocsFoo + numDocsFoo2, 1, 2);
 
-        DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request("metrics-*");
+        DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "metrics-*");
         client().execute(DeleteDataStreamAction.INSTANCE, deleteDataStreamRequest).actionGet();
         getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(0));
@@ -245,7 +251,11 @@ public void testBasicScenario() throws Exception {
     public void testOtherWriteOps() throws Exception {
         putComposableIndexTemplate("id", List.of("metrics-foobar*"));
         String dataStreamName = "metrics-foobar";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         {
@@ -311,9 +321,16 @@ public void testOtherWriteOps() throws Exception {
 
         {
             // TODO: remove when fixing the bug when an index matching a backing index name is created before the data stream is created
-            createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName + "-baz");
+            createDataStreamRequest = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                dataStreamName + "-baz"
+            );
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName + "-baz" });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName + "-baz" }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -451,7 +468,7 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except
         indexDocs(dataStreamName, numDocs);
         verifyDocs(dataStreamName, numDocs, 1, 1);
 
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -485,13 +502,13 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except
         indexDocs(dataStreamName, numDocs2);
         verifyDocs(dataStreamName, numDocs + numDocs2, 1, 2);
 
-        getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
+        getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
         assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName));
         List<Index> backingIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices();
 
-        DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(dataStreamName);
+        DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, dataStreamName);
         client().execute(DeleteDataStreamAction.INSTANCE, deleteDataStreamRequest).actionGet();
         getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(0));
@@ -538,7 +555,11 @@ public void testTimeStampValidationInvalidFieldMapping() throws Exception {
     public void testResolvabilityOfDataStreamsInAPIs() throws Exception {
         putComposableIndexTemplate("id", List.of("logs-*"));
         String dataStreamName = "logs-foobar";
-        CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, request).actionGet();
         IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
         String aliasToDataStream = "logs";
@@ -583,10 +604,12 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception {
         verifyResolvability(dataStreamName, indicesAdmin().prepareOpen(dataStreamName), false);
         verifyResolvability(dataStreamName, indicesAdmin().prepareClose(dataStreamName), true);
         verifyResolvability(aliasToDataStream, indicesAdmin().prepareClose(aliasToDataStream), true);
-        verifyResolvability(dataStreamName, clusterAdmin().prepareSearchShards(dataStreamName), false);
+        verifyResolvability(
+            client().execute(TransportClusterSearchShardsAction.TYPE, new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, dataStreamName))
+        );
         verifyResolvability(client().execute(TransportIndicesShardStoresAction.TYPE, new IndicesShardStoresRequest(dataStreamName)));
 
-        request = new CreateDataStreamAction.Request("logs-barbaz");
+        request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "logs-barbaz");
         client().execute(CreateDataStreamAction.INSTANCE, request).actionGet();
         verifyResolvability(
             "logs-barbaz",
@@ -627,16 +650,29 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception {
         verifyResolvability(wildcardExpression, indicesAdmin().prepareGetIndex().addIndices(wildcardExpression), false);
         verifyResolvability(wildcardExpression, indicesAdmin().prepareOpen(wildcardExpression), false);
         verifyResolvability(wildcardExpression, indicesAdmin().prepareClose(wildcardExpression), false);
-        verifyResolvability(wildcardExpression, clusterAdmin().prepareSearchShards(wildcardExpression), false);
+        verifyResolvability(
+            client().execute(
+                TransportClusterSearchShardsAction.TYPE,
+                new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, wildcardExpression)
+            )
+        );
         verifyResolvability(client().execute(TransportIndicesShardStoresAction.TYPE, new IndicesShardStoresRequest(wildcardExpression)));
     }
 
     public void testCannotDeleteComposableTemplateUsedByDataStream() throws Exception {
         putComposableIndexTemplate("id", List.of("metrics-foobar*"));
         String dataStreamName = "metrics-foobar-baz";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
-        createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName + "-eggplant");
+        createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName + "-eggplant"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         TransportDeleteComposableIndexTemplateAction.Request req = new TransportDeleteComposableIndexTemplateAction.Request("id");
@@ -689,7 +725,11 @@ public void testCannotDeleteComposableTemplateUsedByDataStream() throws Exceptio
     public void testAliasActionsOnDataStreams() throws Exception {
         putComposableIndexTemplate("id1", List.of("metrics-foo*"));
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         AliasActions addAction = new AliasActions(AliasActions.Type.ADD).index(dataStreamName).aliases("foo");
@@ -828,7 +868,11 @@ public void testRandomDataSteamAliasesUpdate() throws Exception {
             .distinct()
             .toArray(String[]::new);
         for (String dataStream : dataStreams) {
-            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStream);
+            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                dataStream
+            );
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         }
         Map<String, Object> indexFilters = Map.of("term", Map.of("type", Map.of("value", "y")));
@@ -867,7 +911,11 @@ public void testDataSteamAliasWithMalformedFilter() throws Exception {
 
         String alias = randomAlphaOfLength(4);
         String dataStream = "log-" + randomAlphaOfLength(4).toLowerCase(Locale.ROOT);
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStream);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStream
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         AliasActions addAction = new AliasActions(AliasActions.Type.ADD).aliases(alias).indices(dataStream);
@@ -890,7 +938,11 @@ public void testDataSteamAliasWithMalformedFilter() throws Exception {
     public void testAliasActionsFailOnDataStreamBackingIndices() throws Exception {
         putComposableIndexTemplate("id1", List.of("metrics-foo*"));
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         String backingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
@@ -915,7 +967,11 @@ public void testAddDataStreamAliasesMixedExpressionValidation() throws Exception
         createIndex("metrics-myindex");
         putComposableIndexTemplate("id1", List.of("metrics-*"));
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         AliasActions addAction = new AliasActions(AliasActions.Type.ADD).index("metrics-*").aliases("my-alias");
@@ -929,7 +985,11 @@ public void testRemoveDataStreamAliasesMixedExpression() throws Exception {
         createIndex("metrics-myindex");
         putComposableIndexTemplate("id1", List.of("metrics-*"));
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
@@ -960,7 +1020,11 @@ public void testRemoveDataStreamAliasesMixedExpression() throws Exception {
     public void testUpdateDataStreamsWithWildcards() throws Exception {
         putComposableIndexTemplate("id1", List.of("metrics-*"));
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         {
             IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
@@ -1003,7 +1067,11 @@ public void testUpdateDataStreamsWithWildcards() throws Exception {
     public void testDataStreamAliasesUnsupportedParametersValidation() throws Exception {
         putComposableIndexTemplate("id1", List.of("metrics-*"));
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         {
             AliasActions addAction = new AliasActions(AliasActions.Type.ADD).index("metrics-*").aliases("my-alias").routing("[routing]");
@@ -1056,9 +1124,16 @@ public void testTimestampFieldCustomAttributes() throws Exception {
                 }""";
         putComposableIndexTemplate("id1", mapping, List.of("logs-foo*"), null, null);
 
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "logs-foobar"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "logs-foobar" });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { "logs-foobar" }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -1073,10 +1148,14 @@ public void testTimestampFieldCustomAttributes() throws Exception {
 
     public void testUpdateMappingViaDataStream() throws Exception {
         putComposableIndexTemplate("id1", List.of("logs-*"));
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "logs-foobar"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet();
 
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -1118,10 +1197,17 @@ public void testUpdateMappingViaDataStream() throws Exception {
 
     public void testUpdateIndexSettingsViaDataStream() throws Exception {
         putComposableIndexTemplate("id1", List.of("logs-*"));
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "logs-foobar"
+        );
 
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet();
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "logs-foobar" });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { "logs-foobar" }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -1255,11 +1341,15 @@ public void testIndexDocsWithCustomRoutingTargetingBackingIndex() throws Excepti
 
     public void testSearchAllResolvesDataStreams() throws Exception {
         putComposableIndexTemplate("id1", List.of("metrics-foo*"));
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "metrics-foo"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         putComposableIndexTemplate("id2", List.of("metrics-bar*"));
-        createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar");
+        createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "metrics-bar");
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         int numDocsBar = randomIntBetween(2, 16);
@@ -1287,7 +1377,7 @@ public void testGetDataStream() throws Exception {
 
         GetDataStreamAction.Response response = client().execute(
             GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(new String[] { "metrics-foo" })
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "metrics-foo" })
         ).actionGet();
         assertThat(response.getDataStreams().size(), is(1));
         DataStreamInfo metricsFooDataStream = response.getDataStreams().get(0);
@@ -1315,7 +1405,11 @@ private static void assertBackingIndex(String backingIndex, String timestampFiel
     public void testNoTimestampInDocument() throws Exception {
         putComposableIndexTemplate("id", List.of("logs-foobar*"));
         String dataStreamName = "logs-foobar";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create").source("{}", XContentType.JSON);
@@ -1326,7 +1420,11 @@ public void testNoTimestampInDocument() throws Exception {
     public void testMultipleTimestampValuesInDocument() throws Exception {
         putComposableIndexTemplate("id", List.of("logs-foobar*"));
         String dataStreamName = "logs-foobar";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create")
@@ -1374,7 +1472,7 @@ public void testMixedAutoCreate() throws Exception {
         bulkResponse = client().bulk(bulkRequest).actionGet();
         assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false));
 
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         GetDataStreamAction.Response getDataStreamsResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         assertThat(getDataStreamsResponse.getDataStreams(), hasSize(4));
@@ -1391,7 +1489,7 @@ public void testMixedAutoCreate() throws Exception {
         assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barfoo2"));
         assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barfoo3"));
 
-        DeleteDataStreamAction.Request deleteDSReq = new DeleteDataStreamAction.Request(new String[] { "*" });
+        DeleteDataStreamAction.Request deleteDSReq = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         client().execute(DeleteDataStreamAction.INSTANCE, deleteDSReq).actionGet();
         TransportDeleteComposableIndexTemplateAction.Request deleteTemplateRequest =
             new TransportDeleteComposableIndexTemplateAction.Request("*");
@@ -1412,7 +1510,7 @@ public void testAutoCreateV1TemplateNoDataStream() {
         BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet();
         assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false));
 
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         GetDataStreamAction.Response getDataStreamsResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         assertThat(getDataStreamsResponse.getDataStreams(), hasSize(0));
@@ -1430,7 +1528,12 @@ public void testCreatingDataStreamAndFirstBackingIndexExistsFails() throws Excep
 
         createIndex(backingIndex);
         putComposableIndexTemplate("id", List.of("logs-*"));
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName, now);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName,
+            now
+        );
         Exception e = expectThrows(
             ElasticsearchStatusException.class,
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest)
@@ -1440,7 +1543,11 @@ public void testCreatingDataStreamAndFirstBackingIndexExistsFails() throws Excep
 
     public void testQueryDataStreamNameInIndexField() throws Exception {
         putComposableIndexTemplate("id1", List.of("metrics-*"));
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "metrics-foo"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs("metrics-foo", 1);
@@ -1454,10 +1561,14 @@ public void testQueryDataStreamNameInIndexField() throws Exception {
     public void testDataStreamMetadata() throws Exception {
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build();
         putComposableIndexTemplate("id1", null, List.of("logs-*"), settings, Map.of("managed_by", "core-features"));
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "logs-foobar"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         getDataStreamResponse.getDataStreams().sort(Comparator.comparing(dataStreamInfo -> dataStreamInfo.getDataStream().getName()));
@@ -1475,7 +1586,11 @@ public void testDataStreamMetadata() throws Exception {
 
     public void testClusterStateIncludeDataStream() throws Exception {
         putComposableIndexTemplate("id1", List.of("metrics-foo*"));
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "metrics-foo"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         // when querying a backing index then the data stream should be included as well.
@@ -1528,7 +1643,10 @@ public void testMultiThreadedRollover() throws Exception {
 
         assertBusy(() -> {
             try {
-                GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "potato-biscuit" });
+                GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    new String[] { "potato-biscuit" }
+                );
                 GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                     .actionGet();
                 String newBackingIndexName = getDataStreamResponse.getDataStreams().get(0).getDataStream().getWriteIndex().getName();
@@ -1552,7 +1670,10 @@ public void testMultiThreadedRollover() throws Exception {
         });
 
         // We should *NOT* have a third index, it should have rolled over *exactly* once
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "potato-biscuit" });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { "potato-biscuit" }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         List<Index> backingIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices();
@@ -1565,7 +1686,11 @@ public void testMultiThreadedRollover() throws Exception {
     public void testSegmentsSortedOnTimestampDesc() throws Exception {
         Settings settings = indexSettings(1, 0).build();
         putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null);
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "metrics-foo"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         // We index data in the increasing order of @timestamp field
@@ -1602,7 +1727,7 @@ public void testCreateDataStreamWithSameNameAsIndexAlias() throws Exception {
         // Important detail: create template with data stream template after the index has been created
         DataStreamIT.putComposableIndexTemplate("my-template", List.of("my-*"));
 
-        var request = new CreateDataStreamAction.Request("my-alias");
+        var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "my-alias");
         var e = expectThrows(IllegalStateException.class, client().execute(CreateDataStreamAction.INSTANCE, request));
         assertThat(e.getMessage(), containsString("[my-alias (alias of ["));
         assertThat(e.getMessage(), containsString("]) conflicts with data stream"));
@@ -1615,7 +1740,7 @@ public void testCreateDataStreamWithSameNameAsIndex() throws Exception {
         // Important detail: create template with data stream template after the index has been created
         DataStreamIT.putComposableIndexTemplate("my-template", List.of("my-*"));
 
-        var request = new CreateDataStreamAction.Request("my-index");
+        var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "my-index");
         var e = expectThrows(IllegalStateException.class, client().execute(CreateDataStreamAction.INSTANCE, request));
         assertThat(e.getMessage(), containsString("data stream [my-index] conflicts with index"));
     }
@@ -1623,18 +1748,20 @@ public void testCreateDataStreamWithSameNameAsIndex() throws Exception {
     public void testCreateDataStreamWithSameNameAsDataStreamAlias() throws Exception {
         {
             DataStreamIT.putComposableIndexTemplate("my-template", List.of("my-*"));
-            var request = new CreateDataStreamAction.Request("my-ds");
+            var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "my-ds");
             assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
             var aliasesAddRequest = new IndicesAliasesRequest();
             aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("my-ds").aliases("my-alias"));
             assertAcked(indicesAdmin().aliases(aliasesAddRequest).actionGet());
 
-            var request2 = new CreateDataStreamAction.Request("my-alias");
+            var request2 = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "my-alias");
             var e = expectThrows(IllegalStateException.class, client().execute(CreateDataStreamAction.INSTANCE, request2));
             assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)"));
         }
         {
-            assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).actionGet());
+            assertAcked(
+                client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).actionGet()
+            );
             DataStreamIT.putComposableIndexTemplate(
                 "my-template",
                 null,
@@ -1645,10 +1772,10 @@ public void testCreateDataStreamWithSameNameAsDataStreamAlias() throws Exception
                 null,
                 false
             );
-            var request = new CreateDataStreamAction.Request("my-ds");
+            var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "my-ds");
             assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
 
-            var request2 = new CreateDataStreamAction.Request("my-alias");
+            var request2 = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "my-alias");
             var e = expectThrows(IllegalStateException.class, client().execute(CreateDataStreamAction.INSTANCE, request2));
             assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)"));
         }
@@ -1660,7 +1787,7 @@ public void testCreateDataStreamAliasWithSameNameAsIndexAlias() throws Exception
             CreateIndexRequest createIndexRequest = new CreateIndexRequest("es-logs").alias(new Alias("logs"));
             assertAcked(indicesAdmin().create(createIndexRequest).actionGet());
 
-            var request = new CreateDataStreamAction.Request("logs-es");
+            var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "logs-es");
             assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
             IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
             aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("logs-es").aliases("logs"));
@@ -1668,7 +1795,9 @@ public void testCreateDataStreamAliasWithSameNameAsIndexAlias() throws Exception
             assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (logs)"));
         }
         {
-            assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).actionGet());
+            assertAcked(
+                client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).actionGet()
+            );
             DataStreamIT.putComposableIndexTemplate(
                 "my-template",
                 null,
@@ -1680,7 +1809,7 @@ public void testCreateDataStreamAliasWithSameNameAsIndexAlias() throws Exception
                 false
             );
 
-            var request = new CreateDataStreamAction.Request("logs-es");
+            var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "logs-es");
             var e = expectThrows(IllegalStateException.class, client().execute(CreateDataStreamAction.INSTANCE, request));
             assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (logs)"));
         }
@@ -1693,7 +1822,7 @@ public void testCreateDataStreamAliasWithSameNameAsIndex() throws Exception {
         assertAcked(indicesAdmin().create(createIndexRequest).actionGet());
 
         {
-            var request = new CreateDataStreamAction.Request("logs-es");
+            var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "logs-es");
             assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
             IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
             aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("logs-es").aliases("logs"));
@@ -1704,7 +1833,9 @@ public void testCreateDataStreamAliasWithSameNameAsIndex() throws Exception {
             );
         }
         {
-            assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).actionGet());
+            assertAcked(
+                client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).actionGet()
+            );
             var e = expectThrows(
                 IllegalArgumentException.class,
                 () -> DataStreamIT.putComposableIndexTemplate(
@@ -1728,7 +1859,7 @@ public void testCreateDataStreamAliasWithSameNameAsIndex() throws Exception {
     public void testCreateIndexWithSameNameAsDataStreamAlias() throws Exception {
         DataStreamIT.putComposableIndexTemplate("my-template", List.of("logs-*"));
 
-        var request = new CreateDataStreamAction.Request("logs-es");
+        var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "logs-es");
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
         IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
         aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("logs-es").aliases("logs"));
@@ -1742,7 +1873,7 @@ public void testCreateIndexWithSameNameAsDataStreamAlias() throws Exception {
     public void testCreateIndexAliasWithSameNameAsDataStreamAlias() throws Exception {
         DataStreamIT.putComposableIndexTemplate("my-template", List.of("logs-*"));
 
-        var request = new CreateDataStreamAction.Request("logs-es");
+        var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "logs-es");
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
         IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
         aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("logs-es").aliases("logs"));
@@ -1766,7 +1897,7 @@ public void testCreateIndexAliasWithSameNameAsDataStreamAlias() throws Exception
     public void testRemoveGhostReference() throws Exception {
         String dataStreamName = "logs-es";
         DataStreamIT.putComposableIndexTemplate("my-template", List.of("logs-*"));
-        var request = new CreateDataStreamAction.Request(dataStreamName);
+        var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName);
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
         assertAcked(indicesAdmin().rolloverIndex(new RolloverRequest(dataStreamName, null)).actionGet());
         var indicesStatsResponse = indicesAdmin().stats(new IndicesStatsRequest()).actionGet();
@@ -1818,7 +1949,11 @@ public void onFailure(Exception e) {
         assertAcked(
             client().execute(
                 ModifyDataStreamsAction.INSTANCE,
-                new ModifyDataStreamsAction.Request(List.of(DataStreamAction.removeBackingIndex(dataStreamName, ghostReference.getName())))
+                new ModifyDataStreamsAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    TEST_REQUEST_TIMEOUT,
+                    List.of(DataStreamAction.removeBackingIndex(dataStreamName, ghostReference.getName()))
+                )
             )
         );
         ClusterState after = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state();
@@ -2042,7 +2177,11 @@ public void testSearchWithRouting() throws IOException, ExecutionException, Inte
             TransportPutComposableIndexTemplateAction.TYPE,
             new TransportPutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)
         ).actionGet();
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("my-logs");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "my-logs"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         assertResponse(prepareSearch("my-logs").setRouting("123"), resp -> { assertEquals(resp.getTotalShards(), 4); });
@@ -2054,7 +2193,7 @@ public void testWriteIndexWriteLoadAndAvgShardSizeIsStoredAfterRollover() throws
         final int numberOfReplicas = randomIntBetween(0, 1);
         final var indexSettings = indexSettings(numberOfShards, numberOfReplicas).build();
         DataStreamIT.putComposableIndexTemplate("my-template", null, List.of("logs-*"), indexSettings, null);
-        final var request = new CreateDataStreamAction.Request(dataStreamName);
+        final var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName);
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
 
         indexDocsAndEnsureThereIsCapturedWriteLoad(dataStreamName);
@@ -2102,7 +2241,7 @@ public void testWriteLoadAndAvgShardSizeIsStoredInABestEffort() throws Exception
         final var indexSettings = indexSettings(2, 1).put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes))
             .build();
         DataStreamIT.putComposableIndexTemplate("my-template", null, List.of("logs-*"), indexSettings, null);
-        final var createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        final var createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName);
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet());
         ensureGreen(dataStreamName);
 
@@ -2173,7 +2312,7 @@ public void testNoShardSizeIsForecastedWhenAllShardStatRequestsFail() throws Exc
 
         final var indexSettings = indexSettings(1, 0).put("index.routing.allocation.require._name", dataOnlyNode).build();
         DataStreamIT.putComposableIndexTemplate("my-template", null, List.of("logs-*"), indexSettings, null);
-        final var createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        final var createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName);
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet());
 
         for (int i = 0; i < 10; i++) {
@@ -2211,7 +2350,7 @@ public void testShardSizeIsForecastedDuringRollover() throws Exception {
         final int numberOfReplicas = randomIntBetween(0, 1);
         final var indexSettings = indexSettings(numberOfShards, numberOfReplicas).build();
         DataStreamIT.putComposableIndexTemplate("my-template", null, List.of("logs-*"), indexSettings, null);
-        final var request = new CreateDataStreamAction.Request(dataStreamName);
+        final var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName);
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
 
         for (int i = 0; i < 4; i++) {
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java
index bebdc085760d4..5b5f3920f019b 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamMigrationIT.java
@@ -71,7 +71,10 @@ public void testBasicMigration() throws Exception {
         assertThat(resolveResponse.getDataStreams().size(), equalTo(0));
         assertThat(resolveResponse.getIndices().size(), equalTo(2));
 
-        client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get();
+        client().execute(
+            MigrateToDataStreamAction.INSTANCE,
+            new MigrateToDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, alias)
+        ).get();
 
         resolveResponse = indicesAdmin().resolveIndex(resolveRequest).get();
         assertThat(resolveResponse.getAliases().size(), equalTo(0));
@@ -112,7 +115,10 @@ public void testMigrationWithoutTemplate() throws Exception {
 
         Exception e = expectThrows(
             Exception.class,
-            () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get()
+            () -> client().execute(
+                MigrateToDataStreamAction.INSTANCE,
+                new MigrateToDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, alias)
+            ).get()
         );
 
         assertTrue(
@@ -144,7 +150,10 @@ public void testMigrationWithoutIndexMappings() throws Exception {
 
         Exception e = expectThrows(
             Exception.class,
-            () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get()
+            () -> client().execute(
+                MigrateToDataStreamAction.INSTANCE,
+                new MigrateToDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, alias)
+            ).get()
         );
 
         assertTrue(throwableOrItsCause(e, IllegalArgumentException.class, "must have mappings for a timestamp field"));
@@ -179,7 +188,10 @@ public void testMigrationWithoutTimestampMapping() throws Exception {
 
         Exception e = expectThrows(
             Exception.class,
-            () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get()
+            () -> client().execute(
+                MigrateToDataStreamAction.INSTANCE,
+                new MigrateToDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, alias)
+            ).get()
         );
 
         assertTrue(throwableOrItsCause(e, IllegalArgumentException.class, "data stream timestamp field [@timestamp] does not exist"));
@@ -214,7 +226,10 @@ public void testMigrationWithoutWriteIndex() throws Exception {
 
         Exception e = expectThrows(
             Exception.class,
-            () -> client().execute(MigrateToDataStreamAction.INSTANCE, new MigrateToDataStreamAction.Request(alias)).get()
+            () -> client().execute(
+                MigrateToDataStreamAction.INSTANCE,
+                new MigrateToDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, alias)
+            ).get()
         );
 
         assertTrue(throwableOrItsCause(e, IllegalArgumentException.class, "alias [" + alias + "] must specify a write index"));
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
index 412410c4e4c7c..d0dd2888f535f 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
@@ -116,15 +116,15 @@ public void setup() throws Exception {
               }
             }""", List.of("with-fs"), null, null, null, null, true);
 
-        CreateDataStreamAction.Request request = new CreateDataStreamAction.Request("ds");
+        CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "ds");
         AcknowledgedResponse response = client.execute(CreateDataStreamAction.INSTANCE, request).get();
         assertTrue(response.isAcknowledged());
 
-        request = new CreateDataStreamAction.Request("other-ds");
+        request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "other-ds");
         response = client.execute(CreateDataStreamAction.INSTANCE, request).get();
         assertTrue(response.isAcknowledged());
 
-        request = new CreateDataStreamAction.Request("with-fs");
+        request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "with-fs");
         response = client.execute(CreateDataStreamAction.INSTANCE, request).get();
         assertTrue(response.isAcknowledged());
 
@@ -140,7 +140,7 @@ public void setup() throws Exception {
 
         // Resolve backing index names after data streams have been created:
         // (these names have a date component, and running around midnight could lead to test failures otherwise)
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
         dsBackingIndexName = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName();
         otherDsBackingIndexName = getDataStreamResponse.getDataStreams().get(1).getDataStream().getIndices().get(0).getName();
@@ -193,7 +193,9 @@ public void testSnapshotAndRestore() throws Exception {
 
         assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" })));
+        assertAcked(
+            client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" }))
+        );
 
         RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
             .cluster()
@@ -213,7 +215,7 @@ public void testSnapshotAndRestore() throws Exception {
 
         GetDataStreamAction.Response ds = client.execute(
             GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(new String[] { "ds" })
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" })
         ).get();
         assertEquals(1, ds.getDataStreams().size());
         assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
@@ -271,7 +273,7 @@ public void testSnapshotAndRestoreAllDataStreamsInPlace() throws Exception {
             assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
         });
 
-        GetDataStreamAction.Request getDataSteamRequest = new GetDataStreamAction.Request(new String[] { "*" });
+        GetDataStreamAction.Request getDataSteamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         GetDataStreamAction.Response ds = client.execute(GetDataStreamAction.INSTANCE, getDataSteamRequest).get();
         assertThat(
             ds.getDataStreams().stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()),
@@ -330,7 +332,7 @@ public void testSnapshotAndRestoreInPlace() {
             assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
         });
 
-        GetDataStreamAction.Request getDataSteamRequest = new GetDataStreamAction.Request(new String[] { "ds" });
+        GetDataStreamAction.Request getDataSteamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" });
         GetDataStreamAction.Response ds = client.execute(GetDataStreamAction.INSTANCE, getDataSteamRequest).actionGet();
         assertThat(
             ds.getDataStreams().stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()),
@@ -363,7 +365,7 @@ public void testFailureStoreSnapshotAndRestore() throws Exception {
 
         assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(fsBackingIndexName, fsFailureIndexName));
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("with-fs")));
+        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "with-fs")));
 
         {
             RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
@@ -377,7 +379,7 @@ public void testFailureStoreSnapshotAndRestore() throws Exception {
 
             GetDataStreamAction.Response ds = client.execute(
                 GetDataStreamAction.INSTANCE,
-                new GetDataStreamAction.Request(new String[] { "with-fs" })
+                new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "with-fs" })
             ).get();
             assertEquals(1, ds.getDataStreams().size());
             assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
@@ -399,7 +401,7 @@ public void testFailureStoreSnapshotAndRestore() throws Exception {
 
             GetDataStreamAction.Response ds = client.execute(
                 GetDataStreamAction.INSTANCE,
-                new GetDataStreamAction.Request(new String[] { "with-fs2" })
+                new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "with-fs2" })
             ).get();
             assertEquals(1, ds.getDataStreams().size());
             assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
@@ -450,7 +452,7 @@ public void testSnapshotAndRestoreAllIncludeSpecificDataStream() throws Exceptio
             );
         }
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).get());
+        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).get());
         assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN));
 
         RestoreSnapshotRequest restoreSnapshotRequest = new RestoreSnapshotRequest(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT);
@@ -472,7 +474,7 @@ public void testSnapshotAndRestoreAllIncludeSpecificDataStream() throws Exceptio
 
         GetDataStreamAction.Response ds = client.execute(
             GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(new String[] { dataStreamToSnapshot })
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamToSnapshot })
         ).get();
         assertEquals(1, ds.getDataStreams().size());
         assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
@@ -494,7 +496,7 @@ public void testSnapshotAndRestoreAllIncludeSpecificDataStream() throws Exceptio
             )
         );
 
-        DeleteDataStreamAction.Request r = new DeleteDataStreamAction.Request(new String[] { dataStreamToSnapshot });
+        DeleteDataStreamAction.Request r = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamToSnapshot });
         assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, r).get());
     }
 
@@ -510,7 +512,10 @@ public void testSnapshotAndRestoreReplaceAll() throws Exception {
             containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName, fsBackingIndexName, fsFailureIndexName)
         );
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get());
+        assertAcked(
+            client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" }))
+                .get()
+        );
         assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN));
 
         var restoreSnapshotRequest = new RestoreSnapshotRequest(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT).waitForCompletion(true)
@@ -528,7 +533,7 @@ public void testSnapshotAndRestoreReplaceAll() throws Exception {
 
         GetDataStreamAction.Response ds = client.execute(
             GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(new String[] { "*" })
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" })
         ).get();
         assertEquals(3, ds.getDataStreams().size());
         assertThat(
@@ -555,7 +560,7 @@ public void testSnapshotAndRestoreReplaceAll() throws Exception {
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
 
-        DeleteDataStreamAction.Request r = new DeleteDataStreamAction.Request(new String[] { "*" });
+        DeleteDataStreamAction.Request r = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, r).get());
     }
 
@@ -571,7 +576,7 @@ public void testSnapshotAndRestoreAll() throws Exception {
             containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName, fsBackingIndexName, fsFailureIndexName)
         );
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).get());
+        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).get());
         assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN));
 
         var restoreSnapshotRequest = new RestoreSnapshotRequest(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT).waitForCompletion(true)
@@ -588,7 +593,7 @@ public void testSnapshotAndRestoreAll() throws Exception {
 
         GetDataStreamAction.Response ds = client.execute(
             GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(new String[] { "*" })
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" })
         ).get();
         assertEquals(3, ds.getDataStreams().size());
         assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
@@ -621,7 +626,12 @@ public void testSnapshotAndRestoreAll() throws Exception {
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
 
-        assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" })).get());
+        assertAcked(
+            client().execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" })
+            ).get()
+        );
     }
 
     public void testSnapshotAndRestoreIncludeAliasesFalse() throws Exception {
@@ -636,7 +646,7 @@ public void testSnapshotAndRestoreIncludeAliasesFalse() throws Exception {
             containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName, fsBackingIndexName, fsFailureIndexName)
         );
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).get());
+        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).get());
         assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN));
 
         var restoreSnapshotRequest = new RestoreSnapshotRequest(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT).waitForCompletion(true)
@@ -654,7 +664,7 @@ public void testSnapshotAndRestoreIncludeAliasesFalse() throws Exception {
 
         GetDataStreamAction.Response ds = client.execute(
             GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(new String[] { "*" })
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" })
         ).get();
         assertEquals(3, ds.getDataStreams().size());
         assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
@@ -668,7 +678,12 @@ public void testSnapshotAndRestoreIncludeAliasesFalse() throws Exception {
 
         GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("*")).actionGet();
         assertThat(getAliasesResponse.getDataStreamAliases(), anEmptyMap());
-        assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" })).get());
+        assertAcked(
+            client().execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" })
+            ).get()
+        );
     }
 
     public void testRename() throws Exception {
@@ -703,7 +718,7 @@ public void testRename() throws Exception {
 
         GetDataStreamAction.Response ds = client.execute(
             GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(new String[] { "ds2" })
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds2" })
         ).get();
         assertEquals(1, ds.getDataStreams().size());
         assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
@@ -761,7 +776,7 @@ public void testRenameWriteDataStream() throws Exception {
 
         GetDataStreamAction.Response ds = client.execute(
             GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(new String[] { "other-ds2" })
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "other-ds2" })
         ).get();
         assertEquals(1, ds.getDataStreams().size());
         assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
@@ -814,7 +829,8 @@ public void testBackingIndexIsNotRenamedWhenRestoringDataStream() {
         );
 
         // delete data stream
-        client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" })).actionGet();
+        client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" }))
+            .actionGet();
 
         // restore data stream attempting to rename the backing index
         RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
@@ -828,7 +844,7 @@ public void testBackingIndexIsNotRenamedWhenRestoringDataStream() {
 
         assertThat(restoreSnapshotResponse.status(), is(RestStatus.OK));
 
-        GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(new String[] { "ds" });
+        GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" });
         GetDataStreamAction.Response response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet();
         assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName));
     }
@@ -867,7 +883,7 @@ public void testDataStreamAndBackingIndicesAreRenamedUsingRegex() {
         assertThat(restoreSnapshotResponse.status(), is(RestStatus.OK));
 
         // assert "ds" was restored as "test-ds" and the backing index has a valid name
-        GetDataStreamAction.Request getRenamedDS = new GetDataStreamAction.Request(new String[] { "test-ds" });
+        GetDataStreamAction.Request getRenamedDS = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "test-ds" });
         GetDataStreamAction.Response response = client.execute(GetDataStreamAction.INSTANCE, getRenamedDS).actionGet();
         assertThat(
             response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(),
@@ -875,7 +891,7 @@ public void testDataStreamAndBackingIndicesAreRenamedUsingRegex() {
         );
 
         // data stream "ds" should still exist in the system
-        GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(new String[] { "ds" });
+        GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" });
         response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet();
         assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName));
     }
@@ -905,7 +921,7 @@ public void testWildcards() throws Exception {
 
         GetDataStreamAction.Response ds = client.execute(
             GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(new String[] { "ds2" })
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds2" })
         ).get();
         assertEquals(1, ds.getDataStreams().size());
         assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
@@ -946,7 +962,9 @@ public void testDataStreamNotRestoredWhenIndexRequested() throws Exception {
         RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
         assertEquals(RestStatus.OK, status);
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" })));
+        assertAcked(
+            client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" }))
+        );
 
         RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
             .cluster()
@@ -957,7 +975,7 @@ public void testDataStreamNotRestoredWhenIndexRequested() throws Exception {
 
         assertEquals(RestStatus.OK, restoreSnapshotResponse.status());
 
-        GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(new String[] { "ds" });
+        GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" });
         expectThrows(ResourceNotFoundException.class, client.execute(GetDataStreamAction.INSTANCE, getRequest));
     }
 
@@ -972,7 +990,12 @@ public void testDataStreamNotIncludedInLimitedSnapshot() throws ExecutionExcepti
             .get();
         assertThat(createSnapshotResponse.getSnapshotInfo().state(), Matchers.is(SnapshotState.SUCCESS));
 
-        assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })));
+        assertAcked(
+            client().execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" })
+            )
+        );
 
         final RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(
             TEST_REQUEST_TIMEOUT,
@@ -1014,7 +1037,10 @@ public void testDeleteDataStreamDuringSnapshot() throws Exception {
         assertDocCount(dataStream, 100L);
         // Resolve backing index name after the data stream has been created because it has a date component,
         // and running around midnight could lead to test failures otherwise
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStream });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStream }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
         String backingIndexName = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName();
 
@@ -1032,7 +1058,10 @@ public void testDeleteDataStreamDuringSnapshot() throws Exception {
         // non-partial snapshots do not allow delete operations on data streams where snapshot has not been completed
         try {
             logger.info("--> delete index while non-partial snapshot is running");
-            client1.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { dataStream })).actionGet();
+            client1.execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStream })
+            ).actionGet();
             fail("Expected deleting index to fail during snapshot");
         } catch (SnapshotInProgressException e) {
             assertThat(e.getMessage(), containsString("Cannot delete data streams that are being snapshotted: [" + dataStream));
@@ -1139,7 +1168,12 @@ public void testSnapshotDSDuringRollover() throws Exception {
         final SnapshotInfo snapshotInfo = assertSuccessful(snapshotFuture);
 
         assertThat(snapshotInfo.dataStreams(), hasItems("ds"));
-        assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" })).get());
+        assertAcked(
+            client().execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ds" })
+            ).get()
+        );
 
         RestoreInfo restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, repoName, snapshotName)
             .setWaitForCompletion(true)
@@ -1179,7 +1213,12 @@ public void testSnapshotDSDuringRolloverAndDeleteOldIndex() throws Exception {
             snapshotInfo.dataStreams(),
             not(hasItems("ds"))
         );
-        assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "other-ds" })));
+        assertAcked(
+            client().execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "other-ds" })
+            )
+        );
 
         RestoreInfo restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, repoName, snapshotName)
             .setWaitForCompletion(true)
@@ -1240,7 +1279,10 @@ public void testRestoreSnapshotFully() throws Exception {
         createIndexWithContent(indexName);
         createFullSnapshot(REPO, snapshotName);
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get());
+        assertAcked(
+            client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" }))
+                .get()
+        );
         assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.lenientExpandOpenHidden()).get());
 
         RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
@@ -1250,7 +1292,7 @@ public void testRestoreSnapshotFully() throws Exception {
             .get();
         assertEquals(RestStatus.OK, restoreSnapshotResponse.status());
 
-        GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(new String[] { "*" });
+        GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         assertThat(client.execute(GetDataStreamAction.INSTANCE, getRequest).get().getDataStreams(), hasSize(3));
         assertNotNull(client.admin().indices().prepareGetIndex().setIndices(indexName).get());
     }
@@ -1258,10 +1300,10 @@ public void testRestoreSnapshotFully() throws Exception {
     public void testRestoreDataStreamAliasWithConflictingDataStream() throws Exception {
         var snapshotName = "test-snapshot";
         createFullSnapshot(REPO, snapshotName);
-        client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).actionGet();
+        client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).actionGet();
         DataStreamIT.putComposableIndexTemplate("my-template", List.of("my-*"));
         try {
-            var request = new CreateDataStreamAction.Request("my-alias");
+            var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "my-alias");
             assertAcked(client.execute(CreateDataStreamAction.INSTANCE, request).actionGet());
             var e = expectThrows(
                 IllegalStateException.class,
@@ -1270,7 +1312,7 @@ public void testRestoreDataStreamAliasWithConflictingDataStream() throws Excepti
             assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)"));
         } finally {
             // Need to remove data streams in order to remove template
-            client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).actionGet();
+            client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).actionGet();
             // Need to remove template, because base class doesn't remove composable index templates after each test (only legacy templates)
             client.execute(
                 TransportDeleteComposableIndexTemplateAction.TYPE,
@@ -1282,7 +1324,7 @@ public void testRestoreDataStreamAliasWithConflictingDataStream() throws Excepti
     public void testRestoreDataStreamAliasWithConflictingIndicesAlias() throws Exception {
         var snapshotName = "test-snapshot";
         createFullSnapshot(REPO, snapshotName);
-        client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).actionGet();
+        client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*")).actionGet();
         CreateIndexRequest createIndexRequest = new CreateIndexRequest("my-index").alias(new Alias("my-alias"));
         assertAcked(client.admin().indices().create(createIndexRequest).actionGet());
 
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java
new file mode 100644
index 0000000000000..a52016e8c7f0b
--- /dev/null
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java
@@ -0,0 +1,428 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.datastreams;
+
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction;
+import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockRequest;
+import org.elasticsearch.action.admin.indices.readonly.TransportAddIndexBlockAction;
+import org.elasticsearch.action.admin.indices.rollover.RolloverAction;
+import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
+import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.FailureStoreMetrics;
+import org.elasticsearch.action.datastreams.CreateDataStreamAction;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Template;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.IngestTestPlugin;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.TestProcessor;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.XContentType;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+/**
+ * An integration test that verifies how different paths/scenarios affect the APM metrics for failure stores.
+ */
+@ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0, scope = ESIntegTestCase.Scope.SUITE)
+public class IngestFailureStoreMetricsIT extends ESIntegTestCase {
+
+    private static final List<String> METRICS = List.of(
+        FailureStoreMetrics.METRIC_TOTAL,
+        FailureStoreMetrics.METRIC_FAILURE_STORE,
+        FailureStoreMetrics.METRIC_REJECTED
+    );
+
+    private String template;
+    private String dataStream;
+    private String pipeline;
+
+    @Before
+    public void initializeRandomNames() {
+        template = "template-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        dataStream = "data-stream-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        pipeline = "pipeline-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        logger.info(
+            "--> running [{}] with generated names data stream [{}], template [{}] and pipeline [{}]",
+            getTestName(),
+            dataStream,
+            template,
+            pipeline
+        );
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(DataStreamsPlugin.class, CustomIngestTestPlugin.class, TestTelemetryPlugin.class, MapperExtrasPlugin.class);
+    }
+
+    public void testNoPipelineNoFailures() throws IOException {
+        putComposableIndexTemplate(true);
+        createDataStream();
+
+        int nrOfDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfDocs, null);
+
+        var measurements = collectTelemetry();
+        assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfDocs, dataStream);
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size());
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size());
+    }
+
+    public void testFailingPipelineNoFailureStore() throws IOException {
+        putComposableIndexTemplate(false);
+        createDataStream();
+        createBasicPipeline("fail");
+
+        int nrOfSuccessfulDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfSuccessfulDocs, null);
+        int nrOfFailingDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfFailingDocs, pipeline);
+
+        var measurements = collectTelemetry();
+        assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfSuccessfulDocs + nrOfFailingDocs, dataStream);
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size());
+        assertMeasurements(
+            measurements.get(FailureStoreMetrics.METRIC_REJECTED),
+            nrOfFailingDocs,
+            dataStream,
+            FailureStoreMetrics.ErrorLocation.PIPELINE,
+            false
+        );
+    }
+
+    public void testFailingPipelineWithFailureStore() throws IOException {
+        putComposableIndexTemplate(true);
+        createDataStream();
+        createBasicPipeline("fail");
+
+        int nrOfSuccessfulDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfSuccessfulDocs, null);
+        int nrOfFailingDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfFailingDocs, pipeline);
+
+        var measurements = collectTelemetry();
+        assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfSuccessfulDocs + nrOfFailingDocs, dataStream);
+        assertMeasurements(
+            measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE),
+            nrOfFailingDocs,
+            dataStream,
+            FailureStoreMetrics.ErrorLocation.PIPELINE
+        );
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size());
+    }
+
+    public void testShardFailureNoFailureStore() throws IOException {
+        putComposableIndexTemplate(false);
+        createDataStream();
+
+        int nrOfSuccessfulDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfSuccessfulDocs, null);
+        int nrOfFailingDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfFailingDocs, "\"foo\"", null);
+
+        var measurements = collectTelemetry();
+        assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfSuccessfulDocs + nrOfFailingDocs, dataStream);
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size());
+        assertMeasurements(
+            measurements.get(FailureStoreMetrics.METRIC_REJECTED),
+            nrOfFailingDocs,
+            dataStream,
+            FailureStoreMetrics.ErrorLocation.SHARD,
+            false
+        );
+    }
+
+    public void testShardFailureWithFailureStore() throws IOException {
+        putComposableIndexTemplate(true);
+        createDataStream();
+
+        int nrOfSuccessfulDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfSuccessfulDocs, null);
+        int nrOfFailingDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfFailingDocs, "\"foo\"", null);
+
+        var measurements = collectTelemetry();
+        assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfSuccessfulDocs + nrOfFailingDocs, dataStream);
+        assertMeasurements(
+            measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE),
+            nrOfFailingDocs,
+            dataStream,
+            FailureStoreMetrics.ErrorLocation.SHARD
+        );
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size());
+    }
+
+    /**
+     * Make sure the rejected counter gets incremented when there were shard-level failures while trying to redirect a document to the
+     * failure store.
+     */
+    public void testRejectionFromFailureStore() throws IOException {
+        putComposableIndexTemplate(true);
+        createDataStream();
+
+        // Initialize failure store.
+        var rolloverRequest = new RolloverRequest(dataStream, null);
+        rolloverRequest.setIndicesOptions(
+            IndicesOptions.builder(rolloverRequest.indicesOptions())
+                .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false))
+                .build()
+        );
+        var rolloverResponse = client().execute(RolloverAction.INSTANCE, rolloverRequest).actionGet();
+        var failureStoreIndex = rolloverResponse.getNewIndex();
+        // Add a write block to the failure store index, which causes shard-level "failures".
+        var addIndexBlockRequest = new AddIndexBlockRequest(IndexMetadata.APIBlock.WRITE, failureStoreIndex);
+        client().execute(TransportAddIndexBlockAction.TYPE, addIndexBlockRequest).actionGet();
+
+        int nrOfSuccessfulDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfSuccessfulDocs, null);
+        int nrOfFailingDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfFailingDocs, "\"foo\"", null);
+
+        var measurements = collectTelemetry();
+        assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfSuccessfulDocs + nrOfFailingDocs, dataStream);
+        assertMeasurements(
+            measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE),
+            nrOfFailingDocs,
+            dataStream,
+            FailureStoreMetrics.ErrorLocation.SHARD
+        );
+        assertMeasurements(
+            measurements.get(FailureStoreMetrics.METRIC_REJECTED),
+            nrOfFailingDocs,
+            dataStream,
+            FailureStoreMetrics.ErrorLocation.SHARD,
+            true
+        );
+    }
+
+    /**
+     * Make sure metrics get the correct <code>data_stream</code> attribute after a reroute.
+     */
+    public void testRerouteSuccessfulCorrectName() throws IOException {
+        putComposableIndexTemplate(false);
+        createDataStream();
+
+        String destination = dataStream + "-destination";
+        final var createDataStreamRequest = new CreateDataStreamAction.Request(destination);
+        assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet());
+        createReroutePipeline(destination);
+
+        int nrOfDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfDocs, pipeline);
+
+        var measurements = collectTelemetry();
+        assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfDocs, destination);
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size());
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size());
+    }
+
+    public void testDropping() throws IOException {
+        putComposableIndexTemplate(true);
+        createDataStream();
+        createBasicPipeline("drop");
+
+        int nrOfDocs = randomIntBetween(5, 10);
+        indexDocs(dataStream, nrOfDocs, pipeline);
+
+        var measurements = collectTelemetry();
+        assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfDocs, dataStream);
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size());
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size());
+    }
+
+    public void testDataStreamAlias() throws IOException {
+        putComposableIndexTemplate(false);
+        createDataStream();
+        var indicesAliasesRequest = new IndicesAliasesRequest();
+        indicesAliasesRequest.addAliasAction(
+            IndicesAliasesRequest.AliasActions.add().alias("some-alias").index(dataStream).writeIndex(true)
+        );
+        client().execute(TransportIndicesAliasesAction.TYPE, indicesAliasesRequest).actionGet();
+
+        int nrOfDocs = randomIntBetween(5, 10);
+        indexDocs("some-alias", nrOfDocs, null);
+
+        var measurements = collectTelemetry();
+        assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfDocs, dataStream);
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size());
+        assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size());
+    }
+
+    private void putComposableIndexTemplate(boolean failureStore) throws IOException {
+        TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(template);
+        request.indexTemplate(
+            ComposableIndexTemplate.builder()
+                .indexPatterns(List.of(dataStream + "*"))
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, failureStore))
+                .template(new Template(null, new CompressedXContent("""
+                    {
+                      "dynamic": false,
+                      "properties": {
+                        "@timestamp": {
+                          "type": "date"
+                        },
+                        "count": {
+                            "type": "long"
+                        }
+                      }
+                    }"""), null))
+                .build()
+        );
+        client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet();
+    }
+
+    private void createDataStream() {
+        final var createDataStreamRequest = new CreateDataStreamAction.Request(dataStream);
+        assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet());
+    }
+
+    private void createBasicPipeline(String processorType) {
+        createPipeline(Strings.format("\"%s\": {}", processorType));
+    }
+
+    private void createReroutePipeline(String destination) {
+        createPipeline(Strings.format("\"reroute\": {\"destination\": \"%s\"}", destination));
+    }
+
+    private void createPipeline(String processor) {
+        String pipelineDefinition = Strings.format("{\"processors\": [{%s}]}", processor);
+        BytesReference bytes = new BytesArray(pipelineDefinition);
+        clusterAdmin().putPipeline(new PutPipelineRequest(pipeline, bytes, XContentType.JSON)).actionGet();
+    }
+
+    private void indexDocs(String dataStream, int numDocs, String pipeline) {
+        indexDocs(dataStream, numDocs, "1", pipeline);
+    }
+
+    private void indexDocs(String dataStream, int numDocs, String value, String pipeline) {
+        BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+        for (int i = 0; i < numDocs; i++) {
+            String time = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis());
+            bulkRequest.add(
+                new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE)
+                    .source(Strings.format("{\"%s\":\"%s\", \"count\": %s}", DEFAULT_TIMESTAMP_FIELD, time, value), XContentType.JSON)
+                    .setPipeline(pipeline)
+            );
+        }
+        client().bulk(bulkRequest).actionGet();
+    }
+
+    private static Map<String, List<Measurement>> collectTelemetry() {
+        Map<String, List<Measurement>> measurements = new HashMap<>();
+        for (PluginsService pluginsService : internalCluster().getInstances(PluginsService.class)) {
+            final TestTelemetryPlugin telemetryPlugin = pluginsService.filterPlugins(TestTelemetryPlugin.class).findFirst().orElseThrow();
+
+            telemetryPlugin.collect();
+
+            for (String metricName : METRICS) {
+                measurements.put(metricName, telemetryPlugin.getLongCounterMeasurement(metricName));
+            }
+        }
+        return measurements;
+    }
+
+    private void assertMeasurements(List<Measurement> measurements, int expectedSize, String expectedDataStream) {
+        assertMeasurements(measurements, expectedSize, expectedDataStream, (Consumer<Measurement>) null);
+    }
+
+    private void assertMeasurements(
+        List<Measurement> measurements,
+        int expectedSize,
+        String expectedDataStream,
+        FailureStoreMetrics.ErrorLocation location
+    ) {
+        assertMeasurements(
+            measurements,
+            expectedSize,
+            expectedDataStream,
+            measurement -> assertEquals(location.name(), measurement.attributes().get("error_location"))
+        );
+    }
+
+    private void assertMeasurements(
+        List<Measurement> measurements,
+        int expectedSize,
+        String expectedDataStream,
+        FailureStoreMetrics.ErrorLocation location,
+        boolean failureStore
+    ) {
+        assertMeasurements(measurements, expectedSize, expectedDataStream, measurement -> {
+            assertEquals(location.name(), measurement.attributes().get("error_location"));
+            assertEquals(failureStore, measurement.attributes().get("failure_store"));
+        });
+    }
+
+    private void assertMeasurements(
+        List<Measurement> measurements,
+        int expectedSize,
+        String expectedDataStream,
+        Consumer<Measurement> customAssertion
+    ) {
+        assertEquals(expectedSize, measurements.size());
+        for (Measurement measurement : measurements) {
+            assertEquals(expectedDataStream, measurement.attributes().get("data_stream"));
+            if (customAssertion != null) {
+                customAssertion.accept(measurement);
+            }
+        }
+    }
+
+    public static class CustomIngestTestPlugin extends IngestTestPlugin {
+        @Override
+        public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
+            Map<String, Processor.Factory> processors = new HashMap<>();
+            processors.put(
+                "drop",
+                (factories, tag, description, config) -> new TestProcessor(tag, "drop", description, ingestDocument -> null)
+            );
+            processors.put("reroute", (factories, tag, description, config) -> {
+                String destination = (String) config.remove("destination");
+                return new TestProcessor(
+                    tag,
+                    "reroute",
+                    description,
+                    (Consumer<IngestDocument>) ingestDocument -> ingestDocument.reroute(destination)
+                );
+            });
+            processors.put(
+                "fail",
+                (processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", description, new RuntimeException())
+            );
+            return processors;
+        }
+    }
+}
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java
index 89d576e74be2f..726f2c21cec7f 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java
@@ -95,11 +95,10 @@ public void onFailure(Exception e) {
     }
 
     private DataStream getDataStream(String dataStreamName) {
-        return client().execute(GetDataStreamAction.INSTANCE, new GetDataStreamAction.Request(new String[] { dataStreamName }))
-            .actionGet()
-            .getDataStreams()
-            .get(0)
-            .getDataStream();
+        return client().execute(
+            GetDataStreamAction.INSTANCE,
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName })
+        ).actionGet().getDataStreams().get(0).getDataStream();
     }
 
     private void createDataStream(String dataStreamName) throws InterruptedException, ExecutionException {
@@ -117,7 +116,11 @@ private void createDataStream(String dataStreamName) throws InterruptedException
         ).actionGet();
         assertThat(putComposableTemplateResponse.isAcknowledged(), is(true));
 
-        final CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        final CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         final AcknowledgedResponse createDataStreamResponse = client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest)
             .get();
         assertThat(createDataStreamResponse.isAcknowledged(), is(true));
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java
index 52ce2a7a33ea6..8db3ed6ddc9c9 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java
@@ -277,7 +277,10 @@ public void testInvalidIndexModeTimeSeriesSwitchWithoutDimensions() throws IOExc
     }
 
     private void assertDataStreamBackingIndicesModes(final String dataStreamName, final List<IndexMode> modes) {
-        final GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+        final GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStreamName }
+        );
         final GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         final DataStream dataStream = getDataStreamResponse.getDataStreams().get(0).getDataStream();
@@ -362,7 +365,11 @@ private void indexTimeSeriesDocuments(
     }
 
     private void createDataStream(final Client client, final String dataStreamName) throws InterruptedException, ExecutionException {
-        final CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        final CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         final AcknowledgedResponse createDataStreamResponse = client.execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest)
             .get();
         assertThat(createDataStreamResponse.isAcknowledged(), is(true));
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java
index eca45b45d1269..ef785086a0ef4 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java
@@ -327,10 +327,14 @@ private Map<String, Object> setupThreeClusters(boolean useAlias) throws IOExcept
                 aliases.put(dataStreamLocalAlias, AliasMetadata.builder(dataStreamLocalAlias).writeIndex(randomBoolean()).build());
             }
             putComposableIndexTemplate(client, "id1", List.of(dataStreamLocal + "*"), aliases);
-            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
+            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                "metrics-foo"
+            );
             assertAcked(client.execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get());
 
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
             GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             DataStream fooDataStream = getDataStreamResponse.getDataStreams().get(0).getDataStream();
@@ -358,10 +362,14 @@ private Map<String, Object> setupThreeClusters(boolean useAlias) throws IOExcept
                 aliases.put(dataStreamRemote1Alias, AliasMetadata.builder(dataStreamRemote1Alias).writeIndex(randomBoolean()).build());
             }
             putComposableIndexTemplate(client, "id2", List.of(dataStreamRemote1 + "*"), aliases);
-            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar");
+            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                "metrics-bar"
+            );
             assertAcked(client.execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get());
 
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
             GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
 
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java
index ec871b201bbdb..a07081e8f0dd9 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java
@@ -344,6 +344,7 @@ public String getFeatureDescription() {
         public void cleanUpFeature(ClusterService clusterService, Client client, ActionListener<ResetFeatureStateStatus> listener) {
             Collection<SystemDataStreamDescriptor> dataStreamDescriptors = getSystemDataStreamDescriptors();
             final DeleteDataStreamAction.Request request = new DeleteDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 dataStreamDescriptors.stream()
                     .map(SystemDataStreamDescriptor::getDataStreamName)
                     .collect(Collectors.toList())
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java
index c147677cf856c..ba9657f71b23e 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java
@@ -58,7 +58,11 @@ public void testSystemDataStreamInGlobalState() throws Exception {
         createRepository(REPO, "fs", location);
 
         {
-            CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(SYSTEM_DATA_STREAM_NAME);
+            CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                SYSTEM_DATA_STREAM_NAME
+            );
             final AcknowledgedResponse response = client().execute(CreateDataStreamAction.INSTANCE, request).get();
             assertTrue(response.isAcknowledged());
         }
@@ -71,7 +75,10 @@ public void testSystemDataStreamInGlobalState() throws Exception {
         assertThat(indexRepsonse.status().getStatus(), oneOf(200, 201));
 
         {
-            GetDataStreamAction.Request request = new GetDataStreamAction.Request(new String[] { SYSTEM_DATA_STREAM_NAME });
+            GetDataStreamAction.Request request = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { SYSTEM_DATA_STREAM_NAME }
+            );
             GetDataStreamAction.Response response = client().execute(GetDataStreamAction.INSTANCE, request).get();
             assertThat(response.getDataStreams(), hasSize(1));
             assertTrue(response.getDataStreams().get(0).getDataStream().isSystem());
@@ -87,7 +94,10 @@ public void testSystemDataStreamInGlobalState() throws Exception {
         // We have to delete the data stream directly, as the feature reset API doesn't clean up system data streams yet
         // See https://github.com/elastic/elasticsearch/issues/75818
         {
-            DeleteDataStreamAction.Request request = new DeleteDataStreamAction.Request(new String[] { SYSTEM_DATA_STREAM_NAME });
+            DeleteDataStreamAction.Request request = new DeleteDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { SYSTEM_DATA_STREAM_NAME }
+            );
             AcknowledgedResponse response = client().execute(DeleteDataStreamAction.INSTANCE, request).get();
             assertTrue(response.isAcknowledged());
         }
@@ -127,7 +137,10 @@ public void testSystemDataStreamInGlobalState() throws Exception {
         assertEquals(restoreSnapshotResponse.getRestoreInfo().totalShards(), restoreSnapshotResponse.getRestoreInfo().successfulShards());
 
         {
-            GetDataStreamAction.Request request = new GetDataStreamAction.Request(new String[] { SYSTEM_DATA_STREAM_NAME });
+            GetDataStreamAction.Request request = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { SYSTEM_DATA_STREAM_NAME }
+            );
             GetDataStreamAction.Response response = client().execute(GetDataStreamAction.INSTANCE, request).get();
             assertThat(response.getDataStreams(), hasSize(1));
             assertTrue(response.getDataStreams().get(0).getDataStream().isSystem());
@@ -144,7 +157,10 @@ public void testSystemDataStreamInGlobalState() throws Exception {
 
     private void assertSystemDataStreamDoesNotExist() {
         try {
-            GetDataStreamAction.Request request = new GetDataStreamAction.Request(new String[] { SYSTEM_DATA_STREAM_NAME });
+            GetDataStreamAction.Request request = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { SYSTEM_DATA_STREAM_NAME }
+            );
             GetDataStreamAction.Response response = client().execute(GetDataStreamAction.INSTANCE, request).get();
             assertThat(response.getDataStreams(), hasSize(0));
         } catch (Exception e) {
@@ -161,7 +177,11 @@ public void testSystemDataStreamInFeatureState() throws Exception {
         createRepository(REPO, "fs", location);
 
         {
-            CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(SYSTEM_DATA_STREAM_NAME);
+            CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                SYSTEM_DATA_STREAM_NAME
+            );
             final AcknowledgedResponse response = client().execute(CreateDataStreamAction.INSTANCE, request).get();
             assertTrue(response.isAcknowledged());
         }
@@ -181,7 +201,10 @@ public void testSystemDataStreamInFeatureState() throws Exception {
         assertThat(indexResponse.status().getStatus(), oneOf(200, 201));
 
         {
-            GetDataStreamAction.Request request = new GetDataStreamAction.Request(new String[] { SYSTEM_DATA_STREAM_NAME });
+            GetDataStreamAction.Request request = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { SYSTEM_DATA_STREAM_NAME }
+            );
             GetDataStreamAction.Response response = client().execute(GetDataStreamAction.INSTANCE, request).get();
             assertThat(response.getDataStreams(), hasSize(1));
             assertTrue(response.getDataStreams().get(0).getDataStream().isSystem());
@@ -201,7 +224,10 @@ public void testSystemDataStreamInFeatureState() throws Exception {
         // We have to delete the data stream directly, as the feature reset API doesn't clean up system data streams yet
         // See https://github.com/elastic/elasticsearch/issues/75818
         {
-            DeleteDataStreamAction.Request request = new DeleteDataStreamAction.Request(new String[] { SYSTEM_DATA_STREAM_NAME });
+            DeleteDataStreamAction.Request request = new DeleteDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { SYSTEM_DATA_STREAM_NAME }
+            );
             AcknowledgedResponse response = client().execute(DeleteDataStreamAction.INSTANCE, request).get();
             assertTrue(response.isAcknowledged());
         }
@@ -221,7 +247,10 @@ public void testSystemDataStreamInFeatureState() throws Exception {
         assertEquals(restoreSnapshotResponse.getRestoreInfo().totalShards(), restoreSnapshotResponse.getRestoreInfo().successfulShards());
 
         {
-            GetDataStreamAction.Request request = new GetDataStreamAction.Request(new String[] { SYSTEM_DATA_STREAM_NAME });
+            GetDataStreamAction.Request request = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { SYSTEM_DATA_STREAM_NAME }
+            );
             GetDataStreamAction.Response response = client().execute(GetDataStreamAction.INSTANCE, request).get();
             assertThat(response.getDataStreams(), hasSize(1));
             assertTrue(response.getDataStreams().get(0).getDataStream().isSystem());
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java
index 24c373df72144..a0a0681dbd245 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.Template;
@@ -35,6 +36,7 @@
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.indices.InvalidIndexTemplateException;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -457,6 +459,16 @@ public void testTrimId() throws Exception {
                 indexName = bulkResponse.getItems()[0].getIndex();
             }
             client().admin().indices().refresh(new RefreshRequest(dataStreamName)).actionGet();
+
+            // In rare cases we can end up with a single segment shard, which means we can't trim away the _id later.
+            // So update an existing doc to create a new segment without adding a new document after force merging:
+            var indexRequest = new IndexRequest(indexName).setIfPrimaryTerm(1L)
+                .setIfSeqNo((numBulkRequests * numDocsPerBulk) - 1)
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+            indexRequest.source(DOC.replace("$time", formatInstant(time.minusMillis(1))), XContentType.JSON);
+            var res = client().index(indexRequest).actionGet();
+            assertThat(res.status(), equalTo(RestStatus.OK));
+            assertThat(res.getVersion(), equalTo(2L));
         }
 
         // Check whether there are multiple segments:
@@ -494,7 +506,7 @@ public void testTrimId() throws Exception {
             assertThat(retentionLeasesStats.retentionLeases().leases(), hasSize(1));
             assertThat(
                 retentionLeasesStats.retentionLeases().leases().iterator().next().retainingSequenceNumber(),
-                equalTo((long) numBulkRequests * numDocsPerBulk)
+                equalTo((long) numBulkRequests * numDocsPerBulk + 1)
             );
         });
 
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java
index b8d7d18dec475..64e4cc2ba9577 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java
@@ -142,10 +142,7 @@ protected Settings nodeSettings() {
     }
 
     public void testIndexingGettingAndSearching() throws Exception {
-        var templateSettings = Settings.builder()
-            .put("index.mode", "time_series")
-            .put("index.number_of_shards", randomIntBetween(2, 10))
-            .put("index.number_of_replicas", 0);
+        var templateSettings = indexSettings(randomIntBetween(2, 10), 0).put("index.mode", "time_series");
 
         var request = new TransportPutComposableIndexTemplateAction.Request("id");
         request.indexTemplate(
@@ -218,10 +215,7 @@ public void testIndexingGettingAndSearching() throws Exception {
 
     public void testIndexingGettingAndSearchingShrunkIndex() throws Exception {
         String dataStreamName = "k8s";
-        var templateSettings = Settings.builder()
-            .put("index.mode", "time_series")
-            .put("index.number_of_shards", 8)
-            .put("index.number_of_replicas", 0);
+        var templateSettings = indexSettings(8, 0).put("index.mode", "time_series");
 
         var request = new TransportPutComposableIndexTemplateAction.Request("id");
         request.indexTemplate(
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java
index 30f6ef313c41c..27d8bdd2ac27c 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java
@@ -43,23 +43,38 @@ public void testGetLifecycle() throws Exception {
         putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
         {
             String dataStreamName = "with-lifecycle-1";
-            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                dataStreamName
+            );
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         }
         {
             String dataStreamName = "with-lifecycle-2";
-            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                dataStreamName
+            );
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         }
         {
             String dataStreamName = "without-lifecycle";
-            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                dataStreamName
+            );
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         }
 
         // Test retrieving all lifecycles
         {
-            GetDataStreamLifecycleAction.Request getDataLifecycleRequest = new GetDataStreamLifecycleAction.Request(new String[] { "*" });
+            GetDataStreamLifecycleAction.Request getDataLifecycleRequest = new GetDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { "*" }
+            );
             GetDataStreamLifecycleAction.Response response = client().execute(
                 GetDataStreamLifecycleAction.INSTANCE,
                 getDataLifecycleRequest
@@ -77,6 +92,7 @@ public void testGetLifecycle() throws Exception {
         // Test retrieving all lifecycles prefixed wildcard
         {
             GetDataStreamLifecycleAction.Request getDataLifecycleRequest = new GetDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { "with-lifecycle*" }
             );
             GetDataStreamLifecycleAction.Response response = client().execute(
@@ -94,6 +110,7 @@ public void testGetLifecycle() throws Exception {
         // Test retrieving concrete data streams
         {
             GetDataStreamLifecycleAction.Request getDataLifecycleRequest = new GetDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { "with-lifecycle-1", "with-lifecycle-2" }
             );
             GetDataStreamLifecycleAction.Response response = client().execute(
@@ -108,6 +125,7 @@ public void testGetLifecycle() throws Exception {
 
         // Test include defaults
         GetDataStreamLifecycleAction.Request getDataLifecycleRequestWithDefaults = new GetDataStreamLifecycleAction.Request(
+            TEST_REQUEST_TIMEOUT,
             new String[] { "*" }
         ).includeDefaults(true);
         GetDataStreamLifecycleAction.Response responseWithRollover = client().execute(
@@ -128,11 +146,16 @@ public void testPutLifecycle() throws Exception {
         putComposableIndexTemplate("id1", null, List.of("my-data-stream*"), null, null, null);
         // Create index without a lifecycle
         String dataStreamName = "my-data-stream";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         {
             GetDataStreamLifecycleAction.Request getDataLifecycleRequest = new GetDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { "my-data-stream" }
             );
             GetDataStreamLifecycleAction.Response response = client().execute(
@@ -149,6 +172,8 @@ public void testPutLifecycle() throws Exception {
         {
             TimeValue dataRetention = randomBoolean() ? null : TimeValue.timeValueMillis(randomMillisUpToYear9999());
             PutDataStreamLifecycleAction.Request putDataLifecycleRequest = new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
                 new String[] { "*" },
                 dataRetention
             );
@@ -157,6 +182,7 @@ public void testPutLifecycle() throws Exception {
                 equalTo(true)
             );
             GetDataStreamLifecycleAction.Request getDataLifecycleRequest = new GetDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { "my-data-stream" }
             );
             GetDataStreamLifecycleAction.Response response = client().execute(
@@ -173,6 +199,8 @@ public void testPutLifecycle() throws Exception {
         {
             TimeValue dataRetention = randomBoolean() ? null : TimeValue.timeValueMillis(randomMillisUpToYear9999());
             PutDataStreamLifecycleAction.Request putDataLifecycleRequest = new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
                 new String[] { "*" },
                 dataRetention,
                 false
@@ -182,6 +210,7 @@ public void testPutLifecycle() throws Exception {
                 equalTo(true)
             );
             GetDataStreamLifecycleAction.Request getDataLifecycleRequest = new GetDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { "my-data-stream" }
             );
             GetDataStreamLifecycleAction.Response response = client().execute(
@@ -201,23 +230,36 @@ public void testDeleteLifecycle() throws Exception {
         putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
         {
             String dataStreamName = "with-lifecycle-1";
-            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                dataStreamName
+            );
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         }
         {
             String dataStreamName = "with-lifecycle-2";
-            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                dataStreamName
+            );
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         }
         {
             String dataStreamName = "with-lifecycle-3";
-            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                dataStreamName
+            );
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         }
 
         // Verify that we have 3 data streams with lifecycles
         {
             GetDataStreamLifecycleAction.Request getDataLifecycleRequest = new GetDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { "with-lifecycle*" }
             );
             GetDataStreamLifecycleAction.Response response = client().execute(
@@ -239,6 +281,7 @@ public void testDeleteLifecycle() throws Exception {
                 equalTo(true)
             );
             GetDataStreamLifecycleAction.Request getDataLifecycleRequest = new GetDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { "with-lifecycle*" }
             );
             GetDataStreamLifecycleAction.Response response = client().execute(
@@ -266,6 +309,7 @@ public void testDeleteLifecycle() throws Exception {
                 equalTo(true)
             );
             GetDataStreamLifecycleAction.Request getDataLifecycleRequest = new GetDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { "with-lifecycle*" }
             );
             GetDataStreamLifecycleAction.Response response = client().execute(
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java
index 9325bb41ab52e..9fafc23a388f4 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java
@@ -233,6 +233,7 @@ public String getFeatureDescription() {
         public void cleanUpFeature(ClusterService clusterService, Client client, ActionListener<ResetFeatureStateStatus> listener) {
             Collection<SystemDataStreamDescriptor> dataStreamDescriptors = getSystemDataStreamDescriptors();
             final DeleteDataStreamAction.Request request = new DeleteDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 dataStreamDescriptors.stream().map(SystemDataStreamDescriptor::getDataStreamName).toList().toArray(Strings.EMPTY_ARRAY)
             );
             request.indicesOptions(
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java
index e7dfdcdaffa05..ee17521ad757d 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java
@@ -51,8 +51,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
-import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamGlobalRetentionAction;
-import org.elasticsearch.datastreams.lifecycle.action.PutDataStreamGlobalRetentionAction;
 import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthIndicatorService;
 import org.elasticsearch.health.Diagnosis;
 import org.elasticsearch.health.GetHealthAction;
@@ -149,13 +147,20 @@ public void testRolloverLifecycle() throws Exception {
 
         putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 1);
 
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -175,13 +180,20 @@ public void testRolloverAndRetention() throws Exception {
         putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
 
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 1);
 
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -206,29 +218,19 @@ public void testSystemDataStreamRetention() throws Exception {
         AtomicLong now = new AtomicLong(clock.millis());
         dataStreamLifecycleServices.forEach(dataStreamLifecycleService -> dataStreamLifecycleService.setNowSupplier(now::get));
         try {
-            // Putting in place a global retention that we expect will be ignored by the system data stream:
-            final int globalRetentionSeconds = 10;
-            client().execute(
-                PutDataStreamGlobalRetentionAction.INSTANCE,
-                new PutDataStreamGlobalRetentionAction.Request(
-                    TEST_REQUEST_TIMEOUT,
-                    TimeValue.timeValueSeconds(globalRetentionSeconds),
-                    TimeValue.timeValueSeconds(globalRetentionSeconds)
-                )
-            ).actionGet();
             try {
 
-                CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(SYSTEM_DATA_STREAM_NAME);
+                CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    TEST_REQUEST_TIMEOUT,
+                    SYSTEM_DATA_STREAM_NAME
+                );
                 client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet();
                 indexDocs(SYSTEM_DATA_STREAM_NAME, 1);
-                /*
-                 * First we advance the time to well beyond the global retention (10s) but well under the configured retention (100d).
-                 * We expect to see that rollover has occurred but that the old index has not been deleted since the global retention is
-                 * ignored.
-                 */
-                now.addAndGet(TimeValue.timeValueSeconds(3 * globalRetentionSeconds).millis());
+                now.addAndGet(TimeValue.timeValueSeconds(30).millis());
                 assertBusy(() -> {
                     GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                        TEST_REQUEST_TIMEOUT,
                         new String[] { SYSTEM_DATA_STREAM_NAME }
                     );
                     GetDataStreamAction.Response getDataStreamResponse = client().execute(
@@ -248,6 +250,7 @@ public void testSystemDataStreamRetention() throws Exception {
                 now.addAndGet(TimeValue.timeValueDays(3 * TestSystemDataStreamPlugin.SYSTEM_DATA_STREAM_RETENTION_DAYS).millis());
                 assertBusy(() -> {
                     GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                        TEST_REQUEST_TIMEOUT,
                         new String[] { SYSTEM_DATA_STREAM_NAME }
                     );
                     GetDataStreamAction.Response getDataStreamResponse = client().execute(
@@ -297,12 +300,12 @@ public void testSystemDataStreamRetention() throws Exception {
                     }
                 });
 
-                client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(SYSTEM_DATA_STREAM_NAME)).actionGet();
-            } finally {
                 client().execute(
-                    DeleteDataStreamGlobalRetentionAction.INSTANCE,
-                    new DeleteDataStreamGlobalRetentionAction.Request(TEST_REQUEST_TIMEOUT)
-                );
+                    DeleteDataStreamAction.INSTANCE,
+                    new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, SYSTEM_DATA_STREAM_NAME)
+                ).actionGet();
+            } finally {
+                // reset properties
             }
         } finally {
             dataStreamLifecycleServices.forEach(dataStreamLifecycleService -> dataStreamLifecycleService.setNowSupplier(clock::millis));
@@ -320,7 +323,11 @@ public void testOriginationDate() throws Exception {
         putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
 
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 1);
@@ -347,7 +354,11 @@ public void testOriginationDate() throws Exception {
         createIndex(indexWithOldOriginationDate, Settings.builder().put(LIFECYCLE_ORIGINATION_DATE, originTimeMillis).build());
         client().execute(
             ModifyDataStreamsAction.INSTANCE,
-            new ModifyDataStreamsAction.Request(List.of(DataStreamAction.addBackingIndex(dataStreamName, indexWithOldOriginationDate)))
+            new ModifyDataStreamsAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                List.of(DataStreamAction.addBackingIndex(dataStreamName, indexWithOldOriginationDate))
+            )
         ).get();
 
         String indexWithNewOriginationDate = "index_new";
@@ -355,11 +366,18 @@ public void testOriginationDate() throws Exception {
         createIndex(indexWithNewOriginationDate, Settings.builder().put(LIFECYCLE_ORIGINATION_DATE, originTimeMillis).build());
         client().execute(
             ModifyDataStreamsAction.INSTANCE,
-            new ModifyDataStreamsAction.Request(List.of(DataStreamAction.addBackingIndex(dataStreamName, indexWithNewOriginationDate)))
+            new ModifyDataStreamsAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                List.of(DataStreamAction.addBackingIndex(dataStreamName, indexWithNewOriginationDate))
+            )
         ).get();
 
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -377,7 +395,11 @@ public void testUpdatingLifecycleAppliesToAllBackingIndices() throws Exception {
         putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
 
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         client().admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).actionGet();
         client().admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).actionGet();
@@ -387,7 +409,10 @@ public void testUpdatingLifecycleAppliesToAllBackingIndices() throws Exception {
         updateLifecycle(dataStreamName, TimeValue.timeValueMillis(1));
         // Verify that the retention has changed for all backing indices
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -416,10 +441,7 @@ public void testAutomaticForceMerge() throws Exception {
             "id1",
             null,
             List.of(dataStreamName + "*"),
-            Settings.builder()
-                .put("index.number_of_replicas", 1)
-                .put("index.number_of_shards", 1)
-                .put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), ONE_HUNDRED_MB)
+            indexSettings(1, 1).put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), ONE_HUNDRED_MB)
                 .put(MergePolicyConfig.INDEX_MERGE_POLICY_MERGE_FACTOR_SETTING.getKey(), TARGET_MERGE_FACTOR_VALUE)
                 .build(),
             null,
@@ -441,7 +463,11 @@ public void testAutomaticForceMerge() throws Exception {
                 });
         }
 
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         int finalGeneration = randomIntBetween(3, 4);
         for (int currentGeneration = 1; currentGeneration < finalGeneration; currentGeneration++) {
@@ -469,7 +495,10 @@ public void testAutomaticForceMerge() throws Exception {
             // run data stream lifecycle once
             dataStreamLifecycleService.run(clusterService.state());
             assertBusy(() -> {
-                GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+                GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    new String[] { dataStreamName }
+                );
                 GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                     .actionGet();
                 assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -524,14 +553,21 @@ public void testErrorRecordingOnRollover() throws Exception {
         );
 
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 1);
 
         // let's allow one rollover to go through
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -565,6 +601,7 @@ public void testErrorRecordingOnRollover() throws Exception {
             assertThat(writeIndexRolloverError.error(), containsString("maximum normal shards open"));
 
             ExplainDataStreamLifecycleAction.Request explainRequest = new ExplainDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { dataStreamName }
             );
             ExplainDataStreamLifecycleAction.Response response = client().execute(ExplainDataStreamLifecycleAction.INSTANCE, explainRequest)
@@ -675,13 +712,20 @@ public void testErrorRecordingOnRetention() throws Exception {
         );
 
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         indexDocs(dataStreamName, 1);
 
         // let's allow one rollover to go through
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -704,7 +748,10 @@ public void testErrorRecordingOnRetention() throws Exception {
             updateLifecycle(dataStreamName, TimeValue.timeValueSeconds(1));
 
             assertBusy(() -> {
-                GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+                GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    new String[] { dataStreamName }
+                );
                 GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                     .actionGet();
                 assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -774,7 +821,10 @@ public void testErrorRecordingOnRetention() throws Exception {
             updateIndexSettings(Settings.builder().put(READ_ONLY.settingName(), false), firstGenerationIndex);
 
             assertBusy(() -> {
-                GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+                GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    new String[] { dataStreamName }
+                );
                 GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                     .actionGet();
                 assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -838,13 +888,20 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception
         );
 
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         indexDocs(dataStreamName, 1);
 
         // let's allow one rollover to go through
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -921,7 +978,11 @@ public void testReenableDataStreamLifecycle() throws Exception {
 
         putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 10);
@@ -932,7 +993,7 @@ public void testReenableDataStreamLifecycle() throws Exception {
 
             ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute(
                 ExplainDataStreamLifecycleAction.INSTANCE,
-                new ExplainDataStreamLifecycleAction.Request(new String[] { writeIndex })
+                new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { writeIndex })
             ).actionGet();
             assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(1));
             for (ExplainIndexDataStreamLifecycle index : dataStreamLifecycleExplainResponse.getIndices()) {
@@ -951,7 +1012,13 @@ public void testReenableDataStreamLifecycle() throws Exception {
 
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
-            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, null, true)
+            new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName },
+                null,
+                true
+            )
         );
 
         assertBusy(() -> {
@@ -994,14 +1061,21 @@ public void testLifecycleAppliedToFailureStore() throws Exception {
             }""", List.of("metrics-fs*"), Settings.builder().put("index.number_of_replicas", 0).build(), null, lifecycle, true);
 
         String dataStreamName = "metrics-fs";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexInvalidFlagDocs(dataStreamName, 1);
 
         // Let's verify the rollover
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -1043,7 +1117,10 @@ public void testLifecycleAppliedToFailureStore() throws Exception {
 
         // And finally apply retention
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -1057,7 +1134,10 @@ public void testLifecycleAppliedToFailureStore() throws Exception {
     }
 
     private static List<String> getBackingIndices(String dataStreamName) {
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStreamName }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -1066,7 +1146,10 @@ private static List<String> getBackingIndices(String dataStreamName) {
     }
 
     private static List<String> getFailureIndices(String dataStreamName) {
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStreamName }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -1147,6 +1230,8 @@ static void putComposableIndexTemplate(
 
     static void updateLifecycle(String dataStreamName, TimeValue dataRetention) {
         PutDataStreamLifecycleAction.Request putDataLifecycleRequest = new PutDataStreamLifecycleAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
             new String[] { dataStreamName },
             dataRetention
         );
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java
index 571bfc05b6464..5a5b451a62c3a 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java
@@ -30,8 +30,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
-import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamGlobalRetentionAction;
-import org.elasticsearch.datastreams.lifecycle.action.PutDataStreamGlobalRetentionAction;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.plugins.Plugin;
@@ -91,13 +89,20 @@ public void testExplainLifecycle() throws Exception {
 
         putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle);
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 1);
 
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -112,6 +117,7 @@ public void testExplainLifecycle() throws Exception {
 
         {
             ExplainDataStreamLifecycleAction.Request explainIndicesRequest = new ExplainDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] {
                     DataStream.getDefaultBackingIndexName(dataStreamName, 1),
                     DataStream.getDefaultBackingIndexName(dataStreamName, 2) }
@@ -152,6 +158,7 @@ public void testExplainLifecycle() throws Exception {
         {
             // let's also explain with include_defaults=true
             ExplainDataStreamLifecycleAction.Request explainIndicesRequest = new ExplainDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] {
                     DataStream.getDefaultBackingIndexName(dataStreamName, 1),
                     DataStream.getDefaultBackingIndexName(dataStreamName, 2) },
@@ -173,6 +180,7 @@ public void testExplainLifecycle() throws Exception {
         {
             // Let's also explain using the data stream name
             ExplainDataStreamLifecycleAction.Request explainIndicesRequest = new ExplainDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { dataStreamName }
             );
             ExplainDataStreamLifecycleAction.Response response = client().execute(
@@ -208,25 +216,22 @@ public void testSystemExplainLifecycle() throws Exception {
          * This test makes sure that for system data streams, we correctly ignore the global retention when calling
          * ExplainDataStreamLifecycle. It is very similar to testExplainLifecycle, but only focuses on the retention for a system index.
          */
-        // Putting in place a global retention that we expect will be ignored by the system data stream:
-        final int globalRetentionSeconds = 10;
-        client().execute(
-            PutDataStreamGlobalRetentionAction.INSTANCE,
-            new PutDataStreamGlobalRetentionAction.Request(
-                TEST_REQUEST_TIMEOUT,
-                TimeValue.timeValueSeconds(globalRetentionSeconds),
-                TimeValue.timeValueSeconds(globalRetentionSeconds)
-            )
-        ).actionGet();
         try {
             String dataStreamName = SYSTEM_DATA_STREAM_NAME;
-            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+            CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                dataStreamName
+            );
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
             indexDocs(dataStreamName, 1);
 
             assertBusy(() -> {
-                GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+                GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    new String[] { dataStreamName }
+                );
                 GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                     .actionGet();
                 assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -240,6 +245,7 @@ public void testSystemExplainLifecycle() throws Exception {
             });
 
             ExplainDataStreamLifecycleAction.Request explainIndicesRequest = new ExplainDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] {
                     DataStream.getDefaultBackingIndexName(dataStreamName, 1),
                     DataStream.getDefaultBackingIndexName(dataStreamName, 2) }
@@ -261,10 +267,7 @@ public void testSystemExplainLifecycle() throws Exception {
                 );
             }
         } finally {
-            client().execute(
-                DeleteDataStreamGlobalRetentionAction.INSTANCE,
-                new DeleteDataStreamGlobalRetentionAction.Request(TEST_REQUEST_TIMEOUT)
-            );
+            // reset properties
         }
     }
 
@@ -275,14 +278,21 @@ public void testExplainLifecycleForIndicesWithErrors() throws Exception {
         putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle);
 
         String dataStreamName = "metrics-foo";
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 1);
 
         // let's allow one rollover to go through
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -303,6 +313,7 @@ public void testExplainLifecycleForIndicesWithErrors() throws Exception {
         String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2);
         assertBusy(() -> {
             ExplainDataStreamLifecycleAction.Request explainIndicesRequest = new ExplainDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { writeIndexName }
             );
             ExplainDataStreamLifecycleAction.Response response = client().execute(
@@ -334,6 +345,7 @@ public void testExplainLifecycleForIndicesWithErrors() throws Exception {
 
         assertBusy(() -> {
             ExplainDataStreamLifecycleAction.Request explainIndicesRequest = new ExplainDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { writeIndexName }
             );
             ExplainDataStreamLifecycleAction.Response response = client().execute(
@@ -363,7 +375,11 @@ public void testExplainDataStreamLifecycleForUnmanagedIndices() throws Exception
             null,
             DataStreamLifecycle.newBuilder().enabled(false).build()
         );
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "metrics-foo"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 4);
@@ -371,6 +387,7 @@ public void testExplainDataStreamLifecycleForUnmanagedIndices() throws Exception
         String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
         assertBusy(() -> {
             ExplainDataStreamLifecycleAction.Request explainIndicesRequest = new ExplainDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 new String[] { writeIndexName }
             );
             ExplainDataStreamLifecycleAction.Response response = client().execute(
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java
index 780864db8b629..1500674e3aee9 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java
@@ -28,6 +28,7 @@
 import java.util.Locale;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 public class LogsDataStreamRestIT extends ESRestTestCase {
@@ -102,19 +103,53 @@ private static void waitForLogs(RestClient client) throws Exception {
 
     private static final String STANDARD_TEMPLATE = """
         {
-            "index_patterns": [ "logs-*-*" ],
-            "data_stream": {},
-            "priority": 201,
-            "template": {
-              "settings": {
-                "index": {
-                  "mode": "standard"
+          "index_patterns": [ "logs-*-*" ],
+          "data_stream": {},
+          "priority": 201,
+          "template": {
+            "settings": {
+              "index": {
+                "mode": "standard"
+              }
+            },
+            "mappings": {
+              "properties": {
+                "@timestamp" : {
+                  "type": "date"
+                },
+                "host.name": {
+                  "type": "keyword"
+                },
+                "pid": {
+                  "type": "long"
+                },
+                "method": {
+                  "type": "keyword"
+                },
+                "ip_address": {
+                  "type": "ip"
                 }
-              },
-              "mappings": {
-                "properties": {
-                  "@timestamp" : {
-                    "type": "date"
+              }
+            }
+          }
+        }""";
+
+    private static final String TIME_SERIES_TEMPLATE = """
+        {
+          "index_patterns": [ "logs-*-*" ],
+          "data_stream": {},
+          "priority": 201,
+          "template": {
+            "settings": {
+              "index": {
+                "mode": "time_series",
+                "look_ahead_time": "5m"
+              }
+            },
+            "mappings": {
+              "properties": {
+                "@timestamp" : {
+                  "type": "date"
                 },
                 "host.name": {
                   "type": "keyword",
@@ -129,6 +164,10 @@ private static void waitForLogs(RestClient client) throws Exception {
                 },
                 "ip_address": {
                   "type": "ip"
+                },
+                "memory_usage_bytes": {
+                  "type": "long",
+                  "time_series_metric": "gauge"
                 }
               }
             }
@@ -142,7 +181,8 @@ private static void waitForLogs(RestClient client) throws Exception {
             "pid": "%d",
             "method": "%s",
             "message": "%s",
-            "ip_address": "%s"
+            "ip_address": "%s",
+            "memory_usage_bytes": "%d"
         }
         """;
 
@@ -158,7 +198,8 @@ public void testLogsIndexing() throws IOException {
                 randomNonNegativeLong(),
                 randomFrom("PUT", "POST", "GET"),
                 randomAlphaOfLength(32),
-                randomIp(randomBoolean())
+                randomIp(randomBoolean()),
+                randomLongBetween(1_000_000L, 2_000_000L)
             )
         );
         assertDataStreamBackingIndexMode("logsdb", 0);
@@ -172,7 +213,8 @@ public void testLogsIndexing() throws IOException {
                 randomNonNegativeLong(),
                 randomFrom("PUT", "POST", "GET"),
                 randomAlphaOfLength(32),
-                randomIp(randomBoolean())
+                randomIp(randomBoolean()),
+                randomLongBetween(1_000_000L, 2_000_000L)
             )
         );
         assertDataStreamBackingIndexMode("logsdb", 1);
@@ -190,7 +232,8 @@ public void testLogsStandardIndexModeSwitch() throws IOException {
                 randomNonNegativeLong(),
                 randomFrom("PUT", "POST", "GET"),
                 randomAlphaOfLength(32),
-                randomIp(randomBoolean())
+                randomIp(randomBoolean()),
+                randomLongBetween(1_000_000L, 2_000_000L)
             )
         );
         assertDataStreamBackingIndexMode("logsdb", 0);
@@ -206,7 +249,8 @@ public void testLogsStandardIndexModeSwitch() throws IOException {
                 randomNonNegativeLong(),
                 randomFrom("PUT", "POST", "GET"),
                 randomAlphaOfLength(64),
-                randomIp(randomBoolean())
+                randomIp(randomBoolean()),
+                randomLongBetween(1_000_000L, 2_000_000L)
             )
         );
         assertDataStreamBackingIndexMode("standard", 1);
@@ -222,7 +266,61 @@ public void testLogsStandardIndexModeSwitch() throws IOException {
                 randomNonNegativeLong(),
                 randomFrom("PUT", "POST", "GET"),
                 randomAlphaOfLength(32),
-                randomIp(randomBoolean())
+                randomIp(randomBoolean()),
+                randomLongBetween(1_000_000L, 2_000_000L)
+            )
+        );
+        assertDataStreamBackingIndexMode("logsdb", 2);
+    }
+
+    public void testLogsTimeSeriesIndexModeSwitch() throws IOException {
+        putTemplate(client, "custom-template", LOGS_TEMPLATE);
+        createDataStream(client, DATA_STREAM_NAME);
+        indexDocument(
+            client,
+            DATA_STREAM_NAME,
+            document(
+                Instant.now(),
+                randomAlphaOfLength(10),
+                randomNonNegativeLong(),
+                randomFrom("PUT", "POST", "GET"),
+                randomAlphaOfLength(32),
+                randomIp(randomBoolean()),
+                randomLongBetween(1_000_000L, 2_000_000L)
+            )
+        );
+        assertDataStreamBackingIndexMode("logsdb", 0);
+
+        putTemplate(client, "custom-template", TIME_SERIES_TEMPLATE);
+        rolloverDataStream(client, DATA_STREAM_NAME);
+        indexDocument(
+            client,
+            DATA_STREAM_NAME,
+            document(
+                Instant.now().plusSeconds(10),
+                randomAlphaOfLength(10),
+                randomNonNegativeLong(),
+                randomFrom("PUT", "POST", "GET"),
+                randomAlphaOfLength(64),
+                randomIp(randomBoolean()),
+                randomLongBetween(1_000_000L, 2_000_000L)
+            )
+        );
+        assertDataStreamBackingIndexMode("time_series", 1);
+
+        putTemplate(client, "custom-template", LOGS_TEMPLATE);
+        rolloverDataStream(client, DATA_STREAM_NAME);
+        indexDocument(
+            client,
+            DATA_STREAM_NAME,
+            document(
+                Instant.now().plusSeconds(320), // 5 mins index.look_ahead_time
+                randomAlphaOfLength(10),
+                randomNonNegativeLong(),
+                randomFrom("PUT", "POST", "GET"),
+                randomAlphaOfLength(32),
+                randomIp(randomBoolean()),
+                randomLongBetween(1_000_000L, 2_000_000L)
             )
         );
         assertDataStreamBackingIndexMode("logsdb", 2);
@@ -238,17 +336,19 @@ private String document(
         long pid,
         final String method,
         final String message,
-        final InetAddress ipAddress
+        final InetAddress ipAddress,
+        long memoryUsageBytes
     ) {
         return String.format(
             Locale.ROOT,
             DOC_TEMPLATE,
-            DateFormatter.forPattern(FormatNames.DATE.getName()).format(timestamp),
+            DateFormatter.forPattern(FormatNames.DATE_TIME.getName()).format(timestamp),
             hostname,
             pid,
             method,
             message,
-            InetAddresses.toAddrString(ipAddress)
+            InetAddresses.toAddrString(ipAddress),
+            memoryUsageBytes
         );
     }
 
@@ -266,7 +366,9 @@ private static void putTemplate(final RestClient client, final String templateNa
     private static void indexDocument(final RestClient client, String dataStreamName, String doc) throws IOException {
         final Request request = new Request("POST", "/" + dataStreamName + "/_doc?refresh=true");
         request.setJsonEntity(doc);
-        assertOK(client.performRequest(request));
+        final Response response = client.performRequest(request);
+        assertOK(response);
+        assertThat(entityAsMap(response).get("result"), equalTo("created"));
     }
 
     private static void rolloverDataStream(final RestClient client, final String dataStreamName) throws IOException {
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java
new file mode 100644
index 0000000000000..514eb6d8742ea
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java
@@ -0,0 +1,190 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.datastreams.lifecycle;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.WarningFailureException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.datastreams.DisabledSecurityDataStreamTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+public class DataStreamGlobalRetentionIT extends DisabledSecurityDataStreamTestCase {
+
+    @Before
+    public void setup() throws IOException {
+        updateClusterSettings(
+            Settings.builder()
+                .put("data_streams.lifecycle.poll_interval", "1s")
+                .put("cluster.lifecycle.default.rollover", "min_docs=1,max_docs=1")
+                .build()
+        );
+        // Create a template with the default lifecycle
+        Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1");
+        putComposableIndexTemplateRequest.setJsonEntity("""
+            {
+              "index_patterns": ["my-data-stream*"],
+              "data_stream": {},
+              "template": {
+                "lifecycle": {}
+              }
+            }
+            """);
+        assertOK(client().performRequest(putComposableIndexTemplateRequest));
+
+        // Create a data streams with one doc
+        Request createDocRequest = new Request("POST", "/my-data-stream/_doc?refresh=true");
+        createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}");
+        assertOK(client().performRequest(createDocRequest));
+    }
+
+    @After
+    public void cleanUp() throws IOException {
+        adminClient().performRequest(new Request("DELETE", "_data_stream/*"));
+        updateClusterSettings(
+            Settings.builder().putNull("data_streams.lifecycle.retention.default").putNull("data_streams.lifecycle.retention.max").build()
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testDataStreamRetention() throws Exception {
+        // Set global retention and add retention to the data stream
+        {
+            updateClusterSettings(
+                Settings.builder()
+                    .put("data_streams.lifecycle.retention.default", "7d")
+                    .put("data_streams.lifecycle.retention.default", "90d")
+                    .build()
+            );
+            Request request = new Request("PUT", "_data_stream/my-data-stream/_lifecycle");
+            request.setJsonEntity("""
+                {
+                  "data_retention": "10s"
+                }""");
+            assertAcknowledged(client().performRequest(request));
+        }
+
+        // Verify that the effective retention matches the default retention
+        {
+            Request request = new Request("GET", "/_data_stream/my-data-stream");
+            Response response = client().performRequest(request);
+            List<Object> dataStreams = (List<Object>) entityAsMap(response).get("data_streams");
+            assertThat(dataStreams.size(), is(1));
+            Map<String, Object> dataStream = (Map<String, Object>) dataStreams.get(0);
+            assertThat(dataStream.get("name"), is("my-data-stream"));
+            Map<String, Object> lifecycle = (Map<String, Object>) dataStream.get("lifecycle");
+            assertThat(lifecycle.get("effective_retention"), is("10s"));
+            assertThat(lifecycle.get("retention_determined_by"), is("data_stream_configuration"));
+            assertThat(lifecycle.get("data_retention"), is("10s"));
+        }
+
+        // Verify that the first generation index was removed
+        assertBusy(() -> {
+            Response response = client().performRequest(new Request("GET", "/_data_stream/my-data-stream"));
+            Map<String, Object> dataStream = ((List<Map<String, Object>>) entityAsMap(response).get("data_streams")).get(0);
+            assertThat(dataStream.get("name"), is("my-data-stream"));
+            List<Object> backingIndices = (List<Object>) dataStream.get("indices");
+            assertThat(backingIndices.size(), is(1));
+            // 2 backing indices created + 1 for the deleted index
+            assertThat(dataStream.get("generation"), is(3));
+        }, 20, TimeUnit.SECONDS);
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testDefaultRetention() throws Exception {
+        // Set default global retention
+        updateClusterSettings(Settings.builder().put("data_streams.lifecycle.retention.default", "10s").build());
+
+        // Verify that the effective retention matches the default retention
+        {
+            Request request = new Request("GET", "/_data_stream/my-data-stream");
+            Response response = client().performRequest(request);
+            List<Object> dataStreams = (List<Object>) entityAsMap(response).get("data_streams");
+            assertThat(dataStreams.size(), is(1));
+            Map<String, Object> dataStream = (Map<String, Object>) dataStreams.get(0);
+            assertThat(dataStream.get("name"), is("my-data-stream"));
+            Map<String, Object> lifecycle = (Map<String, Object>) dataStream.get("lifecycle");
+            assertThat(lifecycle.get("effective_retention"), is("10s"));
+            assertThat(lifecycle.get("retention_determined_by"), is("default_global_retention"));
+            assertThat(lifecycle.get("data_retention"), nullValue());
+        }
+
+        // Verify that the first generation index was removed
+        assertBusy(() -> {
+            Response response = client().performRequest(new Request("GET", "/_data_stream/my-data-stream"));
+            Map<String, Object> dataStream = ((List<Map<String, Object>>) entityAsMap(response).get("data_streams")).get(0);
+            assertThat(dataStream.get("name"), is("my-data-stream"));
+            List<Object> backingIndices = (List<Object>) dataStream.get("indices");
+            assertThat(backingIndices.size(), is(1));
+            // 2 backing indices created + 1 for the deleted index
+            assertThat(dataStream.get("generation"), is(3));
+        }, 20, TimeUnit.SECONDS);
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testMaxRetention() throws Exception {
+        // Set default global retention
+        updateClusterSettings(Settings.builder().put("data_streams.lifecycle.retention.max", "10s").build());
+        boolean withDataStreamLevelRetention = randomBoolean();
+        if (withDataStreamLevelRetention) {
+            try {
+                Request request = new Request("PUT", "_data_stream/my-data-stream/_lifecycle");
+                request.setJsonEntity("""
+                    {
+                      "data_retention": "30d"
+                    }""");
+                assertAcknowledged(client().performRequest(request));
+                fail("Should have returned a warning about data retention exceeding the max retention");
+            } catch (WarningFailureException warningFailureException) {
+                assertThat(
+                    warningFailureException.getMessage(),
+                    containsString("The retention provided [30d] is exceeding the max allowed data retention of this project [10s]")
+                );
+            }
+        }
+
+        // Verify that the effective retention matches the max retention
+        {
+            Request request = new Request("GET", "/_data_stream/my-data-stream");
+            Response response = client().performRequest(request);
+            List<Object> dataStreams = (List<Object>) entityAsMap(response).get("data_streams");
+            assertThat(dataStreams.size(), is(1));
+            Map<String, Object> dataStream = (Map<String, Object>) dataStreams.get(0);
+            assertThat(dataStream.get("name"), is("my-data-stream"));
+            Map<String, Object> lifecycle = (Map<String, Object>) dataStream.get("lifecycle");
+            assertThat(lifecycle.get("effective_retention"), is("10s"));
+            assertThat(lifecycle.get("retention_determined_by"), is("max_global_retention"));
+            if (withDataStreamLevelRetention) {
+                assertThat(lifecycle.get("data_retention"), is("30d"));
+            } else {
+                assertThat(lifecycle.get("data_retention"), nullValue());
+            }
+        }
+
+        // Verify that the first generation index was removed
+        assertBusy(() -> {
+            Response response = client().performRequest(new Request("GET", "/_data_stream/my-data-stream"));
+            Map<String, Object> dataStream = ((List<Map<String, Object>>) entityAsMap(response).get("data_streams")).get(0);
+            assertThat(dataStream.get("name"), is("my-data-stream"));
+            List<Object> backingIndices = (List<Object>) dataStream.get("indices");
+            assertThat(backingIndices.size(), is(1));
+            // 2 backing indices created + 1 for the deleted index
+            assertThat(dataStream.get("generation"), is(3));
+        }, 20, TimeUnit.SECONDS);
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java
index 90e904b554c8d..d3a2867fe2ecd 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java
@@ -111,7 +111,7 @@ public void testConfigureStoredSource() throws IOException {
             e.getMessage(),
             containsString("updating component template [logs@custom] results in invalid composable template [logs]")
         );
-        assertThat(e.getMessage(), containsString("Indices with with index mode [logs] only support synthetic source"));
+        assertThat(e.getMessage(), containsString("Indices with with index mode [logsdb] only support synthetic source"));
 
         assertOK(createDataStream(client, "logs-custom-dev"));
 
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java
index 8ee0e4d715c4c..1d36a04657e9c 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java
@@ -186,25 +186,27 @@ private void deleteTemplates() throws IOException {
         assert deleteContenderTemplate.getStatusLine().getStatusCode() == RestStatus.OK.getStatus();
     }
 
-    private Settings.Builder createSettings(final CheckedConsumer<Settings.Builder, IOException> settingsConsumer) throws IOException {
+    private Settings.Builder createSettings(
+        final CheckedConsumer<Settings.Builder, IOException> settingsConsumer,
+        final CheckedConsumer<Settings.Builder, IOException> commonSettingsConsumer
+    ) throws IOException {
         final Settings.Builder settings = Settings.builder();
         settingsConsumer.accept(settings);
+        commonSettingsConsumer.accept(settings);
         return settings;
     }
 
     private Settings.Builder createBaselineSettings() throws IOException {
-        return createSettings(this::baselineSettings);
+        return createSettings(this::baselineSettings, this::commonSettings);
     }
 
     private Settings.Builder createContenderSettings() throws IOException {
-        return createSettings(this::contenderSettings);
+        return createSettings(this::contenderSettings, this::commonSettings);
     }
 
     private XContentBuilder createMappings(final CheckedConsumer<XContentBuilder, IOException> builderConsumer) throws IOException {
         final XContentBuilder builder = XContentFactory.jsonBuilder();
-        builder.startObject();
         builderConsumer.accept(builder);
-        builder.endObject();
         return builder;
     }
 
@@ -224,14 +226,18 @@ public void baselineSettings(Settings.Builder builder) {}
 
     public void contenderSettings(Settings.Builder builder) {}
 
+    public void commonSettings(Settings.Builder builder) {}
+
     private Response indexDocuments(
         final String dataStreamName,
         final CheckedSupplier<List<XContentBuilder>, IOException> documentsSupplier
     ) throws IOException {
         final StringBuilder sb = new StringBuilder();
+        int id = 0;
         for (var document : documentsSupplier.get()) {
-            sb.append("{ \"create\": {} }").append("\n");
+            sb.append(Strings.format("{ \"create\": { \"_id\" : \"%d\" } }", id)).append("\n");
             sb.append(Strings.toString(document)).append("\n");
+            id++;
         }
         var request = new Request("POST", "/" + dataStreamName + "/_bulk");
         request.setJsonEntity(sb.toString());
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java
index 79e96b7446042..9bf1c394f9105 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.common.time.FormatNames;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.datastreams.logsdb.qa.exceptions.MatcherException;
+import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult;
 import org.elasticsearch.datastreams.logsdb.qa.matchers.Matcher;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
@@ -34,30 +34,85 @@
 
 import java.io.IOException;
 import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+/**
+ * Basic challenge test - we index same documents into an index with standard index mode and an index with logsdb index mode.
+ * Then we verify that results of common operations are the same modulo knows differences like synthetic source modifications.
+ * This test uses simple mapping and document structure in order to allow easier debugging of the test itself.
+ */
 public class StandardVersusLogsIndexModeChallengeRestIT extends AbstractChallengeRestTest {
+    private final int numShards = randomBoolean() ? randomIntBetween(2, 5) : 0;
+    private final int numReplicas = randomBoolean() ? randomIntBetween(1, 3) : 0;
+    private final boolean fullyDynamicMapping = randomBoolean();
 
     public StandardVersusLogsIndexModeChallengeRestIT() {
-        super("logs-apache-baseline", "logs-apache-contender", "baseline-template", "contender-template", 99, 99);
+        super("standard-apache-baseline", "logs-apache-contender", "baseline-template", "contender-template", 101, 101);
     }
 
     @Override
     public void baselineMappings(XContentBuilder builder) throws IOException {
-        mappings(builder);
+        if (fullyDynamicMapping == false) {
+            builder.startObject()
+                .startObject("properties")
+
+                .startObject("@timestamp")
+                .field("type", "date")
+                .endObject()
+
+                .startObject("host.name")
+                .field("type", "keyword")
+                .field("ignore_above", randomIntBetween(1000, 1200))
+                .endObject()
+
+                .startObject("message")
+                .field("type", "keyword")
+                .field("ignore_above", randomIntBetween(1000, 1200))
+                .endObject()
+
+                .startObject("method")
+                .field("type", "keyword")
+                .field("ignore_above", randomIntBetween(1000, 1200))
+                .endObject()
+
+                .startObject("memory_usage_bytes")
+                .field("type", "long")
+                .field("ignore_malformed", randomBoolean())
+                .endObject()
+
+                .endObject()
+
+                .endObject();
+        } else {
+            // We want dynamic mapping, but we need host.name to be a keyword instead of text to support aggregations.
+            builder.startObject()
+                .startObject("properties")
+
+                .startObject("host.name")
+                .field("type", "keyword")
+                .field("ignore_above", randomIntBetween(1000, 1200))
+                .endObject()
+
+                .endObject()
+                .endObject();
+        }
     }
 
     @Override
     public void contenderMappings(XContentBuilder builder) throws IOException {
-        mappings(builder);
-    }
-
-    private static void mappings(final XContentBuilder builder) throws IOException {
+        builder.startObject();
         builder.field("subobjects", false);
-        if (randomBoolean()) {
+
+        if (fullyDynamicMapping == false) {
             builder.startObject("properties")
 
                 .startObject("@timestamp")
@@ -86,27 +141,28 @@ private static void mappings(final XContentBuilder builder) throws IOException {
 
                 .endObject();
         }
+
+        builder.endObject();
     }
 
-    private static void settings(final Settings.Builder settings) {
-        if (randomBoolean()) {
-            settings.put("index.number_of_shards", randomIntBetween(2, 5));
+    @Override
+    public void commonSettings(Settings.Builder builder) {
+        if (numShards > 0) {
+            builder.put("index.number_of_shards", numShards);
         }
-        if (randomBoolean()) {
-            settings.put("index.number_of_replicas", randomIntBetween(1, 3));
+        if (numReplicas > 0) {
+            builder.put("index.number_of_replicas", numReplicas);
         }
+        builder.put("index.mapping.total_fields.limit", 5000);
     }
 
     @Override
     public void contenderSettings(Settings.Builder builder) {
         builder.put("index.mode", "logsdb");
-        settings(builder);
     }
 
     @Override
-    public void baselineSettings(Settings.Builder builder) {
-        settings(builder);
-    }
+    public void baselineSettings(Settings.Builder builder) {}
 
     @Override
     public void beforeStart() throws Exception {
@@ -124,51 +180,45 @@ protected static void waitForLogs(RestClient client) throws Exception {
         });
     }
 
-    @SuppressWarnings("unchecked")
-    public void testMatchAllQuery() throws IOException, MatcherException {
-        final List<XContentBuilder> documents = new ArrayList<>();
+    public void testMatchAllQuery() throws IOException {
         int numberOfDocuments = ESTestCase.randomIntBetween(100, 200);
-        for (int i = 0; i < numberOfDocuments; i++) {
-            documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS)));
-        }
+        final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
 
         assertDocumentIndexing(documents);
 
         final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
             .size(numberOfDocuments);
 
-        Matcher.mappings(getContenderMappings(), getBaselineMappings())
+        final MatchResult matchResult = Matcher.matchSource()
+            .mappings(getContenderMappings(), getBaselineMappings())
             .settings(getContenderSettings(), getBaselineSettings())
             .expected(getQueryHits(queryBaseline(searchSourceBuilder)))
             .ignoringSort(true)
             .isEqualTo(getQueryHits(queryContender(searchSourceBuilder)));
+        assertTrue(matchResult.getMessage(), matchResult.isMatch());
     }
 
-    public void testTermsQuery() throws IOException, MatcherException {
-        final List<XContentBuilder> documents = new ArrayList<>();
-        int numberOfDocuments = randomIntBetween(100, 200);
-        for (int i = 0; i < numberOfDocuments; i++) {
-            documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS)));
-        }
+    public void testTermsQuery() throws IOException {
+        int numberOfDocuments = ESTestCase.randomIntBetween(100, 200);
+        final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
 
         assertDocumentIndexing(documents);
 
         final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.termQuery("method", "put"))
             .size(numberOfDocuments);
 
-        Matcher.mappings(getContenderMappings(), getBaselineMappings())
+        final MatchResult matchResult = Matcher.matchSource()
+            .mappings(getContenderMappings(), getBaselineMappings())
             .settings(getContenderSettings(), getBaselineSettings())
             .expected(getQueryHits(queryBaseline(searchSourceBuilder)))
             .ignoringSort(true)
             .isEqualTo(getQueryHits(queryContender(searchSourceBuilder)));
+        assertTrue(matchResult.getMessage(), matchResult.isMatch());
     }
 
-    public void testHistogramAggregation() throws IOException, MatcherException {
-        final List<XContentBuilder> documents = new ArrayList<>();
-        int numberOfDocuments = randomIntBetween(100, 200);
-        for (int i = 0; i < numberOfDocuments; i++) {
-            documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS)));
-        }
+    public void testHistogramAggregation() throws IOException {
+        int numberOfDocuments = ESTestCase.randomIntBetween(100, 200);
+        final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
 
         assertDocumentIndexing(documents);
 
@@ -176,39 +226,35 @@ public void testHistogramAggregation() throws IOException, MatcherException {
             .size(numberOfDocuments)
             .aggregation(new HistogramAggregationBuilder("agg").field("memory_usage_bytes").interval(100.0D));
 
-        Matcher.mappings(getContenderMappings(), getBaselineMappings())
+        final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings())
             .settings(getContenderSettings(), getBaselineSettings())
             .expected(getAggregationBuckets(queryBaseline(searchSourceBuilder), "agg"))
             .ignoringSort(true)
             .isEqualTo(getAggregationBuckets(queryContender(searchSourceBuilder), "agg"));
+        assertTrue(matchResult.getMessage(), matchResult.isMatch());
     }
 
-    public void testTermsAggregation() throws IOException, MatcherException {
-        final List<XContentBuilder> documents = new ArrayList<>();
-        int numberOfDocuments = randomIntBetween(100, 200);
-        for (int i = 0; i < numberOfDocuments; i++) {
-            documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS)));
-        }
+    public void testTermsAggregation() throws IOException {
+        int numberOfDocuments = ESTestCase.randomIntBetween(100, 200);
+        final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
 
         assertDocumentIndexing(documents);
 
         final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
             .size(0)
-            .aggregation(new TermsAggregationBuilder("agg").field("host.name"));
+            .aggregation(new TermsAggregationBuilder("agg").field("host.name").size(numberOfDocuments));
 
-        Matcher.mappings(getContenderMappings(), getBaselineMappings())
+        final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings())
             .settings(getContenderSettings(), getBaselineSettings())
             .expected(getAggregationBuckets(queryBaseline(searchSourceBuilder), "agg"))
             .ignoringSort(true)
             .isEqualTo(getAggregationBuckets(queryContender(searchSourceBuilder), "agg"));
+        assertTrue(matchResult.getMessage(), matchResult.isMatch());
     }
 
-    public void testDateHistogramAggregation() throws IOException, MatcherException {
-        final List<XContentBuilder> documents = new ArrayList<>();
-        int numberOfDocuments = randomIntBetween(100, 200);
-        for (int i = 0; i < numberOfDocuments; i++) {
-            documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS)));
-        }
+    public void testDateHistogramAggregation() throws IOException {
+        int numberOfDocuments = ESTestCase.randomIntBetween(100, 200);
+        final List<XContentBuilder> documents = generateDocuments(numberOfDocuments);
 
         assertDocumentIndexing(documents);
 
@@ -216,14 +262,26 @@ public void testDateHistogramAggregation() throws IOException, MatcherException
             .aggregation(AggregationBuilders.dateHistogram("agg").field("@timestamp").calendarInterval(DateHistogramInterval.SECOND))
             .size(0);
 
-        Matcher.mappings(getContenderMappings(), getBaselineMappings())
+        final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings())
             .settings(getContenderSettings(), getBaselineSettings())
             .expected(getAggregationBuckets(queryBaseline(searchSourceBuilder), "agg"))
             .ignoringSort(true)
             .isEqualTo(getAggregationBuckets(queryContender(searchSourceBuilder), "agg"));
+        assertTrue(matchResult.getMessage(), matchResult.isMatch());
     }
 
-    private static XContentBuilder generateDocument(final Instant timestamp) throws IOException {
+    private List<XContentBuilder> generateDocuments(int numberOfDocuments) throws IOException {
+        final List<XContentBuilder> documents = new ArrayList<>();
+        // This is static in order to be able to identify documents between test runs.
+        var startingPoint = ZonedDateTime.of(2024, 1, 1, 10, 0, 0, 0, ZoneId.of("UTC")).toInstant();
+        for (int i = 0; i < numberOfDocuments; i++) {
+            documents.add(generateDocument(startingPoint.plus(i, ChronoUnit.SECONDS)));
+        }
+
+        return documents;
+    }
+
+    protected XContentBuilder generateDocument(final Instant timestamp) throws IOException {
         return XContentFactory.jsonBuilder()
             .startObject()
             .field("@timestamp", DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(timestamp))
@@ -238,8 +296,14 @@ private static XContentBuilder generateDocument(final Instant timestamp) throws
     private static List<Map<String, Object>> getQueryHits(final Response response) throws IOException {
         final Map<String, Object> map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true);
         final Map<String, Object> hitsMap = (Map<String, Object>) map.get("hits");
+
         final List<Map<String, Object>> hitsList = (List<Map<String, Object>>) hitsMap.get("hits");
-        return hitsList.stream().map(hit -> (Map<String, Object>) hit.get("_source")).toList();
+        assertThat(hitsList.size(), greaterThan(0));
+
+        return hitsList.stream()
+            .sorted(Comparator.comparingInt((Map<String, Object> hit) -> Integer.parseInt((String) hit.get("_id"))))
+            .map(hit -> (Map<String, Object>) hit.get("_source"))
+            .toList();
     }
 
     @SuppressWarnings("unchecked")
@@ -247,13 +311,22 @@ private static List<Map<String, Object>> getAggregationBuckets(final Response re
         final Map<String, Object> map = XContentHelper.convertToMap(XContentType.JSON.xContent(), response.getEntity().getContent(), true);
         final Map<String, Object> aggs = (Map<String, Object>) map.get("aggregations");
         final Map<String, Object> agg = (Map<String, Object>) aggs.get(aggName);
-        return (List<Map<String, Object>>) agg.get("buckets");
+
+        var buckets = (List<Map<String, Object>>) agg.get("buckets");
+        assertThat(buckets.size(), greaterThan(0));
+
+        return buckets;
     }
 
     private void assertDocumentIndexing(List<XContentBuilder> documents) throws IOException {
         final Tuple<Response, Response> tuple = indexDocuments(() -> documents, () -> documents);
+
         assertThat(tuple.v1().getStatusLine().getStatusCode(), Matchers.equalTo(RestStatus.OK.getStatus()));
+        var baselineResponseBody = entityAsMap(tuple.v1());
+        assertThat("errors in baseline bulk response:\n " + baselineResponseBody, baselineResponseBody.get("errors"), equalTo(false));
+
         assertThat(tuple.v2().getStatusLine().getStatusCode(), Matchers.equalTo(RestStatus.OK.getStatus()));
+        var contenderResponseBody = entityAsMap(tuple.v2());
+        assertThat("errors in contender bulk response:\n " + contenderResponseBody, contenderResponseBody.get("errors"), equalTo(false));
     }
-
 }
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java
new file mode 100644
index 0000000000000..8bd62480f333d
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa;
+
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.FormatNames;
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.index.mapper.ObjectMapper;
+import org.elasticsearch.logsdb.datageneration.DataGenerator;
+import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
+import org.elasticsearch.logsdb.datageneration.fields.PredefinedField;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Challenge test (see {@link StandardVersusLogsIndexModeChallengeRestIT}) that uses randomly generated
+ * mapping and documents in order to cover more code paths and permutations.
+ */
+public class StandardVersusLogsIndexModeRandomDataChallengeRestIT extends StandardVersusLogsIndexModeChallengeRestIT {
+    private final ObjectMapper.Subobjects subobjects;
+
+    private final DataGenerator dataGenerator;
+
+    public StandardVersusLogsIndexModeRandomDataChallengeRestIT() {
+        super();
+        this.subobjects = randomFrom(ObjectMapper.Subobjects.values());
+
+        var specificationBuilder = DataGeneratorSpecification.builder().withFullyDynamicMapping(randomBoolean());
+        if (subobjects != ObjectMapper.Subobjects.ENABLED) {
+            specificationBuilder = specificationBuilder.withNestedFieldsLimit(0);
+        }
+        this.dataGenerator = new DataGenerator(specificationBuilder.withDataSourceHandlers(List.of(new DataSourceHandler() {
+            @Override
+            public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) {
+                if (subobjects == ObjectMapper.Subobjects.ENABLED) {
+                    // Use default behavior
+                    return null;
+                }
+
+                assert request.isNested() == false;
+
+                // "enabled: false" is not compatible with subobjects: false
+                // "dynamic: false/strict/runtime" is not compatible with subobjects: false
+                return new DataSourceResponse.ObjectMappingParametersGenerator(() -> {
+                    var parameters = new HashMap<String, Object>();
+                    parameters.put("subobjects", subobjects.toString());
+                    if (ESTestCase.randomBoolean()) {
+                        parameters.put("dynamic", "true");
+                    }
+                    if (ESTestCase.randomBoolean()) {
+                        parameters.put("enabled", "true");
+                    }
+                    return parameters;
+                });
+            }
+        }))
+            .withPredefinedFields(
+                List.of(
+                    new PredefinedField.WithType("host.name", FieldType.KEYWORD),
+                    // Needed for terms query
+                    new PredefinedField.WithGenerator("method", new FieldDataGenerator() {
+                        @Override
+                        public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+                            return b -> b.startObject().field("type", "keyword").endObject();
+                        }
+
+                        @Override
+                        public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+                            return b -> b.value(randomFrom("put", "post", "get"));
+                        }
+                    }),
+
+                    // Needed for histogram aggregation
+                    new PredefinedField.WithGenerator("memory_usage_bytes", new FieldDataGenerator() {
+                        @Override
+                        public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+                            return b -> b.startObject().field("type", "long").endObject();
+                        }
+
+                        @Override
+                        public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+                            // We can generate this using standard long field but we would get "too many buckets"
+                            return b -> b.value(randomLongBetween(1000, 2000));
+                        }
+                    })
+                )
+            )
+            .build());
+    }
+
+    @Override
+    public void baselineMappings(XContentBuilder builder) throws IOException {
+        dataGenerator.writeMapping(builder);
+    }
+
+    @Override
+    public void contenderMappings(XContentBuilder builder) throws IOException {
+        if (subobjects != ObjectMapper.Subobjects.ENABLED) {
+            dataGenerator.writeMapping(builder, Map.of("subobjects", subobjects.toString()));
+        } else {
+            dataGenerator.writeMapping(builder);
+        }
+    }
+
+    @Override
+    protected XContentBuilder generateDocument(final Instant timestamp) throws IOException {
+        var document = XContentFactory.jsonBuilder();
+        dataGenerator.generateDocument(document, doc -> {
+            doc.field("@timestamp", DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(timestamp));
+        });
+
+        return document;
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/MatcherException.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/MatcherException.java
deleted file mode 100644
index 442e09f67ac1f..0000000000000
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/MatcherException.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.logsdb.qa.exceptions;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-/**
- * Generic base class for all types of mismatch errors.
- */
-public class MatcherException extends Exception {
-
-    public MatcherException(
-        final XContentBuilder actualMappings,
-        final Settings.Builder actualSettings,
-        final XContentBuilder expectedMappings,
-        final Settings.Builder expectedSettings,
-        final String errorMessage
-    ) {
-        super(errorMessage(actualMappings, actualSettings, expectedMappings, expectedSettings, errorMessage));
-    }
-
-    private static String errorMessage(
-        final XContentBuilder actualMappings,
-        final Settings.Builder actualSettings,
-        final XContentBuilder expectedMappings,
-        final Settings.Builder expectedSettings,
-        final String errorMessage
-    ) {
-        return "Error ["
-            + errorMessage
-            + "] "
-            + "actual mappings ["
-            + Strings.toString(actualMappings)
-            + "] "
-            + "actual settings ["
-            + Strings.toString(actualSettings.build())
-            + "] "
-            + "expected mappings ["
-            + Strings.toString(expectedMappings)
-            + "] "
-            + "expected settings ["
-            + Strings.toString(expectedSettings.build())
-            + "] ";
-    }
-}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/MismatchTypeMatcherException.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/MismatchTypeMatcherException.java
deleted file mode 100644
index d0c9b5f544500..0000000000000
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/MismatchTypeMatcherException.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.logsdb.qa.exceptions;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-public class MismatchTypeMatcherException extends MatcherException {
-    public MismatchTypeMatcherException(
-        XContentBuilder actualMappings,
-        Settings.Builder actualSettings,
-        XContentBuilder expectedMappings,
-        Settings.Builder expectedSettings,
-        String errorMessage
-    ) {
-        super(actualMappings, actualSettings, expectedMappings, expectedSettings, errorMessage);
-    }
-}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/NotEqualMatcherException.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/NotEqualMatcherException.java
deleted file mode 100644
index 8cd9946e7f065..0000000000000
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/NotEqualMatcherException.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.logsdb.qa.exceptions;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-public class NotEqualMatcherException extends MatcherException {
-    public NotEqualMatcherException(
-        XContentBuilder actualMappings,
-        Settings.Builder actualSettings,
-        XContentBuilder expectedMappings,
-        Settings.Builder expectedSettings,
-        String errorMessage
-    ) {
-        super(actualMappings, actualSettings, expectedMappings, expectedSettings, errorMessage);
-    }
-}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/UncomparableMatcherException.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/UncomparableMatcherException.java
deleted file mode 100644
index 44455ec2783f6..0000000000000
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/exceptions/UncomparableMatcherException.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.logsdb.qa.exceptions;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-public class UncomparableMatcherException extends MatcherException {
-    public UncomparableMatcherException(
-        XContentBuilder actualMappings,
-        Settings.Builder actualSettings,
-        XContentBuilder expectedMappings,
-        Settings.Builder expectedSettings,
-        String errorMessage
-    ) {
-        super(actualMappings, actualSettings, expectedMappings, expectedSettings, errorMessage);
-    }
-}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ArrayEqualMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ArrayEqualMatcher.java
new file mode 100644
index 0000000000000..a25c525a657d0
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ArrayEqualMatcher.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa.matchers;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintArrays;
+
+class ArrayEqualMatcher extends GenericEqualsMatcher<Object[]> {
+    ArrayEqualMatcher(
+        final XContentBuilder actualMappings,
+        final Settings.Builder actualSettings,
+        final XContentBuilder expectedMappings,
+        final Settings.Builder expectedSettings,
+        final Object[] actual,
+        final Object[] expected,
+        boolean ignoringSort
+    ) {
+        super(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, ignoringSort);
+    }
+
+    @Override
+    public MatchResult match() {
+        return matchArraysEqual(actual, expected, ignoringSort);
+    }
+
+    private MatchResult matchArraysEqual(final Object[] actualArray, final Object[] expectedArray, boolean ignoreSorting) {
+        if (actualArray.length != expectedArray.length) {
+            return MatchResult.noMatch(
+                formatErrorMessage(
+                    actualMappings,
+                    actualSettings,
+                    expectedMappings,
+                    expectedSettings,
+                    "Array lengths do no match, " + prettyPrintArrays(actualArray, expectedArray)
+                )
+            );
+        }
+        if (ignoreSorting) {
+            return matchArraysEqualIgnoringSorting(actualArray, expectedArray)
+                ? MatchResult.match()
+                : MatchResult.noMatch(
+                    formatErrorMessage(
+                        actualMappings,
+                        actualSettings,
+                        expectedMappings,
+                        expectedSettings,
+                        "Arrays do not match when ignoring sort order, " + prettyPrintArrays(actualArray, expectedArray)
+                    )
+                );
+        } else {
+            return matchArraysEqualExact(actualArray, expectedArray)
+                ? MatchResult.match()
+                : MatchResult.noMatch(
+                    formatErrorMessage(
+                        actualMappings,
+                        actualSettings,
+                        expectedMappings,
+                        expectedSettings,
+                        "Arrays do not match exactly, " + prettyPrintArrays(actualArray, expectedArray)
+                    )
+                );
+        }
+    }
+
+    private static boolean matchArraysEqualIgnoringSorting(final Object[] actualArray, final Object[] expectedArray) {
+        final List<Object> actualList = Arrays.asList(actualArray);
+        final List<Object> expectedList = Arrays.asList(expectedArray);
+        return actualList.containsAll(expectedList) && expectedList.containsAll(actualList);
+    }
+
+    private static <T> boolean matchArraysEqualExact(T[] actualArray, T[] expectedArray) {
+        for (int i = 0; i < actualArray.length; i++) {
+            boolean isEqual = actualArray[i].equals(expectedArray[i]);
+            if (isEqual == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/EqualMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/EqualMatcher.java
deleted file mode 100644
index b3621f2596ae4..0000000000000
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/EqualMatcher.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.logsdb.qa.matchers;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.datastreams.logsdb.qa.exceptions.MatcherException;
-import org.elasticsearch.datastreams.logsdb.qa.exceptions.MismatchTypeMatcherException;
-import org.elasticsearch.datastreams.logsdb.qa.exceptions.UncomparableMatcherException;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.util.Arrays;
-import java.util.List;
-
-class EqualMatcher<T> extends Matcher {
-    private final XContentBuilder actualMappings;
-    private final Settings.Builder actualSettings;
-    private final XContentBuilder expectedMappings;
-    private final Settings.Builder expectedSettings;
-    private final T actual;
-    private final T expected;
-    private final boolean ignoringSort;
-
-    EqualMatcher(
-        XContentBuilder actualMappings,
-        Settings.Builder actualSettings,
-        XContentBuilder expectedMappings,
-        Settings.Builder expectedSettings,
-        T actual,
-        T expected,
-        boolean ignoringSort
-    ) {
-        this.actualMappings = actualMappings;
-        this.actualSettings = actualSettings;
-        this.expectedMappings = expectedMappings;
-        this.expectedSettings = expectedSettings;
-        this.actual = actual;
-        this.expected = expected;
-        this.ignoringSort = ignoringSort;
-    }
-
-    @SuppressWarnings("unchecked")
-    public boolean match() throws MatcherException {
-        if (actual == null) {
-            if (expected == null) {
-                throw new UncomparableMatcherException(
-                    actualMappings,
-                    actualSettings,
-                    expectedMappings,
-                    expectedSettings,
-                    "Both 'actual' and 'expected' are null"
-                );
-            }
-            return false;
-        }
-        if (expected == null) {
-            return false;
-        }
-        if (actual.getClass().equals(expected.getClass()) == false) {
-            throw new MismatchTypeMatcherException(
-                actualMappings,
-                actualSettings,
-                expectedMappings,
-                expectedSettings,
-                "Unable to match " + actual.getClass().getSimpleName() + " to " + expected.getClass().getSimpleName()
-            );
-        }
-        if (actual.getClass().isArray()) {
-            return matchArraysEqual((T[]) actual, (T[]) expected, ignoringSort);
-        }
-        if (actual instanceof List<?> act && expected instanceof List<?> exp) {
-            return matchArraysEqual((T[]) (act).toArray(), (T[]) (exp).toArray(), ignoringSort);
-        }
-        return actual.equals(expected);
-    }
-
-    private boolean matchArraysEqual(final T[] actualArray, final T[] expectedArray, boolean ignoreSorting) {
-        if (actualArray.length != expectedArray.length) {
-            return false;
-        }
-        if (ignoreSorting) {
-            return matchArraysEqualIgnoringSorting(actualArray, expectedArray) != false;
-        } else {
-            return matchArraysEqualExact(actualArray, expectedArray) != false;
-        }
-    }
-
-    private static <T> boolean matchArraysEqualIgnoringSorting(T[] actualArray, T[] expectedArray) {
-        final List<T> actualList = Arrays.asList(actualArray);
-        final List<T> expectedList = Arrays.asList(expectedArray);
-        return actualList.containsAll(expectedList) && expectedList.containsAll(actualList);
-    }
-
-    private static <T> boolean matchArraysEqualExact(T[] actualArray, T[] expectedArray) {
-        for (int i = 0; i < actualArray.length; i++) {
-            boolean isEqual = actualArray[i].equals(expectedArray[i]);
-            if (isEqual == false) {
-                return true;
-            }
-        }
-        return false;
-    }
-}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/GenericEqualsMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/GenericEqualsMatcher.java
new file mode 100644
index 0000000000000..6feae5c1ccabc
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/GenericEqualsMatcher.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa.matchers;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.util.List;
+
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
+
+public class GenericEqualsMatcher<T> extends Matcher {
+    protected final XContentBuilder actualMappings;
+    protected final Settings.Builder actualSettings;
+    protected final XContentBuilder expectedMappings;
+    protected final Settings.Builder expectedSettings;
+    protected final T actual;
+    protected final T expected;
+    protected final boolean ignoringSort;
+
+    protected GenericEqualsMatcher(
+        XContentBuilder actualMappings,
+        Settings.Builder actualSettings,
+        XContentBuilder expectedMappings,
+        Settings.Builder expectedSettings,
+        T actual,
+        T expected,
+        boolean ignoringSort
+    ) {
+        this.actualMappings = actualMappings;
+        this.actualSettings = actualSettings;
+        this.expectedMappings = expectedMappings;
+        this.expectedSettings = expectedSettings;
+        this.actual = actual;
+        this.expected = expected;
+        this.ignoringSort = ignoringSort;
+    }
+
+    public MatchResult match() {
+        if (actual == null) {
+            if (expected == null) {
+
+                return MatchResult.noMatch(
+                    formatErrorMessage(
+                        actualMappings,
+                        actualSettings,
+                        expectedMappings,
+                        expectedSettings,
+                        "Both 'actual' and 'expected' are null"
+                    )
+                );
+            }
+            return MatchResult.noMatch(
+                formatErrorMessage(actualMappings, actualSettings, expectedMappings, expectedSettings, "Expected is null but actual is not")
+            );
+        }
+        if (expected == null) {
+            return MatchResult.noMatch(
+                formatErrorMessage(actualMappings, actualSettings, expectedMappings, expectedSettings, "Actual is null but expected is not")
+            );
+        }
+        if (actual.getClass().equals(expected.getClass()) == false) {
+            return MatchResult.noMatch(
+                formatErrorMessage(
+                    actualMappings,
+                    actualSettings,
+                    expectedMappings,
+                    expectedSettings,
+                    "Unable to match " + actual.getClass().getSimpleName() + " to " + expected.getClass().getSimpleName()
+                )
+            );
+        }
+        if (actual.getClass().isArray()) {
+            return new ArrayEqualMatcher(
+                actualMappings,
+                actualSettings,
+                expectedMappings,
+                expectedSettings,
+                (Object[]) actual,
+                (Object[]) expected,
+                ignoringSort
+            ).match();
+        }
+        if (actual instanceof List<?> act && expected instanceof List<?> exp) {
+            return new ListEqualMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings, act, exp, ignoringSort).match();
+        }
+        return new ObjectMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected).match();
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java
new file mode 100644
index 0000000000000..ae18129a77111
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa.matchers;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.util.List;
+
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintCollections;
+
+public class ListEqualMatcher extends GenericEqualsMatcher<List<?>> {
+    public ListEqualMatcher(
+        final XContentBuilder actualMappings,
+        final Settings.Builder actualSettings,
+        final XContentBuilder expectedMappings,
+        final Settings.Builder expectedSettings,
+        final List<?> actual,
+        final List<?> expected,
+        final boolean ignoringSort
+    ) {
+        super(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, ignoringSort);
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public MatchResult match() {
+        return matchListEquals((List<Object>) actual, (List<Object>) expected, ignoringSort);
+    }
+
+    private MatchResult matchListEquals(final List<Object> actualList, final List<Object> expectedList, boolean ignoreSorting) {
+        if (actualList.size() != expectedList.size()) {
+            return MatchResult.noMatch(
+                formatErrorMessage(
+                    actualMappings,
+                    actualSettings,
+                    expectedMappings,
+                    expectedSettings,
+                    "List lengths do not match, " + prettyPrintCollections(actualList, expectedList)
+                )
+            );
+        }
+        if (ignoreSorting) {
+            return matchListsEqualIgnoringSorting(actualList, expectedList)
+                ? MatchResult.match()
+                : MatchResult.noMatch(
+                    formatErrorMessage(
+                        actualMappings,
+                        actualSettings,
+                        expectedMappings,
+                        expectedSettings,
+                        "Lists do not match when ignoring sort order, " + prettyPrintCollections(actualList, expectedList)
+                    )
+                );
+        } else {
+            return matchListsEqualExact(actualList, expectedList)
+                ? MatchResult.match()
+                : MatchResult.noMatch(
+                    formatErrorMessage(
+                        actualMappings,
+                        actualSettings,
+                        expectedMappings,
+                        expectedSettings,
+                        "Lists do not match exactly, " + prettyPrintCollections(actualList, expectedList)
+                    )
+                );
+        }
+    }
+
+    private static boolean matchListsEqualIgnoringSorting(final List<Object> actualList, final List<Object> expectedList) {
+        return actualList.containsAll(expectedList) && expectedList.containsAll(actualList);
+    }
+
+    private static <T> boolean matchListsEqualExact(List<Object> actualList, List<Object> expectedList) {
+        for (int i = 0; i < actualList.size(); i++) {
+            boolean isEqual = actualList.get(i).equals(expectedList.get(i));
+            if (isEqual == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/MatchResult.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/MatchResult.java
new file mode 100644
index 0000000000000..07a57dcca3b71
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/MatchResult.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa.matchers;
+
+import java.util.Objects;
+
+public class MatchResult {
+    private final boolean isMatch;
+    private final String message;
+
+    private MatchResult(boolean isMatch, String message) {
+        this.isMatch = isMatch;
+        this.message = message;
+    }
+
+    public static MatchResult match() {
+        return new MatchResult(true, "Match successful");
+    }
+
+    public static MatchResult noMatch(final String reason) {
+        return new MatchResult(false, reason);
+    }
+
+    public boolean isMatch() {
+        return isMatch;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        MatchResult that = (MatchResult) o;
+        return isMatch == that.isMatch && Objects.equals(message, that.message);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(isMatch, message);
+    }
+
+    @Override
+    public String toString() {
+        return "MatchResult{" + "isMatch=" + isMatch + ", message='" + message + '\'' + '}';
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Matcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Matcher.java
index 50afda072c4d5..fc2aec1c4fa49 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Matcher.java
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Matcher.java
@@ -9,10 +9,12 @@
 package org.elasticsearch.datastreams.logsdb.qa.matchers;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.datastreams.logsdb.qa.exceptions.MatcherException;
-import org.elasticsearch.datastreams.logsdb.qa.exceptions.NotEqualMatcherException;
+import org.elasticsearch.datastreams.logsdb.qa.matchers.source.SourceMatcher;
 import org.elasticsearch.xcontent.XContentBuilder;
 
+import java.util.List;
+import java.util.Map;
+
 /**
  * A base class to be used for the matching logic when comparing query results.
  */
@@ -22,6 +24,14 @@ public static <T> SettingsStep<T> mappings(final XContentBuilder actualMappings,
         return new Builder<>(expectedMappings, actualMappings);
     }
 
+    public static MappingsStep<List<Map<String, Object>>> matchSource() {
+        return new SourceMatcherBuilder();
+    }
+
+    public interface MappingsStep<T> {
+        SettingsStep<T> mappings(XContentBuilder actualMappings, XContentBuilder expectedMappings);
+    }
+
     public interface SettingsStep<T> {
         ExpectedStep<T> settings(Settings.Builder actualSettings, Settings.Builder expectedSettings);
     }
@@ -31,13 +41,12 @@ public interface ExpectedStep<T> {
     }
 
     public interface CompareStep<T> {
-        void isEqualTo(T actual) throws MatcherException;
+        MatchResult isEqualTo(T actual);
 
         CompareStep<T> ignoringSort(boolean ignoringSort);
     }
 
     private static class Builder<T> implements SettingsStep<T>, CompareStep<T>, ExpectedStep<T> {
-
         private final XContentBuilder expectedMappings;
         private final XContentBuilder actualMappings;
         private Settings.Builder expectedSettings;
@@ -63,8 +72,8 @@ private Builder(
         }
 
         @Override
-        public void isEqualTo(T actual) throws MatcherException {
-            boolean match = new EqualMatcher<>(
+        public MatchResult isEqualTo(T actual) {
+            return new GenericEqualsMatcher<>(
                 actualMappings,
                 actualSettings,
                 expectedMappings,
@@ -73,15 +82,6 @@ public void isEqualTo(T actual) throws MatcherException {
                 expected,
                 ignoringSort
             ).match();
-            if (match == false) {
-                throw new NotEqualMatcherException(
-                    actualMappings,
-                    actualSettings,
-                    expectedMappings,
-                    expectedSettings,
-                    "actual [" + actual + "] not equal to [" + expected + "]"
-                );
-            }
         }
 
         @Override
@@ -97,4 +97,54 @@ public CompareStep<T> expected(T expected) {
         }
     }
 
+    private static class SourceMatcherBuilder
+        implements
+            MappingsStep<List<Map<String, Object>>>,
+            SettingsStep<List<Map<String, Object>>>,
+            CompareStep<List<Map<String, Object>>>,
+            ExpectedStep<List<Map<String, Object>>> {
+        private XContentBuilder expectedMappings;
+        private XContentBuilder actualMappings;
+        private Settings.Builder expectedSettings;
+        private Settings.Builder actualSettings;
+        private List<Map<String, Object>> expected;
+        private boolean ignoringSort;
+
+        @Override
+        public ExpectedStep<List<Map<String, Object>>> settings(Settings.Builder actualSettings, Settings.Builder expectedSettings) {
+            this.actualSettings = actualSettings;
+            this.expectedSettings = expectedSettings;
+            return this;
+        }
+
+        private SourceMatcherBuilder() {}
+
+        public SettingsStep<List<Map<String, Object>>> mappings(
+            final XContentBuilder actualMappings,
+            final XContentBuilder expectedMappings
+        ) {
+            this.actualMappings = actualMappings;
+            this.expectedMappings = expectedMappings;
+
+            return this;
+        }
+
+        @Override
+        public MatchResult isEqualTo(List<Map<String, Object>> actual) {
+            return new SourceMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, ignoringSort)
+                .match();
+        }
+
+        @Override
+        public CompareStep<List<Map<String, Object>>> ignoringSort(boolean ignoringSort) {
+            this.ignoringSort = ignoringSort;
+            return this;
+        }
+
+        @Override
+        public CompareStep<List<Map<String, Object>>> expected(List<Map<String, Object>> expected) {
+            this.expected = expected;
+            return this;
+        }
+    }
 }
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Messages.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Messages.java
new file mode 100644
index 0000000000000..db67a8c3d40d0
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/Messages.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa.matchers;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.stream.Collectors;
+
+public class Messages {
+    public static String formatErrorMessage(
+        final XContentBuilder actualMappings,
+        final Settings.Builder actualSettings,
+        final XContentBuilder expectedMappings,
+        final Settings.Builder expectedSettings,
+        final String errorMessage
+    ) {
+        return "Error ["
+            + errorMessage
+            + "] "
+            + "actual mappings ["
+            + Strings.toString(actualMappings)
+            + "] "
+            + "actual settings ["
+            + Strings.toString(actualSettings.build())
+            + "] "
+            + "expected mappings ["
+            + Strings.toString(expectedMappings)
+            + "] "
+            + "expected settings ["
+            + Strings.toString(expectedSettings.build())
+            + "] ";
+    }
+
+    public static String prettyPrintArrays(final Object[] actualArray, final Object[] expectedArray) {
+        return "actual: "
+            + prettyPrintCollection(Arrays.asList(actualArray))
+            + ", expected: "
+            + prettyPrintCollection(Arrays.asList(expectedArray));
+    }
+
+    public static <T> String prettyPrintCollections(final Collection<T> actualList, final Collection<T> expectedList) {
+        return "actual: " + prettyPrintCollection(actualList) + ", expected: " + prettyPrintCollection(expectedList);
+    }
+
+    private static <T> String prettyPrintCollection(final Collection<T> list) {
+        return "[" + list.stream().map(Object::toString).collect(Collectors.joining(", ")) + "]";
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ObjectMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ObjectMatcher.java
new file mode 100644
index 0000000000000..81844131c58a2
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ObjectMatcher.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa.matchers;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
+
+public class ObjectMatcher extends GenericEqualsMatcher<Object> {
+    ObjectMatcher(
+        final XContentBuilder actualMappings,
+        final Settings.Builder actualSettings,
+        final XContentBuilder expectedMappings,
+        final Settings.Builder expectedSettings,
+        final Object actual,
+        final Object expected
+    ) {
+        super(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, true);
+    }
+
+    @Override
+    public MatchResult match() {
+        return actual.equals(expected)
+            ? MatchResult.match()
+            : MatchResult.noMatch(
+                formatErrorMessage(
+                    actualMappings,
+                    actualSettings,
+                    expectedMappings,
+                    expectedSettings,
+                    "Actual does not equal expected, actual: " + actual + ", expected: " + expected
+                )
+            );
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/DynamicFieldMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/DynamicFieldMatcher.java
new file mode 100644
index 0000000000000..dc371aae82ce1
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/DynamicFieldMatcher.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa.matchers.source;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintCollections;
+
+class DynamicFieldMatcher {
+    private final XContentBuilder actualMappings;
+    private final Settings.Builder actualSettings;
+    private final XContentBuilder expectedMappings;
+    private final Settings.Builder expectedSettings;
+
+    DynamicFieldMatcher(
+        XContentBuilder actualMappings,
+        Settings.Builder actualSettings,
+        XContentBuilder expectedMappings,
+        Settings.Builder expectedSettings
+    ) {
+        this.actualMappings = actualMappings;
+        this.actualSettings = actualSettings;
+        this.expectedMappings = expectedMappings;
+        this.expectedSettings = expectedSettings;
+    }
+
+    /**
+     * Performs matching of dynamically mapped field values if they need special treatment.
+     * @return {#{@link MatchResult}} if field values need special treatment by this matcher.
+     * If field values can be matched using generic mapper, returns {@link Optional#empty()}.
+     */
+    public Optional<MatchResult> match(List<Object> actual, List<Object> expected) {
+        if (expected == null) {
+            return Optional.empty();
+        }
+
+        // Floating point values are always mapped as float with dynamic mapping.
+        var isDouble = expected.stream().filter(Objects::nonNull).findFirst().map(o -> o instanceof Double).orElse(false);
+        if (isDouble) {
+            assert expected.stream().allMatch(o -> o == null || o instanceof Double);
+
+            var normalizedActual = normalizeDoubles(actual);
+            var normalizedExpected = normalizeDoubles(expected);
+
+            var matchResult = normalizedActual.equals(normalizedExpected)
+                ? MatchResult.match()
+                : MatchResult.noMatch(
+                    formatErrorMessage(
+                        actualMappings,
+                        actualSettings,
+                        expectedMappings,
+                        expectedSettings,
+                        "Values of dynamically mapped field containing double values don't match after normalization, normalized "
+                            + prettyPrintCollections(normalizedActual, normalizedExpected)
+                    )
+                );
+            return Optional.of(matchResult);
+        }
+
+        return Optional.empty();
+    }
+
+    private static Set<Float> normalizeDoubles(List<Object> values) {
+        if (values == null) {
+            return Set.of();
+        }
+
+        Function<Object, Float> toFloat = (o) -> o instanceof Number n ? n.floatValue() : Float.parseFloat((String) o);
+        return values.stream().filter(Objects::nonNull).map(toFloat).collect(Collectors.toSet());
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java
new file mode 100644
index 0000000000000..253fb4b0e9688
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java
@@ -0,0 +1,226 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa.matchers.source;
+
+import org.apache.lucene.sandbox.document.HalfFloatPoint;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.math.BigInteger;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintCollections;
+
+interface FieldSpecificMatcher {
+    MatchResult match(List<Object> actual, List<Object> expected, Map<String, Object> actualMapping, Map<String, Object> expectedMapping);
+
+    class HalfFloatMatcher implements FieldSpecificMatcher {
+        private final XContentBuilder actualMappings;
+        private final Settings.Builder actualSettings;
+        private final XContentBuilder expectedMappings;
+        private final Settings.Builder expectedSettings;
+
+        HalfFloatMatcher(
+            XContentBuilder actualMappings,
+            Settings.Builder actualSettings,
+            XContentBuilder expectedMappings,
+            Settings.Builder expectedSettings
+        ) {
+            this.actualMappings = actualMappings;
+            this.actualSettings = actualSettings;
+            this.expectedMappings = expectedMappings;
+            this.expectedSettings = expectedSettings;
+        }
+
+        @Override
+        public MatchResult match(
+            List<Object> actual,
+            List<Object> expected,
+            Map<String, Object> actualMapping,
+            Map<String, Object> expectedMapping
+        ) {
+            var actualHalfFloatBytes = normalize(actual);
+            var expectedHalfFloatBytes = normalize(expected);
+
+            return actualHalfFloatBytes.equals(expectedHalfFloatBytes)
+                ? MatchResult.match()
+                : MatchResult.noMatch(
+                    formatErrorMessage(
+                        actualMappings,
+                        actualSettings,
+                        expectedMappings,
+                        expectedSettings,
+                        "Values of type [half_float] don't match after normalization, normalized "
+                            + prettyPrintCollections(actualHalfFloatBytes, expectedHalfFloatBytes)
+                    )
+                );
+        }
+
+        private static Set<Short> normalize(List<Object> values) {
+            if (values == null) {
+                return Set.of();
+            }
+
+            Function<Object, Float> toFloat = (o) -> o instanceof Number n ? n.floatValue() : Float.parseFloat((String) o);
+            return values.stream()
+                .filter(Objects::nonNull)
+                .map(toFloat)
+                // Based on logic in NumberFieldMapper
+                .map(HalfFloatPoint::halfFloatToSortableShort)
+                .collect(Collectors.toSet());
+        }
+    }
+
+    class ScaledFloatMatcher implements FieldSpecificMatcher {
+        private final XContentBuilder actualMappings;
+        private final Settings.Builder actualSettings;
+        private final XContentBuilder expectedMappings;
+        private final Settings.Builder expectedSettings;
+
+        ScaledFloatMatcher(
+            XContentBuilder actualMappings,
+            Settings.Builder actualSettings,
+            XContentBuilder expectedMappings,
+            Settings.Builder expectedSettings
+        ) {
+            this.actualMappings = actualMappings;
+            this.actualSettings = actualSettings;
+            this.expectedMappings = expectedMappings;
+            this.expectedSettings = expectedSettings;
+        }
+
+        @Override
+        public MatchResult match(
+            List<Object> actual,
+            List<Object> expected,
+            Map<String, Object> actualMapping,
+            Map<String, Object> expectedMapping
+        ) {
+            var scalingFactor = actualMapping.get("scaling_factor");
+            var expectedScalingFactor = expectedMapping.get("scaling_factor");
+            if (Objects.equals(scalingFactor, expectedScalingFactor) == false) {
+                throw new IllegalStateException("Scaling factor for scaled_float field does not match between actual and expected mapping");
+            }
+
+            assert scalingFactor instanceof Number;
+            var expectedNormalized = normalizeExpected(expected, ((Number) scalingFactor).doubleValue());
+            var actualNormalized = normalizeActual(actual);
+
+            return actualNormalized.equals(expectedNormalized)
+                ? MatchResult.match()
+                : MatchResult.noMatch(
+                    formatErrorMessage(
+                        actualMappings,
+                        actualSettings,
+                        expectedMappings,
+                        expectedSettings,
+                        "Values of type [scaled_float] don't match after normalization, normalized "
+                            + prettyPrintCollections(actualNormalized, expectedNormalized)
+                    )
+                );
+        }
+
+        private static Set<Double> normalizeExpected(List<Object> values, double scalingFactor) {
+            if (values == null) {
+                return Set.of();
+            }
+
+            return values.stream()
+                .filter(Objects::nonNull)
+                .map(ScaledFloatMatcher::toDouble)
+                // Based on logic in ScaledFloatFieldMapper
+                .map(v -> {
+                    var encoded = Math.round(v * scalingFactor);
+                    return encoded / scalingFactor;
+                })
+                .collect(Collectors.toSet());
+        }
+
+        private static Set<Double> normalizeActual(List<Object> values) {
+            if (values == null) {
+                return Set.of();
+            }
+
+            return values.stream().filter(Objects::nonNull).map(ScaledFloatMatcher::toDouble).collect(Collectors.toSet());
+        }
+
+        private static double toDouble(Object value) {
+            return ((Number) value).doubleValue();
+        }
+    }
+
+    class UnsignedLongMatcher implements FieldSpecificMatcher {
+        private final XContentBuilder actualMappings;
+        private final Settings.Builder actualSettings;
+        private final XContentBuilder expectedMappings;
+        private final Settings.Builder expectedSettings;
+
+        UnsignedLongMatcher(
+            XContentBuilder actualMappings,
+            Settings.Builder actualSettings,
+            XContentBuilder expectedMappings,
+            Settings.Builder expectedSettings
+        ) {
+            this.actualMappings = actualMappings;
+            this.actualSettings = actualSettings;
+            this.expectedMappings = expectedMappings;
+            this.expectedSettings = expectedSettings;
+        }
+
+        @Override
+        public MatchResult match(
+            List<Object> actual,
+            List<Object> expected,
+            Map<String, Object> actualMapping,
+            Map<String, Object> expectedMapping
+        ) {
+            var expectedNormalized = normalize(expected);
+            var actualNormalized = normalize(actual);
+
+            return actualNormalized.equals(expectedNormalized)
+                ? MatchResult.match()
+                : MatchResult.noMatch(
+                    formatErrorMessage(
+                        actualMappings,
+                        actualSettings,
+                        expectedMappings,
+                        expectedSettings,
+                        "Values of type [unsigned_long] don't match after normalization, normalized "
+                            + prettyPrintCollections(actualNormalized, expectedNormalized)
+                    )
+                );
+        }
+
+        private static Set<BigInteger> normalize(List<Object> values) {
+            if (values == null) {
+                return Set.of();
+            }
+
+            return values.stream().filter(Objects::nonNull).map(UnsignedLongMatcher::toBigInteger).collect(Collectors.toSet());
+        }
+
+        private static BigInteger toBigInteger(Object value) {
+            if (value instanceof String s) {
+                return new BigInteger(s, 10);
+            }
+            if (value instanceof Long l) {
+                return BigInteger.valueOf(l);
+            }
+
+            return (BigInteger) value;
+        }
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/MappingTransforms.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/MappingTransforms.java
new file mode 100644
index 0000000000000..eade6f10e48fe
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/MappingTransforms.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa.matchers.source;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+class MappingTransforms {
+    /**
+     * Container for mapping of a field. Contains field mapping parameters and mapping parameters of parent fields (if present)
+     * in order of increasing distance (direct parent first).
+     * This is needed because some parent mapping parameters influence how source of the field is stored (e.g. `enabled: false`).
+     * @param mappingParameters
+     * @param parentMappingParameters
+     */
+    record FieldMapping(Map<String, Object> mappingParameters, List<Map<String, Object>> parentMappingParameters) {}
+
+    /**
+     * Normalize mapping to have the same structure as normalized source and enable field mapping lookup.
+     * Similar to {@link SourceTransforms#normalize(Map)} but needs to get rid of intermediate nodes
+     * and collect results into a different data structure.
+     *
+     * @param map raw mapping document converted to map
+     * @return map from normalized field name (like a.b.c) to a map of mapping parameters (like type)
+     */
+    public static Map<String, FieldMapping> normalizeMapping(Map<String, Object> map) {
+        var flattened = new HashMap<String, FieldMapping>();
+
+        descend(null, map, flattened);
+
+        return flattened;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void descend(String pathFromRoot, Map<String, Object> currentLevel, Map<String, FieldMapping> flattened) {
+        for (var entry : currentLevel.entrySet()) {
+            if (entry.getKey().equals("_doc") || entry.getKey().equals("properties")) {
+                descend(pathFromRoot, (Map<String, Object>) entry.getValue(), flattened);
+            } else {
+                if (entry.getValue() instanceof Map<?, ?> map) {
+                    var pathToField = pathFromRoot == null ? entry.getKey() : pathFromRoot + "." + entry.getKey();
+
+                    // Descending to subobject, we need to remember parent mapping
+                    if (pathFromRoot != null) {
+                        var parentMapping = flattened.computeIfAbsent(
+                            pathFromRoot,
+                            k -> new FieldMapping(new HashMap<>(), new ArrayList<>())
+                        );
+                        var childMapping = flattened.computeIfAbsent(
+                            pathToField,
+                            k -> new FieldMapping(new HashMap<>(), new ArrayList<>())
+                        );
+                        childMapping.parentMappingParameters.add(parentMapping.mappingParameters);
+                        childMapping.parentMappingParameters.addAll(parentMapping.parentMappingParameters);
+                    }
+
+                    descend(pathToField, (Map<String, Object>) map, flattened);
+                } else {
+                    var pathToField = pathFromRoot == null ? "_doc" : pathFromRoot;
+                    // We are either at the lowest level of mapping or it's a leaf field of top level object
+                    flattened.computeIfAbsent(pathToField, k -> new FieldMapping(new HashMap<>(), new ArrayList<>())).mappingParameters.put(
+                        entry.getKey(),
+                        entry.getValue()
+                    );
+                }
+            }
+        }
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java
new file mode 100644
index 0000000000000..5eb93cee67d74
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java
@@ -0,0 +1,198 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa.matchers.source;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.datastreams.logsdb.qa.matchers.GenericEqualsMatcher;
+import org.elasticsearch.datastreams.logsdb.qa.matchers.ListEqualMatcher;
+import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintCollections;
+
+public class SourceMatcher extends GenericEqualsMatcher<List<Map<String, Object>>> {
+    private final Map<String, MappingTransforms.FieldMapping> actualNormalizedMapping;
+    private final Map<String, MappingTransforms.FieldMapping> expectedNormalizedMapping;
+
+    private final Map<String, FieldSpecificMatcher> fieldSpecificMatchers;
+    private final DynamicFieldMatcher dynamicFieldMatcher;
+
+    public SourceMatcher(
+        final XContentBuilder actualMappings,
+        final Settings.Builder actualSettings,
+        final XContentBuilder expectedMappings,
+        final Settings.Builder expectedSettings,
+        final List<Map<String, Object>> actual,
+        final List<Map<String, Object>> expected,
+        final boolean ignoringSort
+    ) {
+        super(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, ignoringSort);
+
+        var actualMappingAsMap = XContentHelper.convertToMap(BytesReference.bytes(actualMappings), false, actualMappings.contentType())
+            .v2();
+        this.actualNormalizedMapping = MappingTransforms.normalizeMapping(actualMappingAsMap);
+
+        var expectedMappingAsMap = XContentHelper.convertToMap(BytesReference.bytes(expectedMappings), false, actualMappings.contentType())
+            .v2();
+        this.expectedNormalizedMapping = MappingTransforms.normalizeMapping(expectedMappingAsMap);
+
+        this.fieldSpecificMatchers = Map.of(
+            "half_float",
+            new FieldSpecificMatcher.HalfFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings),
+            "scaled_float",
+            new FieldSpecificMatcher.ScaledFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings),
+            "unsigned_long",
+            new FieldSpecificMatcher.UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)
+        );
+        this.dynamicFieldMatcher = new DynamicFieldMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings);
+    }
+
+    @Override
+    public MatchResult match() {
+        if (actual.size() != expected.size()) {
+            return MatchResult.noMatch(
+                formatErrorMessage(
+                    actualMappings,
+                    actualSettings,
+                    expectedMappings,
+                    expectedSettings,
+                    "Number of documents does not match, " + prettyPrintCollections(actual, expected)
+                )
+            );
+        }
+
+        var sortedAndFlattenedActual = actual.stream().map(SourceTransforms::normalize).toList();
+        var sortedAndFlattenedExpected = expected.stream().map(SourceTransforms::normalize).toList();
+
+        for (int i = 0; i < sortedAndFlattenedActual.size(); i++) {
+            var actual = sortedAndFlattenedActual.get(i);
+            var expected = sortedAndFlattenedExpected.get(i);
+
+            var result = compareSource(actual, expected);
+            if (result.isMatch() == false) {
+                var message = "Source matching failed at document id [" + i + "]. " + result.getMessage();
+                return MatchResult.noMatch(message);
+            }
+        }
+
+        return MatchResult.match();
+    }
+
+    private MatchResult compareSource(Map<String, List<Object>> actual, Map<String, List<Object>> expected) {
+        for (var expectedFieldEntry : expected.entrySet()) {
+            var name = expectedFieldEntry.getKey();
+
+            var actualValues = actual.get(name);
+            var expectedValues = expectedFieldEntry.getValue();
+
+            // There are cases when field values are stored in ignored source
+            // so we try to match them as is first and then apply field specific matcher.
+            // This is temporary, we should be able to tell when source is exact using mappings.
+            // See #111916.
+            var genericMatchResult = matchWithGenericMatcher(actualValues, expectedValues);
+            if (genericMatchResult.isMatch()) {
+                return genericMatchResult;
+            }
+
+            var matchIncludingFieldSpecificMatchers = matchWithFieldSpecificMatcher(name, actualValues, expectedValues).orElse(
+                genericMatchResult
+            );
+            if (matchIncludingFieldSpecificMatchers.isMatch() == false) {
+                var message = "Source documents don't match for field [" + name + "]: " + matchIncludingFieldSpecificMatchers.getMessage();
+                return MatchResult.noMatch(message);
+            }
+        }
+
+        return MatchResult.match();
+    }
+
+    private Optional<MatchResult> matchWithFieldSpecificMatcher(String fieldName, List<Object> actualValues, List<Object> expectedValues) {
+        var actualFieldMapping = actualNormalizedMapping.get(fieldName);
+        if (actualFieldMapping == null) {
+            if (expectedNormalizedMapping.get(fieldName) != null
+                // Special cases due to fields being defined in default mapping for logsdb index mode
+                && fieldName.equals("@timestamp") == false
+                && fieldName.equals("host.name") == false) {
+                throw new IllegalStateException(
+                    "Leaf field [" + fieldName + "] is present in expected mapping but absent in actual mapping"
+                );
+            }
+
+            // Field is dynamically mapped
+            return dynamicFieldMatcher.match(actualValues, expectedValues);
+        }
+
+        var actualFieldType = (String) actualFieldMapping.mappingParameters().get("type");
+        if (actualFieldType == null) {
+            throw new IllegalStateException("Field type is missing from leaf field Leaf field [" + fieldName + "] mapping parameters");
+        }
+
+        var expectedFieldMapping = expectedNormalizedMapping.get(fieldName);
+        if (expectedFieldMapping == null) {
+            throw new IllegalStateException("Leaf field [" + fieldName + "] is present in actual mapping but absent in expected mapping");
+        } else {
+            var expectedFieldType = expectedFieldMapping.mappingParameters().get("type");
+            if (Objects.equals(actualFieldType, expectedFieldType) == false) {
+                throw new IllegalStateException(
+                    "Leaf field ["
+                        + fieldName
+                        + "] has type ["
+                        + actualFieldType
+                        + "] in actual mapping but a different type ["
+                        + expectedFieldType
+                        + "] in expected mapping"
+                );
+            }
+        }
+
+        if (sourceMatchesExactly(expectedFieldMapping, expectedValues)) {
+            return Optional.empty();
+        }
+
+        var fieldSpecificMatcher = fieldSpecificMatchers.get(actualFieldType);
+        if (fieldSpecificMatcher == null) {
+            return Optional.empty();
+        }
+
+        MatchResult matched = fieldSpecificMatcher.match(
+            actualValues,
+            expectedValues,
+            actualFieldMapping.mappingParameters(),
+            expectedFieldMapping.mappingParameters()
+        );
+        return Optional.of(matched);
+    }
+
+    // Checks for scenarios when source is stored exactly and therefore can be compared without special logic.
+    private boolean sourceMatchesExactly(MappingTransforms.FieldMapping mapping, List<Object> expectedValues) {
+        return mapping.parentMappingParameters().stream().anyMatch(m -> m.getOrDefault("enabled", "true").equals("false"));
+    }
+
+    private MatchResult matchWithGenericMatcher(List<Object> actualValues, List<Object> expectedValues) {
+        var genericListMatcher = new ListEqualMatcher(
+            actualMappings,
+            actualSettings,
+            expectedMappings,
+            expectedSettings,
+            SourceTransforms.normalizeValues(actualValues),
+            SourceTransforms.normalizeValues(expectedValues),
+            true
+        );
+
+        return genericListMatcher.match();
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceTransforms.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceTransforms.java
new file mode 100644
index 0000000000000..d7d5bf369b018
--- /dev/null
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceTransforms.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams.logsdb.qa.matchers.source;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+class SourceTransforms {
+    /**
+     * This preprocessing step makes it easier to match the document using a unified structure.
+     * It performs following modifications:
+     * <ul>
+     * <li> Flattens all nested maps into top level map with full field path as key (e.g. "a.b.c.d") </li>
+     * <li> Transforms all field values to arrays of length >= 1 </li>
+     * </ul>
+     * <p>
+     * It also makes it possible to work with subobjects: false/auto settings.
+     *
+     * @return flattened map
+     */
+    public static Map<String, List<Object>> normalize(Map<String, Object> map) {
+        var flattened = new HashMap<String, List<Object>>();
+
+        descend(null, map, flattened);
+
+        return flattened;
+    }
+
+    public static <T> List<T> normalizeValues(List<T> values) {
+        if (values == null) {
+            return Collections.emptyList();
+        }
+
+        return normalizeValues(values, Function.identity());
+    }
+
+    public static <T, U> List<U> normalizeValues(List<T> values, Function<T, U> transform) {
+        if (values == null) {
+            return Collections.emptyList();
+        }
+
+        // Synthetic source modifications:
+        // * null values are not present
+        // * duplicates are removed
+        return new ArrayList<>(
+            values.stream().filter(v -> v != null && Objects.equals(v, "null") == false).map(transform).collect(Collectors.toSet())
+        );
+    }
+
+    private static void descend(String pathFromRoot, Map<String, Object> currentLevel, Map<String, List<Object>> flattened) {
+        for (var entry : currentLevel.entrySet()) {
+            var pathToCurrentField = pathFromRoot == null ? entry.getKey() : pathFromRoot + "." + entry.getKey();
+            if (entry.getValue() instanceof List<?> list) {
+                for (var fieldValue : list) {
+                    handleField(pathToCurrentField, fieldValue, flattened);
+                }
+            } else {
+                handleField(pathToCurrentField, entry.getValue(), flattened);
+            }
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void handleField(String pathToCurrentField, Object currentField, Map<String, List<Object>> flattened) {
+        if (currentField instanceof Map<?, ?> map) {
+            descend(pathToCurrentField, (Map<String, Object>) map, flattened);
+        } else {
+            flattened.computeIfAbsent(pathToCurrentField, k -> new ArrayList<>()).add(currentField);
+        }
+    }
+}
diff --git a/modules/data-streams/src/javaRestTest/resources/roles.yml b/modules/data-streams/src/javaRestTest/resources/roles.yml
index 63e506dff8d39..74c238fdae4f2 100644
--- a/modules/data-streams/src/javaRestTest/resources/roles.yml
+++ b/modules/data-streams/src/javaRestTest/resources/roles.yml
@@ -16,10 +16,4 @@ under_privilged:
         - read
         - write
         - view_index_metadata
-manage_data_stream_global_retention:
-  cluster:
-    - manage_data_stream_global_retention
-monitor_data_stream_global_retention:
-  cluster:
-    - monitor_data_stream_global_retention
-no_privilege:
\ No newline at end of file
+no_privilege:
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
index 02663731a1716..615c0006a4ce6 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
@@ -22,9 +22,6 @@
 import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction;
 import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction;
 import org.elasticsearch.client.internal.OriginSettingClient;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.NamedDiff;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -44,12 +41,8 @@
 import org.elasticsearch.datastreams.action.PromoteDataStreamTransportAction;
 import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore;
 import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService;
-import org.elasticsearch.datastreams.lifecycle.UpdateDataStreamGlobalRetentionService;
-import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamGlobalRetentionAction;
 import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamLifecycleAction;
-import org.elasticsearch.datastreams.lifecycle.action.GetDataStreamGlobalRetentionAction;
 import org.elasticsearch.datastreams.lifecycle.action.GetDataStreamLifecycleStatsAction;
-import org.elasticsearch.datastreams.lifecycle.action.PutDataStreamGlobalRetentionAction;
 import org.elasticsearch.datastreams.lifecycle.action.TransportDeleteDataStreamLifecycleAction;
 import org.elasticsearch.datastreams.lifecycle.action.TransportExplainDataStreamLifecycleAction;
 import org.elasticsearch.datastreams.lifecycle.action.TransportGetDataStreamLifecycleAction;
@@ -139,7 +132,6 @@ public static TimeValue getLookAheadTime(Settings settings) {
     private final SetOnce<DataStreamLifecycleService> dataLifecycleInitialisationService = new SetOnce<>();
     private final SetOnce<DataStreamLifecycleHealthInfoPublisher> dataStreamLifecycleErrorsPublisher = new SetOnce<>();
     private final SetOnce<DataStreamLifecycleHealthIndicatorService> dataStreamLifecycleHealthIndicatorService = new SetOnce<>();
-    private final SetOnce<UpdateDataStreamGlobalRetentionService> dataStreamGlobalRetentionService = new SetOnce<>();
     private final Settings settings;
 
     public DataStreamsPlugin(Settings settings) {
@@ -209,19 +201,15 @@ public Collection<?> createComponents(PluginServices services) {
                 errorStoreInitialisationService.get(),
                 services.allocationService(),
                 dataStreamLifecycleErrorsPublisher.get(),
-                services.dataStreamGlobalRetentionResolver()
+                services.dataStreamGlobalRetentionSettings()
             )
         );
         dataLifecycleInitialisationService.get().init();
         dataStreamLifecycleHealthIndicatorService.set(new DataStreamLifecycleHealthIndicatorService());
-        dataStreamGlobalRetentionService.set(
-            new UpdateDataStreamGlobalRetentionService(services.clusterService(), services.dataStreamGlobalRetentionResolver())
-        );
 
         components.add(errorStoreInitialisationService.get());
         components.add(dataLifecycleInitialisationService.get());
         components.add(dataStreamLifecycleErrorsPublisher.get());
-        components.add(dataStreamGlobalRetentionService.get());
         return components;
     }
 
@@ -240,24 +228,6 @@ public Collection<?> createComponents(PluginServices services) {
         actions.add(new ActionHandler<>(DeleteDataStreamLifecycleAction.INSTANCE, TransportDeleteDataStreamLifecycleAction.class));
         actions.add(new ActionHandler<>(ExplainDataStreamLifecycleAction.INSTANCE, TransportExplainDataStreamLifecycleAction.class));
         actions.add(new ActionHandler<>(GetDataStreamLifecycleStatsAction.INSTANCE, TransportGetDataStreamLifecycleStatsAction.class));
-        actions.add(
-            new ActionHandler<>(
-                PutDataStreamGlobalRetentionAction.INSTANCE,
-                PutDataStreamGlobalRetentionAction.TransportPutDataStreamGlobalRetentionAction.class
-            )
-        );
-        actions.add(
-            new ActionHandler<>(
-                GetDataStreamGlobalRetentionAction.INSTANCE,
-                GetDataStreamGlobalRetentionAction.TransportGetDataStreamGlobalSettingsAction.class
-            )
-        );
-        actions.add(
-            new ActionHandler<>(
-                DeleteDataStreamGlobalRetentionAction.INSTANCE,
-                DeleteDataStreamGlobalRetentionAction.TransportDeleteDataStreamGlobalRetentionAction.class
-            )
-        );
         return actions;
     }
 
@@ -293,14 +263,6 @@ public List<RestHandler> getRestHandlers(
         return handlers;
     }
 
-    @Override
-    public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
-        return List.of(
-            new NamedWriteableRegistry.Entry(ClusterState.Custom.class, DataStreamGlobalRetention.TYPE, DataStreamGlobalRetention::read),
-            new NamedWriteableRegistry.Entry(NamedDiff.class, DataStreamGlobalRetention.TYPE, DataStreamGlobalRetention::readDiffFrom)
-        );
-    }
-
     @Override
     public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) {
         return List.of(new DataStreamIndexSettingsProvider(parameters.mapperServiceFactory()));
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java
index 36f5ecaadd446..77e9d20e34c0e 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/CreateDataStreamTransportAction.java
@@ -18,10 +18,10 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.SystemDataStreamDescriptor;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
index 1b18f8b799f4d..d85522d7099b6 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.Index;
@@ -34,6 +33,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java
index a614a2dc40e25..8f50f07c358fc 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java
@@ -26,12 +26,12 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.SnapshotInProgressException;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.tasks.Task;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java
index 8017b1c72f862..dcca32355082b 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java
@@ -21,14 +21,13 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -37,6 +36,7 @@
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.indices.SystemDataStreamDescriptor;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -57,7 +57,7 @@ public class GetDataStreamsTransportAction extends TransportMasterNodeReadAction
     private static final Logger LOGGER = LogManager.getLogger(GetDataStreamsTransportAction.class);
     private final SystemIndices systemIndices;
     private final ClusterSettings clusterSettings;
-    private final DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver;
+    private final DataStreamGlobalRetentionSettings globalRetentionSettings;
 
     @Inject
     public GetDataStreamsTransportAction(
@@ -67,7 +67,7 @@ public GetDataStreamsTransportAction(
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
         SystemIndices systemIndices,
-        DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver
+        DataStreamGlobalRetentionSettings globalRetentionSettings
     ) {
         super(
             GetDataStreamAction.NAME,
@@ -81,7 +81,7 @@ public GetDataStreamsTransportAction(
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.systemIndices = systemIndices;
-        this.dataStreamGlobalRetentionResolver = dataStreamGlobalRetentionResolver;
+        this.globalRetentionSettings = globalRetentionSettings;
         clusterSettings = clusterService.getClusterSettings();
     }
 
@@ -93,7 +93,7 @@ protected void masterOperation(
         ActionListener<GetDataStreamAction.Response> listener
     ) throws Exception {
         listener.onResponse(
-            innerOperation(state, request, indexNameExpressionResolver, systemIndices, clusterSettings, dataStreamGlobalRetentionResolver)
+            innerOperation(state, request, indexNameExpressionResolver, systemIndices, clusterSettings, globalRetentionSettings)
         );
     }
 
@@ -103,7 +103,7 @@ static GetDataStreamAction.Response innerOperation(
         IndexNameExpressionResolver indexNameExpressionResolver,
         SystemIndices systemIndices,
         ClusterSettings clusterSettings,
-        DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver
+        DataStreamGlobalRetentionSettings globalRetentionSettings
     ) {
         List<DataStream> dataStreams = getDataStreams(state, indexNameExpressionResolver, request);
         List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = new ArrayList<>(dataStreams.size());
@@ -223,7 +223,7 @@ public int compareTo(IndexInfo o) {
         return new GetDataStreamAction.Response(
             dataStreamInfos,
             request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null,
-            dataStreamGlobalRetentionResolver.resolve(state)
+            globalRetentionSettings.get()
         );
     }
 
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java
index adbbfe7b28541..cdb342a3ddf3b 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/MigrateToDataStreamTransportAction.java
@@ -19,9 +19,9 @@
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataMigrateToDataStreamService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java
index 97f081575d748..777d239e0ace5 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java
index e048393494139..c901b7a707b6c 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java
@@ -22,10 +22,10 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java
index aabe865f9fe1d..99d4f8bb7cd28 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java
@@ -44,7 +44,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -95,8 +95,6 @@
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.WRITE;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.DownsampleTaskStatus.STARTED;
-import static org.elasticsearch.cluster.metadata.IndexMetadata.DownsampleTaskStatus.SUCCESS;
-import static org.elasticsearch.cluster.metadata.IndexMetadata.DownsampleTaskStatus.UNKNOWN;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_DOWNSAMPLE_STATUS;
 import static org.elasticsearch.datastreams.DataStreamsPlugin.LIFECYCLE_CUSTOM_INDEX_METADATA_KEY;
 
@@ -164,7 +162,7 @@ public class DataStreamLifecycleService implements ClusterStateListener, Closeab
     final ResultDeduplicator<TransportRequest, Void> transportActionsDeduplicator;
     final ResultDeduplicator<String, Void> clusterStateChangesDeduplicator;
     private final DataStreamLifecycleHealthInfoPublisher dslHealthInfoPublisher;
-    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
+    private final DataStreamGlobalRetentionSettings globalRetentionSettings;
     private LongSupplier nowSupplier;
     private final Clock clock;
     private final DataStreamLifecycleErrorStore errorStore;
@@ -213,7 +211,7 @@ public DataStreamLifecycleService(
         DataStreamLifecycleErrorStore errorStore,
         AllocationService allocationService,
         DataStreamLifecycleHealthInfoPublisher dataStreamLifecycleHealthInfoPublisher,
-        DataStreamGlobalRetentionResolver globalRetentionResolver
+        DataStreamGlobalRetentionSettings globalRetentionSettings
     ) {
         this.settings = settings;
         this.client = client;
@@ -224,7 +222,7 @@ public DataStreamLifecycleService(
         this.clusterStateChangesDeduplicator = new ResultDeduplicator<>(threadPool.getThreadContext());
         this.nowSupplier = nowSupplier;
         this.errorStore = errorStore;
-        this.globalRetentionResolver = globalRetentionResolver;
+        this.globalRetentionSettings = globalRetentionSettings;
         this.scheduledJob = null;
         this.pollInterval = DATA_STREAM_LIFECYCLE_POLL_INTERVAL_SETTING.get(settings);
         this.targetMergePolicyFloorSegment = DATA_STREAM_MERGE_POLICY_TARGET_FLOOR_SEGMENT_SETTING.get(settings);
@@ -298,13 +296,13 @@ public void close() {
 
     @Override
     public void triggered(SchedulerEngine.Event event) {
-        if (event.getJobName().equals(LIFECYCLE_JOB_NAME)) {
+        if (event.jobName().equals(LIFECYCLE_JOB_NAME)) {
             if (this.isMaster) {
                 logger.trace(
                     "Data stream lifecycle job triggered: {}, {}, {}",
-                    event.getJobName(),
-                    event.getScheduledTime(),
-                    event.getTriggeredTime()
+                    event.jobName(),
+                    event.scheduledTime(),
+                    event.triggeredTime()
                 );
                 run(clusterService.state());
                 dslHealthInfoPublisher.publishDslErrorEntries(new ActionListener<>() {
@@ -821,8 +819,7 @@ private Index maybeExecuteRollover(ClusterState state, DataStream dataStream, bo
                 RolloverRequest rolloverRequest = getDefaultRolloverRequest(
                     rolloverConfiguration,
                     dataStream.getName(),
-                    dataStream.getLifecycle()
-                        .getEffectiveDataRetention(dataStream.isSystem() ? null : globalRetentionResolver.resolve(state)),
+                    dataStream.getLifecycle().getEffectiveDataRetention(globalRetentionSettings.get(), dataStream.isInternal()),
                     rolloverFailureStore
                 );
                 transportActionsDeduplicator.executeOnce(
@@ -874,7 +871,7 @@ private Index maybeExecuteRollover(ClusterState state, DataStream dataStream, bo
      */
     Set<Index> maybeExecuteRetention(ClusterState state, DataStream dataStream, Set<Index> indicesToExcludeForRemainingRun) {
         Metadata metadata = state.metadata();
-        DataStreamGlobalRetention globalRetention = dataStream.isSystem() ? null : globalRetentionResolver.resolve(state);
+        DataStreamGlobalRetention globalRetention = dataStream.isSystem() ? null : globalRetentionSettings.get();
         List<Index> backingIndicesOlderThanRetention = dataStream.getIndicesPastRetention(metadata::index, nowSupplier, globalRetention);
         if (backingIndicesOlderThanRetention.isEmpty()) {
             return Set.of();
@@ -882,7 +879,7 @@ Set<Index> maybeExecuteRetention(ClusterState state, DataStream dataStream, Set<
         Set<Index> indicesToBeRemoved = new HashSet<>();
         // We know that there is lifecycle and retention because there are indices to be deleted
         assert dataStream.getLifecycle() != null;
-        TimeValue effectiveDataRetention = dataStream.getLifecycle().getEffectiveDataRetention(globalRetention);
+        TimeValue effectiveDataRetention = dataStream.getLifecycle().getEffectiveDataRetention(globalRetention, dataStream.isInternal());
         for (Index index : backingIndicesOlderThanRetention) {
             if (indicesToExcludeForRemainingRun.contains(index) == false) {
                 IndexMetadata backingIndex = metadata.index(index);
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionService.java
deleted file mode 100644
index a18095c555f12..0000000000000
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionService.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.lifecycle;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateAckListener;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor;
-import org.elasticsearch.cluster.ClusterStateTaskListener;
-import org.elasticsearch.cluster.SimpleBatchedAckListenerTaskExecutor;
-import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
-import org.elasticsearch.common.Priority;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.Tuple;
-import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamGlobalRetentionAction;
-import org.elasticsearch.datastreams.lifecycle.action.PutDataStreamGlobalRetentionAction;
-import org.elasticsearch.datastreams.lifecycle.action.UpdateDataStreamGlobalRetentionResponse;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Objects;
-
-/**
- * This service manages the global retention configuration, it provides an API to set or remove global retention
- * from the cluster state.
- */
-public class UpdateDataStreamGlobalRetentionService {
-
-    private static final Logger logger = LogManager.getLogger(UpdateDataStreamGlobalRetentionService.class);
-
-    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
-    private final MasterServiceTaskQueue<UpsertGlobalDataStreamMetadataTask> taskQueue;
-
-    public UpdateDataStreamGlobalRetentionService(
-        ClusterService clusterService,
-        DataStreamGlobalRetentionResolver globalRetentionResolver
-    ) {
-        this.globalRetentionResolver = globalRetentionResolver;
-        ClusterStateTaskExecutor<UpsertGlobalDataStreamMetadataTask> executor = new SimpleBatchedAckListenerTaskExecutor<>() {
-
-            @Override
-            public Tuple<ClusterState, ClusterStateAckListener> executeTask(
-                UpsertGlobalDataStreamMetadataTask task,
-                ClusterState clusterState
-            ) {
-                return new Tuple<>(updateGlobalRetention(clusterState, task.globalRetention()), task);
-            }
-        };
-        this.taskQueue = clusterService.createTaskQueue("data-stream-global-retention", Priority.HIGH, executor);
-
-    }
-
-    public void updateGlobalRetention(
-        PutDataStreamGlobalRetentionAction.Request request,
-        List<UpdateDataStreamGlobalRetentionResponse.AffectedDataStream> affectedDataStreams,
-        final ActionListener<UpdateDataStreamGlobalRetentionResponse> listener
-    ) {
-        taskQueue.submitTask(
-            "update-data-stream-global-retention",
-            new UpsertGlobalDataStreamMetadataTask(
-                request.getGlobalRetention(),
-                affectedDataStreams,
-                listener,
-                request.masterNodeTimeout()
-            ),
-            request.masterNodeTimeout()
-        );
-    }
-
-    public void removeGlobalRetention(
-        DeleteDataStreamGlobalRetentionAction.Request request,
-        List<UpdateDataStreamGlobalRetentionResponse.AffectedDataStream> affectedDataStreams,
-        final ActionListener<UpdateDataStreamGlobalRetentionResponse> listener
-    ) {
-        taskQueue.submitTask(
-            "remove-data-stream-global-retention",
-            new UpsertGlobalDataStreamMetadataTask(null, affectedDataStreams, listener, request.masterNodeTimeout()),
-            request.masterNodeTimeout()
-        );
-    }
-
-    public List<UpdateDataStreamGlobalRetentionResponse.AffectedDataStream> determineAffectedDataStreams(
-        @Nullable DataStreamGlobalRetention newGlobalRetention,
-        ClusterState clusterState
-    ) {
-        var previousGlobalRetention = globalRetentionResolver.resolve(clusterState);
-        if (Objects.equals(newGlobalRetention, previousGlobalRetention)) {
-            return List.of();
-        }
-        List<UpdateDataStreamGlobalRetentionResponse.AffectedDataStream> affectedDataStreams = new ArrayList<>();
-        for (DataStream dataStream : clusterState.metadata().dataStreams().values()) {
-            if (dataStream.getLifecycle() != null) {
-                TimeValue previousEffectiveRetention = dataStream.getLifecycle()
-                    .getEffectiveDataRetention(dataStream.isSystem() ? null : previousGlobalRetention);
-                TimeValue newEffectiveRetention = dataStream.getLifecycle()
-                    .getEffectiveDataRetention(dataStream.isSystem() ? null : newGlobalRetention);
-                if (Objects.equals(previousEffectiveRetention, newEffectiveRetention) == false) {
-                    affectedDataStreams.add(
-                        new UpdateDataStreamGlobalRetentionResponse.AffectedDataStream(
-                            dataStream.getName(),
-                            newEffectiveRetention,
-                            previousEffectiveRetention
-                        )
-                    );
-                }
-            }
-        }
-        affectedDataStreams.sort(Comparator.comparing(UpdateDataStreamGlobalRetentionResponse.AffectedDataStream::dataStreamName));
-        return affectedDataStreams;
-    }
-
-    // Visible for testing
-    ClusterState updateGlobalRetention(ClusterState clusterState, @Nullable DataStreamGlobalRetention retentionFromRequest) {
-        // Detecting if this update will result in a change in the cluster state, requires to use only the global retention from
-        // the cluster state and not the factory retention.
-        final var initialRetentionFromClusterState = DataStreamGlobalRetention.getFromClusterState(clusterState);
-        // Avoid storing empty retention in the cluster state
-        final var newRetention = DataStreamGlobalRetention.EMPTY.equals(retentionFromRequest) ? null : retentionFromRequest;
-        if (Objects.equals(newRetention, initialRetentionFromClusterState)) {
-            return clusterState;
-        }
-        if (newRetention == null) {
-            return clusterState.copyAndUpdate(b -> b.removeCustom(DataStreamGlobalRetention.TYPE));
-        }
-        return clusterState.copyAndUpdate(b -> b.putCustom(DataStreamGlobalRetention.TYPE, newRetention));
-    }
-
-    /**
-     * A base class for the task updating the global retention in the cluster state.
-     */
-    record UpsertGlobalDataStreamMetadataTask(
-        @Nullable DataStreamGlobalRetention globalRetention,
-        List<UpdateDataStreamGlobalRetentionResponse.AffectedDataStream> affectedDataStreams,
-        ActionListener<UpdateDataStreamGlobalRetentionResponse> listener,
-        TimeValue ackTimeout
-    ) implements ClusterStateTaskListener, ClusterStateAckListener {
-
-        @Override
-        public void onFailure(Exception e) {
-            listener.onFailure(e);
-        }
-
-        @Override
-        public boolean mustAck(DiscoveryNode discoveryNode) {
-            return true;
-        }
-
-        @Override
-        public void onAllNodesAcked() {
-            listener.onResponse(new UpdateDataStreamGlobalRetentionResponse(true, affectedDataStreams));
-        }
-
-        @Override
-        public void onAckFailure(Exception e) {
-            logger.debug("Failed to update global retention [{}] with error [{}]", globalRetention, e.getMessage());
-            listener.onResponse(UpdateDataStreamGlobalRetentionResponse.FAILED);
-        }
-
-        @Override
-        public void onAckTimeout() {
-            logger.debug("Failed to update global retention [{}] because timeout was reached", globalRetention);
-            listener.onResponse(UpdateDataStreamGlobalRetentionResponse.FAILED);
-        }
-    }
-}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamGlobalRetentionAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamGlobalRetentionAction.java
deleted file mode 100644
index 92cb855b7cb4e..0000000000000
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamGlobalRetentionAction.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.lifecycle.action;
-
-import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.master.MasterNodeRequest;
-import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.datastreams.lifecycle.UpdateDataStreamGlobalRetentionService;
-import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
-
-/**
- * Deletes the global retention for data streams (if it's not a dry run) and it returns the affected data streams.
- */
-public class DeleteDataStreamGlobalRetentionAction {
-
-    public static final ActionType<UpdateDataStreamGlobalRetentionResponse> INSTANCE = new ActionType<>(
-        "cluster:admin/data_stream/global_retention/delete"
-    );
-
-    private DeleteDataStreamGlobalRetentionAction() {/* no instances */}
-
-    public static final class Request extends MasterNodeRequest<Request> {
-        private boolean dryRun = false;
-
-        public Request(StreamInput in) throws IOException {
-            super(in);
-            dryRun = in.readBoolean();
-        }
-
-        @Override
-        public ActionRequestValidationException validate() {
-            return null;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeBoolean(dryRun);
-        }
-
-        public Request(TimeValue masterNodeTimeout) {
-            super(masterNodeTimeout);
-        }
-
-        public boolean dryRun() {
-            return dryRun;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            DeleteDataStreamGlobalRetentionAction.Request request = (DeleteDataStreamGlobalRetentionAction.Request) o;
-            return dryRun == request.dryRun;
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(dryRun);
-        }
-
-        public void dryRun(boolean dryRun) {
-            this.dryRun = dryRun;
-        }
-    }
-
-    public static class TransportDeleteDataStreamGlobalRetentionAction extends TransportMasterNodeAction<
-        Request,
-        UpdateDataStreamGlobalRetentionResponse> {
-
-        private final UpdateDataStreamGlobalRetentionService globalRetentionService;
-        private final FeatureService featureService;
-
-        @Inject
-        public TransportDeleteDataStreamGlobalRetentionAction(
-            TransportService transportService,
-            ClusterService clusterService,
-            ThreadPool threadPool,
-            ActionFilters actionFilters,
-            IndexNameExpressionResolver indexNameExpressionResolver,
-            UpdateDataStreamGlobalRetentionService globalRetentionService,
-            FeatureService featureService
-        ) {
-            super(
-                INSTANCE.name(),
-                transportService,
-                clusterService,
-                threadPool,
-                actionFilters,
-                Request::new,
-                indexNameExpressionResolver,
-                UpdateDataStreamGlobalRetentionResponse::new,
-                threadPool.executor(ThreadPool.Names.MANAGEMENT)
-            );
-            this.globalRetentionService = globalRetentionService;
-            this.featureService = featureService;
-        }
-
-        @Override
-        protected void masterOperation(
-            Task task,
-            Request request,
-            ClusterState state,
-            ActionListener<UpdateDataStreamGlobalRetentionResponse> listener
-        ) throws Exception {
-            if (featureService.clusterHasFeature(state, DataStreamGlobalRetention.GLOBAL_RETENTION) == false) {
-                listener.onFailure(
-                    new ResourceNotFoundException(
-                        "Data stream global retention feature not found, please ensure all nodes have the feature "
-                            + DataStreamGlobalRetention.GLOBAL_RETENTION.id()
-                    )
-                );
-                return;
-            }
-            List<UpdateDataStreamGlobalRetentionResponse.AffectedDataStream> affectedDataStreams = globalRetentionService
-                .determineAffectedDataStreams(null, state);
-            if (request.dryRun()) {
-                listener.onResponse(new UpdateDataStreamGlobalRetentionResponse(false, true, affectedDataStreams));
-            } else {
-                globalRetentionService.removeGlobalRetention(request, affectedDataStreams, listener);
-            }
-        }
-
-        @Override
-        protected ClusterBlockException checkBlock(Request request, ClusterState state) {
-            return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
-        }
-    }
-}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamGlobalRetentionAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamGlobalRetentionAction.java
deleted file mode 100644
index 1d1064dd42b1a..0000000000000
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamGlobalRetentionAction.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.lifecycle.action;
-
-import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
-import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Objects;
-
-/**
- * Retrieves the global retention for data streams.
- */
-public class GetDataStreamGlobalRetentionAction {
-
-    public static final ActionType<Response> INSTANCE = new ActionType<>("cluster:monitor/data_stream/global_retention/get");
-
-    private GetDataStreamGlobalRetentionAction() {/* no instances */}
-
-    public static final class Request extends MasterNodeReadRequest<Request> {
-
-        public Request(StreamInput in) throws IOException {
-            super(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-        }
-
-        @Override
-        public ActionRequestValidationException validate() {
-            return null;
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            return super.equals(obj);
-        }
-
-        @Override
-        public int hashCode() {
-            return super.hashCode();
-        }
-    }
-
-    public static class Response extends ActionResponse implements ToXContentObject {
-
-        private final DataStreamGlobalRetention globalRetention;
-
-        public Response(DataStreamGlobalRetention globalRetention) {
-            this.globalRetention = globalRetention;
-        }
-
-        public Response(StreamInput in) throws IOException {
-            super(in);
-            globalRetention = DataStreamGlobalRetention.read(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            globalRetention.writeTo(out);
-        }
-
-        @Override
-        public String toString() {
-            return "Response{" + "globalRetention=" + globalRetention + '}';
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            globalRetention.toXContentFragment(builder, params);
-            builder.endObject();
-            return builder;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            Response that = (Response) o;
-            return Objects.equals(globalRetention, that.globalRetention);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(globalRetention);
-        }
-    }
-
-    public static class TransportGetDataStreamGlobalSettingsAction extends TransportMasterNodeReadAction<Request, Response> {
-
-        private final FeatureService featureService;
-        private final DataStreamGlobalRetentionResolver globalRetentionResolver;
-
-        @Inject
-        public TransportGetDataStreamGlobalSettingsAction(
-            TransportService transportService,
-            ClusterService clusterService,
-            ThreadPool threadPool,
-            ActionFilters actionFilters,
-            IndexNameExpressionResolver indexNameExpressionResolver,
-            FeatureService featureService,
-            DataStreamGlobalRetentionResolver globalRetentionResolver
-        ) {
-            super(
-                INSTANCE.name(),
-                transportService,
-                clusterService,
-                threadPool,
-                actionFilters,
-                Request::new,
-                indexNameExpressionResolver,
-                Response::new,
-                threadPool.executor(ThreadPool.Names.MANAGEMENT)
-            );
-            this.featureService = featureService;
-            this.globalRetentionResolver = globalRetentionResolver;
-        }
-
-        @Override
-        protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
-            if (featureService.clusterHasFeature(state, DataStreamGlobalRetention.GLOBAL_RETENTION) == false) {
-                listener.onFailure(
-                    new ResourceNotFoundException(
-                        "Data stream global retention feature not found, please ensure all nodes have the feature "
-                            + DataStreamGlobalRetention.GLOBAL_RETENTION.id()
-                    )
-                );
-                return;
-            }
-            DataStreamGlobalRetention globalRetention = globalRetentionResolver.resolve(state);
-            listener.onResponse(new Response(globalRetention == null ? DataStreamGlobalRetention.EMPTY : globalRetention));
-        }
-
-        @Override
-        protected ClusterBlockException checkBlock(Request request, ClusterState state) {
-            return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
-        }
-    }
-}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java
index 6e930defd4e0b..71f07c8cac668 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java
@@ -76,7 +76,7 @@ public Response(StreamInput in) throws IOException {
         public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalVLong(runDuration);
             out.writeOptionalVLong(timeBetweenStarts);
-            out.writeCollection(dataStreamStats, (o, v) -> v.writeTo(o));
+            out.writeCollection(dataStreamStats, StreamOutput::writeWriteable);
         }
 
         public Long getRunDuration() {
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamGlobalRetentionAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamGlobalRetentionAction.java
deleted file mode 100644
index cd9156ad8b2c8..0000000000000
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamGlobalRetentionAction.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.lifecycle.action;
-
-import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.ValidateActions;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.master.MasterNodeRequest;
-import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.datastreams.lifecycle.UpdateDataStreamGlobalRetentionService;
-import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
-
-/**
- * Sets the global retention for data streams (if it's not a dry run) and it returns the affected data streams.
- */
-public class PutDataStreamGlobalRetentionAction {
-
-    public static final ActionType<UpdateDataStreamGlobalRetentionResponse> INSTANCE = new ActionType<>(
-        "cluster:admin/data_stream/global_retention/put"
-    );
-
-    private PutDataStreamGlobalRetentionAction() {/* no instances */}
-
-    public static final class Request extends MasterNodeRequest<Request> {
-
-        private final DataStreamGlobalRetention globalRetention;
-        private boolean dryRun = false;
-
-        public Request(StreamInput in) throws IOException {
-            super(in);
-            globalRetention = DataStreamGlobalRetention.read(in);
-            dryRun = in.readBoolean();
-        }
-
-        @Override
-        public ActionRequestValidationException validate() {
-            ActionRequestValidationException validationException = null;
-            if (globalRetention.equals(DataStreamGlobalRetention.EMPTY)) {
-                return ValidateActions.addValidationError(
-                    "At least one of 'default_retention' or 'max_retention' should be defined."
-                        + " If you want to remove the configuration please use the DELETE method",
-                    validationException
-                );
-            }
-            return validationException;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            globalRetention.writeTo(out);
-            out.writeBoolean(dryRun);
-        }
-
-        public Request(TimeValue masterNodeTimeout, @Nullable TimeValue defaultRetention, @Nullable TimeValue maxRetention) {
-            super(masterNodeTimeout);
-            this.globalRetention = new DataStreamGlobalRetention(defaultRetention, maxRetention);
-        }
-
-        public DataStreamGlobalRetention getGlobalRetention() {
-            return globalRetention;
-        }
-
-        public boolean dryRun() {
-            return dryRun;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            PutDataStreamGlobalRetentionAction.Request request = (PutDataStreamGlobalRetentionAction.Request) o;
-            return Objects.equals(globalRetention, request.globalRetention) && dryRun == request.dryRun;
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(globalRetention, dryRun);
-        }
-
-        public void dryRun(boolean dryRun) {
-            this.dryRun = dryRun;
-        }
-    }
-
-    public static class TransportPutDataStreamGlobalRetentionAction extends TransportMasterNodeAction<
-        Request,
-        UpdateDataStreamGlobalRetentionResponse> {
-
-        private final UpdateDataStreamGlobalRetentionService globalRetentionService;
-        private final FeatureService featureService;
-
-        @Inject
-        public TransportPutDataStreamGlobalRetentionAction(
-            TransportService transportService,
-            ClusterService clusterService,
-            ThreadPool threadPool,
-            ActionFilters actionFilters,
-            IndexNameExpressionResolver indexNameExpressionResolver,
-            UpdateDataStreamGlobalRetentionService globalRetentionService,
-            FeatureService featureService
-        ) {
-            super(
-                INSTANCE.name(),
-                transportService,
-                clusterService,
-                threadPool,
-                actionFilters,
-                Request::new,
-                indexNameExpressionResolver,
-                UpdateDataStreamGlobalRetentionResponse::new,
-                threadPool.executor(ThreadPool.Names.MANAGEMENT)
-            );
-            this.globalRetentionService = globalRetentionService;
-            this.featureService = featureService;
-        }
-
-        @Override
-        protected void masterOperation(
-            Task task,
-            Request request,
-            ClusterState state,
-            ActionListener<UpdateDataStreamGlobalRetentionResponse> listener
-        ) throws Exception {
-            if (featureService.clusterHasFeature(state, DataStreamGlobalRetention.GLOBAL_RETENTION) == false) {
-                listener.onFailure(
-                    new ResourceNotFoundException(
-                        "Data stream global retention feature not found, please ensure all nodes have the feature "
-                            + DataStreamGlobalRetention.GLOBAL_RETENTION.id()
-                    )
-                );
-                return;
-            }
-            List<UpdateDataStreamGlobalRetentionResponse.AffectedDataStream> affectedDataStreams = globalRetentionService
-                .determineAffectedDataStreams(request.globalRetention, state);
-            if (request.dryRun()) {
-                listener.onResponse(new UpdateDataStreamGlobalRetentionResponse(false, true, affectedDataStreams));
-            } else {
-                globalRetentionService.updateGlobalRetention(request, affectedDataStreams, listener);
-            }
-        }
-
-        @Override
-        protected ClusterBlockException checkBlock(Request request, ClusterState state) {
-            return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
-        }
-    }
-}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java
index 9683588bdcae3..9dd5794d980f6 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java
@@ -18,9 +18,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java
index fe5b3a1a378ff..0ffb5809c2f0f 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java
@@ -18,17 +18,17 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -44,7 +44,7 @@ public class TransportExplainDataStreamLifecycleAction extends TransportMasterNo
     ExplainDataStreamLifecycleAction.Response> {
 
     private final DataStreamLifecycleErrorStore errorStore;
-    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
+    private final DataStreamGlobalRetentionSettings globalRetentionSettings;
 
     @Inject
     public TransportExplainDataStreamLifecycleAction(
@@ -54,7 +54,7 @@ public TransportExplainDataStreamLifecycleAction(
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
         DataStreamLifecycleErrorStore dataLifecycleServiceErrorStore,
-        DataStreamGlobalRetentionResolver globalRetentionResolver
+        DataStreamGlobalRetentionSettings globalRetentionSettings
     ) {
         super(
             ExplainDataStreamLifecycleAction.INSTANCE.name(),
@@ -68,7 +68,7 @@ public TransportExplainDataStreamLifecycleAction(
             threadPool.executor(ThreadPool.Names.MANAGEMENT)
         );
         this.errorStore = dataLifecycleServiceErrorStore;
-        this.globalRetentionResolver = globalRetentionResolver;
+        this.globalRetentionSettings = globalRetentionSettings;
     }
 
     @Override
@@ -103,7 +103,7 @@ protected void masterOperation(
             ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle(
                 index,
                 true,
-                parentDataStream.isSystem(),
+                parentDataStream.isInternal(),
                 idxMetadata.getCreationDate(),
                 rolloverInfo == null ? null : rolloverInfo.getTime(),
                 generationDate,
@@ -118,7 +118,7 @@ protected void masterOperation(
             new ExplainDataStreamLifecycleAction.Response(
                 explainIndices,
                 request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null,
-                globalRetentionResolver.resolve(state)
+                globalRetentionSettings.get()
             )
         );
     }
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java
index 7ac9eaae41a50..452295aab0ce9 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java
@@ -16,13 +16,13 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -40,7 +40,7 @@ public class TransportGetDataStreamLifecycleAction extends TransportMasterNodeRe
     GetDataStreamLifecycleAction.Request,
     GetDataStreamLifecycleAction.Response> {
     private final ClusterSettings clusterSettings;
-    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
+    private final DataStreamGlobalRetentionSettings globalRetentionSettings;
 
     @Inject
     public TransportGetDataStreamLifecycleAction(
@@ -49,7 +49,7 @@ public TransportGetDataStreamLifecycleAction(
         ThreadPool threadPool,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
-        DataStreamGlobalRetentionResolver globalRetentionResolver
+        DataStreamGlobalRetentionSettings globalRetentionSettings
     ) {
         super(
             GetDataStreamLifecycleAction.INSTANCE.name(),
@@ -63,7 +63,7 @@ public TransportGetDataStreamLifecycleAction(
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         clusterSettings = clusterService.getClusterSettings();
-        this.globalRetentionResolver = globalRetentionResolver;
+        this.globalRetentionSettings = globalRetentionSettings;
     }
 
     @Override
@@ -96,7 +96,7 @@ protected void masterOperation(
                     .sorted(Comparator.comparing(GetDataStreamLifecycleAction.Response.DataStreamLifecycle::dataStreamName))
                     .toList(),
                 request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null,
-                globalRetentionResolver.resolve(state)
+                globalRetentionSettings.get()
             )
         );
     }
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java
index 03bc1d129eaba..3a5ef0e86e889 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java
@@ -17,10 +17,10 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java
index 11ecf85b1ac26..ed9f263bdf03a 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java
@@ -19,9 +19,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/UpdateDataStreamGlobalRetentionResponse.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/UpdateDataStreamGlobalRetentionResponse.java
deleted file mode 100644
index d0ab707b91f20..0000000000000
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/UpdateDataStreamGlobalRetentionResponse.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.lifecycle.action;
-
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
-import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * This response is used by {@link PutDataStreamGlobalRetentionAction} and {@link DeleteDataStreamGlobalRetentionAction} to
- * communicate to the user the result of a global retention update and the affected data streams.
- */
-public final class UpdateDataStreamGlobalRetentionResponse extends ActionResponse implements ChunkedToXContentObject {
-
-    public static final UpdateDataStreamGlobalRetentionResponse FAILED = new UpdateDataStreamGlobalRetentionResponse(
-        false,
-        false,
-        List.of()
-    );
-
-    private final boolean acknowledged;
-    private final boolean dryRun;
-    private final List<AffectedDataStream> affectedDataStreams;
-
-    public UpdateDataStreamGlobalRetentionResponse(StreamInput in) throws IOException {
-        super(in);
-        acknowledged = in.readBoolean();
-        dryRun = in.readBoolean();
-        affectedDataStreams = in.readCollectionAsImmutableList(AffectedDataStream::read);
-    }
-
-    public UpdateDataStreamGlobalRetentionResponse(boolean acknowledged, List<AffectedDataStream> affectedDataStreams) {
-        this(acknowledged, false, affectedDataStreams);
-    }
-
-    public UpdateDataStreamGlobalRetentionResponse(boolean acknowledged, boolean dryRun, List<AffectedDataStream> affectedDataStreams) {
-        this.acknowledged = acknowledged;
-        this.dryRun = dryRun;
-        this.affectedDataStreams = affectedDataStreams;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeBoolean(acknowledged);
-        out.writeBoolean(dryRun);
-        out.writeCollection(affectedDataStreams);
-    }
-
-    @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-        return Iterators.concat(ChunkedToXContentHelper.startObject(), Iterators.single(((builder, params1) -> {
-            builder.field("acknowledged", acknowledged);
-            builder.field("dry_run", dryRun);
-            return builder;
-        })),
-            ChunkedToXContentHelper.startArray("affected_data_streams"),
-            Iterators.map(affectedDataStreams.iterator(), affectedDataStream -> affectedDataStream::toXContent),
-            ChunkedToXContentHelper.endArray(),
-            ChunkedToXContentHelper.endObject()
-        );
-    }
-
-    public record AffectedDataStream(String dataStreamName, TimeValue newEffectiveRetention, TimeValue previousEffectiveRetention)
-        implements
-            Writeable,
-            ToXContentObject {
-
-        public static AffectedDataStream read(StreamInput in) throws IOException {
-            return new AffectedDataStream(in.readString(), in.readOptionalTimeValue(), in.readOptionalTimeValue());
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(dataStreamName);
-            out.writeOptionalTimeValue(newEffectiveRetention);
-            out.writeOptionalTimeValue(previousEffectiveRetention);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field("name", dataStreamName);
-            builder.field("new_effective_retention", newEffectiveRetention == null ? "infinite" : newEffectiveRetention.getStringRep());
-            builder.field(
-                "previous_effective_retention",
-                previousEffectiveRetention == null ? "infinite" : previousEffectiveRetention.getStringRep()
-            );
-            builder.endObject();
-            return builder;
-        }
-    }
-
-    public boolean isAcknowledged() {
-        return acknowledged;
-    }
-
-    public boolean isDryRun() {
-        return dryRun;
-    }
-
-    public List<AffectedDataStream> getAffectedDataStreams() {
-        return affectedDataStreams;
-    }
-}
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java
index 048ef0bab8e0c..82350130e57af 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.datastreams.lifecycle.ExplainDataStreamLifecycleAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -19,6 +20,7 @@
 import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
 
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
@@ -39,10 +41,12 @@ public List<Route> routes() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
         String[] indices = Strings.splitStringByCommaToArray(restRequest.param("index"));
-        ExplainDataStreamLifecycleAction.Request explainRequest = new ExplainDataStreamLifecycleAction.Request(indices);
+        ExplainDataStreamLifecycleAction.Request explainRequest = new ExplainDataStreamLifecycleAction.Request(
+            getMasterNodeTimeout(restRequest),
+            indices
+        );
         explainRequest.includeDefaults(restRequest.paramAsBoolean("include_defaults", false));
         explainRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, IndicesOptions.strictExpandOpen()));
-        explainRequest.masterNodeTimeout(getMasterNodeTimeout(restRequest));
         return channel -> client.execute(
             ExplainDataStreamLifecycleAction.INSTANCE,
             explainRequest,
@@ -54,4 +58,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
     public boolean allowSystemIndexAccessByDefault() {
         return true;
     }
+
+    @Override
+    public Set<String> supportedCapabilities() {
+        return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY);
+    }
 }
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java
index 3d802d483fd8c..00f9d4da88301 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java
@@ -10,14 +10,17 @@
 import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
 
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
@@ -37,6 +40,7 @@ public List<Route> routes() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
         GetDataStreamLifecycleAction.Request getDataLifecycleRequest = new GetDataStreamLifecycleAction.Request(
+            RestUtils.getMasterNodeTimeout(request),
             Strings.splitStringByCommaToArray(request.param("name"))
         );
         getDataLifecycleRequest.includeDefaults(request.paramAsBoolean("include_defaults", false));
@@ -52,4 +56,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
     public boolean allowSystemIndexAccessByDefault() {
         return true;
     }
+
+    @Override
+    public Set<String> supportedCapabilities() {
+        return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY);
+    }
 }
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java
index 59d7099e27b52..4b285eaf59b97 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java
@@ -41,10 +41,17 @@ public List<Route> routes() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         try (XContentParser parser = request.contentParser()) {
-            PutDataStreamLifecycleAction.Request putLifecycleRequest = PutDataStreamLifecycleAction.Request.parseRequest(parser);
-            putLifecycleRequest.indices(Strings.splitStringByCommaToArray(request.param("name")));
-            putLifecycleRequest.masterNodeTimeout(getMasterNodeTimeout(request));
-            putLifecycleRequest.ackTimeout(getAckTimeout(request));
+            PutDataStreamLifecycleAction.Request putLifecycleRequest = PutDataStreamLifecycleAction.Request.parseRequest(
+                parser,
+                (dataRetention, enabled, downsampling) -> new PutDataStreamLifecycleAction.Request(
+                    getMasterNodeTimeout(request),
+                    getAckTimeout(request),
+                    Strings.splitStringByCommaToArray(request.param("name")),
+                    dataRetention,
+                    enabled,
+                    downsampling
+                )
+            );
             putLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(request, putLifecycleRequest.indicesOptions()));
             return channel -> client.execute(
                 PutDataStreamLifecycleAction.INSTANCE,
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestCreateDataStreamAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestCreateDataStreamAction.java
index b747f413e0432..a6dc9cfc9fb18 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestCreateDataStreamAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestCreateDataStreamAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -35,7 +36,11 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        CreateDataStreamAction.Request putDataStreamRequest = new CreateDataStreamAction.Request(request.param("name"));
+        CreateDataStreamAction.Request putDataStreamRequest = new CreateDataStreamAction.Request(
+            RestUtils.getMasterNodeTimeout(request),
+            RestUtils.getAckTimeout(request),
+            request.param("name")
+        );
         return channel -> client.execute(CreateDataStreamAction.INSTANCE, putDataStreamRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDeleteDataStreamAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDeleteDataStreamAction.java
index df6f490c50764..551d14ced5104 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDeleteDataStreamAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestDeleteDataStreamAction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -37,6 +38,7 @@ public List<Route> routes() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
         DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(
+            RestUtils.getMasterNodeTimeout(request),
             Strings.splitStringByCommaToArray(request.param("name"))
         );
         deleteDataStreamRequest.indicesOptions(IndicesOptions.fromRequest(request, deleteDataStreamRequest.indicesOptions()));
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java
index fbd4b9acf747e..c3178208d51c2 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java
@@ -10,14 +10,17 @@
 import org.elasticsearch.action.datastreams.GetDataStreamAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
@@ -37,6 +40,7 @@ public List<Route> routes() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
         GetDataStreamAction.Request getDataStreamsRequest = new GetDataStreamAction.Request(
+            RestUtils.getMasterNodeTimeout(request),
             Strings.splitStringByCommaToArray(request.param("name"))
         );
         getDataStreamsRequest.includeDefaults(request.paramAsBoolean("include_defaults", false));
@@ -48,4 +52,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
     public boolean allowSystemIndexAccessByDefault() {
         return true;
     }
+
+    @Override
+    public Set<String> supportedCapabilities() {
+        return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY);
+    }
 }
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestMigrateToDataStreamAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestMigrateToDataStreamAction.java
index 072801c3f9107..7c935bf825095 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestMigrateToDataStreamAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestMigrateToDataStreamAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -35,7 +36,11 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        MigrateToDataStreamAction.Request req = new MigrateToDataStreamAction.Request(request.param("name"));
+        MigrateToDataStreamAction.Request req = new MigrateToDataStreamAction.Request(
+            RestUtils.getMasterNodeTimeout(request),
+            RestUtils.getAckTimeout(request),
+            request.param("name")
+        );
         return channel -> client.execute(MigrateToDataStreamAction.INSTANCE, req, new RestToXContentListener<>(channel));
     }
 }
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestModifyDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestModifyDataStreamsAction.java
index 957ba1b3db8a0..0b33e06eb5940 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestModifyDataStreamsAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestModifyDataStreamsAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -20,8 +21,6 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
-import static org.elasticsearch.rest.RestUtils.getAckTimeout;
-import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestModifyDataStreamsAction extends BaseRestHandler {
@@ -40,13 +39,18 @@ public List<Route> routes() {
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         ModifyDataStreamsAction.Request modifyDsRequest;
         try (XContentParser parser = request.contentParser()) {
-            modifyDsRequest = ModifyDataStreamsAction.Request.PARSER.parse(parser, null);
+            modifyDsRequest = ModifyDataStreamsAction.Request.PARSER.parse(
+                parser,
+                actions -> new ModifyDataStreamsAction.Request(
+                    RestUtils.getMasterNodeTimeout(request),
+                    RestUtils.getAckTimeout(request),
+                    actions
+                )
+            );
         }
         if (modifyDsRequest.getActions() == null || modifyDsRequest.getActions().isEmpty()) {
             throw new IllegalArgumentException("no data stream actions specified, at least one must be specified");
         }
-        modifyDsRequest.masterNodeTimeout(getMasterNodeTimeout(request));
-        modifyDsRequest.ackTimeout(getAckTimeout(request));
         return channel -> client.execute(ModifyDataStreamsAction.INSTANCE, modifyDsRequest, new RestToXContentListener<>(channel));
     }
 
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestPromoteDataStreamAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestPromoteDataStreamAction.java
index 23c744eaaed2c..50ee1e193d5c8 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestPromoteDataStreamAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestPromoteDataStreamAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
@@ -31,7 +32,10 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
-        PromoteDataStreamAction.Request request = new PromoteDataStreamAction.Request(restRequest.param("name"));
+        PromoteDataStreamAction.Request request = new PromoteDataStreamAction.Request(
+            RestUtils.getMasterNodeTimeout(restRequest),
+            restRequest.param("name")
+        );
         return channel -> client.execute(PromoteDataStreamAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java
index 2204c82670e69..bc313d145c17e 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java
@@ -236,7 +236,12 @@ private String createDataStream(boolean hidden) throws Exception {
                 new TransportPutComposableIndexTemplateAction.Request(dataStreamName + "_template").indexTemplate(template)
             )
         );
-        assertAcked(client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)));
+        assertAcked(
+            client().execute(
+                CreateDataStreamAction.INSTANCE,
+                new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName)
+            )
+        );
         createdDataStreams.add(dataStreamName);
         return dataStreamName;
     }
@@ -277,7 +282,12 @@ private DataStreamsStatsAction.Response getDataStreamsStats(boolean includeHidde
     }
 
     private void deleteDataStream(String dataStreamName) {
-        assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { dataStreamName })));
+        assertAcked(
+            client().execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName })
+            )
+        );
         assertAcked(
             client().execute(
                 TransportDeleteComposableIndexTemplateAction.TYPE,
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java
index 022e33621c4f4..d5356e371f497 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
@@ -216,7 +216,10 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() {
             xContentRegistry(),
             EmptySystemIndices.INSTANCE,
             indexSettingProviders,
-            new DataStreamGlobalRetentionResolver(DataStreamFactoryRetention.emptyFactoryRetention())
+            DataStreamGlobalRetentionSettings.create(
+                ClusterSettings.createBuiltInClusterSettings(),
+                DataStreamFactoryRetention.emptyFactoryRetention()
+            )
         );
     }
 
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java
index 97959fa385241..eb35c44d30331 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java
@@ -61,7 +61,7 @@ public void testGetTimestampFieldTypeForTsdbDataStream() throws IOException {
         DocWriteResponse indexResponse = indexDoc();
 
         var indicesService = getInstanceFromNode(IndicesService.class);
-        var result = indicesService.getTimestampFieldType(indexResponse.getShardId().getIndex());
+        var result = indicesService.getTimestampFieldTypeInfo(indexResponse.getShardId().getIndex());
         assertThat(result, notNullValue());
     }
 
@@ -70,7 +70,7 @@ public void testGetTimestampFieldTypeForDataStream() throws IOException {
         DocWriteResponse indexResponse = indexDoc();
 
         var indicesService = getInstanceFromNode(IndicesService.class);
-        var result = indicesService.getTimestampFieldType(indexResponse.getShardId().getIndex());
+        var result = indicesService.getTimestampFieldTypeInfo(indexResponse.getShardId().getIndex());
         assertThat(result, nullValue());
     }
 
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java
index bff238d7f16c3..7d0359a946c2e 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java
@@ -25,7 +25,7 @@ protected Writeable.Reader<Request> instanceReader() {
 
     @Override
     protected Request createTestInstance() {
-        return new Request(randomAlphaOfLength(8));
+        return new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomAlphaOfLength(8));
     }
 
     @Override
@@ -34,13 +34,17 @@ protected Request mutateInstance(Request instance) {
     }
 
     public void testValidateRequest() {
-        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request("my-data-stream");
+        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "my-data-stream"
+        );
         ActionRequestValidationException e = req.validate();
         assertNull(e);
     }
 
     public void testValidateRequestWithoutName() {
-        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request("");
+        CreateDataStreamAction.Request req = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "");
         ActionRequestValidationException e = req.validate();
         assertNotNull(e);
         assertThat(e.validationErrors().size(), equalTo(1));
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java
index d1b3bde331524..056af78846f3e 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java
@@ -25,7 +25,7 @@ protected Writeable.Reader<Request> instanceReader() {
 
     @Override
     protected Request createTestInstance() {
-        return new Request(randomArray(1, 3, String[]::new, () -> randomAlphaOfLength(6)));
+        return new Request(TEST_REQUEST_TIMEOUT, randomArray(1, 3, String[]::new, () -> randomAlphaOfLength(6)));
     }
 
     @Override
@@ -34,13 +34,13 @@ protected Request mutateInstance(Request instance) {
     }
 
     public void testValidateRequest() {
-        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { "my-data-stream" });
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "my-data-stream" });
         ActionRequestValidationException e = req.validate();
         assertNull(e);
     }
 
     public void testValidateRequestWithoutName() {
-        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[0]);
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[0]);
         ActionRequestValidationException e = req.validate();
         assertNotNull(e);
         assertThat(e.validationErrors().size(), equalTo(1));
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportActionTests.java
index d394db9523cce..cdfad60fcfcba 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportActionTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportActionTests.java
@@ -47,7 +47,7 @@ public void testDeleteDataStream() {
         final List<String> otherIndices = randomSubsetOf(List.of("foo", "bar", "baz"));
 
         ClusterState cs = DataStreamTestHelper.getClusterStateWithDataStreams(List.of(new Tuple<>(dataStreamName, 2)), otherIndices);
-        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { dataStreamName });
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName });
         ClusterState newState = DeleteDataStreamTransportAction.removeDataStream(iner, cs, req, validator, Settings.EMPTY);
         assertThat(newState.metadata().dataStreams().size(), equalTo(0));
         assertThat(newState.metadata().indices().size(), equalTo(otherIndices.size()));
@@ -71,7 +71,7 @@ public void testDeleteDataStreamWithFailureStore() {
             false,
             true
         );
-        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { dataStreamName });
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName });
         ClusterState newState = DeleteDataStreamTransportAction.removeDataStream(iner, cs, req, validator, Settings.EMPTY);
         assertThat(newState.metadata().dataStreams().size(), equalTo(0));
         assertThat(newState.metadata().indices().size(), equalTo(otherIndices.size()));
@@ -92,7 +92,7 @@ public void testDeleteMultipleDataStreams() {
             List.of()
         );
 
-        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { "ba*", "eggplant" });
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "ba*", "eggplant" });
         ClusterState newState = DeleteDataStreamTransportAction.removeDataStream(iner, cs, req, validator, Settings.EMPTY);
         assertThat(newState.metadata().dataStreams().size(), equalTo(1));
         DataStream remainingDataStream = newState.metadata().dataStreams().get(dataStreamNames[0]);
@@ -116,7 +116,7 @@ public void testDeleteSnapshottingDataStream() {
             .withAddedEntry(createEntry(dataStreamName2, "repo2", true));
         ClusterState snapshotCs = ClusterState.builder(cs).putCustom(SnapshotsInProgress.TYPE, snapshotsInProgress).build();
 
-        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { dataStreamName });
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName });
         SnapshotInProgressException e = expectThrows(
             SnapshotInProgressException.class,
             () -> DeleteDataStreamTransportAction.removeDataStream(iner, snapshotCs, req, validator, Settings.EMPTY)
@@ -167,13 +167,16 @@ public void testDeleteNonexistentDataStream() {
             () -> DeleteDataStreamTransportAction.removeDataStream(
                 iner,
                 cs,
-                new DeleteDataStreamAction.Request(new String[] { dataStreamName }),
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName }),
                 validator,
                 Settings.EMPTY
             )
         );
 
-        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { dataStreamName + "*" });
+        DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStreamName + "*" }
+        );
         ClusterState newState = DeleteDataStreamTransportAction.removeDataStream(iner, cs, req, validator, Settings.EMPTY);
         assertThat(newState, sameInstance(cs));
         assertThat(newState.metadata().dataStreams().size(), equalTo(cs.metadata().dataStreams().size()));
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java
index 5e6a1ee4bc9d7..58bb3919d5a7c 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java
@@ -20,7 +20,7 @@ protected Writeable.Reader<Request> instanceReader() {
 
     @Override
     protected Request createTestInstance() {
-        return new Request(switch (randomIntBetween(1, 4)) {
+        return new Request(TEST_REQUEST_TIMEOUT, switch (randomIntBetween(1, 4)) {
             case 1 -> generateRandomStringArray(3, 8, false, false);
             case 2 -> {
                 String[] parameters = generateRandomStringArray(3, 8, false, false);
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportActionTests.java
index 58ab69d383464..80d867ec7745e 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportActionTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportActionTests.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention;
 import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -45,14 +45,15 @@ public class GetDataStreamsTransportActionTests extends ESTestCase {
 
     private final IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance();
     private final SystemIndices systemIndices = new SystemIndices(List.of());
-    private final DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver = new DataStreamGlobalRetentionResolver(
+    private final DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings = DataStreamGlobalRetentionSettings.create(
+        ClusterSettings.createBuiltInClusterSettings(),
         DataStreamFactoryRetention.emptyFactoryRetention()
     );
 
     public void testGetDataStream() {
         final String dataStreamName = "my-data-stream";
         ClusterState cs = getClusterStateWithDataStreams(List.of(new Tuple<>(dataStreamName, 1)), List.of());
-        GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[] { dataStreamName });
+        GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName });
         List<DataStream> dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req);
         assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamName)));
     }
@@ -64,19 +65,22 @@ public void testGetDataStreamsWithWildcards() {
             List.of()
         );
 
-        GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[] { dataStreamNames[1].substring(0, 5) + "*" });
+        GetDataStreamAction.Request req = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStreamNames[1].substring(0, 5) + "*" }
+        );
         List<DataStream> dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req);
         assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[1])));
 
-        req = new GetDataStreamAction.Request(new String[] { "*" });
+        req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" });
         dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req);
         assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[1], dataStreamNames[0])));
 
-        req = new GetDataStreamAction.Request((String[]) null);
+        req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, (String[]) null);
         dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req);
         assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[1], dataStreamNames[0])));
 
-        req = new GetDataStreamAction.Request(new String[] { "matches-none*" });
+        req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "matches-none*" });
         dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req);
         assertThat(dataStreams, empty());
     }
@@ -88,19 +92,22 @@ public void testGetDataStreamsWithoutWildcards() {
             List.of()
         );
 
-        GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[] { dataStreamNames[0], dataStreamNames[1] });
+        GetDataStreamAction.Request req = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStreamNames[0], dataStreamNames[1] }
+        );
         List<DataStream> dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req);
         assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[1], dataStreamNames[0])));
 
-        req = new GetDataStreamAction.Request(new String[] { dataStreamNames[1] });
+        req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamNames[1] });
         dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req);
         assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[1])));
 
-        req = new GetDataStreamAction.Request(new String[] { dataStreamNames[0] });
+        req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamNames[0] });
         dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req);
         assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[0])));
 
-        GetDataStreamAction.Request req2 = new GetDataStreamAction.Request(new String[] { "foo" });
+        GetDataStreamAction.Request req2 = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "foo" });
         IndexNotFoundException e = expectThrows(
             IndexNotFoundException.class,
             () -> GetDataStreamsTransportAction.getDataStreams(cs, resolver, req2)
@@ -111,7 +118,7 @@ public void testGetDataStreamsWithoutWildcards() {
     public void testGetNonexistentDataStream() {
         final String dataStreamName = "my-data-stream";
         ClusterState cs = ClusterState.builder(new ClusterName("_name")).build();
-        GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[] { dataStreamName });
+        GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName });
         IndexNotFoundException e = expectThrows(
             IndexNotFoundException.class,
             () -> GetDataStreamsTransportAction.getDataStreams(cs, resolver, req)
@@ -152,14 +159,14 @@ public void testGetTimeSeriesDataStream() {
             state = ClusterState.builder(new ClusterName("_name")).metadata(mBuilder).build();
         }
 
-        var req = new GetDataStreamAction.Request(new String[] {});
+        var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
         var response = GetDataStreamsTransportAction.innerOperation(
             state,
             req,
             resolver,
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
-            dataStreamGlobalRetentionResolver
+            dataStreamGlobalRetentionSettings
         );
         assertThat(
             response.getDataStreams(),
@@ -189,7 +196,7 @@ public void testGetTimeSeriesDataStream() {
             resolver,
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
-            dataStreamGlobalRetentionResolver
+            dataStreamGlobalRetentionSettings
         );
         assertThat(
             response.getDataStreams(),
@@ -232,14 +239,14 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() {
             state = ClusterState.builder(new ClusterName("_name")).metadata(mBuilder).build();
         }
 
-        var req = new GetDataStreamAction.Request(new String[] {});
+        var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
         var response = GetDataStreamsTransportAction.innerOperation(
             state,
             req,
             resolver,
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
-            dataStreamGlobalRetentionResolver
+            dataStreamGlobalRetentionSettings
         );
         assertThat(
             response.getDataStreams(),
@@ -275,14 +282,14 @@ public void testGetTimeSeriesMixedDataStream() {
             state = ClusterState.builder(new ClusterName("_name")).metadata(mBuilder).build();
         }
 
-        var req = new GetDataStreamAction.Request(new String[] {});
+        var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
         var response = GetDataStreamsTransportAction.innerOperation(
             state,
             req,
             resolver,
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
-            dataStreamGlobalRetentionResolver
+            dataStreamGlobalRetentionSettings
         );
 
         var name1 = DataStream.getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli());
@@ -320,28 +327,39 @@ public void testPassingGlobalRetention() {
             state = ClusterState.builder(new ClusterName("_name")).metadata(mBuilder).build();
         }
 
-        var req = new GetDataStreamAction.Request(new String[] {});
+        var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
         var response = GetDataStreamsTransportAction.innerOperation(
             state,
             req,
             resolver,
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
-            dataStreamGlobalRetentionResolver
+            dataStreamGlobalRetentionSettings
         );
         assertThat(response.getGlobalRetention(), nullValue());
         DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(
             TimeValue.timeValueDays(randomIntBetween(1, 5)),
             TimeValue.timeValueDays(randomIntBetween(5, 10))
         );
-        state = ClusterState.builder(state).putCustom(DataStreamGlobalRetention.TYPE, globalRetention).build();
+        DataStreamGlobalRetentionSettings withGlobalRetentionSettings = DataStreamGlobalRetentionSettings.create(
+            ClusterSettings.createBuiltInClusterSettings(
+                Settings.builder()
+                    .put(
+                        DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING.getKey(),
+                        globalRetention.defaultRetention()
+                    )
+                    .put(DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING.getKey(), globalRetention.maxRetention())
+                    .build()
+            ),
+            DataStreamFactoryRetention.emptyFactoryRetention()
+        );
         response = GetDataStreamsTransportAction.innerOperation(
             state,
             req,
             resolver,
             systemIndices,
             ClusterSettings.createBuiltInClusterSettings(),
-            dataStreamGlobalRetentionResolver
+            withGlobalRetentionSettings
         );
         assertThat(response.getGlobalRetention(), equalTo(globalRetention));
     }
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
index c965eb2ba2536..8cb27fd9fd282 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
@@ -37,7 +37,7 @@
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle.Downsampling;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle.Downsampling.Round;
@@ -138,7 +138,8 @@ public class DataStreamLifecycleServiceTests extends ESTestCase {
     private List<TransportRequest> clientSeenRequests;
     private DoExecuteDelegate clientDelegate;
     private ClusterService clusterService;
-    private final DataStreamGlobalRetentionResolver globalRetentionResolver = new DataStreamGlobalRetentionResolver(
+    private final DataStreamGlobalRetentionSettings globalRetentionSettings = DataStreamGlobalRetentionSettings.create(
+        ClusterSettings.createBuiltInClusterSettings(),
         DataStreamFactoryRetention.emptyFactoryRetention()
     );
 
@@ -187,7 +188,7 @@ public void setupServices() {
                 errorStore,
                 new FeatureService(List.of(new DataStreamFeatures()))
             ),
-            globalRetentionResolver
+            globalRetentionSettings
         );
         clientDelegate = null;
         dataStreamLifecycleService.init();
@@ -1392,13 +1393,7 @@ public void testTimeSeriesIndicesStillWithinTimeBounds() {
         {
             // non time_series indices are not within time bounds (they don't have any)
             IndexMetadata indexMeta = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30))
-                .settings(
-                    Settings.builder()
-                        .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                        .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-                        .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current())
-                        .build()
-                )
+                .settings(indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()))
                 .build();
 
             Metadata newMetadata = Metadata.builder(clusterState.metadata()).put(indexMeta, true).build();
@@ -1432,7 +1427,7 @@ public void testTrackingTimeStats() {
                 errorStore,
                 new FeatureService(List.of(new DataStreamFeatures()))
             ),
-            globalRetentionResolver
+            globalRetentionSettings
         );
         assertThat(service.getLastRunDuration(), is(nullValue()));
         assertThat(service.getTimeBetweenStarts(), is(nullValue()));
@@ -1596,12 +1591,14 @@ private ClusterState createClusterState(String indexName, Map<String, String> cu
         var routingTableBuilder = RoutingTable.builder();
         Metadata.Builder metadataBuilder = Metadata.builder();
         Map<String, IndexMetadata> indices = new HashMap<>();
-        Settings indexSettings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 10))
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 3))
-            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current())
-            .build();
-        IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName).version(randomLong()).settings(indexSettings);
+        IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName)
+            .version(randomLong())
+            .settings(
+                indexSettings(randomIntBetween(1, 10), randomIntBetween(0, 3)).put(
+                    IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(),
+                    IndexVersion.current()
+                )
+            );
         if (customDataStreamLifecycleMetadata != null) {
             indexMetadataBuilder.putCustom(LIFECYCLE_CUSTOM_INDEX_METADATA_KEY, customDataStreamLifecycleMetadata);
         }
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionServiceTests.java
deleted file mode 100644
index 41d00d063955d..0000000000000
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/UpdateDataStreamGlobalRetentionServiceTests.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.datastreams.lifecycle;
-
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
-import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
-import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.test.ClusterServiceUtils;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.Matchers.empty;
-
-public class UpdateDataStreamGlobalRetentionServiceTests extends ESTestCase {
-    private static TestThreadPool threadPool;
-    private ClusterService clusterService;
-    private UpdateDataStreamGlobalRetentionService service;
-
-    @BeforeClass
-    public static void setupThreadPool() {
-        threadPool = new TestThreadPool(getTestClass().getName());
-    }
-
-    @Before
-    public void setupServices() {
-        clusterService = ClusterServiceUtils.createClusterService(threadPool);
-        service = new UpdateDataStreamGlobalRetentionService(
-            clusterService,
-            new DataStreamGlobalRetentionResolver(DataStreamFactoryRetention.emptyFactoryRetention())
-        );
-    }
-
-    @After
-    public void closeClusterService() {
-        clusterService.close();
-    }
-
-    @AfterClass
-    public static void tearDownThreadPool() {
-        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
-        threadPool = null;
-    }
-
-    public void testUpdateClusterState() {
-        // Removing from a cluster state without global retention
-        {
-            assertThat(service.updateGlobalRetention(ClusterState.EMPTY_STATE, null), equalTo(ClusterState.EMPTY_STATE));
-            assertThat(
-                service.updateGlobalRetention(ClusterState.EMPTY_STATE, DataStreamGlobalRetention.EMPTY),
-                equalTo(ClusterState.EMPTY_STATE)
-            );
-        }
-
-        // Removing from a cluster state with global retention
-        {
-            ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
-                .putCustom(DataStreamGlobalRetention.TYPE, randomNonEmptyGlobalRetention())
-                .build();
-            DataStreamGlobalRetention updatedRetention = DataStreamGlobalRetention.getFromClusterState(
-                service.updateGlobalRetention(clusterState, null)
-            );
-            assertThat(updatedRetention, nullValue());
-            updatedRetention = DataStreamGlobalRetention.getFromClusterState(
-                service.updateGlobalRetention(clusterState, DataStreamGlobalRetention.EMPTY)
-            );
-            assertThat(updatedRetention, nullValue());
-        }
-
-        // Updating retention
-        {
-            var initialRetention = randomNonEmptyGlobalRetention();
-            ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
-                .putCustom(DataStreamGlobalRetention.TYPE, initialRetention)
-                .build();
-            var expectedRetention = randomValueOtherThan(
-                initialRetention,
-                UpdateDataStreamGlobalRetentionServiceTests::randomNonEmptyGlobalRetention
-            );
-            var updatedRetention = DataStreamGlobalRetention.getFromClusterState(
-                service.updateGlobalRetention(clusterState, expectedRetention)
-            );
-            assertThat(updatedRetention, equalTo(expectedRetention));
-        }
-
-        // No change means no new cluster state
-        {
-            UpdateDataStreamGlobalRetentionService serviceWithRandomFactoryRetention = new UpdateDataStreamGlobalRetentionService(
-                clusterService,
-                new DataStreamGlobalRetentionResolver(
-                    randomBoolean() ? DataStreamFactoryRetention.emptyFactoryRetention() : randomNonEmptyFactoryRetention()
-                )
-            );
-            var retention = randomBoolean() ? null : randomNonEmptyGlobalRetention();
-            ClusterState clusterState = retention == null
-                ? ClusterState.EMPTY_STATE
-                : ClusterState.builder(ClusterName.DEFAULT).putCustom(DataStreamGlobalRetention.TYPE, retention).build();
-            var updatedClusterState = serviceWithRandomFactoryRetention.updateGlobalRetention(clusterState, retention);
-            assertThat(updatedClusterState == clusterState, is(true));
-        }
-    }
-
-    public void testDetermineAffectedDataStreams() {
-        Metadata.Builder builder = Metadata.builder();
-        DataStream dataStreamWithoutLifecycle = newDataStreamInstance(
-            "ds-no-lifecycle",
-            List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))),
-            1,
-            null,
-            false,
-            null,
-            List.of()
-        );
-        builder.put(dataStreamWithoutLifecycle);
-        String dataStreamNoRetention = "ds-no-retention";
-        DataStream dataStreamWithLifecycleNoRetention = newDataStreamInstance(
-            dataStreamNoRetention,
-            List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))),
-            1,
-            null,
-            false,
-            DataStreamLifecycle.DEFAULT,
-            List.of()
-        );
-
-        builder.put(dataStreamWithLifecycleNoRetention);
-        DataStream dataStreamWithLifecycleShortRetention = newDataStreamInstance(
-            "ds-no-short-retention",
-            List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))),
-            1,
-            null,
-            false,
-            DataStreamLifecycle.newBuilder().dataRetention(TimeValue.timeValueDays(7)).build(),
-            List.of()
-        );
-        builder.put(dataStreamWithLifecycleShortRetention);
-        String dataStreamLongRetention = "ds-long-retention";
-        DataStream dataStreamWithLifecycleLongRetention = newDataStreamInstance(
-            dataStreamLongRetention,
-            List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))),
-            1,
-            null,
-            false,
-            DataStreamLifecycle.newBuilder().dataRetention(TimeValue.timeValueDays(365)).build(),
-            List.of()
-        );
-        builder.put(dataStreamWithLifecycleLongRetention);
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(builder).build();
-        // No global retention
-        {
-            var affectedDataStreams = service.determineAffectedDataStreams(null, clusterState);
-            assertThat(affectedDataStreams.isEmpty(), is(true));
-        }
-        // No difference in global retention
-        {
-            var globalRetention = randomNonEmptyGlobalRetention();
-            var clusterStateWithRetention = ClusterState.builder(clusterState)
-                .putCustom(DataStreamGlobalRetention.TYPE, globalRetention)
-                .build();
-            var affectedDataStreams = service.determineAffectedDataStreams(globalRetention, clusterStateWithRetention);
-            assertThat(affectedDataStreams.isEmpty(), is(true));
-        }
-        // Default retention in effect
-        {
-            var globalRetention = new DataStreamGlobalRetention(TimeValue.timeValueDays(randomIntBetween(1, 10)), null);
-            var affectedDataStreams = service.determineAffectedDataStreams(globalRetention, clusterState);
-            if (dataStreamWithLifecycleNoRetention.isSystem()) {
-                assertThat(affectedDataStreams.size(), is(0));
-            } else {
-                assertThat(affectedDataStreams.size(), is(1));
-                var dataStream = affectedDataStreams.get(0);
-                assertThat(dataStream.dataStreamName(), equalTo(dataStreamNoRetention));
-                assertThat(dataStream.previousEffectiveRetention(), nullValue());
-                assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getDefaultRetention()));
-            }
-        }
-        // Max retention in effect
-        {
-            var globalRetention = new DataStreamGlobalRetention(null, TimeValue.timeValueDays(randomIntBetween(10, 90)));
-            var affectedDataStreams = service.determineAffectedDataStreams(globalRetention, clusterState);
-            if (dataStreamWithLifecycleLongRetention.isSystem() && dataStreamWithLifecycleNoRetention.isSystem()) {
-                assertThat(affectedDataStreams.size(), is(0));
-            } else if (dataStreamWithLifecycleLongRetention.isSystem() == false && dataStreamWithLifecycleNoRetention.isSystem() == false) {
-                assertThat(affectedDataStreams.size(), is(2));
-                var dataStream = affectedDataStreams.get(0);
-                assertThat(dataStream.dataStreamName(), equalTo(dataStreamLongRetention));
-                assertThat(dataStream.previousEffectiveRetention(), notNullValue());
-                assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getMaxRetention()));
-                dataStream = affectedDataStreams.get(1);
-                assertThat(dataStream.dataStreamName(), equalTo(dataStreamNoRetention));
-                assertThat(dataStream.previousEffectiveRetention(), nullValue());
-                assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getMaxRetention()));
-            } else if (dataStreamWithLifecycleLongRetention.isSystem() == false) {
-                assertThat(affectedDataStreams.size(), is(1));
-                var dataStream = affectedDataStreams.get(0);
-                assertThat(dataStream.dataStreamName(), equalTo(dataStreamLongRetention));
-                assertThat(dataStream.previousEffectiveRetention(), notNullValue());
-                assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getMaxRetention()));
-            } else {
-                assertThat(affectedDataStreams.size(), is(1));
-                var dataStream = affectedDataStreams.get(0);
-                assertThat(dataStream.dataStreamName(), equalTo(dataStreamNoRetention));
-                assertThat(dataStream.previousEffectiveRetention(), nullValue());
-                assertThat(dataStream.newEffectiveRetention(), equalTo(globalRetention.getMaxRetention()));
-            }
-        }
-
-        // Requested global retention match the factory retention, so no affected data streams
-        {
-            DataStreamFactoryRetention factoryRetention = randomNonEmptyFactoryRetention();
-            UpdateDataStreamGlobalRetentionService serviceWithRandomFactoryRetention = new UpdateDataStreamGlobalRetentionService(
-                clusterService,
-                new DataStreamGlobalRetentionResolver(factoryRetention)
-            );
-            var globalRetention = new DataStreamGlobalRetention(factoryRetention.getDefaultRetention(), factoryRetention.getMaxRetention());
-            var affectedDataStreams = serviceWithRandomFactoryRetention.determineAffectedDataStreams(globalRetention, clusterState);
-            assertThat(affectedDataStreams, is(empty()));
-        }
-    }
-
-    private static DataStream newDataStreamInstance(
-        String name,
-        List<Index> indices,
-        long generation,
-        Map<String, Object> metadata,
-        boolean replicated,
-        @Nullable DataStreamLifecycle lifecycle,
-        List<Index> failureStores
-    ) {
-        DataStream.Builder builder = DataStream.builder(name, indices)
-            .setGeneration(generation)
-            .setMetadata(metadata)
-            .setReplicated(replicated)
-            .setLifecycle(lifecycle)
-            .setFailureStoreEnabled(failureStores.isEmpty() == false)
-            .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build());
-        if (randomBoolean()) {
-            builder.setSystem(true);
-            builder.setHidden(true);
-        }
-        return builder.build();
-    }
-
-    private static DataStreamGlobalRetention randomNonEmptyGlobalRetention() {
-        boolean withDefault = randomBoolean();
-        return new DataStreamGlobalRetention(
-            withDefault ? TimeValue.timeValueDays(randomIntBetween(1, 1000)) : null,
-            withDefault == false || randomBoolean() ? TimeValue.timeValueDays(randomIntBetween(1000, 2000)) : null
-        );
-    }
-
-    private static DataStreamFactoryRetention randomNonEmptyFactoryRetention() {
-        boolean withDefault = randomBoolean();
-        TimeValue defaultRetention = withDefault ? TimeValue.timeValueDays(randomIntBetween(10, 20)) : null;
-        TimeValue maxRetention = withDefault && randomBoolean() ? null : TimeValue.timeValueDays(randomIntBetween(50, 200));
-        return new DataStreamFactoryRetention() {
-            @Override
-            public TimeValue getMaxRetention() {
-                return maxRetention;
-            }
-
-            @Override
-            public TimeValue getDefaultRetention() {
-                return defaultRetention;
-            }
-
-            @Override
-            public void init(ClusterSettings clusterSettings) {
-
-            }
-        };
-    }
-}
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml
index 35e3f38d55c26..ec0c82365a681 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml
@@ -1150,3 +1150,96 @@ setup:
         name: simple-data-stream2
   - is_true: acknowledged
 
+---
+"Create data stream with match all template":
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "data streams supoprt for match all templates only supported in 8.16"
+
+  - do:
+      allowed_warnings:
+        - "index template [match-all-template] has index patterns [*] matching patterns from existing older templates [.monitoring-logstash,.monitoring-es,.monitoring-beats,.monitoring-alerts-7,global,.monitoring-kibana] with patterns (.monitoring-logstash => [.monitoring-logstash-7-*],.monitoring-es => [.monitoring-es-7-*],.monitoring-beats => [.monitoring-beats-7-*],.monitoring-alerts-7 => [.monitoring-alerts-7],global => [*],.monitoring-kibana => [.monitoring-kibana-7-*]); this template [match-all-template] will take precedence during new index creation"
+        - "index template [match-all-template] has index patterns [*] matching patterns from existing older templates [.monitoring-logstash,.monitoring-es,.monitoring-beats,.monitoring-alerts-7,.monitoring-kibana] with patterns (.monitoring-logstash => [.monitoring-logstash-7-*],.monitoring-es => [.monitoring-es-7-*],.monitoring-beats => [.monitoring-beats-7-*],.monitoring-alerts-7 => [.monitoring-alerts-7],.monitoring-kibana => [.monitoring-kibana-7-*]); this template [match-all-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: match-all-template
+        body:
+          index_patterns: [ "*" ]
+          priority: 1
+          data_stream: {}
+
+  - do:
+      indices.create_data_stream:
+        name: match-all-data-stream
+  - is_true: acknowledged
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+  - do:
+      indices.get_data_stream:
+        name: "*"
+  - match: { data_streams.0.name: match-all-data-stream }
+  - match: { data_streams.0.generation: 1 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-match-all-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.status: 'GREEN' }
+  - match: { data_streams.0.template: 'match-all-template' }
+  - match: { data_streams.0.hidden: false }
+
+  - do:
+      indices.delete_data_stream:
+        name: match-all-data-stream
+  - is_true: acknowledged
+
+  - do:
+      indices.delete_index_template:
+        name: match-all-template
+  - is_true: acknowledged
+
+---
+"Create hidden data stream with match all template":
+  - requires:
+      cluster_features: [ "gte_v8.16.0" ]
+      reason: "data streams supoprt for match all templates only supported in 8.16"
+  - do:
+      allowed_warnings:
+        - "index template [match-all-hidden-template] has index patterns [*] matching patterns from existing older templates [.monitoring-logstash,.monitoring-es,.monitoring-beats,.monitoring-alerts-7,global,.monitoring-kibana] with patterns (.monitoring-logstash => [.monitoring-logstash-7-*],.monitoring-es => [.monitoring-es-7-*],.monitoring-beats => [.monitoring-beats-7-*],.monitoring-alerts-7 => [.monitoring-alerts-7],global => [*],.monitoring-kibana => [.monitoring-kibana-7-*]); this template [match-all-hidden-template] will take precedence during new index creation"
+        - "index template [match-all-hidden-template] has index patterns [*] matching patterns from existing older templates [.monitoring-logstash,.monitoring-es,.monitoring-beats,.monitoring-alerts-7,.monitoring-kibana] with patterns (.monitoring-logstash => [.monitoring-logstash-7-*],.monitoring-es => [.monitoring-es-7-*],.monitoring-beats => [.monitoring-beats-7-*],.monitoring-alerts-7 => [.monitoring-alerts-7],.monitoring-kibana => [.monitoring-kibana-7-*]); this template [match-all-hidden-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: match-all-hidden-template
+        body:
+          index_patterns: [ "*" ]
+          priority: 1
+          data_stream:
+            hidden: true
+  - do:
+      indices.create_data_stream:
+        name: match-all-hidden-data-stream
+  - is_true: acknowledged
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+  - do:
+      indices.get_data_stream:
+        name: "*"
+  - length: { data_streams: 0 }
+
+  - do:
+      indices.get_data_stream:
+        name: ['*']
+        expand_wildcards: hidden
+  - length: { data_streams: 1 }
+  - match: { data_streams.0.name: match-all-hidden-data-stream }
+  - match: { data_streams.0.hidden: true }
+
+  - do:
+      indices.delete_data_stream:
+        name: match-all-hidden-data-stream
+  - is_true: acknowledged
+
+  - do:
+      indices.delete_index_template:
+        name: match-all-hidden-template
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
index 8c23232bb457c..d20231a6d6cf2 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
@@ -232,13 +232,13 @@ dynamic templates:
         refresh: true
         body:
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:08.138Z", "data": "10", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5" }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z", "data": "10", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5", "attributes.another.dim3": true }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:09.138Z", "data": "20", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5" }'
+          - '{ "@timestamp": "2023-09-01T13:03:09.138Z", "data": "20", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5", "attributes.another.dim3": true }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:10.138Z", "data": "30", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5" }'
+          - '{ "@timestamp": "2023-09-01T13:03:10.138Z", "data": "30", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5", "attributes.another.dim3": false }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:10.238Z", "data": "40", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5" }'
+          - '{ "@timestamp": "2023-09-01T13:03:10.238Z", "data": "40", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5", "attributes.another.dim3": false }'
 
   - do:
       search:
@@ -264,7 +264,7 @@ dynamic templates:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -283,7 +283,7 @@ dynamic templates:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -302,7 +302,7 @@ dynamic templates:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -321,7 +321,25 @@ dynamic templates:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" }
+  - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
+  - do:
+      search:
+        index: k9s
+        body:
+          size: 0
+          aggs:
+            filterA:
+              filter:
+                term:
+                  another.dim3: true
+              aggs:
+                tsids:
+                  terms:
+                    field: _tsid
+
+  - length: { aggregations.filterA.tsids.buckets: 1 }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "NOHjOAVWLTVWZM4CXLoraZYgYpiKqVppKnpcfycX2dfFiw707uoshWIGVb-ie-ZDQ7hwqiw" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
 ---
@@ -554,13 +572,13 @@ dynamic templates with nesting:
         refresh: true
         body:
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5",  "attributes.a.much.deeper.nested.dim": "AC" }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5", "attributes.another.dim3": true, "attributes.a.much.deeper.nested.dim": "AC" }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5",  "attributes.a.much.deeper.nested.dim": "AC" }'
+          - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5", "attributes.another.dim3": true, "attributes.a.much.deeper.nested.dim": "AC" }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5",  "attributes.a.much.deeper.nested.dim": "BD" }'
+          - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5", "attributes.another.dim3": false, "attributes.a.much.deeper.nested.dim": "BD" }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
-          - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5",  "attributes.a.much.deeper.nested.dim": "BD" }'
+          - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5", "attributes.another.dim3": false, "attributes.a.much.deeper.nested.dim": "BD" }'
 
   - do:
       search:
@@ -586,7 +604,7 @@ dynamic templates with nesting:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -605,7 +623,7 @@ dynamic templates with nesting:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -624,7 +642,7 @@ dynamic templates with nesting:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -643,7 +661,26 @@ dynamic templates with nesting:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" }
+  - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
+
+  - do:
+      search:
+        index: k9s
+        body:
+          size: 0
+          aggs:
+            filterA:
+              filter:
+                term:
+                  another.dim3: true
+              aggs:
+                tsids:
+                  terms:
+                    field: _tsid
+
+  - length: { aggregations.filterA.tsids.buckets: 1 }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
   - do:
@@ -662,7 +699,7 @@ dynamic templates with nesting:
                     field: _tsid
 
   - length: { aggregations.filterA.tsids.buckets: 1 }
-  - match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "OFP9EtCzqs8Sp7Rn2I9NahMBkssYqVppKnpcfycgYpiKiw707hfZ18UMdd8dUGmp6bH35LX6Gni-" }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
 ---
@@ -763,6 +800,19 @@ dynamic templates with incremental indexing:
           - '{ "@timestamp": "2023-09-01T13:06:10.138Z","data": "330",  "attributes.a.much.deeper.nested.dim": "BD" }'
           - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
           - '{ "@timestamp": "2023-09-01T13:06:10.238Z","data": "340",  "attributes.a.much.deeper.nested.dim": "BD" }'
+  - do:
+      bulk:
+        index: k9s
+        refresh: true
+        body:
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:05:08.138Z","data": "210", "resource.attributes.another.deeper.dim3": true }'
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:05:09.138Z","data": "220", "resource.attributes.another.deeper.dim3": true }'
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:05:10.138Z","data": "230", "resource.attributes.another.deeper.dim3": false }'
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:05:10.238Z","data": "240", "resource.attributes.another.deeper.dim3": false }'
 
   - do:
       search:
@@ -770,7 +820,7 @@ dynamic templates with incremental indexing:
         body:
           size: 0
 
-  - match: { hits.total.value: 16 }
+  - match: { hits.total.value: 20 }
 
   - do:
       search:
@@ -862,6 +912,24 @@ dynamic templates with incremental indexing:
   - length: { aggregations.filterA.tsids.buckets: 1 }
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
+  - do:
+      search:
+        index: k9s
+        body:
+          size: 0
+          aggs:
+            filterA:
+              filter:
+                term:
+                  another.deeper.dim3: true
+              aggs:
+                tsids:
+                  terms:
+                    field: _tsid
+
+  - length: { aggregations.filterA.tsids.buckets: 1 }
+  - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
+
 ---
 subobject in passthrough object auto flatten:
   - requires:
@@ -1052,3 +1120,113 @@ dimensions with ignore_malformed and ignore_above:
   - length: { aggregations.keyword_dims.buckets: 1 }
   - match: { aggregations.keyword_dims.buckets.0.key: "foo" }
   - match: { aggregations.keyword_dims.buckets.0.doc_count: 2 }
+
+---
+non string dimension fields:
+  - requires:
+      cluster_features: ["mapper.pass_through_priority", "routing.boolean_routing_path", "mapper.boolean_dimension"]
+      reason: support for priority in passthrough objects
+  - do:
+      allowed_warnings:
+        - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-dynamic-template
+        body:
+          index_patterns: [k9s*]
+          data_stream: {}
+          template:
+            settings:
+              index:
+                number_of_shards: 1
+                mode: time_series
+                time_series:
+                  start_time: 2023-08-31T13:03:08.138Z
+
+            mappings:
+              properties:
+                attributes:
+                  type: passthrough
+                  dynamic: true
+                  time_series_dimension: true
+                  priority: 0
+                metrics:
+                  type: passthrough
+                  priority: 1
+              dynamic_templates:
+                - counter_metric:
+                    mapping:
+                      type: integer
+                      time_series_metric: counter
+                - ip_attributes:
+                    path_match: "*.ip"
+                    match_mapping_type: string
+                    mapping:
+                      type: ip
+                - strings_as_keywords:
+                    match_mapping_type: string
+                    mapping:
+                      type: keyword
+                - double_as_double:
+                    match_mapping_type: double
+                    mapping:
+                      type: double
+
+  - do:
+      bulk:
+        index: k9s
+        refresh: true
+        body:
+          - '{ "create": { "dynamic_templates": { "metrics.data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z", "metrics.data": "10", "attributes.string": "foo", "attributes.boolean": true, "attributes.integer": 1, "attributes.double": 1.1, "attributes.host.ip": "127.0.0.1" }'
+  - is_false: errors
+
+  - do:
+      search:
+        index: k9s
+        body:
+          query:
+            bool:
+              must:
+                - term:
+                    attributes.string:
+                      value: foo
+                - term:
+                    attributes.boolean:
+                      value: true
+                - term:
+                    attributes.integer:
+                      value: 1
+                - term:
+                    attributes.double:
+                      value: 1.1
+                - term:
+                    attributes.host.ip:
+                      value: 127.0.0.1
+          size: 1
+          fields: [ "*" ]
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.attributes\.string: [ "foo" ] }
+  - match: { hits.hits.0.fields.attributes\.boolean: [ true ] }
+  - match: { hits.hits.0.fields.attributes\.integer: [ 1 ] }
+  - match: { hits.hits.0.fields.attributes\.double: [ 1.1 ] }
+  - match: { hits.hits.0.fields.attributes\.host\.ip: [ "127.0.0.1" ] }
+
+  - do:
+      indices.get_data_stream:
+        name: k9s
+  - set: { data_streams.0.indices.0.index_name: idx0name }
+
+  - do:
+      indices.get_mapping:
+        index: $idx0name
+        expand_wildcards: hidden
+  - match: { .$idx0name.mappings.properties.attributes.properties.string.type: 'keyword' }
+  - match: { .$idx0name.mappings.properties.attributes.properties.string.time_series_dimension: true }
+  - match: { .$idx0name.mappings.properties.attributes.properties.boolean.type: 'boolean' }
+  - match: { .$idx0name.mappings.properties.attributes.properties.boolean.time_series_dimension: true }
+  - match: { .$idx0name.mappings.properties.attributes.properties.integer.type: 'long' }
+  - match: { .$idx0name.mappings.properties.attributes.properties.integer.time_series_dimension: true }
+  - match: { .$idx0name.mappings.properties.attributes.properties.double.type: 'double' }
+  - match: { .$idx0name.mappings.properties.attributes.properties.double.time_series_dimension: true }
+  - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'ip' }
+  - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.time_series_dimension: true }
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml
index 54ce32eb13207..991504b27f65f 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml
@@ -123,7 +123,7 @@ teardown:
   - match: { hits.hits.0._source.document.source.foo: 'bar' }
   - match: { hits.hits.0._source.error.type: 'fail_processor_exception' }
   - match: { hits.hits.0._source.error.message: 'error_message' }
-  - contains: { hits.hits.0._source.error.stack_trace: 'org.elasticsearch.ingest.common.FailProcessorException: error_message' }
+  - contains: { hits.hits.0._source.error.stack_trace: 'error_message' }
   - length: { hits.hits.0._source.error.pipeline_trace: 2 }
   - match: { hits.hits.0._source.error.pipeline_trace.0: 'parent_failing_pipeline' }
   - match: { hits.hits.0._source.error.pipeline_trace.1: 'failing_pipeline' }
@@ -207,7 +207,6 @@ teardown:
   - match: { hits.hits.0._source.error.type: 'document_parsing_exception' }
   - contains: { hits.hits.0._source.error.message: "failed to parse field [count] of type [long] in document with id " }
   - contains: { hits.hits.0._source.error.message: "Preview of field's value: 'invalid value'" }
-  - contains: { hits.hits.0._source.error.stack_trace: "org.elasticsearch.index.mapper.DocumentParsingException: " }
   - contains: { hits.hits.0._source.error.stack_trace: "failed to parse field [count] of type [long] in document with id" }
   - contains: { hits.hits.0._source.error.stack_trace: "Preview of field's value: 'invalid value'" }
 
@@ -319,4 +318,433 @@ teardown:
         index: .fs-destination-*
   - length:   { hits.hits: 1  }
   - match: { hits.hits.0._index: "/\\.fs-destination-data-stream-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" }
-  - match: { hits.hits.0._source.document.index: 'destination-data-stream' }
+  - match: { hits.hits.0._source.document.index: 'logs-foobar' }
+
+---
+"Failure redirects to original failure store during index change if self referenced":
+  - requires:
+      cluster_features: [ "gte_v8.15.0" ]
+      reason: "data stream failure stores REST structure changed in 8.15+"
+      test_runner_features: [ allowed_warnings, contains ]
+
+  - do:
+      ingest.put_pipeline:
+        id: "failing_pipeline"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set": {
+                  "field": "_index",
+                  "value": "logs-elsewhere"
+                }
+              },
+              {
+                "script": {
+                  "source": "ctx.object.data = ctx.object"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      allowed_warnings:
+        - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: generic_logs_template
+        body:
+          index_patterns: logs-*
+          data_stream:
+            failure_store: true
+          template:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+              index:
+                default_pipeline: "failing_pipeline"
+
+  - do:
+      index:
+        index: logs-foobar
+        refresh: true
+        body:
+          '@timestamp': '2020-12-12'
+          object:
+            data:
+              field: 'someValue'
+
+  - do:
+      indices.get_data_stream:
+        name: logs-foobar
+  - match: { data_streams.0.name: logs-foobar }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+
+  - do:
+      search:
+        index: logs-foobar
+        body: { query: { match_all: { } } }
+  - length: { hits.hits: 0 }
+
+  - do:
+      search:
+        index: .fs-logs-foobar-*
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._index: "/\\.fs-logs-foobar-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }
+  - exists: hits.hits.0._source.@timestamp
+  - not_exists: hits.hits.0._source.foo
+  - not_exists: hits.hits.0._source.document.id
+  - match: { hits.hits.0._source.document.index: 'logs-foobar' }
+  - match: { hits.hits.0._source.document.source.@timestamp: '2020-12-12' }
+  - match: { hits.hits.0._source.document.source.object.data.field: 'someValue' }
+  - match: { hits.hits.0._source.error.type: 'illegal_argument_exception' }
+  - contains: { hits.hits.0._source.error.message: 'Failed to generate the source document for ingest pipeline' }
+  - contains: { hits.hits.0._source.error.stack_trace: 'Failed to generate the source document for ingest pipeline' }
+  - match: { hits.hits.0._source.error.pipeline_trace.0: 'failing_pipeline' }
+  - match: { hits.hits.0._source.error.pipeline: 'failing_pipeline' }
+
+  - do:
+      indices.delete_data_stream:
+        name: logs-foobar
+  - is_true: acknowledged
+
+  - do:
+      indices.delete:
+        index: .fs-logs-foobar-*
+  - is_true: acknowledged
+
+---
+"Failure redirects to original failure store during index change if final pipeline changes target":
+  - requires:
+      cluster_features: [ "gte_v8.15.0" ]
+      reason: "data stream failure stores REST structure changed in 8.15+"
+      test_runner_features: [ allowed_warnings, contains ]
+
+  - do:
+      ingest.put_pipeline:
+        id: "change_index_pipeline"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set": {
+                  "field": "_index",
+                  "value": "logs-elsewhere"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      allowed_warnings:
+        - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: generic_logs_template
+        body:
+          index_patterns: logs-*
+          data_stream:
+            failure_store: true
+          template:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+              index:
+                final_pipeline: "change_index_pipeline"
+
+  - do:
+      index:
+        index: logs-foobar
+        refresh: true
+        body:
+          '@timestamp': '2020-12-12'
+          foo: bar
+
+  - do:
+      indices.get_data_stream:
+        name: logs-foobar
+  - match: { data_streams.0.name: logs-foobar }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+
+  - do:
+      search:
+        index: logs-foobar
+        body: { query: { match_all: { } } }
+  - length: { hits.hits: 0 }
+
+  - do:
+      search:
+        index: .fs-logs-foobar-*
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._index: "/\\.fs-logs-foobar-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }
+  - exists: hits.hits.0._source.@timestamp
+  - not_exists: hits.hits.0._source.foo
+  - not_exists: hits.hits.0._source.document.id
+  - match: { hits.hits.0._source.document.index: 'logs-foobar' }
+  - match: { hits.hits.0._source.document.source.@timestamp: '2020-12-12' }
+  - match: { hits.hits.0._source.document.source.foo: 'bar' }
+  - match: { hits.hits.0._source.error.type: 'illegal_state_exception' }
+  - contains: { hits.hits.0._source.error.message: "final pipeline [change_index_pipeline] can't change the target index" }
+  - contains: { hits.hits.0._source.error.stack_trace: "final pipeline [change_index_pipeline] can't change the target index" }
+  - match: { hits.hits.0._source.error.pipeline_trace.0: 'change_index_pipeline' }
+  - match: { hits.hits.0._source.error.pipeline: 'change_index_pipeline' }
+
+  - do:
+      indices.delete_data_stream:
+        name: logs-foobar
+  - is_true: acknowledged
+
+  - do:
+      indices.delete:
+        index: .fs-logs-foobar-*
+  - is_true: acknowledged
+
+---
+"Failure redirects to correct failure store when index loop is detected":
+  - requires:
+      cluster_features: [ "gte_v8.15.0" ]
+      reason: "data stream failure stores REST structure changed in 8.15+"
+      test_runner_features: [ allowed_warnings, contains ]
+
+  - do:
+      ingest.put_pipeline:
+        id: "send_to_destination"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "reroute": {
+                  "tag": "reroute-tag-1",
+                  "destination": "destination-data-stream"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.put_pipeline:
+        id: "send_back_to_original"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "reroute": {
+                  "tag": "reroute-tag-2",
+                  "destination": "logs-foobar"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      allowed_warnings:
+        - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: generic_logs_template
+        body:
+          index_patterns: logs-*
+          data_stream:
+            failure_store: true
+          template:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+              index:
+                default_pipeline: "send_to_destination"
+
+  - do:
+      allowed_warnings:
+        - "index template [destination_logs_template] has index patterns [destination-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [destination_logs_template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: destination_logs_template
+        body:
+          index_patterns: destination-*
+          data_stream:
+            failure_store: true
+          template:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+              index:
+                default_pipeline: "send_back_to_original"
+
+  - do:
+      index:
+        index: logs-foobar
+        refresh: true
+        body:
+          '@timestamp': '2020-12-12'
+          foo: bar
+
+
+  - do:
+      indices.get_data_stream:
+        name: destination-data-stream
+  - match: { data_streams.0.name: destination-data-stream }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-destination-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-destination-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+
+  - do:
+      search:
+        index: destination-data-stream
+        body: { query: { match_all: { } } }
+  - length: { hits.hits: 0 }
+
+  - do:
+      search:
+        index: .fs-destination-data-stream-*
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._index: "/\\.fs-destination-data-stream-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }
+  - exists: hits.hits.0._source.@timestamp
+  - not_exists: hits.hits.0._source.foo
+  - not_exists: hits.hits.0._source.document.id
+  - match: { hits.hits.0._source.document.index: 'logs-foobar' }
+  - match: { hits.hits.0._source.document.source.@timestamp: '2020-12-12' }
+  - match: { hits.hits.0._source.document.source.foo: 'bar' }
+  - match: { hits.hits.0._source.error.type: 'illegal_state_exception' }
+  - contains: { hits.hits.0._source.error.message: 'index cycle detected' }
+  - contains: { hits.hits.0._source.error.stack_trace: 'index cycle detected' }
+  - match: { hits.hits.0._source.error.pipeline_trace.0: 'send_back_to_original' }
+  - match: { hits.hits.0._source.error.pipeline: 'send_back_to_original' }
+
+  - do:
+      indices.delete_data_stream:
+        name: destination-data-stream
+  - is_true: acknowledged
+
+  - do:
+      indices.delete:
+        index: .fs-destination-data-stream-*
+  - is_true: acknowledged
+
+---
+"Failure redirects to correct failure store when pipeline loop is detected":
+  - requires:
+      cluster_features: [ "gte_v8.15.0" ]
+      reason: "data stream failure stores REST structure changed in 8.15+"
+      test_runner_features: [ allowed_warnings, contains ]
+
+  - do:
+      ingest.put_pipeline:
+        id: "step_1"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "pipeline": {
+                  "tag": "step-1",
+                  "name": "step_2"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.put_pipeline:
+        id: "step_2"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "pipeline": {
+                  "tag": "step-2",
+                  "name": "step_1"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      allowed_warnings:
+        - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: generic_logs_template
+        body:
+          index_patterns: logs-*
+          data_stream:
+            failure_store: true
+          template:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+              index:
+                default_pipeline: "step_1"
+
+  - do:
+      index:
+        index: logs-foobar
+        refresh: true
+        body:
+          '@timestamp': '2020-12-12'
+          foo: bar
+
+  - do:
+      indices.get_data_stream:
+        name: logs-foobar
+  - match: { data_streams.0.name: logs-foobar }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { data_streams.0.failure_store.enabled: true }
+  - length: { data_streams.0.failure_store.indices: 1 }
+  - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+
+  - do:
+      search:
+        index: logs-foobar
+        body: { query: { match_all: { } } }
+  - length: { hits.hits: 0 }
+
+  - do:
+      search:
+        index: .fs-logs-foobar-*
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._index: "/\\.fs-logs-foobar-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" }
+  - exists: hits.hits.0._source.@timestamp
+  - not_exists: hits.hits.0._source.foo
+  - not_exists: hits.hits.0._source.document.id
+  - match: { hits.hits.0._source.document.index: 'logs-foobar' }
+  - match: { hits.hits.0._source.document.source.@timestamp: '2020-12-12' }
+  - match: { hits.hits.0._source.document.source.foo: 'bar' }
+  - match: { hits.hits.0._source.error.type: 'graph_structure_exception' }
+  - contains: { hits.hits.0._source.error.message: 'Cycle detected for pipeline: step_1' }
+  - contains: { hits.hits.0._source.error.stack_trace: 'Cycle detected for pipeline: step_1' }
+  - match: { hits.hits.0._source.error.pipeline_trace.0: 'step_1' }
+  - match: { hits.hits.0._source.error.pipeline_trace.1: 'step_2' }
+  - match: { hits.hits.0._source.error.pipeline: 'step_2' }
+  - match: { hits.hits.0._source.error.processor_tag: 'step-2' }
+  - match: { hits.hits.0._source.error.processor_type: 'pipeline' }
+
+  - do:
+      indices.delete_data_stream:
+        name: logs-foobar
+  - is_true: acknowledged
+
+  - do:
+      indices.delete:
+        index: .fs-logs-foobar-*
+  - is_true: acknowledged
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/40_effective_retention.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/40_effective_retention.yml
new file mode 100644
index 0000000000000..ef36f283fe237
--- /dev/null
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/40_effective_retention.yml
@@ -0,0 +1,104 @@
+setup:
+  - requires:
+      cluster_features: [ "gte_v8.11.0" ]
+      reason: "Data stream lifecycle was released as tech preview in 8.11"
+      test_runner_features: allowed_warnings
+  - do:
+      allowed_warnings:
+        - "index template [template-with-lifecycle] has index patterns [managed-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [template-with-lifecycle] will take precedence during new index creation"
+      indices.put_index_template:
+        name: template-with-lifecycle
+        body:
+          index_patterns: [ managed-data-stream ]
+          template:
+            settings:
+              index.number_of_replicas: 0
+            lifecycle:
+              data_retention: "30d"
+          data_stream: { }
+  - do:
+      indices.create_data_stream:
+        name: managed-data-stream
+---
+teardown:
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            data_streams.lifecycle.retention.max: null
+            data_streams.lifecycle.retention.default: null
+
+---
+"Retrieve effective retention via the data stream API":
+  - requires:
+      reason: "Effective retention was exposed in 8.16+"
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: GET
+          path: /_data_stream/{index}
+          capabilities: [ 'data_stream_lifecycle_effective_retention' ]
+  - do:
+      indices.get_data_stream:
+        name: "managed-data-stream"
+  - match: { data_streams.0.name: managed-data-stream }
+  - match: { data_streams.0.lifecycle.data_retention: '30d' }
+  - match: { data_streams.0.lifecycle.effective_retention: '30d'}
+  - match: { data_streams.0.lifecycle.retention_determined_by: 'data_stream_configuration'}
+
+---
+"Retrieve effective retention with explain":
+  - requires:
+      reason: "Effective retention was exposed in 8.16+"
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: GET
+          path: /{index}/_lifecycle/explain
+          capabilities: [ 'data_stream_lifecycle_effective_retention' ]
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            data_streams.lifecycle.retention.max: "7d"
+  - is_true: acknowledged
+  - do:
+      indices.get_data_stream:
+        name: "managed-data-stream"
+  - match: { data_streams.0.name: managed-data-stream }
+  - set:
+      data_streams.0.indices.0.index_name: backing_index
+
+  - do:
+      indices.explain_data_lifecycle:
+        index: managed-data-stream
+        include_defaults: true
+  - match: { indices.$backing_index.managed_by_lifecycle: true }
+  - match: { indices.$backing_index.lifecycle.data_retention: '30d' }
+  - match: { indices.$backing_index.lifecycle.effective_retention: '7d' }
+  - match: { indices.$backing_index.lifecycle.retention_determined_by: 'max_global_retention' }
+
+---
+"Retrieve effective retention with data stream lifecycle":
+  - requires:
+      reason: "Effective retention was exposed in 8.16+"
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: GET
+          path: /_data_stream/{index}/_lifecycle
+          capabilities: [ 'data_stream_lifecycle_effective_retention' ]
+  - do:
+      indices.put_data_lifecycle:
+        name: "managed-data-stream"
+        body: {}
+  - is_true: acknowledged
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            data_streams.lifecycle.retention.default: "7d"
+  - do:
+      indices.get_data_lifecycle:
+        name: "managed-data-stream"
+  - length: { data_streams: 1}
+  - match: { data_streams.0.name: managed-data-stream }
+  - match: { data_streams.0.lifecycle.effective_retention: '7d' }
+  - match: { data_streams.0.lifecycle.retention_determined_by: 'default_global_retention' }
diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java
index 0ff34cf687500..f1c592e6e8345 100644
--- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java
+++ b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java
@@ -8,14 +8,23 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.internal.Requests;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.gateway.GatewayService;
@@ -26,6 +35,7 @@
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.util.Arrays;
@@ -33,11 +43,16 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.stream.IntStream;
 
+import static org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters.Metric.INGEST;
+import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.putJsonStoredScript;
 import static org.elasticsearch.test.NodeRoles.onlyRole;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 // Ideally I like this test to live in the server module, but otherwise a large part of the ScriptProcessor
@@ -179,9 +194,9 @@ public Settings onNodeStopped(String nodeName) {
     public void testPipelineWithScriptProcessorThatHasStoredScript() throws Exception {
         internalCluster().startNode();
 
-        clusterAdmin().preparePutStoredScript().setId("1").setContent(new BytesArray(Strings.format("""
+        putJsonStoredScript("1", Strings.format("""
             {"script": {"lang": "%s", "source": "my_script"} }
-            """, MockScriptEngine.NAME)), XContentType.JSON).get();
+            """, MockScriptEngine.NAME));
         BytesReference pipeline = new BytesArray("""
             {
               "processors" : [
@@ -326,4 +341,89 @@ public boolean validateClusterForming() {
         source = client().prepareGet("index", "fails").get(timeout).getSource();
         assertNull(source);
     }
+
+    /**
+     * This test is for confirming that forwarded bulk requests do not use system_write thread pool
+     * for non-system indexes. Before this fix, we were using system_write thread pool for all forwarded
+     * bulk requests causing the system_write thread pool to get overloaded.
+     */
+    public void testForwardBulkWithSystemWritePoolDisabled() throws Exception {
+        // Create a node with master only role and a node with ingest role
+        final String masterOnlyNode = internalCluster().startMasterOnlyNode();
+        final String ingestNode = internalCluster().startNode();
+
+        ensureStableCluster(2);
+
+        // Create Bulk Request
+        createIndex("index");
+
+        BytesReference source = new BytesArray("""
+            {
+              "processors" : [
+                  {"set" : {"field": "y", "value": 0}}
+              ]
+            }""");
+
+        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON);
+        clusterAdmin().putPipeline(putPipelineRequest).get();
+
+        int numRequests = scaledRandomIntBetween(32, 128);
+        BulkRequest bulkRequest = new BulkRequest();
+        BulkResponse response;
+        for (int i = 0; i < numRequests; i++) {
+            IndexRequest indexRequest = new IndexRequest("index").id(Integer.toString(i)).setPipeline("_id");
+            indexRequest.source(Requests.INDEX_CONTENT_TYPE, "x", 1);
+            bulkRequest.add(indexRequest);
+        }
+        assertThat(numRequests, equalTo(bulkRequest.requests().size()));
+
+        // Block system_write thread pool on the ingest node
+        final ThreadPool ingestNodeThreadPool = internalCluster().getInstance(ThreadPool.class, ingestNode);
+        final var blockingLatch = new CountDownLatch(1);
+        try {
+            blockSystemWriteThreadPool(blockingLatch, ingestNodeThreadPool);
+            // Send bulk request to master only node, so it will forward it to the ingest node.
+            response = safeGet(client(masterOnlyNode).bulk(bulkRequest));
+        } finally {
+            blockingLatch.countDown();
+        }
+
+        // Make sure the requests are processed (even though we blocked system_write thread pool above).
+        assertThat(response.getItems().length, equalTo(numRequests));
+        assertFalse(response.hasFailures());
+
+        // Check Node Ingest stats
+        NodesStatsResponse nodesStatsResponse = clusterAdmin().nodesStats(new NodesStatsRequest(ingestNode).addMetric(INGEST)).actionGet();
+        assertThat(nodesStatsResponse.getNodes().size(), equalTo(1));
+
+        NodeStats stats = nodesStatsResponse.getNodes().get(0);
+        assertThat(stats.getIngestStats().totalStats().ingestCount(), equalTo((long) numRequests));
+        assertThat(stats.getIngestStats().totalStats().ingestFailedCount(), equalTo(0L));
+        final var pipelineStats = stats.getIngestStats().pipelineStats().get(0);
+        assertThat(pipelineStats.pipelineId(), equalTo("_id"));
+        assertThat(pipelineStats.stats().ingestCount(), equalTo((long) numRequests));
+
+        MultiGetResponse docListResponse = safeGet(
+            client().prepareMultiGet().addIds("index", IntStream.range(0, numRequests).mapToObj(String::valueOf).toList()).execute()
+        );
+
+        assertThat(docListResponse.getResponses().length, equalTo(numRequests));
+        Map<String, Object> document;
+        for (int i = 0; i < numRequests; i++) {
+            document = docListResponse.getResponses()[i].getResponse().getSourceAsMap();
+            assertThat(document.get("y"), equalTo(0));
+        }
+    }
+
+    private void blockSystemWriteThreadPool(CountDownLatch blockingLatch, ThreadPool threadPool) {
+        assertThat(blockingLatch.getCount(), greaterThan(0L));
+        final var executor = threadPool.executor(ThreadPool.Names.SYSTEM_WRITE);
+        // Add tasks repeatedly until we get an EsRejectedExecutionException which indicates that the threadpool and its queue are full.
+        expectThrows(EsRejectedExecutionException.class, () -> {
+            // noinspection InfiniteLoopStatement
+            while (true) {
+                executor.execute(() -> safeAwait(blockingLatch));
+            }
+        });
+    }
 }
diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java
deleted file mode 100644
index 16a8013ae9c4a..0000000000000
--- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.plugins.internal;
-
-import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.ingest.PutPipelineRequest;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.ingest.common.IngestCommonPlugin;
-import org.elasticsearch.plugins.IngestPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.xcontent.FilterXContentParserWrapper;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentType;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
-import static org.hamcrest.Matchers.equalTo;
-
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
-public class DocumentSizeObserverWithPipelinesIT extends ESIntegTestCase {
-
-    private static String TEST_INDEX_NAME = "test-index-name";
-    // the assertions are done in plugin which is static and will be created by ES server.
-    // hence a static flag to make sure it is indeed used
-    public static volatile boolean hasWrappedParser;
-    public static AtomicLong providedFixedSize = new AtomicLong();
-
-    public void testDocumentIsReportedWithPipelines() throws Exception {
-        hasWrappedParser = false;
-        // pipeline adding fields, changing destination is not affecting reporting
-        final BytesReference pipelineBody = new BytesArray("""
-            {
-              "processors": [
-                {
-                   "set": {
-                     "field": "my-text-field",
-                     "value": "xxxx"
-                   }
-                 },
-                 {
-                   "set": {
-                     "field": "my-boolean-field",
-                     "value": true
-                   }
-                 }
-              ]
-            }
-            """);
-        clusterAdmin().putPipeline(new PutPipelineRequest("pipeline", pipelineBody, XContentType.JSON)).actionGet();
-
-        client().index(
-            new IndexRequest(TEST_INDEX_NAME).setPipeline("pipeline")
-                .id("1")
-                .source(jsonBuilder().startObject().field("test", "I am sam i am").endObject())
-        ).actionGet();
-        assertBusy(() -> {
-            // ingest node has used an observer that was counting #map operations
-            // and passed that info to newFixedSize observer in TransportShardBulkAction
-            assertTrue(hasWrappedParser);
-            assertThat(providedFixedSize.get(), equalTo(1L));
-        });
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return List.of(TestDocumentParsingProviderPlugin.class, IngestCommonPlugin.class);
-    }
-
-    public static class TestDocumentParsingProviderPlugin extends Plugin implements DocumentParsingProviderPlugin, IngestPlugin {
-
-        public TestDocumentParsingProviderPlugin() {}
-
-        @Override
-        public DocumentParsingProvider getDocumentParsingProvider() {
-            // returns a static instance, because we want to assert that the wrapping is called only once
-            return new DocumentParsingProvider() {
-                @Override
-                public <T> DocumentSizeObserver newDocumentSizeObserver(DocWriteRequest<T> request) {
-                    if (request instanceof IndexRequest indexRequest && indexRequest.getNormalisedBytesParsed() > 0) {
-                        long normalisedBytesParsed = indexRequest.getNormalisedBytesParsed();
-                        providedFixedSize.set(normalisedBytesParsed);
-                        return new TestDocumentSizeObserver(normalisedBytesParsed);
-                    }
-                    return new TestDocumentSizeObserver(0L);
-                }
-
-                @Override
-                public DocumentSizeReporter newDocumentSizeReporter(
-                    String indexName,
-                    MapperService mapperService,
-                    DocumentSizeAccumulator documentSizeAccumulator
-                ) {
-                    return DocumentSizeReporter.EMPTY_INSTANCE;
-                }
-            };
-        }
-    }
-
-    public static class TestDocumentSizeObserver implements DocumentSizeObserver {
-        long mapCounter = 0;
-        long wrapperCounter = 0;
-
-        public TestDocumentSizeObserver(long mapCounter) {
-            this.mapCounter = mapCounter;
-        }
-
-        @Override
-        public XContentParser wrapParser(XContentParser xContentParser) {
-            wrapperCounter++;
-            hasWrappedParser = true;
-            return new FilterXContentParserWrapper(xContentParser) {
-
-                @Override
-                public Map<String, Object> map() throws IOException {
-                    mapCounter++;
-                    return super.map();
-                }
-            };
-        }
-
-        @Override
-        public long normalisedBytesParsed() {
-            return mapCounter;
-        }
-
-    }
-
-}
diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorWithPipelinesIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorWithPipelinesIT.java
new file mode 100644
index 0000000000000..7f0910ea5cc4d
--- /dev/null
+++ b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorWithPipelinesIT.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.internal;
+
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.ingest.common.IngestCommonPlugin;
+import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.FilterXContentParserWrapper;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class XContentMeteringParserDecoratorWithPipelinesIT extends ESIntegTestCase {
+
+    private static String TEST_INDEX_NAME = "test-index-name";
+    // the assertions are done in plugin which is static and will be created by ES server.
+    // hence a static flag to make sure it is indeed used
+    public static volatile boolean hasWrappedParser;
+    public static AtomicLong providedFixedSize = new AtomicLong();
+
+    public void testDocumentIsReportedWithPipelines() throws Exception {
+        hasWrappedParser = false;
+        // pipeline adding fields, changing destination is not affecting reporting
+        final BytesReference pipelineBody = new BytesArray("""
+            {
+              "processors": [
+                {
+                   "set": {
+                     "field": "my-text-field",
+                     "value": "xxxx"
+                   }
+                 },
+                 {
+                   "set": {
+                     "field": "my-boolean-field",
+                     "value": true
+                   }
+                 }
+              ]
+            }
+            """);
+        clusterAdmin().putPipeline(new PutPipelineRequest("pipeline", pipelineBody, XContentType.JSON)).actionGet();
+
+        client().index(
+            new IndexRequest(TEST_INDEX_NAME).setPipeline("pipeline")
+                .id("1")
+                .source(jsonBuilder().startObject().field("test", "I am sam i am").endObject())
+        ).actionGet();
+        assertBusy(() -> {
+            // ingest node has used an observer that was counting #map operations
+            // and passed that info to newFixedSize observer in TransportShardBulkAction
+            assertTrue(hasWrappedParser);
+            assertThat(providedFixedSize.get(), equalTo(1L));
+        });
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(TestDocumentParsingProviderPlugin.class, IngestCommonPlugin.class);
+    }
+
+    public static class TestDocumentParsingProviderPlugin extends Plugin implements DocumentParsingProviderPlugin, IngestPlugin {
+
+        public TestDocumentParsingProviderPlugin() {}
+
+        @Override
+        public DocumentParsingProvider getDocumentParsingProvider() {
+            // returns a static instance, because we want to assert that the wrapping is called only once
+            return new DocumentParsingProvider() {
+                @Override
+                public <T> XContentMeteringParserDecorator newMeteringParserDecorator(DocWriteRequest<T> request) {
+                    if (request instanceof IndexRequest indexRequest && indexRequest.getNormalisedBytesParsed() > 0) {
+                        long normalisedBytesParsed = indexRequest.getNormalisedBytesParsed();
+                        providedFixedSize.set(normalisedBytesParsed);
+                        return new TestXContentMeteringParserDecorator(normalisedBytesParsed);
+                    }
+                    return new TestXContentMeteringParserDecorator(0L);
+                }
+
+                @Override
+                public DocumentSizeReporter newDocumentSizeReporter(
+                    String indexName,
+                    MapperService mapperService,
+                    DocumentSizeAccumulator documentSizeAccumulator
+                ) {
+                    return DocumentSizeReporter.EMPTY_INSTANCE;
+                }
+            };
+        }
+    }
+
+    public static class TestXContentMeteringParserDecorator implements XContentMeteringParserDecorator {
+        long mapCounter = 0;
+
+        public TestXContentMeteringParserDecorator(long mapCounter) {
+            this.mapCounter = mapCounter;
+        }
+
+        @Override
+        public XContentParser decorate(XContentParser xContentParser) {
+            hasWrappedParser = true;
+            return new FilterXContentParserWrapper(xContentParser) {
+
+                @Override
+                public Map<String, Object> map() throws IOException {
+                    mapCounter++;
+                    return super.map();
+                }
+            };
+        }
+
+        @Override
+        public ParsedDocument.DocumentSize meteredDocumentSize() {
+            return new ParsedDocument.DocumentSize(mapCounter, 0);
+        }
+    }
+
+}
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java
index 27ef5a10dd5c2..0377da53846d5 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java
@@ -225,7 +225,7 @@ private static Flow buildFlow(
         }
         flow.protocol = Transport.fromObject(protocol);
 
-        switch (flow.protocol) {
+        switch (flow.protocol.getType()) {
             case Tcp, Udp, Sctp -> {
                 flow.sourcePort = parseIntFromObjectOrString(sourcePort.get(), "source port");
                 if (flow.sourcePort < 1 || flow.sourcePort > 65535) {
@@ -336,12 +336,12 @@ public CommunityIdProcessor create(
      */
     public static final class Flow {
 
-        private static final List<Transport> TRANSPORTS_WITH_PORTS = List.of(
-            Transport.Tcp,
-            Transport.Udp,
-            Transport.Sctp,
-            Transport.Icmp,
-            Transport.IcmpIpV6
+        private static final List<Transport.Type> TRANSPORTS_WITH_PORTS = List.of(
+            Transport.Type.Tcp,
+            Transport.Type.Udp,
+            Transport.Type.Sctp,
+            Transport.Type.Icmp,
+            Transport.Type.IcmpIpV6
         );
 
         InetAddress source;
@@ -362,20 +362,21 @@ boolean isOrdered() {
         }
 
         byte[] toBytes() {
-            boolean hasPort = TRANSPORTS_WITH_PORTS.contains(protocol);
+            Transport.Type protoType = protocol.getType();
+            boolean hasPort = TRANSPORTS_WITH_PORTS.contains(protoType);
             int len = source.getAddress().length + destination.getAddress().length + 2 + (hasPort ? 4 : 0);
             ByteBuffer bb = ByteBuffer.allocate(len);
 
             boolean isOneWay = false;
-            if (protocol == Transport.Icmp || protocol == Transport.IcmpIpV6) {
+            if (protoType == Transport.Type.Icmp || protoType == Transport.Type.IcmpIpV6) {
                 // ICMP protocols populate port fields with ICMP data
-                Integer equivalent = IcmpType.codeEquivalent(icmpType, protocol == Transport.IcmpIpV6);
+                Integer equivalent = IcmpType.codeEquivalent(icmpType, protoType == Transport.Type.IcmpIpV6);
                 isOneWay = equivalent == null;
                 sourcePort = icmpType;
                 destinationPort = equivalent == null ? icmpCode : equivalent;
             }
 
-            boolean keepOrder = isOrdered() || ((protocol == Transport.Icmp || protocol == Transport.IcmpIpV6) && isOneWay);
+            boolean keepOrder = isOrdered() || ((protoType == Transport.Type.Icmp || protoType == Transport.Type.IcmpIpV6) && isOneWay);
             bb.put(keepOrder ? source.getAddress() : destination.getAddress());
             bb.put(keepOrder ? destination.getAddress() : source.getAddress());
             bb.put(toUint16(protocol.getTransportNumber() << 8));
@@ -397,39 +398,63 @@ String toCommunityId(byte[] seed) {
         }
     }
 
-    public enum Transport {
-        Icmp(1),
-        Igmp(2),
-        Tcp(6),
-        Udp(17),
-        Gre(47),
-        IcmpIpV6(58),
-        Eigrp(88),
-        Ospf(89),
-        Pim(103),
-        Sctp(132);
-
-        private final int transportNumber;
+    static class Transport {
+        public enum Type {
+            Unknown(-1),
+            Icmp(1),
+            Igmp(2),
+            Tcp(6),
+            Udp(17),
+            Gre(47),
+            IcmpIpV6(58),
+            Eigrp(88),
+            Ospf(89),
+            Pim(103),
+            Sctp(132);
+
+            private final int transportNumber;
+
+            private static final Map<String, Type> TRANSPORT_NAMES;
+
+            static {
+                TRANSPORT_NAMES = new HashMap<>();
+                TRANSPORT_NAMES.put("icmp", Icmp);
+                TRANSPORT_NAMES.put("igmp", Igmp);
+                TRANSPORT_NAMES.put("tcp", Tcp);
+                TRANSPORT_NAMES.put("udp", Udp);
+                TRANSPORT_NAMES.put("gre", Gre);
+                TRANSPORT_NAMES.put("ipv6-icmp", IcmpIpV6);
+                TRANSPORT_NAMES.put("icmpv6", IcmpIpV6);
+                TRANSPORT_NAMES.put("eigrp", Eigrp);
+                TRANSPORT_NAMES.put("ospf", Ospf);
+                TRANSPORT_NAMES.put("pim", Pim);
+                TRANSPORT_NAMES.put("sctp", Sctp);
+            }
 
-        private static final Map<String, Transport> TRANSPORT_NAMES;
+            Type(int transportNumber) {
+                this.transportNumber = transportNumber;
+            }
 
-        static {
-            TRANSPORT_NAMES = new HashMap<>();
-            TRANSPORT_NAMES.put("icmp", Icmp);
-            TRANSPORT_NAMES.put("igmp", Igmp);
-            TRANSPORT_NAMES.put("tcp", Tcp);
-            TRANSPORT_NAMES.put("udp", Udp);
-            TRANSPORT_NAMES.put("gre", Gre);
-            TRANSPORT_NAMES.put("ipv6-icmp", IcmpIpV6);
-            TRANSPORT_NAMES.put("icmpv6", IcmpIpV6);
-            TRANSPORT_NAMES.put("eigrp", Eigrp);
-            TRANSPORT_NAMES.put("ospf", Ospf);
-            TRANSPORT_NAMES.put("pim", Pim);
-            TRANSPORT_NAMES.put("sctp", Sctp);
+            public int getTransportNumber() {
+                return transportNumber;
+            }
         }
 
-        Transport(int transportNumber) {
+        private Type type;
+        private int transportNumber;
+
+        Transport(int transportNumber, Type type) { // Change constructor to public
             this.transportNumber = transportNumber;
+            this.type = type;
+        }
+
+        Transport(Type type) { // Change constructor to public
+            this.transportNumber = type.getTransportNumber();
+            this.type = type;
+        }
+
+        public Type getType() {
+            return this.type;
         }
 
         public int getTransportNumber() {
@@ -437,19 +462,26 @@ public int getTransportNumber() {
         }
 
         public static Transport fromNumber(int transportNumber) {
-            return switch (transportNumber) {
-                case 1 -> Icmp;
-                case 2 -> Igmp;
-                case 6 -> Tcp;
-                case 17 -> Udp;
-                case 47 -> Gre;
-                case 58 -> IcmpIpV6;
-                case 88 -> Eigrp;
-                case 89 -> Ospf;
-                case 103 -> Pim;
-                case 132 -> Sctp;
-                default -> throw new IllegalArgumentException("unknown transport protocol number [" + transportNumber + "]");
+            if (transportNumber < 0 || transportNumber >= 255) {
+                // transport numbers range https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml
+                throw new IllegalArgumentException("invalid transport protocol number [" + transportNumber + "]");
+            }
+
+            Type type = switch (transportNumber) {
+                case 1 -> Type.Icmp;
+                case 2 -> Type.Igmp;
+                case 6 -> Type.Tcp;
+                case 17 -> Type.Udp;
+                case 47 -> Type.Gre;
+                case 58 -> Type.IcmpIpV6;
+                case 88 -> Type.Eigrp;
+                case 89 -> Type.Ospf;
+                case 103 -> Type.Pim;
+                case 132 -> Type.Sctp;
+                default -> Type.Unknown;
             };
+
+            return new Transport(transportNumber, type);
         }
 
         public static Transport fromObject(Object o) {
@@ -457,8 +489,8 @@ public static Transport fromObject(Object o) {
                 return fromNumber(number.intValue());
             } else if (o instanceof String protocolStr) {
                 // check if matches protocol name
-                if (TRANSPORT_NAMES.containsKey(protocolStr.toLowerCase(Locale.ROOT))) {
-                    return TRANSPORT_NAMES.get(protocolStr.toLowerCase(Locale.ROOT));
+                if (Type.TRANSPORT_NAMES.containsKey(protocolStr.toLowerCase(Locale.ROOT))) {
+                    return new Transport(Type.TRANSPORT_NAMES.get(protocolStr.toLowerCase(Locale.ROOT)));
                 }
 
                 // check if convertible to protocol number
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
index 23bbaf62cec07..ae3de1f1c446f 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
@@ -30,6 +30,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ConcurrentMap;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.Supplier;
 
@@ -45,7 +46,7 @@ public final class DateProcessor extends AbstractProcessor {
     private final String field;
     private final String targetField;
     private final List<String> formats;
-    private final List<Function<Map<String, Object>, Function<String, ZonedDateTime>>> dateParsers;
+    private final List<BiFunction<ZoneId, Locale, Function<String, ZonedDateTime>>> dateParsers;
     private final String outputFormat;
 
     DateProcessor(
@@ -80,14 +81,12 @@ public final class DateProcessor extends AbstractProcessor {
 
         for (String format : formats) {
             DateFormat dateFormat = DateFormat.fromString(format);
-            dateParsers.add((params) -> {
-                var documentTimezone = timezone == null ? null : timezone.newInstance(params).execute();
-                var documentLocale = locale == null ? null : locale.newInstance(params).execute();
-                return Cache.INSTANCE.getOrCompute(
+            dateParsers.add(
+                (documentTimezone, documentLocale) -> Cache.INSTANCE.getOrCompute(
                     new Cache.Key(format, documentTimezone, documentLocale),
-                    () -> dateFormat.getFunction(format, newDateTimeZone(documentTimezone), newLocale(documentLocale))
-                );
-            });
+                    () -> dateFormat.getFunction(format, documentTimezone, documentLocale)
+                )
+            );
         }
         this.outputFormat = outputFormat;
         formatter = DateFormatter.forPattern(this.outputFormat);
@@ -106,15 +105,27 @@ public IngestDocument execute(IngestDocument ingestDocument) {
         Object obj = ingestDocument.getFieldValue(field, Object.class);
         String value = null;
         if (obj != null) {
-            // Not use Objects.toString(...) here, because null gets changed to "null" which may confuse some date parsers
+            // Don't use Objects.toString(...) here, because null gets changed to "null" which may confuse some date parsers
             value = obj.toString();
         }
 
+        // run (potential) mustache application just a single time for this document in order to
+        // extract the timezone and locale to use for date parsing
+        final ZoneId documentTimezone;
+        final Locale documentLocale;
+        final Map<String, Object> sourceAndMetadata = ingestDocument.getSourceAndMetadata();
+        try {
+            documentTimezone = newDateTimeZone(timezone == null ? null : timezone.newInstance(sourceAndMetadata).execute());
+            documentLocale = newLocale(locale == null ? null : locale.newInstance(sourceAndMetadata).execute());
+        } catch (Exception e) {
+            throw new IllegalArgumentException("unable to parse date [" + value + "]", e);
+        }
+
         ZonedDateTime dateTime = null;
         Exception lastException = null;
-        for (Function<Map<String, Object>, Function<String, ZonedDateTime>> dateParser : dateParsers) {
+        for (BiFunction<ZoneId, Locale, Function<String, ZonedDateTime>> dateParser : dateParsers) {
             try {
-                dateTime = dateParser.apply(ingestDocument.getSourceAndMetadata()).apply(value);
+                dateTime = dateParser.apply(documentTimezone, documentLocale).apply(value);
                 break;
             } catch (Exception e) {
                 // try the next parser and keep track of the exceptions
@@ -255,6 +266,6 @@ Function<String, ZonedDateTime> getOrCompute(Key key, Supplier<Function<String,
             return fn;
         }
 
-        record Key(String format, String zoneId, String locale) {}
+        record Key(String format, ZoneId zoneId, Locale locale) {}
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
index 8add2b0d05535..1e53ea5cda682 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
@@ -16,12 +16,12 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.grok.GrokBuiltinPatterns;
 import org.elasticsearch.grok.PatternBank;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java
index ca9b3f3d81bd9..3848f4531adcb 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java
@@ -166,14 +166,30 @@ public void testBeatsProtocolNumber() throws Exception {
         testCommunityIdProcessor(event, "1:D3t8Q1aFA6Ev0A/AO4i9PnU3AeI=");
     }
 
-    public void testBeatsIanaNumber() throws Exception {
+    public void testBeatsIanaNumberProtocolTCP() throws Exception {
         @SuppressWarnings("unchecked")
         var network = (Map<String, Object>) event.get("network");
         network.remove("transport");
-        network.put("iana_number", CommunityIdProcessor.Transport.Tcp.getTransportNumber());
+        network.put("iana_number", CommunityIdProcessor.Transport.Type.Tcp.getTransportNumber());
         testCommunityIdProcessor(event, "1:LQU9qZlK+B5F3KDmev6m5PMibrg=");
     }
 
+    public void testBeatsIanaNumberProtocolIPv4() throws Exception {
+        @SuppressWarnings("unchecked")
+        var network = (Map<String, Object>) event.get("network");
+        network.put("iana_number", "4");
+        network.remove("transport");
+        @SuppressWarnings("unchecked")
+        var source = (Map<String, Object>) event.get("source");
+        source.put("ip", "192.168.1.2");
+        source.remove("port");
+        @SuppressWarnings("unchecked")
+        var destination = (Map<String, Object>) event.get("destination");
+        destination.put("ip", "10.1.2.3");
+        destination.remove("port");
+        testCommunityIdProcessor(event, "1:KXQzmk3bdsvD6UXj7dvQ4bM6Zvw=");
+    }
+
     public void testIpv6() throws Exception {
         @SuppressWarnings("unchecked")
         var source = (Map<String, Object>) event.get("source");
@@ -201,10 +217,10 @@ public void testStringAndNumber() throws Exception {
         @SuppressWarnings("unchecked")
         var network = (Map<String, Object>) event.get("network");
         network.remove("transport");
-        network.put("iana_number", CommunityIdProcessor.Transport.Tcp.getTransportNumber());
+        network.put("iana_number", CommunityIdProcessor.Transport.Type.Tcp.getTransportNumber());
         testCommunityIdProcessor(event, "1:LQU9qZlK+B5F3KDmev6m5PMibrg=");
 
-        network.put("iana_number", Integer.toString(CommunityIdProcessor.Transport.Tcp.getTransportNumber()));
+        network.put("iana_number", Integer.toString(CommunityIdProcessor.Transport.Type.Tcp.getTransportNumber()));
         testCommunityIdProcessor(event, "1:LQU9qZlK+B5F3KDmev6m5PMibrg=");
 
         // protocol number
@@ -359,8 +375,13 @@ private void testCommunityIdProcessor(Map<String, Object> source, int seed, Stri
     }
 
     public void testTransportEnum() {
-        for (CommunityIdProcessor.Transport t : CommunityIdProcessor.Transport.values()) {
-            assertThat(CommunityIdProcessor.Transport.fromNumber(t.getTransportNumber()), equalTo(t));
+        for (CommunityIdProcessor.Transport.Type t : CommunityIdProcessor.Transport.Type.values()) {
+            if (t == CommunityIdProcessor.Transport.Type.Unknown) {
+                expectThrows(IllegalArgumentException.class, () -> CommunityIdProcessor.Transport.fromNumber(t.getTransportNumber()));
+                continue;
+            }
+
+            assertThat(CommunityIdProcessor.Transport.fromNumber(t.getTransportNumber()).getType(), equalTo(t));
         }
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
index e5773c6c553df..532bf575bda74 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
@@ -29,6 +29,8 @@
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.atMost;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -372,8 +374,8 @@ public void testCacheIsEvictedAfterReachMaxCapacity() {
         Function<String, ZonedDateTime> zonedDateTimeFunction1 = str -> ZonedDateTime.now();
         Function<String, ZonedDateTime> zonedDateTimeFunction2 = str -> ZonedDateTime.now();
         var cache = new DateProcessor.Cache(1);
-        var key1 = new DateProcessor.Cache.Key("format-1", ZoneId.systemDefault().toString(), Locale.ROOT.toString());
-        var key2 = new DateProcessor.Cache.Key("format-2", ZoneId.systemDefault().toString(), Locale.ROOT.toString());
+        var key1 = new DateProcessor.Cache.Key("format-1", ZoneId.systemDefault(), Locale.ROOT);
+        var key2 = new DateProcessor.Cache.Key("format-2", ZoneId.systemDefault(), Locale.ROOT);
 
         when(supplier1.get()).thenReturn(zonedDateTimeFunction1);
         when(supplier2.get()).thenReturn(zonedDateTimeFunction2);
@@ -391,4 +393,36 @@ public void testCacheIsEvictedAfterReachMaxCapacity() {
         verify(supplier1, times(3)).get();
         verify(supplier2, times(2)).get();
     }
+
+    public void testMustacheTemplateExecutesAtMostTwiceWithMultipleFormats() {
+        final TemplateScript.Factory factory = mock(TemplateScript.Factory.class);
+        final TemplateScript compiledScript = mock(TemplateScript.class);
+        when(factory.newInstance(any())).thenReturn(compiledScript);
+        when(compiledScript.execute()).thenReturn(null);
+
+        final List<String> matchFormats = List.of(
+            "dd/MM/yyyy",
+            "dd-MM-yyyy",
+            "uuuu-dd-MM",
+            "uuuu-MM-dd",
+            "TAI64N",
+            "epoch_millis",
+            "yyyy dd MM"
+        );
+        DateProcessor dateProcessor = new DateProcessor(
+            randomAlphaOfLength(10),
+            null,
+            factory,
+            factory,
+            "date_as_string",
+            matchFormats,
+            "date_as_date"
+        );
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("date_as_string", "2010 12 06");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        dateProcessor.execute(ingestDocument);
+        verify(compiledScript, atMost(2)).execute();
+    }
 }
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
index 5bdb6da5c7b29..bc5bb165cd0d2 100644
--- a/modules/ingest-geoip/build.gradle
+++ b/modules/ingest-geoip/build.gradle
@@ -88,3 +88,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task ->
   task.skipTestsByFilePattern("**/ingest_geoip/20_geoip_processor.yml", "from 8.0 yaml rest tests use geoip test fixture and default geoip are no longer packaged. In 7.x yaml tests used default databases which makes tests results very different, so skipping these tests")
   // task.skipTest("lang_mustache/50_multi_search_template/Multi-search template with errors", "xxx")
 }
+
+artifacts {
+  restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
+}
diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java
index ae811db226b06..92ec911dbf451 100644
--- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java
+++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java
@@ -16,17 +16,14 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.StreamsUtils;
 import org.junit.ClassRule;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collection;
 import java.util.List;
 
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
+
 public abstract class AbstractGeoIpIT extends ESIntegTestCase {
     private static final boolean useFixture = Booleans.parseBoolean(System.getProperty("geoip_use_service", "false")) == false;
 
@@ -45,23 +42,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     @Override
     protected Settings nodeSettings(final int nodeOrdinal, final Settings otherSettings) {
         final Path databasePath = createTempDir();
-        try {
-            Files.createDirectories(databasePath);
-            Files.copy(
-                new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb")),
-                databasePath.resolve("GeoLite2-City.mmdb")
-            );
-            Files.copy(
-                new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb")),
-                databasePath.resolve("GeoLite2-Country.mmdb")
-            );
-            Files.copy(
-                new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-ASN.mmdb")),
-                databasePath.resolve("GeoLite2-ASN.mmdb")
-            );
-        } catch (final IOException e) {
-            throw new UncheckedIOException(e);
-        }
+        copyDefaultDatabases(databasePath);
         return Settings.builder()
             .put("ingest.geoip.database_path", databasePath)
             .put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), false)
diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java
index 2d068373717d8..cc757c413713d 100644
--- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java
+++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.ingest.EnterpriseGeoIpTask;
 import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration;
@@ -54,13 +53,12 @@
 public class EnterpriseGeoIpDownloaderIT extends ESIntegTestCase {
 
     private static final String DATABASE_TYPE = "GeoIP2-City";
-    private static final boolean useFixture = Booleans.parseBoolean(System.getProperty("geoip_use_service", "false")) == false;
 
     @ClassRule
-    public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture(useFixture, DATABASE_TYPE);
+    public static final EnterpriseGeoIpHttpFixture fixture = new EnterpriseGeoIpHttpFixture(DATABASE_TYPE);
 
     protected String getEndpoint() {
-        return useFixture ? fixture.getAddress() : null;
+        return fixture.getAddress();
     }
 
     @Override
@@ -71,11 +69,9 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         builder.setSecureSettings(secureSettings)
             .put(super.nodeSettings(nodeOrdinal, otherSettings))
             .put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), true);
-        if (getEndpoint() != null) {
-            // note: this is using the enterprise fixture for the regular downloader, too, as
-            // a slightly hacky way of making the regular downloader not actually download any files
-            builder.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), getEndpoint());
-        }
+        // note: this is using the enterprise fixture for the regular downloader, too, as
+        // a slightly hacky way of making the regular downloader not actually download any files
+        builder.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), getEndpoint());
         return builder.build();
     }
 
@@ -94,9 +90,7 @@ public void testEnterpriseDownloaderTask() throws Exception {
          * was updated with information from the database.
          * Note that the "enterprise database" is actually just a geolite database being loaded by the GeoIpHttpFixture.
          */
-        if (getEndpoint() != null) {
-            EnterpriseGeoIpDownloader.DEFAULT_MAXMIND_ENDPOINT = getEndpoint();
-        }
+        EnterpriseGeoIpDownloader.DEFAULT_MAXMIND_ENDPOINT = getEndpoint();
         final String pipelineName = "enterprise_geoip_pipeline";
         final String indexName = "enterprise_geoip_test_index";
         final String sourceField = "ip";
diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
index f7ab384c69bf1..d994bd70eb7a0 100644
--- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
+++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
@@ -66,6 +66,7 @@
 import java.util.zip.GZIPInputStream;
 
 import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.hamcrest.Matchers.anEmptyMap;
@@ -688,12 +689,7 @@ private void setupDatabasesInConfigDirectory() throws Exception {
             .forEach(path -> {
                 try {
                     Files.createDirectories(path);
-                    Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-City.mmdb"), path.resolve("GeoLite2-City.mmdb"));
-                    Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), path.resolve("GeoLite2-ASN.mmdb"));
-                    Files.copy(
-                        GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-Country.mmdb"),
-                        path.resolve("GeoLite2-Country.mmdb")
-                    );
+                    copyDefaultDatabases(path);
                 } catch (IOException e) {
                     throw new UncheckedIOException(e);
                 }
diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java
index 8d8b0b4215b3f..87daefab7b428 100644
--- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java
+++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java
@@ -22,10 +22,8 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -34,7 +32,8 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
@@ -68,8 +67,8 @@ public void test() throws Exception {
         when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE);
         DatabaseNodeService databaseNodeService = createRegistry(geoIpConfigDir, geoIpTmpDir, clusterService);
         GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
+        copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
+        copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
         databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
         databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
         lazyLoadReaders(databaseNodeService);
@@ -138,18 +137,14 @@ public void test() throws Exception {
                             assertThat(previous1.current(), equalTo(-1));
                         });
                     } else {
-                        Files.copy(
-                            ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"),
-                            geoIpTmpDir.resolve("GeoLite2-City.mmdb"),
-                            StandardCopyOption.REPLACE_EXISTING
-                        );
+                        copyDatabase("GeoLite2-City-Test.mmdb", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
                         databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
                     }
                     DatabaseReaderLazyLoader previous2 = databaseNodeService.get("GeoLite2-City-Test.mmdb");
-                    InputStream source = ConfigDatabases.class.getResourceAsStream(
-                        i % 2 == 0 ? "/GeoIP2-City-Test.mmdb" : "/GeoLite2-City-Test.mmdb"
+                    copyDatabase(
+                        i % 2 == 0 ? "GeoIP2-City-Test.mmdb" : "GeoLite2-City-Test.mmdb",
+                        geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")
                     );
-                    Files.copy(source, geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"), StandardCopyOption.REPLACE_EXISTING);
                     databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
 
                     DatabaseReaderLazyLoader current1 = databaseNodeService.get("GeoLite2-City.mmdb");
@@ -194,7 +189,7 @@ private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoI
         throws IOException {
         GeoIpCache cache = new GeoIpCache(0);
         ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache);
-        copyDatabaseFiles(geoIpConfigDir, configDatabases);
+        copyDefaultDatabases(geoIpConfigDir, configDatabases);
         DatabaseNodeService databaseNodeService = new DatabaseNodeService(
             geoIpTmpDir,
             mock(Client.class),
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index e39705a71f56c..82b9e930280b7 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -592,7 +592,7 @@ private Map<String, Object> retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabas
                 }
                 case ISP_ORGANIZATION_NAME -> {
                     if (ispOrganization != null) {
-                        geoData.put("isp_organization", ispOrganization);
+                        geoData.put("isp_organization_name", ispOrganization);
                     }
                 }
                 case MOBILE_COUNTRY_CODE -> {
@@ -660,7 +660,7 @@ private Map<String, Object> retrieveIspGeoData(GeoIpDatabase geoIpDatabase, Inet
                 }
                 case ISP_ORGANIZATION_NAME -> {
                     if (ispOrganization != null) {
-                        geoData.put("isp_organization", ispOrganization);
+                        geoData.put("isp_organization_name", ispOrganization);
                     }
                 }
                 case MOBILE_COUNTRY_CODE -> {
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java
index 43aacee956279..4f9b9062332e4 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java
@@ -25,12 +25,12 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata;
 import org.elasticsearch.ingest.geoip.direct.DeleteDatabaseConfigurationAction.Request;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java
index a14a143e3f404..ae090dc4c64f6 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java
@@ -17,10 +17,10 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java
index 540be68671d38..490a9edbec89a 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java
@@ -24,13 +24,13 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata;
 import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction.Request;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java
index c7a1337f20e59..81836cda29568 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java
@@ -166,7 +166,7 @@ public static class NodeResponse extends BaseNodeResponse {
         protected NodeResponse(StreamInput in) throws IOException {
             super(in);
             downloaderStats = in.readBoolean() ? new GeoIpDownloaderStats(in) : null;
-            if (in.getTransportVersion().onOrAfter(TransportVersions.GEOIP_CACHE_STATS)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 cacheStats = in.readBoolean() ? new CacheStats(in) : null;
             } else {
                 cacheStats = null;
@@ -217,7 +217,7 @@ public void writeTo(StreamOutput out) throws IOException {
             if (downloaderStats != null) {
                 downloaderStats.writeTo(out);
             }
-            if (out.getTransportVersion().onOrAfter(TransportVersions.GEOIP_CACHE_STATS)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 out.writeBoolean(cacheStats != null);
                 if (cacheStats != null) {
                     cacheStats.writeTo(out);
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java
index 6a3aa81f82e9e..7a4b1fec900e9 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.ingest.geoip.DatabaseNodeService;
 import org.elasticsearch.ingest.geoip.GeoIpDownloader;
@@ -22,6 +21,7 @@
 import org.elasticsearch.ingest.geoip.stats.GeoIpStatsAction.NodeResponse;
 import org.elasticsearch.ingest.geoip.stats.GeoIpStatsAction.Request;
 import org.elasticsearch.ingest.geoip.stats.GeoIpStatsAction.Response;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java
index 01d7cdc9b9d5c..7b962fed0ca83 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java
@@ -20,12 +20,11 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.io.IOException;
-import java.nio.file.CopyOption;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
@@ -62,8 +61,8 @@ public void testLocalDatabasesEmptyConfig() throws Exception {
 
     public void testDatabasesConfigDir() throws Exception {
         Path configDir = createTempDir();
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb"));
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"));
+        copyDatabase("GeoIP2-City-Test.mmdb", configDir.resolve("GeoIP2-City.mmdb"));
+        copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));
 
         ConfigDatabases configDatabases = new ConfigDatabases(configDir, new GeoIpCache(0));
         configDatabases.initialize(resourceWatcherService);
@@ -92,9 +91,9 @@ public void testDatabasesDynamicUpdateConfigDir() throws Exception {
             assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country"));
         }
 
-        CopyOption option = StandardCopyOption.REPLACE_EXISTING;
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb"));
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"), option);
+        copyDatabase("GeoIP2-City-Test.mmdb", configDir.resolve("GeoIP2-City.mmdb"));
+        copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));
+
         assertBusy(() -> {
             assertThat(configDatabases.getConfigDatabases().size(), equalTo(4));
             DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-ASN.mmdb");
@@ -116,7 +115,8 @@ public void testDatabasesDynamicUpdateConfigDir() throws Exception {
 
     public void testDatabasesUpdateExistingConfDatabase() throws Exception {
         Path configDir = createTempDir();
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), configDir.resolve("GeoLite2-City.mmdb"));
+        copyDatabase("GeoLite2-City.mmdb", configDir);
+
         GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload
         ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache);
         configDatabases.initialize(resourceWatcherService);
@@ -131,11 +131,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception {
             assertThat(cache.count(), equalTo(1));
         }
 
-        Files.copy(
-            ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"),
-            configDir.resolve("GeoLite2-City.mmdb"),
-            StandardCopyOption.REPLACE_EXISTING
-        );
+        copyDatabase("GeoLite2-City-Test.mmdb", configDir.resolve("GeoLite2-City.mmdb"));
         assertBusy(() -> {
             assertThat(configDatabases.getConfigDatabases().size(), equalTo(1));
             assertThat(cache.count(), equalTo(0));
@@ -154,11 +150,9 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception {
         });
     }
 
-    private static Path prepareConfigDir() throws IOException {
+    private static Path prepareConfigDir() {
         Path dir = createTempDir();
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), dir.resolve("GeoLite2-ASN.mmdb"));
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), dir.resolve("GeoLite2-City.mmdb"));
-        Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-Country.mmdb"), dir.resolve("GeoLite2-Country.mmdb"));
+        copyDefaultDatabases(dir);
         return dir;
     }
 
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java
index 34d5429142cec..1579c7020c58a 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java
@@ -83,7 +83,7 @@
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
-import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
 import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask;
 import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.TYPE;
 import static org.hamcrest.Matchers.empty;
@@ -117,10 +117,9 @@ public class DatabaseNodeServiceTests extends ESTestCase {
     @Before
     public void setup() throws IOException {
         final Path geoIpConfigDir = createTempDir();
-        Files.createDirectories(geoIpConfigDir);
         GeoIpCache cache = new GeoIpCache(1000);
         ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache);
-        copyDatabaseFiles(geoIpConfigDir, configDatabases);
+        copyDefaultDatabases(geoIpConfigDir, configDatabases);
 
         threadPool = new TestThreadPool(ConfigDatabases.class.getSimpleName());
         Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build();
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java
index 58cb566165db2..1676ce14698a9 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -39,6 +40,7 @@
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentType;
 import org.hamcrest.Matchers;
@@ -85,7 +87,11 @@ public void setup() throws IOException {
             "e4a3411cdd7b21eaf18675da5a7f9f360d33c6882363b2c19c38715834c9e836  GeoIP2-City_20240709.tar.gz".getBytes(StandardCharsets.UTF_8)
         );
         clusterService = mock(ClusterService.class);
-        threadPool = new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), MeterRegistry.NOOP);
+        threadPool = new ThreadPool(
+            Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(),
+            MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders()
+        );
         when(clusterService.getClusterSettings()).thenReturn(
             new ClusterSettings(Settings.EMPTY, Set.of(GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING))
         );
@@ -476,7 +482,9 @@ public void testUpdateDatabasesWriteBlock() {
                 "index ["
                     + geoIpIndex
                     + "] blocked by: [TOO_MANY_REQUESTS/12/disk usage exceeded flood-stage watermark, "
-                    + "index has read-only-allow-delete block];"
+                    + "index has read-only-allow-delete block; for more information, see "
+                    + ReferenceDocs.FLOOD_STAGE_WATERMARK
+                    + "];"
             )
         );
         verifyNoInteractions(httpClient);
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java
index 06b2605bd6d41..f213868fb65a1 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
@@ -44,6 +45,7 @@
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
@@ -91,7 +93,11 @@ public void setup() throws IOException {
         httpClient = mock(HttpClient.class);
         when(httpClient.getBytes(anyString())).thenReturn("[]".getBytes(StandardCharsets.UTF_8));
         clusterService = mock(ClusterService.class);
-        threadPool = new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), MeterRegistry.NOOP);
+        threadPool = new ThreadPool(
+            Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(),
+            MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders()
+        );
         when(clusterService.getClusterSettings()).thenReturn(
             new ClusterSettings(
                 Settings.EMPTY,
@@ -590,7 +596,9 @@ public void testUpdateDatabasesWriteBlock() {
                 "index ["
                     + geoIpIndex
                     + "] blocked by: [TOO_MANY_REQUESTS/12/disk usage exceeded flood-stage watermark, "
-                    + "index has read-only-allow-delete block];"
+                    + "index has read-only-allow-delete block; for more information, see "
+                    + ReferenceDocs.FLOOD_STAGE_WATERMARK
+                    + "];"
             )
         );
         verifyNoInteractions(httpClient);
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
index 663ae1152246a..a0541df0d4d8a 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -25,18 +25,15 @@
 import org.elasticsearch.ingest.geoip.Database.Property;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.StreamsUtils;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.junit.After;
 import org.junit.Before;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -45,6 +42,9 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.DEFAULT_DATABASES;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
+import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDefaultDatabases;
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
@@ -57,8 +57,6 @@
 
 public class GeoIpProcessorFactoryTests extends ESTestCase {
 
-    static Set<String> DEFAULT_DATABASE_FILENAMES = Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb");
-
     private Path geoipTmpDir;
     private Path geoIpConfigDir;
     private ConfigDatabases configDatabases;
@@ -74,7 +72,7 @@ public void loadDatabaseReaders() throws IOException {
         Client client = mock(Client.class);
         GeoIpCache cache = new GeoIpCache(1000);
         configDatabases = new ConfigDatabases(geoIpConfigDir, new GeoIpCache(1000));
-        copyDatabaseFiles(geoIpConfigDir, configDatabases);
+        copyDefaultDatabases(geoIpConfigDir, configDatabases);
         geoipTmpDir = createTempDir();
         clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE);
@@ -181,7 +179,7 @@ public void testBuildDbFile() throws Exception {
         assertFalse(processor.isIgnoreMissing());
     }
 
-    public void testBuildWithCountryDbAndAsnFields() throws Exception {
+    public void testBuildWithCountryDbAndAsnFields() {
         GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
@@ -201,7 +199,7 @@ public void testBuildWithCountryDbAndAsnFields() throws Exception {
         );
     }
 
-    public void testBuildWithAsnDbAndCityFields() throws Exception {
+    public void testBuildWithAsnDbAndCityFields() {
         GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
@@ -218,10 +216,7 @@ public void testBuildWithAsnDbAndCityFields() throws Exception {
     }
 
     public void testBuildNonExistingDbFile() throws Exception {
-        Files.copy(
-            GeoIpProcessorFactoryTests.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"),
-            geoipTmpDir.resolve("GeoLite2-City.mmdb")
-        );
+        copyDatabase("GeoLite2-City-Test.mmdb", geoipTmpDir.resolve("GeoLite2-City.mmdb"));
         databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City.mmdb"));
         GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
 
@@ -234,11 +229,11 @@ public void testBuildNonExistingDbFile() throws Exception {
 
     public void testBuildBuiltinDatabaseMissing() throws Exception {
         GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
-        cleanDatabaseFiles(geoIpConfigDir, configDatabases);
+        cleanDatabases(geoIpConfigDir, configDatabases);
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("database_file", randomFrom(DEFAULT_DATABASE_FILENAMES));
+        config.put("database_file", randomFrom(DEFAULT_DATABASES));
         Processor processor = factory.create(null, null, null, config);
         assertThat(processor, instanceOf(GeoIpProcessor.DatabaseUnavailableProcessor.class));
     }
@@ -267,7 +262,7 @@ public void testBuildFields() throws Exception {
         assertFalse(processor.isIgnoreMissing());
     }
 
-    public void testBuildIllegalFieldOption() throws Exception {
+    public void testBuildIllegalFieldOption() {
         GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
 
         Map<String, Object> config1 = new HashMap<>();
@@ -324,14 +319,13 @@ public void testBuildNullDatabase() throws Exception {
         assertThat(e.getMessage(), equalTo("[database_file] Unsupported database type [null] for file [GeoLite2-City.mmdb]"));
     }
 
-    @SuppressWarnings("HiddenField")
     public void testLazyLoading() throws Exception {
         final Path configDir = createTempDir();
         final Path geoIpConfigDir = configDir.resolve("ingest-geoip");
         Files.createDirectories(geoIpConfigDir);
         GeoIpCache cache = new GeoIpCache(1000);
         ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache);
-        copyDatabaseFiles(geoIpConfigDir, configDatabases);
+        copyDefaultDatabases(geoIpConfigDir, configDatabases);
 
         // Loading another database reader instances, because otherwise we can't test lazy loading as the
         // database readers used at class level are reused between tests. (we want to keep that otherwise running this
@@ -358,7 +352,7 @@ public void testLazyLoading() throws Exception {
         config.put("database_file", "GeoLite2-City.mmdb");
         final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config);
 
-        // these are lazy loaded until first use so we expect null here
+        // these are lazy loaded until first use, so we expect null here
         assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-City.mmdb").databaseReader.get());
         city.execute(document);
         // the first ingest should trigger a database load
@@ -369,7 +363,7 @@ public void testLazyLoading() throws Exception {
         config.put("database_file", "GeoLite2-Country.mmdb");
         final GeoIpProcessor country = (GeoIpProcessor) factory.create(null, "_tag", null, config);
 
-        // these are lazy loaded until first use so we expect null here
+        // these are lazy loaded until first use, so we expect null here
         assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-Country.mmdb").databaseReader.get());
         country.execute(document);
         // the first ingest should trigger a database load
@@ -380,22 +374,21 @@ public void testLazyLoading() throws Exception {
         config.put("database_file", "GeoLite2-ASN.mmdb");
         final GeoIpProcessor asn = (GeoIpProcessor) factory.create(null, "_tag", null, config);
 
-        // these are lazy loaded until first use so we expect null here
+        // these are lazy loaded until first use, so we expect null here
         assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-ASN.mmdb").databaseReader.get());
         asn.execute(document);
         // the first ingest should trigger a database load
         assertNotNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-ASN.mmdb").databaseReader.get());
     }
 
-    @SuppressWarnings("HiddenField")
     public void testLoadingCustomDatabase() throws IOException {
         final Path configDir = createTempDir();
         final Path geoIpConfigDir = configDir.resolve("ingest-geoip");
         Files.createDirectories(geoIpConfigDir);
         ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, new GeoIpCache(1000));
-        copyDatabaseFiles(geoIpConfigDir, configDatabases);
+        copyDefaultDatabases(geoIpConfigDir, configDatabases);
         // fake the GeoIP2-City database
-        copyDatabaseFile(geoIpConfigDir, "GeoLite2-City.mmdb");
+        copyDatabase("GeoLite2-City.mmdb", geoIpConfigDir);
         Files.move(geoIpConfigDir.resolve("GeoLite2-City.mmdb"), geoIpConfigDir.resolve("GeoIP2-City.mmdb"));
 
         /*
@@ -428,7 +421,7 @@ public void testLoadingCustomDatabase() throws IOException {
         config.put("database_file", "GeoIP2-City.mmdb");
         final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config);
 
-        // these are lazy loaded until first use so we expect null here
+        // these are lazy loaded until first use, so we expect null here
         assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoIP2-City.mmdb").databaseReader.get());
         city.execute(document);
         // the first ingest should trigger a database load
@@ -490,7 +483,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception {
             assertThat(geoData.get("city_name"), equalTo("Tumba"));
         }
         {
-            copyDatabaseFile(geoipTmpDir, "GeoLite2-City-Test.mmdb");
+            copyDatabase("GeoLite2-City-Test.mmdb", geoipTmpDir);
             IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
             databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City-Test.mmdb"));
             processor.execute(ingestDocument);
@@ -498,7 +491,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception {
             assertThat(geoData.get("city_name"), equalTo("Linköping"));
         }
         {
-            // No databases are available, so assume that databases still need to be downloaded and therefor not fail:
+            // No databases are available, so assume that databases still need to be downloaded and therefore not fail:
             IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
             databaseNodeService.removeStaleEntries(List.of("GeoLite2-City.mmdb"));
             configDatabases.updateDatabase(geoIpConfigDir.resolve("GeoLite2-City.mmdb"), false);
@@ -507,7 +500,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception {
             assertThat(geoData, nullValue());
         }
         {
-            // There are database available, but not the right one, so tag:
+            // There are databases available, but not the right one, so tag:
             databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City-Test.mmdb"));
             IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
             processor.execute(ingestDocument);
@@ -517,7 +510,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception {
 
     public void testDatabaseNotReadyYet() throws Exception {
         GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
-        cleanDatabaseFiles(geoIpConfigDir, configDatabases);
+        cleanDatabases(geoIpConfigDir, configDatabases);
 
         {
             Map<String, Object> config = new HashMap<>();
@@ -542,7 +535,7 @@ public void testDatabaseNotReadyYet() throws Exception {
             );
         }
 
-        copyDatabaseFile(geoipTmpDir, "GeoLite2-City-Test.mmdb");
+        copyDatabase("GeoLite2-City-Test.mmdb", geoipTmpDir);
         databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City-Test.mmdb"));
 
         {
@@ -562,25 +555,9 @@ public void testDatabaseNotReadyYet() throws Exception {
         }
     }
 
-    private static void copyDatabaseFile(final Path path, final String databaseFilename) throws IOException {
-        Files.copy(
-            new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/" + databaseFilename)),
-            path.resolve(databaseFilename),
-            StandardCopyOption.REPLACE_EXISTING
-        );
-    }
-
-    static void copyDatabaseFiles(final Path path, ConfigDatabases configDatabases) throws IOException {
-        for (final String databaseFilename : DEFAULT_DATABASE_FILENAMES) {
-            copyDatabaseFile(path, databaseFilename);
-            configDatabases.updateDatabase(path.resolve(databaseFilename), true);
+    private static void cleanDatabases(final Path directory, ConfigDatabases configDatabases) {
+        for (final String database : DEFAULT_DATABASES) {
+            configDatabases.updateDatabase(directory.resolve(database), false);
         }
     }
-
-    static void cleanDatabaseFiles(final Path path, ConfigDatabases configDatabases) throws IOException {
-        for (final String databaseFilename : DEFAULT_DATABASE_FILENAMES) {
-            configDatabases.updateDatabase(path.resolve(databaseFilename), false);
-        }
-    }
-
 }
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
index 6276155d9f083..762818a7c65db 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
@@ -134,7 +134,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception {
         assertIngestDocument(originalIngestDocument, ingestDocument);
     }
 
-    public void testNullWithoutIgnoreMissing() throws Exception {
+    public void testNullWithoutIgnoreMissing() {
         GeoIpProcessor processor = new GeoIpProcessor(
             randomAlphaOfLength(10),
             null,
@@ -156,7 +156,7 @@ public void testNullWithoutIgnoreMissing() throws Exception {
         assertThat(exception.getMessage(), equalTo("field [source_field] is null, cannot extract geoip information."));
     }
 
-    public void testNonExistentWithoutIgnoreMissing() throws Exception {
+    public void testNonExistentWithoutIgnoreMissing() {
         GeoIpProcessor processor = new GeoIpProcessor(
             randomAlphaOfLength(10),
             null,
@@ -463,7 +463,7 @@ public void testEnterprise() throws Exception {
         assertThat(geoData.get("residential_proxy"), equalTo(false));
         assertThat(geoData.get("domain"), equalTo("frpt.net"));
         assertThat(geoData.get("isp"), equalTo("Fairpoint Communications"));
-        assertThat(geoData.get("isp_organization"), equalTo("Fairpoint Communications"));
+        assertThat(geoData.get("isp_organization_name"), equalTo("Fairpoint Communications"));
         assertThat(geoData.get("user_type"), equalTo("residential"));
         assertThat(geoData.get("connection_type"), equalTo("Cable/DSL"));
     }
@@ -497,7 +497,7 @@ public void testIsp() throws Exception {
         assertThat(geoData.get("organization_name"), equalTo("CELLCO-PART"));
         assertThat(geoData.get("network"), equalTo("149.101.100.0/28"));
         assertThat(geoData.get("isp"), equalTo("Verizon Wireless"));
-        assertThat(geoData.get("isp_organization"), equalTo("Verizon Wireless"));
+        assertThat(geoData.get("isp_organization_name"), equalTo("Verizon Wireless"));
         assertThat(geoData.get("mobile_network_code"), equalTo("004"));
         assertThat(geoData.get("mobile_country_code"), equalTo("310"));
     }
@@ -526,7 +526,7 @@ public void testAddressIsNotInTheDatabase() throws Exception {
     /**
      * Don't silently do DNS lookups or anything trappy on bogus data
      */
-    public void testInvalid() throws Exception {
+    public void testInvalid() {
         GeoIpProcessor processor = new GeoIpProcessor(
             randomAlphaOfLength(10),
             null,
@@ -803,7 +803,7 @@ long databaseFileSize() throws IOException {
             }
 
             @Override
-            InputStream databaseInputStream() throws IOException {
+            InputStream databaseInputStream() {
                 return databaseInputStreamSupplier.get();
             }
 
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java
new file mode 100644
index 0000000000000..a3d72aca2295c
--- /dev/null
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.ingest.geoip;
+
+import org.elasticsearch.core.SuppressForbidden;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Set;
+
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+
+public final class GeoIpTestUtils {
+
+    private GeoIpTestUtils() {
+        // utility class
+    }
+
+    public static final Set<String> DEFAULT_DATABASES = Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb");
+
+    @SuppressForbidden(reason = "uses java.io.File")
+    private static boolean isDirectory(final Path path) {
+        return path.toFile().isDirectory();
+    }
+
+    public static void copyDatabase(final String databaseName, final Path destination) {
+        try (InputStream is = GeoIpTestUtils.class.getResourceAsStream("/" + databaseName)) {
+            if (is == null) {
+                throw new FileNotFoundException("Resource [" + databaseName + "] not found in classpath");
+            }
+
+            Files.copy(is, isDirectory(destination) ? destination.resolve(databaseName) : destination, REPLACE_EXISTING);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public static void copyDefaultDatabases(final Path directory) {
+        for (final String database : DEFAULT_DATABASES) {
+            copyDatabase(database, directory);
+        }
+    }
+
+    public static void copyDefaultDatabases(final Path directory, ConfigDatabases configDatabases) {
+        for (final String database : DEFAULT_DATABASES) {
+            copyDatabase(database, directory);
+            configDatabases.updateDatabase(directory.resolve(database), true);
+        }
+    }
+}
diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-Anonymous-IP-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-Anonymous-IP-Test.mmdb
index 17fc3715090ae..1b142d0001b9c 100644
Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-Anonymous-IP-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-Anonymous-IP-Test.mmdb differ
diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-City-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-City-Test.mmdb
index 7ed43d616a85d..04220ff4b6411 100644
Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-City-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-City-Test.mmdb differ
diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-Connection-Type-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-Connection-Type-Test.mmdb
index 7bfae78964df0..c49ca3ad48f39 100644
Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-Connection-Type-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-Connection-Type-Test.mmdb differ
diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-Domain-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-Domain-Test.mmdb
index d21c2a93df7d4..596a96617f241 100644
Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-Domain-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-Domain-Test.mmdb differ
diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-Enterprise-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-Enterprise-Test.mmdb
index 837b725e9c154..16c1acf800260 100644
Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-Enterprise-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-Enterprise-Test.mmdb differ
diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-ISP-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-ISP-Test.mmdb
index d16b0eee4c5e5..a4277d0a55c47 100644
Binary files a/modules/ingest-geoip/src/test/resources/GeoIP2-ISP-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoIP2-ISP-Test.mmdb differ
diff --git a/modules/ingest-geoip/src/test/resources/GeoLite2-City-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoLite2-City-Test.mmdb
index 0809201619b59..393efe464b610 100644
Binary files a/modules/ingest-geoip/src/test/resources/GeoLite2-City-Test.mmdb and b/modules/ingest-geoip/src/test/resources/GeoLite2-City-Test.mmdb differ
diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java
index 121a6b01ea792..c41fea6b86ceb 100644
--- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java
+++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java
@@ -8,7 +8,6 @@
 
 package org.elasticsearch.script.expression;
 
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
@@ -18,10 +17,10 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xcontent.XContentType;
 
-import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 
+import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.putJsonStoredScript;
 import static org.hamcrest.Matchers.containsString;
 
 //TODO: please convert to unit tests!
@@ -38,9 +37,9 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singleton(ExpressionPlugin.class);
     }
 
-    public void testAllOpsDisabledIndexedScripts() throws IOException {
-        clusterAdmin().preparePutStoredScript().setId("script1").setContent(new BytesArray("""
-            {"script": {"lang": "expression", "source": "2"} }"""), XContentType.JSON).get();
+    public void testAllOpsDisabledIndexedScripts() {
+        putJsonStoredScript("script1", """
+            {"script": {"lang": "expression", "source": "2"} }""");
         prepareIndex("test").setId("1").setSource("{\"theField\":\"foo\"}", XContentType.JSON).get();
         try {
             client().prepareUpdate("test", "1").setScript(new Script(ScriptType.STORED, null, "script1", Collections.emptyMap())).get();
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java
index 159851affd004..622a1bd4afd25 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java
@@ -42,6 +42,12 @@ public boolean needs_score() {
         return needsScores;
     }
 
+    @Override
+    public boolean needs_termStats() {
+        // _termStats is not available for expressions
+        return false;
+    }
+
     @Override
     public ScoreScript newInstance(final DocReader reader) throws IOException {
         // Use DocReader to get the leaf context while transitioning to DocReader for Painless. DocReader for expressions should follow.
diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
index e17fb4b26cd28..bfac939721122 100644
--- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
+++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
@@ -8,7 +8,13 @@
 package org.elasticsearch.script.mustache;
 
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
+import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -155,8 +161,8 @@ public void testTemplateQueryAsEscapedStringWithConditionalClauseAtEnd() throws
         );
     }
 
-    public void testIndexedTemplateClient() throws Exception {
-        assertAcked(clusterAdmin().preparePutStoredScript().setId("testTemplate").setContent(new BytesArray("""
+    public void testIndexedTemplateClient() {
+        putJsonStoredScript("testTemplate", """
             {
               "script": {
                 "lang": "mustache",
@@ -168,9 +174,12 @@ public void testIndexedTemplateClient() throws Exception {
                   }
                 }
               }
-            }"""), XContentType.JSON));
+            }""");
 
-        GetStoredScriptResponse getResponse = clusterAdmin().prepareGetStoredScript("testTemplate").get();
+        GetStoredScriptResponse getResponse = safeExecute(
+            GetStoredScriptAction.INSTANCE,
+            new GetStoredScriptRequest(TEST_REQUEST_TIMEOUT, "testTemplate")
+        );
         assertNotNull(getResponse.getSource());
 
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
@@ -193,9 +202,14 @@ public void testIndexedTemplateClient() throws Exception {
             4
         );
 
-        assertAcked(clusterAdmin().prepareDeleteStoredScript("testTemplate"));
+        assertAcked(
+            safeExecute(
+                TransportDeleteStoredScriptAction.TYPE,
+                new DeleteStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "testTemplate")
+            )
+        );
 
-        getResponse = clusterAdmin().prepareGetStoredScript("testTemplate").get();
+        getResponse = safeExecute(GetStoredScriptAction.INSTANCE, new GetStoredScriptRequest(TEST_REQUEST_TIMEOUT, "testTemplate"));
         assertNull(getResponse.getSource());
     }
 
@@ -250,7 +264,7 @@ public void testBadTemplate() {
         }
     }
 
-    public void testIndexedTemplate() throws Exception {
+    public void testIndexedTemplate() {
 
         String script = """
             {
@@ -267,9 +281,9 @@ public void testIndexedTemplate() throws Exception {
             }
             """;
 
-        assertAcked(clusterAdmin().preparePutStoredScript().setId("1a").setContent(new BytesArray(script), XContentType.JSON));
-        assertAcked(clusterAdmin().preparePutStoredScript().setId("2").setContent(new BytesArray(script), XContentType.JSON));
-        assertAcked(clusterAdmin().preparePutStoredScript().setId("3").setContent(new BytesArray(script), XContentType.JSON));
+        putJsonStoredScript("1a", script);
+        putJsonStoredScript("2", script);
+        putJsonStoredScript("3", script);
 
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
         bulkRequestBuilder.add(prepareIndex("test").setId("1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
@@ -335,13 +349,12 @@ public void testIndexedTemplateOverwrite() throws Exception {
               }
             }""";
         for (int i = 1; i < iterations; i++) {
-            assertAcked(
-                clusterAdmin().preparePutStoredScript()
-                    .setId("git01")
-                    .setContent(new BytesArray(query.replace("{{slop}}", Integer.toString(-1))), XContentType.JSON)
-            );
+            putJsonStoredScript("git01", query.replace("{{slop}}", Integer.toString(-1)));
 
-            GetStoredScriptResponse getResponse = clusterAdmin().prepareGetStoredScript("git01").get();
+            GetStoredScriptResponse getResponse = safeExecute(
+                GetStoredScriptAction.INSTANCE,
+                new GetStoredScriptRequest(TEST_REQUEST_TIMEOUT, "git01")
+            );
             assertNotNull(getResponse.getSource());
 
             Map<String, Object> templateParams = new HashMap<>();
@@ -357,11 +370,8 @@ public void testIndexedTemplateOverwrite() throws Exception {
             );
             assertThat(e.getMessage(), containsString("No negative slop allowed"));
 
-            assertAcked(
-                clusterAdmin().preparePutStoredScript()
-                    .setId("git01")
-                    .setContent(new BytesArray(query.replace("{{slop}}", Integer.toString(0))), XContentType.JSON)
-            );
+            putJsonStoredScript("git01", query.replace("{{slop}}", Integer.toString(0)));
+
             assertHitCount(
                 new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest("testindex"))
                     .setScript("git01")
@@ -373,8 +383,8 @@ public void testIndexedTemplateOverwrite() throws Exception {
         }
     }
 
-    public void testIndexedTemplateWithArray() throws Exception {
-        String multiQuery = """
+    public void testIndexedTemplateWithArray() {
+        putJsonStoredScript("4", """
             {
               "script": {
                 "lang": "mustache",
@@ -390,8 +400,8 @@ public void testIndexedTemplateWithArray() throws Exception {
                   }
                 }
               }
-            }""";
-        assertAcked(clusterAdmin().preparePutStoredScript().setId("4").setContent(new BytesArray(multiQuery), XContentType.JSON));
+            }""");
+
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
         bulkRequestBuilder.add(prepareIndex("test").setId("1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
         bulkRequestBuilder.add(prepareIndex("test").setId("2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
@@ -454,4 +464,14 @@ public void testCCSCheckCompatibility() throws Exception {
     public static void assertHitCount(SearchTemplateRequestBuilder requestBuilder, long expectedHitCount) {
         assertResponse(requestBuilder, response -> ElasticsearchAssertions.assertHitCount(response.getResponse(), expectedHitCount));
     }
+
+    private void putJsonStoredScript(String id, String jsonContent) {
+        assertAcked(
+            safeExecute(
+                TransportPutStoredScriptAction.TYPE,
+                new PutStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).id(id)
+                    .content(new BytesArray(jsonContent), XContentType.JSON)
+            )
+        );
+    }
 }
diff --git a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchUsageStatsIT.java b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchUsageStatsIT.java
index b14ca7ea7cfa2..e51a4822c67ba 100644
--- a/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchUsageStatsIT.java
+++ b/modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchUsageStatsIT.java
@@ -10,16 +10,14 @@
 
 import org.elasticsearch.action.admin.cluster.stats.SearchUsageStats;
 import org.elasticsearch.client.Request;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.putJsonStoredScript;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
 public class SearchUsageStatsIT extends ESIntegTestCase {
@@ -62,7 +60,7 @@ public void testSearchUsageStats() throws IOException {
             getRestClient().performRequest(request);
         }
         {
-            assertAcked(clusterAdmin().preparePutStoredScript().setId("testTemplate").setContent(new BytesArray("""
+            putJsonStoredScript("testTemplate", """
                 {
                   "script": {
                     "lang": "mustache",
@@ -74,7 +72,7 @@ public void testSearchUsageStats() throws IOException {
                       }
                     }
                   }
-                }"""), XContentType.JSON));
+                }""");
             Request request = new Request("GET", "/_search/template");
             request.setJsonEntity("""
                 {
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java
index b97d1b00573f4..2e50d14b24d72 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java
@@ -142,12 +142,12 @@ public static byte[] writeMultiLineFormat(MultiSearchTemplateRequest multiSearch
                 MultiSearchRequest.writeSearchRequestParams(searchRequest, xContentBuilder);
                 BytesReference.bytes(xContentBuilder).writeTo(output);
             }
-            output.write(xContent.streamSeparator());
+            output.write(xContent.bulkSeparator());
             try (XContentBuilder xContentBuilder = XContentBuilder.builder(xContent)) {
                 templateRequest.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
                 BytesReference.bytes(xContentBuilder).writeTo(output);
             }
-            output.write(xContent.streamSeparator());
+            output.write(xContent.bulkSeparator());
         }
         return output.toByteArray();
     }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
index d9466536ac46c..f7f68324907ff 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
@@ -94,7 +94,7 @@ public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, b
     }
 
     @Override
-    public boolean supportsContentStream() {
+    public boolean supportsBulkContent() {
         return true;
     }
 
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java
index ba1d4dbaba012..cac278563d944 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportMultiSearchTemplateAction.java
@@ -20,10 +20,10 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
index 74e4705447b64..35d3bfdf6446d 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
@@ -16,11 +16,11 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java
index 3c183830afa6d..b11ebd2e652cb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java
@@ -16,11 +16,11 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.painless.PainlessScriptEngine;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.rest.BaseRestHandler;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
index 0736fd4ef4a87..919f2659e1f92 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
@@ -40,7 +40,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeometryFormatterFactory;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -68,6 +67,7 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.painless.spi.PainlessTestScript;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt
index 5082d5f1c7bdb..0dab7dcbadfb5 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt
@@ -13,6 +13,23 @@ class org.elasticsearch.script.ScoreScript @no_import {
 class org.elasticsearch.script.ScoreScript$Factory @no_import {
 }
 
+class org.elasticsearch.script.StatsSummary {
+    double getMin()
+    double getMax()
+    double getAverage()
+    double getSum()
+    long getCount()
+}
+
+class org.elasticsearch.script.ScriptTermStats {
+    int uniqueTermsCount()
+    int matchedTermsCount()
+    StatsSummary docFreq()
+    StatsSummary totalTermFreq()
+    StatsSummary termFreq()
+    StatsSummary termPositions()
+}
+
 static_import {
     double saturation(double, double) from_class org.elasticsearch.script.ScoreScriptUtils
     double sigmoid(double, double, double) from_class org.elasticsearch.script.ScoreScriptUtils
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreScriptTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreScriptTests.java
new file mode 100644
index 0000000000000..08b55fdf3bcc3
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreScriptTests.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.painless.spi.Whitelist;
+import org.elasticsearch.painless.spi.WhitelistLoader;
+import org.elasticsearch.script.ScoreScript;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.painless.ScriptTestCase.PAINLESS_BASE_WHITELIST;
+
+public class ScoreScriptTests extends ESSingleNodeTestCase {
+    /**
+     * Test that needTermStats() is reported correctly depending on whether _termStats is used
+     */
+    public void testNeedsTermStats() {
+        IndexService index = createIndex("test", Settings.EMPTY, "type", "d", "type=double");
+
+        Map<ScriptContext<?>, List<Whitelist>> contexts = new HashMap<>();
+        List<Whitelist> whitelists = new ArrayList<>(PAINLESS_BASE_WHITELIST);
+        whitelists.add(WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, "org.elasticsearch.script.score.txt"));
+        contexts.put(ScoreScript.CONTEXT, whitelists);
+        PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts);
+
+        SearchExecutionContext searchExecutionContext = index.newSearchExecutionContext(0, 0, null, () -> 0, null, emptyMap());
+
+        ScoreScript.Factory factory = service.compile(null, "1.2", ScoreScript.CONTEXT, Collections.emptyMap());
+        ScoreScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), searchExecutionContext.lookup());
+        assertFalse(ss.needs_termStats());
+
+        factory = service.compile(null, "doc['d'].value", ScoreScript.CONTEXT, Collections.emptyMap());
+        ss = factory.newFactory(Collections.emptyMap(), searchExecutionContext.lookup());
+        assertFalse(ss.needs_termStats());
+
+        factory = service.compile(null, "1/_termStats.totalTermFreq().getAverage()", ScoreScript.CONTEXT, Collections.emptyMap());
+        ss = factory.newFactory(Collections.emptyMap(), searchExecutionContext.lookup());
+        assertTrue(ss.needs_termStats());
+
+        factory = service.compile(null, "doc['d'].value * _termStats.docFreq().getSum()", ScoreScript.CONTEXT, Collections.emptyMap());
+        ss = factory.newFactory(Collections.emptyMap(), searchExecutionContext.lookup());
+        assertTrue(ss.needs_termStats());
+    }
+}
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml
new file mode 100644
index 0000000000000..f82b844f01588
--- /dev/null
+++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml
@@ -0,0 +1,612 @@
+setup:
+  - requires:
+      cluster_features: ["script.term_stats"]
+      reason: "support for term stats has been added in 8.16"
+
+  - do:
+      indices.create:
+        index: test-index
+        body:
+          settings:
+            number_of_shards: "2"
+          mappings:
+            properties:
+              title:
+                type: text
+              genre:
+                type: text
+                fields:
+                  keyword:
+                    type: keyword
+
+  - do:
+      index: { refresh: true, index: test-index, id: "1", routing: 0, body: {"title": "Star wars", "genre": "Sci-fi"} }
+  - do:
+      index: { refresh: true, index: test-index, id: "2", routing: 1, body: {"title": "Star trek", "genre": "Sci-fi"} }
+  - do:
+      index: { refresh: true, index: test-index, id: "3", routing: 1, body: {"title": "Rambo", "genre": "War movie"} }
+  - do:
+      index: { refresh: true, index: test-index, id: "4", routing: 1, body: {"title": "Rambo II", "genre": "War movie"} }
+
+---
+"match query: uniqueTermsCount without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.uniqueTermsCount()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 2 }
+  - match: { hits.hits.1._score: 2 }
+
+---
+"match query: uniqueTermsCount with DFS":
+  - do:
+      search:
+        search_type: dfs_query_then_fetch
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.uniqueTermsCount()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 2 }
+  - match: { hits.hits.1._score: 2 }
+
+---
+"match query: matchedTermsCount without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.matchedTermsCount()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 2 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"match query: matchedTermsCount with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.matchedTermsCount()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 2 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"match query: docFreq min without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.docFreq().getMin()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 0 }
+
+---
+"match query: docFreq min with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.docFreq().getMin()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"match query: docFreq max without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.docFreq().getMax()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"match query: docFreq max with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.docFreq().getMax()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 2 }
+  - match: { hits.hits.1._score: 2 }
+
+---
+"match query: totalTermFreq sum without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.totalTermFreq().getSum()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 2 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"match query: totalTermFreq sum with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.totalTermFreq().getSum()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 3 }
+  - match: { hits.hits.1._score: 3 }
+
+---
+"match query: termFreq sum without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.termFreq().getSum()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 2 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"match query: termFreq sum with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.termFreq().getSum()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 2 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"match query: termPositions avg without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.termPositions().getAverage()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1.5 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"match query: termPositions avg with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { match: { "title": "Star wars" } }
+              script:
+                source: "return _termStats.termPositions().getAverage()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1.5 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"term query: uniqueTermsCount without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.uniqueTermsCount()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"term query: uniqueTermsCount with DFS":
+  - do:
+      search:
+        search_type: dfs_query_then_fetch
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.uniqueTermsCount()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"term query: matchedTermsCount without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.matchedTermsCount()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"term query: matchedTermsCount with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.matchedTermsCount()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"term query: docFreq min without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.docFreq().getMin()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"term query: docFreq min with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.docFreq().getMin()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 2 }
+  - match: { hits.hits.1._score: 2 }
+
+---
+"term query: docFreq max without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.docFreq().getMax()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"term query: docFreq max with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.docFreq().getMax()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 2 }
+  - match: { hits.hits.1._score: 2 }
+
+---
+"term query: totalTermFreq sum without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.totalTermFreq().getSum()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"term query: totalTermFreq sum with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.totalTermFreq().getSum()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 2 }
+  - match: { hits.hits.1._score: 2 }
+
+---
+"term query: termFreq sum without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.termFreq().getSum()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"term query: termFreq sum with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.termFreq().getSum()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 1 }
+  - match: { hits.hits.1._score: 1 }
+
+---
+"term query: termPositions avg without DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.termPositions().getAverage()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 0 }
+  - match: { hits.hits.1._score: 0 }
+
+---
+"term query: termPositions avg with DFS":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        search_type: dfs_query_then_fetch
+        index: test-index
+        body:
+          query:
+            script_score:
+              query: { term: { "genre.keyword": "Sci-fi" } }
+              script:
+                source: "return _termStats.termPositions().getAverage()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 0 }
+  - match: { hits.hits.1._score: 0 }
+
+---
+"Complex bool query: uniqueTermsCount":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query:
+                bool:
+                  must:
+                    match: { "title": "star wars" }
+                  should:
+                    term: { "genre.keyword": "Sci-fi" }
+                  filter:
+                    match: { "genre" : "sci"}
+                  must_not:
+                    term: { "genre.keyword": "War" }
+              script:
+                source: "return _termStats.uniqueTermsCount()"
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._score: 4 }
+  - match: { hits.hits.1._score: 4 }
+
+
+---
+"match_all query: uniqueTermsCount":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query:
+                match_all: {}
+              script:
+                source: "return _termStats.uniqueTermsCount()"
+  - match: { hits.total: 4 }
+  - match: { hits.hits.0._score: 0 }
+  - match: { hits.hits.1._score: 0 }
+  - match: { hits.hits.2._score: 0 }
+  - match: { hits.hits.3._score: 0 }
+
+---
+"match_all query: docFreq":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query:
+                match_all: {}
+              script:
+                source: "return _termStats.docFreq().getMax()"
+  - match: { hits.total: 4 }
+  - match: { hits.hits.0._score: 0 }
+  - match: { hits.hits.1._score: 0 }
+  - match: { hits.hits.2._score: 0 }
+  - match: { hits.hits.3._score: 0 }
+
+---
+"match_all query: totalTermFreq":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query:
+                match_all: {}
+              script:
+                source: "return _termStats.totalTermFreq().getSum()"
+  - match: { hits.total: 4 }
+  - match: { hits.hits.0._score: 0 }
+  - match: { hits.hits.1._score: 0 }
+  - match: { hits.hits.2._score: 0 }
+  - match: { hits.hits.3._score: 0 }
+
+---
+"match_all query: termFreq":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query:
+                match_all: {}
+              script:
+                source: "return _termStats.termFreq().getMax()"
+  - match: { hits.total: 4 }
+  - match: { hits.hits.0._score: 0 }
+  - match: { hits.hits.1._score: 0 }
+  - match: { hits.hits.2._score: 0 }
+  - match: { hits.hits.3._score: 0 }
+
+---
+"match_all query: termPositions":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            script_score:
+              query:
+                match_all: {}
+              script:
+                source: "return _termStats.termPositions().getSum()"
+  - match: { hits.total: 4 }
+  - match: { hits.hits.0._score: 0 }
+  - match: { hits.hits.1._score: 0 }
+  - match: { hits.hits.2._score: 0 }
+  - match: { hits.hits.3._score: 0 }
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml
index 67457e64c874e..b1505368a5c2b 100644
--- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml
+++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml
@@ -1,130 +1,145 @@
 ---
 "binary":
-    - skip:
-        features: ["headers"]
-    - do:
-        indices.create:
-            index: test
-            body:
-                mappings:
-                    properties:
-                        binary:
-                            type: binary
-                            doc_values: true
+  - skip:
+      features: ["headers"]
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              binary:
+                type: binary
+                doc_values: true
+              sort_field:
+                type: keyword
+                doc_values: true
 
-    - do:
-        #set the header so we won't randomize it
-        headers:
-            Content-Type: application/json
-        index:
-            index: test
-            id: "1"
-            body:
-                binary: "U29tZSBiaW5hcnkgYmxvYg=="
+  - do:
+      #set the header so we won't randomize it
+      headers:
+        Content-Type: application/json
+      index:
+        index: test
+        id: "1"
+        body:
+          binary: "U29tZSBiaW5hcnkgYmxvYg=="
+          sort_field: "1"
 
-    - do:
-        #set the header so we won't randomize it
-        headers:
-          Content-Type: application/json
-        index:
-          index: test
-          id: "2"
-          body:
-            binary: [
-              "U29tZSBiaW5hcnkgYmxvYg==",
-              "MTIzNA==",
-              "dGVzdA=="
-            ]
+  - do:
+      #set the header so we won't randomize it
+      headers:
+        Content-Type: application/json
+      index:
+        index: test
+        id: "2"
+        body:
+          binary: [
+            "U29tZSBiaW5hcnkgYmxvYg==",
+            "MTIzNA==",
+            "dGVzdA=="
+          ]
+          sort_field: "2"
 
-    - do:
-        #set the header so we won't randomize it
-        headers:
-          Content-Type: application/json
-        index:
-          index: test
-          id: "3"
-          body: {}
+  - do:
+      #set the header so we won't randomize it
+      headers:
+        Content-Type: application/json
+      index:
+        index: test
+        id: "3"
+        body:
+          sort_field: "3"
 
-    - do:
-        indices.refresh: {}
+  - do:
+      indices.refresh: {}
 
-    - do:
-        search:
-            body:
-                script_fields:
-                    field1:
-                        script:
-                            source: "if (doc['binary'].size() == 0) {return 'empty'} doc['binary'].get(0).utf8ToString()"
-                    field2:
-                        script:
-                            source: "if (doc['binary'].size() == 0) {return 'empty'} doc['binary'].value.utf8ToString()"
-    - match: { hits.hits.0.fields.field1.0: "Some binary blob" }
-    - match: { hits.hits.0.fields.field2.0: "Some binary blob" }
+  - do:
+      search:
+        body:
+          sort: sort_field
+          script_fields:
+            field1:
+              script:
+                source: "if (doc['binary'].size() == 0) {return 'empty'} doc['binary'].get(0).utf8ToString()"
+            field2:
+              script:
+                source: "if (doc['binary'].size() == 0) {return 'empty'} doc['binary'].value.utf8ToString()"
+  - match: { hits.hits.0.fields.field1.0: "Some binary blob" }
+  - match: { hits.hits.0.fields.field2.0: "Some binary blob" }
+  - match: { hits.hits.1.fields.field1.0: "1234" }
+  - match: { hits.hits.1.fields.field2.0: "1234" }
+  - match: { hits.hits.2.fields.field1.0: "empty" }
+  - match: { hits.hits.2.fields.field2.0: "empty" }
 
-    - do:
-        search:
-          body:
-            script_fields:
-              field1:
-                script:
-                  source: "ByteBuffer bb = field('binary').get(null); if (bb == null) {return -1;} return bb.get(0)"
-              field2:
-                script:
-                  source: "ByteBuffer bb = field('binary').get(0, null); if (bb == null) {return -1;} return bb.get(0)"
-              field3:
-                script:
-                  source: "int total = 0; for (value in field('binary')) {total += value.get(0)} total"
-    - match: { hits.hits.0.fields.field1.0: 83 }
-    - match: { hits.hits.0.fields.field2.0: 83 }
-    - match: { hits.hits.0.fields.field3.0: 83 }
-    - match: { hits.hits.1.fields.field1.0: 49 }
-    - match: { hits.hits.1.fields.field2.0: 49 }
-    - match: { hits.hits.1.fields.field3.0: 248 }
-    - match: { hits.hits.2.fields.field1.0: -1 }
-    - match: { hits.hits.2.fields.field2.0: -1 }
-    - match: { hits.hits.2.fields.field3.0: 0 }
+  - do:
+      search:
+        body:
+          sort: sort_field
+          script_fields:
+            field1:
+              script:
+                source: "ByteBuffer bb = field('binary').get(null); if (bb == null) {return -1;} return bb.get(0)"
+            field2:
+              script:
+                source: "ByteBuffer bb = field('binary').get(0, null); if (bb == null) {return -1;} return bb.get(0)"
+            field3:
+              script:
+                source: "int total = 0; for (value in field('binary')) {total += value.get(0)} total"
+  - match: { hits.hits.0.fields.field1.0: 83 }
+  - match: { hits.hits.0.fields.field2.0: 83 }
+  - match: { hits.hits.0.fields.field3.0: 83 }
+  - match: { hits.hits.1.fields.field1.0: 49 }
+  - match: { hits.hits.1.fields.field2.0: 49 }
+  - match: { hits.hits.1.fields.field3.0: 248 }
+  - match: { hits.hits.2.fields.field1.0: -1 }
+  - match: { hits.hits.2.fields.field2.0: -1 }
+  - match: { hits.hits.2.fields.field3.0: 0 }
 
-    - do:
-        search:
-          body:
-            script_fields:
-              field1:
-                script:
-                  source: "ByteBuffer bb = field('binary').get(null); if (bb == null) {return -1;} return bb.limit()"
-              field2:
-                script:
-                  source: "ByteBuffer bb = field('binary').get(0, null); if (bb == null) {return -1;} return bb.limit()"
-              field3:
-                script:
-                  source: "int total = 0; for (ByteBuffer value : field('binary')) {total += value.limit()} total"
-    - match: { hits.hits.0.fields.field1.0: 16 }
-    - match: { hits.hits.0.fields.field2.0: 16 }
-    - match: { hits.hits.0.fields.field3.0: 16 }
-    - match: { hits.hits.1.fields.field1.0: 4 }
-    - match: { hits.hits.1.fields.field2.0: 4 }
-    - match: { hits.hits.1.fields.field3.0: 24 }
-    - match: { hits.hits.2.fields.field1.0: -1 }
-    - match: { hits.hits.2.fields.field2.0: -1 }
-    - match: { hits.hits.2.fields.field3.0: 0 }
+  - do:
+      search:
+        body:
+          sort: sort_field
+          script_fields:
+            field1:
+              script:
+                source: "ByteBuffer bb = field('binary').get(null); if (bb == null) {return -1;} return bb.limit()"
+            field2:
+              script:
+                source: "ByteBuffer bb = field('binary').get(0, null); if (bb == null) {return -1;} return bb.limit()"
+            field3:
+              script:
+                source: "int total = 0; for (ByteBuffer value : field('binary')) {total += value.limit()} total"
+  - match: { hits.hits.0.fields.field1.0: 16 }
+  - match: { hits.hits.0.fields.field2.0: 16 }
+  - match: { hits.hits.0.fields.field3.0: 16 }
+  - match: { hits.hits.1.fields.field1.0: 4 }
+  - match: { hits.hits.1.fields.field2.0: 4 }
+  - match: { hits.hits.1.fields.field3.0: 24 }
+  - match: { hits.hits.2.fields.field1.0: -1 }
+  - match: { hits.hits.2.fields.field2.0: -1 }
+  - match: { hits.hits.2.fields.field3.0: 0 }
 
-    - do:
-        search:
-          body:
-            script_fields:
-              field1:
-                script:
-                  source: "ByteBuffer bb = $('binary', null); if (bb == null) {return -1;} return bb.get(0)"
-    - match: { hits.hits.0.fields.field1.0: 83 }
-    - match: { hits.hits.1.fields.field1.0: 49 }
-    - match: { hits.hits.2.fields.field1.0: -1 }
+  - do:
+      search:
+        body:
+          sort: sort_field
+          script_fields:
+            field1:
+              script:
+                source: "ByteBuffer bb = $('binary', null); if (bb == null) {return -1;} return bb.get(0)"
+  - match: { hits.hits.0.fields.field1.0: 83 }
+  - match: { hits.hits.1.fields.field1.0: 49 }
+  - match: { hits.hits.2.fields.field1.0: -1 }
 
-    - do:
-        search:
-          body:
-            script_fields:
-              field1:
-                script:
-                  source: "ByteBuffer bb = $('binary', null); if (bb == null) {return -1;} return bb.limit()"
-    - match: { hits.hits.0.fields.field1.0: 16 }
-    - match: { hits.hits.1.fields.field1.0: 4 }
-    - match: { hits.hits.2.fields.field1.0: -1 }
+  - do:
+      search:
+        body:
+          sort: sort_field
+          script_fields:
+            field1:
+              script:
+                source: "ByteBuffer bb = $('binary', null); if (bb == null) {return -1;} return bb.limit()"
+  - match: { hits.hits.0.fields.field1.0: 16 }
+  - match: { hits.hits.1.fields.field1.0: 4 }
+  - match: { hits.hits.2.fields.field1.0: -1 }
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
index 899cc42fea1e0..b3cd3586fca54 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
@@ -447,7 +447,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
                 "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        return new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), leafName(), null) {
+        return new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), leafName()) {
             @Override
             protected void write(XContentBuilder b, Object value) throws IOException {
                 b.value((String) value);
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
index ad1b54afd985d..258eb4ba3e22a 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
@@ -20,12 +20,12 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.TemplateScript;
diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java
new file mode 100644
index 0000000000000..e7d26b0808a48
--- /dev/null
+++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java
@@ -0,0 +1,216 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.reindex.BulkIndexByScrollResponseMatcher;
+import org.elasticsearch.reindex.ReindexPlugin;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.reindex.DeleteByQueryMetrics.DELETE_BY_QUERY_TIME_HISTOGRAM;
+import static org.elasticsearch.reindex.ReindexMetrics.REINDEX_TIME_HISTOGRAM;
+import static org.elasticsearch.reindex.UpdateByQueryMetrics.UPDATE_BY_QUERY_TIME_HISTOGRAM;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0, scope = ESIntegTestCase.Scope.TEST)
+public class ReindexPluginMetricsIT extends ESIntegTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(ReindexPlugin.class, TestTelemetryPlugin.class);
+    }
+
+    protected ReindexRequestBuilder reindex() {
+        return new ReindexRequestBuilder(client());
+    }
+
+    protected UpdateByQueryRequestBuilder updateByQuery() {
+        return new UpdateByQueryRequestBuilder(client());
+    }
+
+    protected DeleteByQueryRequestBuilder deleteByQuery() {
+        return new DeleteByQueryRequestBuilder(client());
+    }
+
+    public static BulkIndexByScrollResponseMatcher matcher() {
+        return new BulkIndexByScrollResponseMatcher();
+    }
+
+    public void testReindexMetrics() throws Exception {
+        final String dataNodeName = internalCluster().startNode();
+
+        indexRandom(
+            true,
+            prepareIndex("source").setId("1").setSource("foo", "a"),
+            prepareIndex("source").setId("2").setSource("foo", "a"),
+            prepareIndex("source").setId("3").setSource("foo", "b"),
+            prepareIndex("source").setId("4").setSource("foo", "c")
+        );
+        assertHitCount(prepareSearch("source").setSize(0), 4);
+
+        final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName)
+            .filterPlugins(TestTelemetryPlugin.class)
+            .findFirst()
+            .orElseThrow();
+
+        // Copy all the docs
+        reindex().source("source").destination("dest").get();
+        // Use assertBusy to wait for all threads to complete so we get deterministic results
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(1));
+        });
+
+        // Now none of them
+        createIndex("none");
+        reindex().source("source").destination("none").filter(termQuery("foo", "no_match")).get();
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(2));
+        });
+
+        // Now half of them
+        reindex().source("source").destination("dest_half").filter(termQuery("foo", "a")).get();
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(3));
+        });
+
+        // Limit with maxDocs
+        reindex().source("source").destination("dest_size_one").maxDocs(1).get();
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(4));
+        });
+    }
+
+    public void testDeleteByQueryMetrics() throws Exception {
+        final String dataNodeName = internalCluster().startNode();
+
+        indexRandom(
+            true,
+            prepareIndex("test").setId("1").setSource("foo", "a"),
+            prepareIndex("test").setId("2").setSource("foo", "a"),
+            prepareIndex("test").setId("3").setSource("foo", "b"),
+            prepareIndex("test").setId("4").setSource("foo", "c"),
+            prepareIndex("test").setId("5").setSource("foo", "d"),
+            prepareIndex("test").setId("6").setSource("foo", "e"),
+            prepareIndex("test").setId("7").setSource("foo", "f")
+        );
+
+        assertHitCount(prepareSearch("test").setSize(0), 7);
+
+        final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName)
+            .filterPlugins(TestTelemetryPlugin.class)
+            .findFirst()
+            .orElseThrow();
+
+        // Deletes two docs that matches "foo:a"
+        deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).get();
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(1));
+        });
+
+        // Deletes the two first docs with limit by size
+        DeleteByQueryRequestBuilder request = deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).size(2).refresh(true);
+        request.source().addSort("foo.keyword", SortOrder.ASC);
+        request.get();
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(2));
+        });
+
+        // Deletes but match no docs
+        deleteByQuery().source("test").filter(termQuery("foo", "no_match")).refresh(true).get();
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(3));
+        });
+
+        // Deletes all remaining docs
+        deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true).get();
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(4));
+        });
+    }
+
+    public void testUpdateByQueryMetrics() throws Exception {
+        final String dataNodeName = internalCluster().startNode();
+
+        indexRandom(
+            true,
+            prepareIndex("test").setId("1").setSource("foo", "a"),
+            prepareIndex("test").setId("2").setSource("foo", "a"),
+            prepareIndex("test").setId("3").setSource("foo", "b"),
+            prepareIndex("test").setId("4").setSource("foo", "c")
+        );
+        assertHitCount(prepareSearch("test").setSize(0), 4);
+        assertEquals(1, client().prepareGet("test", "1").get().getVersion());
+        assertEquals(1, client().prepareGet("test", "4").get().getVersion());
+
+        final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName)
+            .filterPlugins(TestTelemetryPlugin.class)
+            .findFirst()
+            .orElseThrow();
+
+        // Reindex all the docs
+        updateByQuery().source("test").refresh(true).get();
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(1));
+        });
+
+        // Now none of them
+        updateByQuery().source("test").filter(termQuery("foo", "no_match")).refresh(true).get();
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(2));
+        });
+
+        // Now half of them
+        updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).get();
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(3));
+        });
+
+        // Limit with size
+        UpdateByQueryRequestBuilder request = updateByQuery().source("test").size(3).refresh(true);
+        request.source().addSort("foo.keyword", SortOrder.ASC);
+        request.get();
+        assertBusy(() -> {
+            testTelemetryPlugin.collect();
+            List<Measurement> measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM);
+            assertThat(measurements.size(), equalTo(4));
+        });
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java
index e4f734fa5721c..82c42f5d680ff 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelper.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -114,12 +115,14 @@ static <Request extends AbstractBulkByScrollRequest<Request>> void initTaskState
     ) {
         int configuredSlices = request.getSlices();
         if (configuredSlices == AbstractBulkByScrollRequest.AUTO_SLICES) {
-            ClusterSearchShardsRequest shardsRequest = new ClusterSearchShardsRequest();
-            shardsRequest.indices(request.getSearchRequest().indices());
-            client.admin().cluster().searchShards(shardsRequest, listener.safeMap(response -> {
-                setWorkerCount(request, task, countSlicesBasedOnShards(response));
-                return null;
-            }));
+            client.execute(
+                TransportClusterSearchShardsAction.TYPE,
+                new ClusterSearchShardsRequest(request.getTimeout(), request.getSearchRequest().indices()),
+                listener.safeMap(response -> {
+                    setWorkerCount(request, task, countSlicesBasedOnShards(response));
+                    return null;
+                })
+            );
         } else {
             setWorkerCount(request, task, configuredSlices);
             listener.onResponse(null);
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java
new file mode 100644
index 0000000000000..2cedf0d5f5823
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.reindex;
+
+import org.elasticsearch.telemetry.metric.LongHistogram;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+public class DeleteByQueryMetrics {
+    public static final String DELETE_BY_QUERY_TIME_HISTOGRAM = "es.delete_by_query.duration.histogram";
+
+    private final LongHistogram deleteByQueryTimeSecsHistogram;
+
+    public DeleteByQueryMetrics(MeterRegistry meterRegistry) {
+        this(
+            meterRegistry.registerLongHistogram(DELETE_BY_QUERY_TIME_HISTOGRAM, "Time taken to execute Delete by Query request", "seconds")
+        );
+    }
+
+    private DeleteByQueryMetrics(LongHistogram deleteByQueryTimeSecsHistogram) {
+        this.deleteByQueryTimeSecsHistogram = deleteByQueryTimeSecsHistogram;
+    }
+
+    public long recordTookTime(long tookTime) {
+        deleteByQueryTimeSecsHistogram.record(tookTime);
+        return tookTime;
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java
new file mode 100644
index 0000000000000..3025357aa6538
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.reindex;
+
+import org.elasticsearch.telemetry.metric.LongHistogram;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+public class ReindexMetrics {
+
+    public static final String REINDEX_TIME_HISTOGRAM = "es.reindex.duration.histogram";
+
+    private final LongHistogram reindexTimeSecsHistogram;
+
+    public ReindexMetrics(MeterRegistry meterRegistry) {
+        this(meterRegistry.registerLongHistogram(REINDEX_TIME_HISTOGRAM, "Time to reindex by search", "millis"));
+    }
+
+    private ReindexMetrics(LongHistogram reindexTimeSecsHistogram) {
+        this.reindexTimeSecsHistogram = reindexTimeSecsHistogram;
+    }
+
+    public long recordTookTime(long tookTime) {
+        reindexTimeSecsHistogram.record(tookTime);
+        return tookTime;
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java
index 1a40f77250e5f..3169d4c4ee1fb 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java
@@ -34,7 +34,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
@@ -85,8 +84,11 @@ public List<RestHandler> getRestHandlers(
 
     @Override
     public Collection<?> createComponents(PluginServices services) {
-        return Collections.singletonList(
-            new ReindexSslConfig(services.environment().settings(), services.environment(), services.resourceWatcherService())
+        return List.of(
+            new ReindexSslConfig(services.environment().settings(), services.environment(), services.resourceWatcherService()),
+            new ReindexMetrics(services.telemetryProvider().getMeterRegistry()),
+            new UpdateByQueryMetrics(services.telemetryProvider().getMeterRegistry()),
+            new DeleteByQueryMetrics(services.telemetryProvider().getMeterRegistry())
         );
     }
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java
index dbe1968bb076a..cb393a42f52a1 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
@@ -65,6 +66,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiFunction;
 import java.util.function.LongSupplier;
@@ -82,19 +84,22 @@ public class Reindexer {
     private final ThreadPool threadPool;
     private final ScriptService scriptService;
     private final ReindexSslConfig reindexSslConfig;
+    private final ReindexMetrics reindexMetrics;
 
     Reindexer(
         ClusterService clusterService,
         Client client,
         ThreadPool threadPool,
         ScriptService scriptService,
-        ReindexSslConfig reindexSslConfig
+        ReindexSslConfig reindexSslConfig,
+        @Nullable ReindexMetrics reindexMetrics
     ) {
         this.clusterService = clusterService;
         this.client = client;
         this.threadPool = threadPool;
         this.scriptService = scriptService;
         this.reindexSslConfig = reindexSslConfig;
+        this.reindexMetrics = reindexMetrics;
     }
 
     public void initTask(BulkByScrollTask task, ReindexRequest request, ActionListener<Void> listener) {
@@ -102,6 +107,8 @@ public void initTask(BulkByScrollTask task, ReindexRequest request, ActionListen
     }
 
     public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkClient, ActionListener<BulkByScrollResponse> listener) {
+        long startTime = System.nanoTime();
+
         BulkByScrollParallelizationHelper.executeSlicedAction(
             task,
             request,
@@ -122,7 +129,12 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl
                     clusterService.state(),
                     reindexSslConfig,
                     request,
-                    listener
+                    ActionListener.runAfter(listener, () -> {
+                        long elapsedTime = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime);
+                        if (reindexMetrics != null) {
+                            reindexMetrics.recordTookTime(elapsedTime);
+                        }
+                    })
                 );
                 searchAction.start();
             }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java
index 530246db35c93..53381c33d7f78 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java
@@ -14,23 +14,27 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.BulkByScrollTask;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.concurrent.TimeUnit;
+
 public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteByQueryRequest, BulkByScrollResponse> {
 
     private final ThreadPool threadPool;
     private final Client client;
     private final ScriptService scriptService;
     private final ClusterService clusterService;
+    private final DeleteByQueryMetrics deleteByQueryMetrics;
 
     @Inject
     public TransportDeleteByQueryAction(
@@ -39,18 +43,21 @@ public TransportDeleteByQueryAction(
         Client client,
         TransportService transportService,
         ScriptService scriptService,
-        ClusterService clusterService
+        ClusterService clusterService,
+        @Nullable DeleteByQueryMetrics deleteByQueryMetrics
     ) {
         super(DeleteByQueryAction.NAME, transportService, actionFilters, DeleteByQueryRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.threadPool = threadPool;
         this.client = client;
         this.scriptService = scriptService;
         this.clusterService = clusterService;
+        this.deleteByQueryMetrics = deleteByQueryMetrics;
     }
 
     @Override
     public void doExecute(Task task, DeleteByQueryRequest request, ActionListener<BulkByScrollResponse> listener) {
         BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task;
+        long startTime = System.nanoTime();
         BulkByScrollParallelizationHelper.startSlicedAction(
             request,
             bulkByScrollTask,
@@ -64,8 +71,20 @@ public void doExecute(Task task, DeleteByQueryRequest request, ActionListener<Bu
                     clusterService.localNode(),
                     bulkByScrollTask
                 );
-                new AsyncDeleteByQueryAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, listener)
-                    .start();
+                new AsyncDeleteByQueryAction(
+                    bulkByScrollTask,
+                    logger,
+                    assigningClient,
+                    threadPool,
+                    request,
+                    scriptService,
+                    ActionListener.runAfter(listener, () -> {
+                        long elapsedTime = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime);
+                        if (deleteByQueryMetrics != null) {
+                            deleteByQueryMetrics.recordTookTime(elapsedTime);
+                        }
+                    })
+                ).start();
             }
         );
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java
index f19d359ddeb44..821a137ac7566 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java
@@ -15,15 +15,16 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.BulkByScrollTask;
 import org.elasticsearch.index.reindex.ReindexAction;
 import org.elasticsearch.index.reindex.ReindexRequest;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -53,7 +54,8 @@ public TransportReindexAction(
         AutoCreateIndex autoCreateIndex,
         Client client,
         TransportService transportService,
-        ReindexSslConfig sslConfig
+        ReindexSslConfig sslConfig,
+        @Nullable ReindexMetrics reindexMetrics
     ) {
         this(
             ReindexAction.NAME,
@@ -66,7 +68,8 @@ public TransportReindexAction(
             autoCreateIndex,
             client,
             transportService,
-            sslConfig
+            sslConfig,
+            reindexMetrics
         );
     }
 
@@ -81,12 +84,13 @@ protected TransportReindexAction(
         AutoCreateIndex autoCreateIndex,
         Client client,
         TransportService transportService,
-        ReindexSslConfig sslConfig
+        ReindexSslConfig sslConfig,
+        @Nullable ReindexMetrics reindexMetrics
     ) {
         super(name, transportService, actionFilters, ReindexRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.client = client;
         this.reindexValidator = new ReindexValidator(settings, clusterService, indexNameExpressionResolver, autoCreateIndex);
-        this.reindexer = new Reindexer(clusterService, client, threadPool, scriptService, sslConfig);
+        this.reindexer = new Reindexer(clusterService, client, threadPool, scriptService, sslConfig, reindexMetrics);
     }
 
     @Override
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java
index e5df42a484172..4d09253e046d4 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.reindex.BulkByScrollTask;
 import org.elasticsearch.index.reindex.LeaderBulkByScrollTaskState;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java
index de84d74d05ee8..997d4d32fe042 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java
@@ -17,14 +17,15 @@
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.BulkByScrollTask;
 import org.elasticsearch.index.reindex.ScrollableHitSource;
 import org.elasticsearch.index.reindex.UpdateByQueryAction;
 import org.elasticsearch.index.reindex.UpdateByQueryRequest;
 import org.elasticsearch.index.reindex.WorkerBulkByScrollTaskState;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.CtxMap;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
@@ -35,6 +36,7 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.function.BiFunction;
 import java.util.function.LongSupplier;
 
@@ -44,6 +46,7 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
     private final Client client;
     private final ScriptService scriptService;
     private final ClusterService clusterService;
+    private final UpdateByQueryMetrics updateByQueryMetrics;
 
     @Inject
     public TransportUpdateByQueryAction(
@@ -52,18 +55,21 @@ public TransportUpdateByQueryAction(
         Client client,
         TransportService transportService,
         ScriptService scriptService,
-        ClusterService clusterService
+        ClusterService clusterService,
+        @Nullable UpdateByQueryMetrics updateByQueryMetrics
     ) {
         super(UpdateByQueryAction.NAME, transportService, actionFilters, UpdateByQueryRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.threadPool = threadPool;
         this.client = client;
         this.scriptService = scriptService;
         this.clusterService = clusterService;
+        this.updateByQueryMetrics = updateByQueryMetrics;
     }
 
     @Override
     protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener<BulkByScrollResponse> listener) {
         BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task;
+        long startTime = System.nanoTime();
         BulkByScrollParallelizationHelper.startSlicedAction(
             request,
             bulkByScrollTask,
@@ -78,8 +84,21 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener
                     clusterService.localNode(),
                     bulkByScrollTask
                 );
-                new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, scriptService, request, state, listener)
-                    .start();
+                new AsyncIndexBySearchAction(
+                    bulkByScrollTask,
+                    logger,
+                    assigningClient,
+                    threadPool,
+                    scriptService,
+                    request,
+                    state,
+                    ActionListener.runAfter(listener, () -> {
+                        long elapsedTime = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime);
+                        if (updateByQueryMetrics != null) {
+                            updateByQueryMetrics.recordTookTime(elapsedTime);
+                        }
+                    })
+                ).start();
             }
         );
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java
new file mode 100644
index 0000000000000..6ca52769a1ba9
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.reindex;
+
+import org.elasticsearch.telemetry.metric.LongHistogram;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+public class UpdateByQueryMetrics {
+    public static final String UPDATE_BY_QUERY_TIME_HISTOGRAM = "es.update_by_query.duration.histogram";
+
+    private final LongHistogram updateByQueryTimeSecsHistogram;
+
+    public UpdateByQueryMetrics(MeterRegistry meterRegistry) {
+        this(
+            meterRegistry.registerLongHistogram(UPDATE_BY_QUERY_TIME_HISTOGRAM, "Time taken to execute Update by Query request", "seconds")
+        );
+    }
+
+    private UpdateByQueryMetrics(LongHistogram updateByQueryTimeSecsHistogram) {
+        this.updateByQueryTimeSecsHistogram = updateByQueryTimeSecsHistogram;
+    }
+
+    public long recordTookTime(long tookTime) {
+        updateByQueryTimeSecsHistogram.record(tookTime);
+        return tookTime;
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java
index c40a4f72bc133..47505919ba7d2 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.TimeValue;
@@ -992,7 +993,7 @@ private static class DummyTransportAsyncBulkByScrollAction extends TransportActi
         BulkByScrollResponse> {
 
         protected DummyTransportAsyncBulkByScrollAction(String actionName, ActionFilters actionFilters, TaskManager taskManager) {
-            super(actionName, actionFilters, taskManager);
+            super(actionName, actionFilters, taskManager, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         }
 
         @Override
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java
new file mode 100644
index 0000000000000..58adc6aebaa9b
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.reindex;
+
+import org.elasticsearch.telemetry.InstrumentType;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.RecordingMeterRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.List;
+
+import static org.elasticsearch.reindex.DeleteByQueryMetrics.DELETE_BY_QUERY_TIME_HISTOGRAM;
+
+public class DeleteByQueryMetricsTests extends ESTestCase {
+    private RecordingMeterRegistry recordingMeterRegistry;
+    private DeleteByQueryMetrics metrics;
+
+    @Before
+    public void createMetrics() {
+        recordingMeterRegistry = new RecordingMeterRegistry();
+        metrics = new DeleteByQueryMetrics(recordingMeterRegistry);
+    }
+
+    public void testRecordTookTime() {
+        int secondsTaken = randomIntBetween(1, 50);
+        metrics.recordTookTime(secondsTaken);
+        List<Measurement> measurements = recordingMeterRegistry.getRecorder()
+            .getMeasurements(InstrumentType.LONG_HISTOGRAM, DELETE_BY_QUERY_TIME_HISTOGRAM);
+        assertEquals(measurements.size(), 1);
+        assertEquals(measurements.get(0).getLong(), secondsTaken);
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java
new file mode 100644
index 0000000000000..4711530585817
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.reindex;
+
+import org.elasticsearch.telemetry.InstrumentType;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.RecordingMeterRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.List;
+
+import static org.elasticsearch.reindex.ReindexMetrics.REINDEX_TIME_HISTOGRAM;
+
+public class ReindexMetricsTests extends ESTestCase {
+
+    private RecordingMeterRegistry recordingMeterRegistry;
+    private ReindexMetrics metrics;
+
+    @Before
+    public void createMetrics() {
+        recordingMeterRegistry = new RecordingMeterRegistry();
+        metrics = new ReindexMetrics(recordingMeterRegistry);
+    }
+
+    public void testRecordTookTime() {
+        int secondsTaken = randomIntBetween(1, 50);
+        metrics.recordTookTime(secondsTaken);
+        List<Measurement> measurements = recordingMeterRegistry.getRecorder()
+            .getMeasurements(InstrumentType.LONG_HISTOGRAM, REINDEX_TIME_HISTOGRAM);
+        assertEquals(measurements.size(), 1);
+        assertEquals(measurements.get(0).getLong(), secondsTaken);
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java
new file mode 100644
index 0000000000000..548d18d202984
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.reindex;
+
+import org.elasticsearch.telemetry.InstrumentType;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.RecordingMeterRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.List;
+
+import static org.elasticsearch.reindex.UpdateByQueryMetrics.UPDATE_BY_QUERY_TIME_HISTOGRAM;
+
+public class UpdateByQueryMetricsTests extends ESTestCase {
+
+    private RecordingMeterRegistry recordingMeterRegistry;
+    private UpdateByQueryMetrics metrics;
+
+    @Before
+    public void createMetrics() {
+        recordingMeterRegistry = new RecordingMeterRegistry();
+        metrics = new UpdateByQueryMetrics(recordingMeterRegistry);
+    }
+
+    public void testRecordTookTime() {
+        int secondsTaken = randomIntBetween(1, 50);
+        metrics.recordTookTime(secondsTaken);
+        List<Measurement> measurements = recordingMeterRegistry.getRecorder()
+            .getMeasurements(InstrumentType.LONG_HISTOGRAM, UPDATE_BY_QUERY_TIME_HISTOGRAM);
+        assertEquals(measurements.size(), 1);
+        assertEquals(measurements.get(0).getLong(), secondsTaken);
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java
index 876ddefda161b..c4d591f804750 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java
@@ -60,6 +60,7 @@ protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptSer
             null,
             transportService,
             scriptService,
+            null,
             null
         );
         return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(
diff --git a/modules/repository-azure/build.gradle b/modules/repository-azure/build.gradle
index d093816acd45f..6334e5ae6a195 100644
--- a/modules/repository-azure/build.gradle
+++ b/modules/repository-azure/build.gradle
@@ -1,5 +1,7 @@
 import org.apache.tools.ant.filters.ReplaceTokens
+import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
+import org.elasticsearch.gradle.internal.test.RestIntegTestTask
 
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
@@ -17,65 +19,69 @@ esplugin {
 }
 
 versions << [
-  'azure': '12.20.1',
-  'azureCommon': '12.19.1',
-  'azureCore': '1.34.0',
-  'azureCoreHttpNetty': '1.12.7',
-  'azureJackson': '2.15.4',
-  'azureJacksonDatabind': '2.13.4.2',
-  'azureAvro': '12.5.3',
-
-  'jakartaActivation': '1.2.1',
-  'jakartaXMLBind': '2.3.2',
-  'stax2API': '4.2.1',
-  'woodstox': '6.4.0',
-
-  'reactorNetty': '1.0.39',
-  'reactorCore': '3.4.34',
-  'reactiveStreams': '1.0.4',
+  'azureReactorNetty': '1.0.45',
 ]
 
 dependencies {
-  api "com.azure:azure-storage-blob:${versions.azure}"
-  api "com.azure:azure-storage-common:${versions.azureCommon}"
-  api "com.azure:azure-core-http-netty:${versions.azureCoreHttpNetty}"
-  api "com.azure:azure-core:${versions.azureCore}"
-  // jackson
-  api "com.fasterxml.jackson.core:jackson-core:${versions.azureJackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.azureJacksonDatabind}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.azureJackson}"
-
-  // jackson xml
-  api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.azureJackson}"
-  api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.azureJackson}"
-  api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.azureJackson}"
-  api "jakarta.activation:jakarta.activation-api:${versions.jakartaActivation}"
-  // The SDK uses javax.xml bindings
-  api "jakarta.xml.bind:jakarta.xml.bind-api:${versions.jakartaXMLBind}"
-  api "org.codehaus.woodstox:stax2-api:${versions.stax2API}"
-  api "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}"
-
-  // netty
+  // Microsoft
+  api "com.azure:azure-core-http-netty:1.15.3"
+  api "com.azure:azure-core:1.51.0"
+  api "com.azure:azure-identity:1.13.2"
+  api "com.azure:azure-json:1.2.0"
+  api "com.azure:azure-storage-blob:12.27.1"
+  api "com.azure:azure-storage-common:12.26.1"
+  api "com.azure:azure-storage-internal-avro:12.12.1"
+  api "com.azure:azure-xml:1.1.0"
+  api "com.microsoft.azure:msal4j-persistence-extension:1.3.0"
+  api "com.microsoft.azure:msal4j:1.16.2"
+
+  // Jackson
+  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
+  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+  api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.jackson}"
+  api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
+  api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}"
+
+  // Netty
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-http2:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
   api "io.netty:netty-handler-proxy:${versions.netty}"
   api "io.netty:netty-resolver-dns:${versions.netty}"
 
-  // reactor
-  api "io.projectreactor.netty:reactor-netty-core:${versions.reactorNetty}"
-  api "io.projectreactor.netty:reactor-netty-http:${versions.reactorNetty}"
-  api "io.projectreactor:reactor-core:${versions.reactorCore}"
-  api "org.reactivestreams:reactive-streams:${versions.reactiveStreams}"
+  // Reactor
+  api "io.projectreactor.netty:reactor-netty-core:${versions.azureReactorNetty}"
+  api "io.projectreactor.netty:reactor-netty-http:${versions.azureReactorNetty}"
+  api "io.projectreactor:reactor-core:3.4.38"
+  api "org.reactivestreams:reactive-streams:1.0.4"
+
+  // Others
+  api "com.fasterxml.woodstox:woodstox-core:6.7.0"
+  api "com.github.stephenc.jcip:jcip-annotations:1.0-1"
+  api "com.nimbusds:content-type:2.3"
+  api "com.nimbusds:lang-tag:1.7"
+  api "com.nimbusds:nimbus-jose-jwt:9.37.3"
+  api "com.nimbusds:oauth2-oidc-sdk:11.9.1"
+  api "jakarta.activation:jakarta.activation-api:1.2.1"
+  api "jakarta.xml.bind:jakarta.xml.bind-api:2.3.3"
+  api "net.java.dev.jna:jna-platform:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version
+  api "net.java.dev.jna:jna:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version
+  api "net.minidev:accessors-smart:2.5.0"
+  api "net.minidev:json-smart:2.5.0"
+  api "org.codehaus.woodstox:stax2-api:4.2.2"
+  api "org.ow2.asm:asm:9.3"
+
+  runtimeOnly "com.google.crypto.tink:tink:1.14.0"
+  runtimeOnly "com.google.protobuf:protobuf-java:4.27.0"
+  runtimeOnly "com.google.code.gson:gson:2.11.0"
+  runtimeOnly "org.cryptomator:siv-mode:1.5.2"
 
   implementation project(":modules:transport-netty4")
   implementation("org.slf4j:slf4j-api:${versions.slf4j}")
   runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}"
 //  runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}") https://github.com/elastic/elasticsearch/issues/93714
 
-
-  runtimeOnly "com.azure:azure-storage-internal-avro:${versions.azureAvro}"
-
   testImplementation project(':test:fixtures:azure-fixture')
   yamlRestTestImplementation project(':test:fixtures:azure-fixture')
 }
@@ -165,12 +171,94 @@ tasks.named("thirdPartyAudit").configure {
     'com.ctc.wstx.shaded.msv_core.driver.textui.Driver',
     // [missing classes] SLF4j includes an optional class that depends on an extension class. see Log4jLogger#createConverter
     //    'org.slf4j.ext.EventData' - bring back when https://github.com/elastic/elasticsearch/issues/93714 is done
+
+    // Optional dependency of tink
+    'com.google.api.client.http.HttpHeaders',
+    'com.google.api.client.http.HttpRequest',
+    'com.google.api.client.http.HttpRequestFactory',
+    'com.google.api.client.http.HttpResponse',
+    'com.google.api.client.http.HttpTransport',
+    'com.google.api.client.http.javanet.NetHttpTransport',
+    'com.google.api.client.http.javanet.NetHttpTransport$Builder',
+
+    // Optional dependency of nimbus-jose-jwt and oauth2-oidc-sdk
+    'org.bouncycastle.asn1.pkcs.PrivateKeyInfo',
+    'org.bouncycastle.asn1.x509.AlgorithmIdentifier',
+    'org.bouncycastle.asn1.x509.SubjectPublicKeyInfo',
+    'org.bouncycastle.cert.X509CertificateHolder',
+    'org.bouncycastle.cert.jcajce.JcaX509CertificateHolder',
+    'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
+    'org.bouncycastle.crypto.InvalidCipherTextException',
+    'org.bouncycastle.crypto.engines.AESEngine',
+    'org.bouncycastle.crypto.modes.GCMBlockCipher',
+    'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider',
+    'org.bouncycastle.jce.provider.BouncyCastleProvider',
+    'org.bouncycastle.openssl.PEMKeyPair',
+    'org.bouncycastle.openssl.PEMParser',
+    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
+    'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+
+    // OAuth servlet support is optional and not required
+    'jakarta.servlet.ServletRequest',
+    'jakarta.servlet.http.HttpServletRequest',
+    'jakarta.servlet.http.HttpServletResponse',
+    'javax.servlet.ServletRequest',
+    'javax.servlet.http.HttpServletRequest',
+    'javax.servlet.http.HttpServletResponse',
+
+    // OpenSAML support is optional
+    'org.joda.time.DateTime',
+    'net.shibboleth.utilities.java.support.xml.SerializeSupport',
+    'org.opensaml.core.config.InitializationException',
+    'org.opensaml.core.config.InitializationService',
+    'org.opensaml.core.xml.XMLObject',
+    'org.opensaml.core.xml.XMLObjectBuilder',
+    'org.opensaml.core.xml.XMLObjectBuilderFactory',
+    'org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport',
+    'org.opensaml.core.xml.io.Marshaller',
+    'org.opensaml.core.xml.io.MarshallerFactory',
+    'org.opensaml.core.xml.io.MarshallingException',
+    'org.opensaml.core.xml.io.Unmarshaller',
+    'org.opensaml.core.xml.io.UnmarshallerFactory',
+    'org.opensaml.core.xml.schema.XSString',
+    'org.opensaml.core.xml.schema.impl.XSStringBuilder',
+    'org.opensaml.saml.saml2.core.Assertion',
+    'org.opensaml.saml.saml2.core.Attribute',
+    'org.opensaml.saml.saml2.core.AttributeStatement',
+    'org.opensaml.saml.saml2.core.AttributeValue',
+    'org.opensaml.saml.saml2.core.Audience',
+    'org.opensaml.saml.saml2.core.AudienceRestriction',
+    'org.opensaml.saml.saml2.core.AuthnContext',
+    'org.opensaml.saml.saml2.core.AuthnContextClassRef',
+    'org.opensaml.saml.saml2.core.AuthnStatement',
+    'org.opensaml.saml.saml2.core.Conditions',
+    'org.opensaml.saml.saml2.core.Issuer',
+    'org.opensaml.saml.saml2.core.NameID',
+    'org.opensaml.saml.saml2.core.Subject',
+    'org.opensaml.saml.saml2.core.SubjectConfirmation',
+    'org.opensaml.saml.saml2.core.SubjectConfirmationData',
+    'org.opensaml.saml.security.impl.SAMLSignatureProfileValidator',
+    'org.opensaml.security.credential.BasicCredential',
+    'org.opensaml.security.credential.Credential',
+    'org.opensaml.security.credential.UsageType',
+    'org.opensaml.xmlsec.signature.Signature',
+    'org.opensaml.xmlsec.signature.support.SignatureException',
+    'org.opensaml.xmlsec.signature.support.SignatureValidator',
+    'org.opensaml.xmlsec.signature.support.Signer',
   )
 
   ignoreViolations(
     'javax.activation.MailcapCommandMap',
     'javax.activation.MimetypesFileTypeMap',
     'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException',
+
+    'com.google.protobuf.MessageSchema',
+    'com.google.protobuf.UnsafeUtil',
+    'com.google.protobuf.UnsafeUtil$1',
+    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$MemoryAccessor',
   )
 }
 
@@ -180,6 +268,8 @@ String azureKey = System.getenv("azure_storage_key")
 String azureContainer = System.getenv("azure_storage_container")
 String azureBasePath = System.getenv("azure_storage_base_path")
 String azureSasToken = System.getenv("azure_storage_sas_token")
+String azureTenantId = System.getenv("azure_storage_tenant_id")
+String azureClientId = System.getenv("azure_storage_client_id")
 
 if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
   azureAccount = 'azure_integration_test_account'
@@ -187,6 +277,8 @@ if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSas
   azureContainer = 'container'
   azureBasePath = ''
   azureSasToken = ''
+  azureTenantId = ''
+  azureClientId = ''
   useFixture = true
 }
 
@@ -203,6 +295,7 @@ tasks.named("processYamlRestTestResources") {
 tasks.named("internalClusterTest") {
   // this is tested explicitly in a separate test task
   exclude '**/AzureStorageCleanupThirdPartyTests.class'
+  systemProperty "AZURE_POD_IDENTITY_AUTHORITY_HOST", "127.0.0.1:1" // ensure a fast failure
 }
 
 tasks.named("yamlRestTest") {
@@ -213,6 +306,33 @@ tasks.named("yamlRestTest") {
   systemProperty 'test.azure.sas_token', azureSasToken
 }
 
+tasks.register("managedIdentityYamlRestTest", RestIntegTestTask) {
+  testClassesDirs = sourceSets.yamlRestTest.output.classesDirs
+  classpath = sourceSets.yamlRestTest.runtimeClasspath
+  systemProperty 'test.azure.fixture', Boolean.toString(useFixture)
+  systemProperty 'test.azure.account', azureAccount
+  systemProperty 'test.azure.container', azureContainer
+  // omitting key and sas_token so that we use a bearer token from the metadata service
+}
+
+tasks.register("workloadIdentityYamlRestTest", RestIntegTestTask) {
+  testClassesDirs = sourceSets.yamlRestTest.output.classesDirs
+  classpath = sourceSets.yamlRestTest.runtimeClasspath
+  systemProperty 'test.azure.fixture', Boolean.toString(useFixture)
+  systemProperty 'test.azure.account', azureAccount
+  systemProperty 'test.azure.container', azureContainer
+  // random uuids to satisfy format requirements -- the actual values are not important
+  systemProperty 'test.azure.tenant_id', azureTenantId ?: "583d4f71-148a-4163-bad5-2311e13c60dc"
+  systemProperty 'test.azure.client_id', azureClientId ?: "86dd1b33-96c1-4a2e-92ac-b844404fc691"
+  // omitting key and sas_token so that we use a bearer token from workload identity
+}
+
+if (BuildParams.inFipsJvm) {
+  // Cannot override the trust store in FIPS mode, and these tasks require a HTTPS fixture
+  tasks.named("managedIdentityYamlRestTest").configure { enabled = false }
+  tasks.named("workloadIdentityYamlRestTest").configure { enabled = false }
+}
+
 tasks.register("azureThirdPartyUnitTest", Test) {
   SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
   SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
@@ -233,4 +353,6 @@ tasks.register('azureThirdPartyTest') {
 
 tasks.named("check") {
   dependsOn("azureThirdPartyUnitTest")
+  dependsOn("managedIdentityYamlRestTest")
+  dependsOn("workloadIdentityYamlRestTest")
 }
diff --git a/x-pack/plugin/security/licenses/nimbus-jose-jwt-LICENSE.txt b/modules/repository-azure/licenses/accessors-smart-LICENSE.txt
similarity index 100%
rename from x-pack/plugin/security/licenses/nimbus-jose-jwt-LICENSE.txt
rename to modules/repository-azure/licenses/accessors-smart-LICENSE.txt
diff --git a/modules/repository-azure/licenses/accessors-smart-NOTICE.txt b/modules/repository-azure/licenses/accessors-smart-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/repository-azure/licenses/asm-LICENSE.txt b/modules/repository-azure/licenses/asm-LICENSE.txt
new file mode 100644
index 0000000000000..22649e87adc9b
--- /dev/null
+++ b/modules/repository-azure/licenses/asm-LICENSE.txt
@@ -0,0 +1,80 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>ASM - License</title>
+  <link rel="stylesheet" type="text/css" href="style.css">
+</head>
+<body>
+  <input id="nav-trigger" type="checkbox" hidden>
+<div class="header">
+  <label for="nav-trigger" class="nav-trigger">
+    <img src="burger-icon.svg" class="burger-icon">
+  </label>
+  <a href="index.html">
+    <img src="asm-logo.svg" class="asm-icon">
+  </a>
+  <span class="asm-title">ASM</span>
+  <a href="https://gitlab.ow2.org/asm/asm">
+    <img src="gitlab-logo.svg" class="gitlab-icon">
+  </a>
+</div>
+<label for="nav-trigger" class="nav-overlay"></label>
+<nav>
+  <ul>
+    <li><a href="index.html">Home</a></li>
+    <li>
+      <a href="documentation.html">Documentation</a>
+      <ul>
+        <li><a href="asm4-guide.pdf">User guide</a></li>
+        <li><a href="javadoc/overview-summary.html">Javadoc</a></li>
+        <li><a href="faq.html">FAQ</a></li>
+        <li><a href="developer-guide.html">Developer guide</a></li>
+      </ul>
+    </li>
+    <li>
+      <a href="about.html">About</a>
+      <ul>
+        <li><a href="license.html">License</a></li>
+        <li><a href="publications.html">Publications</a></li>
+        <li><a href="versions.html">Versions</a></li>
+      </ul>
+    </li>
+  </ul>
+</nav>
+
+  <div class="page-content">
+    <h1>License</h1>
+    <p>ASM is released under the following 3-Clause BSD License:</p>
+    <pre>ASM: a very small and fast Java bytecode manipulation framework
+Copyright (c) 2000-2011 INRIA, France Telecom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holders nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
+</pre>
+  </div>
+</body>
+</html>
diff --git a/modules/repository-azure/licenses/asm-NOTICE.txt b/modules/repository-azure/licenses/asm-NOTICE.txt
new file mode 100644
index 0000000000000..8d1c8b69c3fce
--- /dev/null
+++ b/modules/repository-azure/licenses/asm-NOTICE.txt
@@ -0,0 +1 @@
+ 
diff --git a/modules/repository-azure/licenses/content-type-LICENSE.txt b/modules/repository-azure/licenses/content-type-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/modules/repository-azure/licenses/content-type-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/modules/repository-azure/licenses/content-type-NOTICE.txt b/modules/repository-azure/licenses/content-type-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/repository-azure/licenses/gson-LICENSE.txt b/modules/repository-azure/licenses/gson-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/modules/repository-azure/licenses/gson-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/modules/repository-azure/licenses/gson-NOTICE.txt b/modules/repository-azure/licenses/gson-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/repository-azure/licenses/jcip-annotations-LICENSE.txt b/modules/repository-azure/licenses/jcip-annotations-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/modules/repository-azure/licenses/jcip-annotations-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/modules/repository-azure/licenses/jcip-annotations-NOTICE.txt b/modules/repository-azure/licenses/jcip-annotations-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/repository-azure/licenses/jna-LICENSE.txt b/modules/repository-azure/licenses/jna-LICENSE.txt
new file mode 100644
index 0000000000000..f433b1a53f5b8
--- /dev/null
+++ b/modules/repository-azure/licenses/jna-LICENSE.txt
@@ -0,0 +1,177 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
diff --git a/modules/repository-azure/licenses/jna-NOTICE.txt b/modules/repository-azure/licenses/jna-NOTICE.txt
new file mode 100644
index 0000000000000..8d1c8b69c3fce
--- /dev/null
+++ b/modules/repository-azure/licenses/jna-NOTICE.txt
@@ -0,0 +1 @@
+ 
diff --git a/modules/repository-azure/licenses/jna-platform-LICENSE.txt b/modules/repository-azure/licenses/jna-platform-LICENSE.txt
new file mode 100644
index 0000000000000..f433b1a53f5b8
--- /dev/null
+++ b/modules/repository-azure/licenses/jna-platform-LICENSE.txt
@@ -0,0 +1,177 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
diff --git a/modules/repository-azure/licenses/jna-platform-NOTICE.txt b/modules/repository-azure/licenses/jna-platform-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/repository-azure/licenses/json-smart-LICENSE.txt b/modules/repository-azure/licenses/json-smart-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/modules/repository-azure/licenses/json-smart-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/modules/repository-azure/licenses/json-smart-NOTICE.txt b/modules/repository-azure/licenses/json-smart-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/repository-azure/licenses/lang-tag-LICENSE.txt b/modules/repository-azure/licenses/lang-tag-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/modules/repository-azure/licenses/lang-tag-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/modules/repository-azure/licenses/lang-tag-NOTICE.txt b/modules/repository-azure/licenses/lang-tag-NOTICE.txt
new file mode 100644
index 0000000000000..37a85f6850d57
--- /dev/null
+++ b/modules/repository-azure/licenses/lang-tag-NOTICE.txt
@@ -0,0 +1,14 @@
+Nimbus Language Tags
+
+Copyright 2012-2016, Connect2id Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
diff --git a/modules/repository-azure/licenses/msal4j-LICENSE.txt b/modules/repository-azure/licenses/msal4j-LICENSE.txt
new file mode 100644
index 0000000000000..21071075c2459
--- /dev/null
+++ b/modules/repository-azure/licenses/msal4j-LICENSE.txt
@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
diff --git a/modules/repository-azure/licenses/msal4j-NOTICE.txt b/modules/repository-azure/licenses/msal4j-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt b/modules/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt
new file mode 100644
index 0000000000000..4b1ad51b2f0ef
--- /dev/null
+++ b/modules/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt
@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
diff --git a/modules/repository-azure/licenses/msal4j-persistence-extension-NOTICE.txt b/modules/repository-azure/licenses/msal4j-persistence-extension-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt b/modules/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/modules/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/security/licenses/nimbus-jose-jwt-NOTICE.txt b/modules/repository-azure/licenses/nimbus-jose-jwt-NOTICE.txt
similarity index 100%
rename from x-pack/plugin/security/licenses/nimbus-jose-jwt-NOTICE.txt
rename to modules/repository-azure/licenses/nimbus-jose-jwt-NOTICE.txt
diff --git a/modules/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt b/modules/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/modules/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/modules/repository-azure/licenses/oauth2-oidc-sdk-NOTICE.txt b/modules/repository-azure/licenses/oauth2-oidc-sdk-NOTICE.txt
new file mode 100644
index 0000000000000..5e111b04cfc45
--- /dev/null
+++ b/modules/repository-azure/licenses/oauth2-oidc-sdk-NOTICE.txt
@@ -0,0 +1,14 @@
+Nimbus OAuth 2.0 SDK with OpenID Connect extensions
+
+Copyright 2012-2018, Connect2id Ltd and contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
diff --git a/modules/repository-azure/licenses/protobuf-java-LICENSE.txt b/modules/repository-azure/licenses/protobuf-java-LICENSE.txt
new file mode 100644
index 0000000000000..19b305b00060a
--- /dev/null
+++ b/modules/repository-azure/licenses/protobuf-java-LICENSE.txt
@@ -0,0 +1,32 @@
+Copyright 2008 Google Inc.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Code generated by the Protocol Buffer compiler is owned by the owner
+of the input file used when generating it.  This code is not
+standalone and requires a support library to be linked with it.  This
+support library is itself covered by the above license.
diff --git a/modules/repository-azure/licenses/protobuf-java-NOTICE.txt b/modules/repository-azure/licenses/protobuf-java-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/repository-azure/licenses/siv-mode-LICENSE.txt b/modules/repository-azure/licenses/siv-mode-LICENSE.txt
new file mode 100644
index 0000000000000..edfa1ebd93281
--- /dev/null
+++ b/modules/repository-azure/licenses/siv-mode-LICENSE.txt
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2015-2017 Cryptomator
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/modules/repository-azure/licenses/siv-mode-NOTICE.txt b/modules/repository-azure/licenses/siv-mode-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/repository-azure/licenses/tink-LICENSE.txt b/modules/repository-azure/licenses/tink-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/modules/repository-azure/licenses/tink-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/modules/repository-azure/licenses/tink-NOTICE.txt b/modules/repository-azure/licenses/tink-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
index e916b02e62b8e..15d47f6bec800 100644
--- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
+++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
@@ -153,9 +153,8 @@ long getUploadBlockSize() {
 
     @SuppressForbidden(reason = "this test uses a HttpHandler to emulate an Azure endpoint")
     private static class AzureBlobStoreHttpHandler extends AzureHttpHandler implements BlobStoreHttpHandler {
-
         AzureBlobStoreHttpHandler(final String account, final String container) {
-            super(account, container);
+            super(account, container, null /* no auth header validation - sometimes it's omitted in these tests (TODO why?) */);
         }
     }
 
diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureRepositoryMissingCredentialsIT.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureRepositoryMissingCredentialsIT.java
new file mode 100644
index 0000000000000..0ce8b7a3e8ea4
--- /dev/null
+++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureRepositoryMissingCredentialsIT.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories.azure;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
+import org.elasticsearch.action.admin.cluster.repositories.put.TransportPutRepositoryAction;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.RepositoryVerificationException;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Collection;
+
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsString;
+
+public class AzureRepositoryMissingCredentialsIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopyNoNullElements(super.nodePlugins(), AzureRepositoryPlugin.class);
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString(AzureStorageSettings.ACCOUNT_SETTING.getConcreteSettingForNamespace("default").getKey(), "test-account");
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)).setSecureSettings(secureSettings).build();
+    }
+
+    public void testMissingCredentialsException() {
+        assertThat(
+            asInstanceOf(
+                RepositoryVerificationException.class,
+                ExceptionsHelper.unwrapCause(
+                    safeAwaitFailure(
+                        AcknowledgedResponse.class,
+                        l -> client().execute(
+                            TransportPutRepositoryAction.TYPE,
+                            new PutRepositoryRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "test-repo").type("azure"),
+                            l
+                        )
+                    )
+                )
+            ).getCause().getMessage(),
+            allOf(
+                containsString("EnvironmentCredential authentication unavailable"),
+                containsString("WorkloadIdentityCredential authentication unavailable"),
+                containsString("Managed Identity authentication is not available"),
+                containsString("SharedTokenCacheCredential authentication unavailable")
+            )
+        );
+    }
+}
diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
index 5436802bff71a..80bd788da9a9f 100644
--- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
+++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
@@ -10,10 +10,12 @@
 
 import fixture.azure.AzureHttpFixture;
 
+import com.azure.core.exception.HttpResponseException;
 import com.azure.storage.blob.BlobContainerClient;
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.blob.models.BlobStorageException;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -44,11 +46,16 @@
 public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
     private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.azure.fixture", "true"));
 
+    private static final String AZURE_ACCOUNT = System.getProperty("test.azure.account");
+
     @ClassRule
     public static AzureHttpFixture fixture = new AzureHttpFixture(
-        USE_FIXTURE,
-        System.getProperty("test.azure.account"),
-        System.getProperty("test.azure.container")
+        USE_FIXTURE ? AzureHttpFixture.Protocol.HTTP : AzureHttpFixture.Protocol.NONE,
+        AZURE_ACCOUNT,
+        System.getProperty("test.azure.container"),
+        System.getProperty("test.azure.tenant_id"),
+        System.getProperty("test.azure.client_id"),
+        AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_ACCOUNT)
     );
 
     @Override
@@ -183,8 +190,8 @@ public void testMultiBlockUpload() throws Exception {
 
     public void testReadFromPositionLargerThanBlobLength() {
         testReadFromPositionLargerThanBlobLength(
-            e -> asInstanceOf(BlobStorageException.class, e.getCause()).getStatusCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED
-                .getStatus()
+            e -> asInstanceOf(BlobStorageException.class, ExceptionsHelper.unwrap(e, HttpResponseException.class))
+                .getStatusCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()
         );
     }
 }
diff --git a/modules/repository-azure/src/main/java/module-info.java b/modules/repository-azure/src/main/java/module-info.java
index cc34996ee2d1f..bef2a175a112f 100644
--- a/modules/repository-azure/src/main/java/module-info.java
+++ b/modules/repository-azure/src/main/java/module-info.java
@@ -21,6 +21,7 @@
     requires com.azure.http.netty;
     requires com.azure.storage.blob;
     requires com.azure.storage.common;
+    requires com.azure.identity;
 
     requires io.netty.buffer;
     requires io.netty.transport;
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
index d271c8d1e99a1..eb643b3c1460d 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
@@ -8,11 +8,11 @@
 
 package org.elasticsearch.repositories.azure;
 
-import com.azure.storage.blob.models.BlobStorageException;
+import com.azure.core.exception.HttpResponseException;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.core.util.Throwables;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobContainer;
@@ -69,13 +69,13 @@ private InputStream openInputStream(OperationPurpose purpose, String blobName, l
         try {
             return blobStore.getInputStream(blobKey, position, length);
         } catch (Exception e) {
-            Throwable rootCause = Throwables.getRootCause(e);
-            if (rootCause instanceof BlobStorageException blobStorageException) {
-                if (blobStorageException.getStatusCode() == RestStatus.NOT_FOUND.getStatus()) {
+            if (ExceptionsHelper.unwrap(e, HttpResponseException.class) instanceof HttpResponseException httpResponseException) {
+                final var httpStatusCode = httpResponseException.getResponse().getStatusCode();
+                if (httpStatusCode == RestStatus.NOT_FOUND.getStatus()) {
                     throw new NoSuchFileException("Blob [" + blobKey + "] not found");
                 }
-                if (blobStorageException.getStatusCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()) {
-                    throw new RequestedRangeNotSatisfiedException(blobKey, position, length == null ? -1 : length, blobStorageException);
+                if (httpStatusCode == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()) {
+                    throw new RequestedRangeNotSatisfiedException(blobKey, position, length == null ? -1 : length, e);
                 }
             }
             throw new IOException("Unable to get input stream for blob [" + blobKey + "]", e);
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java
index cdfd83b79b370..911d1c167db7f 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java
@@ -28,6 +28,7 @@
 import com.azure.core.http.ProxyOptions;
 import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
 import com.azure.core.http.policy.HttpPipelinePolicy;
+import com.azure.identity.DefaultAzureCredentialBuilder;
 import com.azure.storage.blob.BlobServiceAsyncClient;
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.blob.BlobServiceClientBuilder;
@@ -62,6 +63,17 @@ class AzureClientProvider extends AbstractLifecycleComponent {
     private static final int DEFAULT_EVENT_LOOP_THREAD_COUNT = 1;
     private static final int PENDING_CONNECTION_QUEUE_SIZE = -1; // see ConnectionProvider.ConnectionPoolSpec.pendingAcquireMaxCount
 
+    /**
+     * Test-only system property to disable instance discovery for workload identity authentication in the Azure SDK.
+     * This is necessary since otherwise the SDK will attempt to verify identities via a real host
+     * (e.g. <a href="https://login.microsoft.com/">https://login.microsoft.com/</a>) for
+     * workload identity authentication. This is incompatible with our test environment.
+     */
+    private static final boolean DISABLE_INSTANCE_DISCOVERY = System.getProperty(
+        "tests.azure.credentials.disable_instance_discovery",
+        "false"
+    ).equals("true");
+
     static final Setting<Integer> EVENT_LOOP_THREAD_COUNT = Setting.intSetting(
         "repository.azure.http_client.event_loop_executor_thread_count",
         DEFAULT_EVENT_LOOP_THREAD_COUNT,
@@ -168,6 +180,14 @@ AzureBlobServiceClient createClient(
             .httpClient(httpClient)
             .retryOptions(retryOptions);
 
+        if (settings.hasCredentials() == false) {
+            final DefaultAzureCredentialBuilder credentialBuilder = new DefaultAzureCredentialBuilder().executorService(eventLoopGroup);
+            if (DISABLE_INSTANCE_DISCOVERY) {
+                credentialBuilder.disableInstanceDiscovery();
+            }
+            builder.credential(credentialBuilder.build());
+        }
+
         if (successfulRequestConsumer != null) {
             builder.addPolicy(new SuccessfulRequestTracker(successfulRequestConsumer));
         }
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 388474acc75ea..c8c0b15db5ebe 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -26,6 +26,7 @@
 
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 import static org.elasticsearch.core.Strings.format;
@@ -175,4 +176,9 @@ protected ByteSizeValue chunkSize() {
     public boolean isReadOnly() {
         return readonly;
     }
+
+    @Override
+    protected Set<String> getExtraUsageFeatures() {
+        return storageService.getExtraUsageFeatures(Repository.CLIENT_NAME.get(getMetadata().settings()));
+    }
 }
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
index 73d969ee31b19..f6b1c2775926c 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
@@ -27,6 +27,8 @@
 import org.elasticsearch.threadpool.ScalingExecutorBuilder;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -42,9 +44,8 @@ public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin, R
     public static final String NETTY_EVENT_LOOP_THREAD_POOL_NAME = "azure_event_loop";
 
     static {
-        // Trigger static initialization with the plugin class loader
-        // so we have access to the proper xml parser
-        JacksonAdapter.createDefaultSerializerAdapter();
+        // Trigger static initialization with the plugin class loader so we have access to the proper xml parser
+        AccessController.doPrivileged((PrivilegedAction<Object>) JacksonAdapter::createDefaultSerializerAdapter);
     }
 
     // protected for testing
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
index 0d6cd7bf3d246..09088004759a8 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
@@ -24,6 +24,7 @@
 import java.net.Proxy;
 import java.net.URL;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.BiConsumer;
 
 import static java.util.Collections.emptyMap;
@@ -165,4 +166,15 @@ public void refreshSettings(Map<String, AzureStorageSettings> clientsSettings) {
         this.storageSettings = Map.copyOf(clientsSettings);
         // clients are built lazily by {@link client(String, LocationMode)}
     }
+
+    /**
+     * For Azure repositories, we report the different kinds of credentials in use in the telemetry.
+     */
+    public Set<String> getExtraUsageFeatures(String clientName) {
+        try {
+            return getClientSettings(clientName).credentialsUsageFeatures();
+        } catch (Exception e) {
+            return Set.of();
+        }
+    }
 }
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java
index 93a388dea98ae..2333a1fdb9e93 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java
@@ -29,6 +29,7 @@
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
 final class AzureStorageSettings {
 
@@ -124,15 +125,13 @@ final class AzureStorageSettings {
     );
 
     private final String account;
-
-    @Nullable
-    private final String sasToken;
-
     private final String connectString;
     private final String endpointSuffix;
     private final TimeValue timeout;
     private final int maxRetries;
     private final Proxy proxy;
+    private final boolean hasCredentials;
+    private final Set<String> credentialsUsageFeatures;
 
     private AzureStorageSettings(
         String account,
@@ -148,11 +147,17 @@ private AzureStorageSettings(
         String secondaryEndpoint
     ) {
         this.account = account;
-        this.sasToken = sasToken;
         this.connectString = buildConnectString(account, key, sasToken, endpointSuffix, endpoint, secondaryEndpoint);
+        this.hasCredentials = Strings.hasText(key) || Strings.hasText(sasToken);
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
+        this.credentialsUsageFeatures = Strings.hasText(key) ? Set.of("uses_key_credentials")
+            : Strings.hasText(sasToken) ? Set.of("uses_sas_token")
+            : SocketAccess.doPrivilegedException(() -> System.getenv("AZURE_FEDERATED_TOKEN_FILE")) == null
+                ? Set.of("uses_default_credentials", "uses_managed_identity")
+            : Set.of("uses_default_credentials", "uses_workload_identity");
+
         // Register the proxy if we have any
         // Validate proxy settings
         if (proxyType.equals(Proxy.Type.DIRECT) && ((proxyPort != 0) || Strings.hasText(proxyHost))) {
@@ -203,33 +208,32 @@ private static String buildConnectString(
     ) {
         final boolean hasSasToken = Strings.hasText(sasToken);
         final boolean hasKey = Strings.hasText(key);
-        if (hasSasToken == false && hasKey == false) {
-            throw new SettingsException("Neither a secret key nor a shared access token was set.");
-        }
         if (hasSasToken && hasKey) {
-            throw new SettingsException("Both a secret as well as a shared access token were set.");
+            throw new SettingsException("Both a secret as well as a shared access token were set for account [" + account + "]");
         }
         final StringBuilder connectionStringBuilder = new StringBuilder();
         connectionStringBuilder.append("DefaultEndpointsProtocol=https").append(";AccountName=").append(account);
         if (hasKey) {
             connectionStringBuilder.append(";AccountKey=").append(key);
-        } else {
+        } else if (hasSasToken) {
             connectionStringBuilder.append(";SharedAccessSignature=").append(sasToken);
+        } else {
+            connectionStringBuilder.append(";AccountKey=none"); // required for validation, but ignored
         }
         final boolean hasEndpointSuffix = Strings.hasText(endpointSuffix);
         final boolean hasEndpoint = Strings.hasText(endpoint);
         final boolean hasSecondaryEndpoint = Strings.hasText(secondaryEndpoint);
 
         if (hasEndpointSuffix && hasEndpoint) {
-            throw new SettingsException("Both an endpoint suffix as well as a primary endpoint were set");
+            throw new SettingsException("Both an endpoint suffix as well as a primary endpoint were set for account [" + account + "]");
         }
 
         if (hasEndpointSuffix && hasSecondaryEndpoint) {
-            throw new SettingsException("Both an endpoint suffix as well as a secondary endpoint were set");
+            throw new SettingsException("Both an endpoint suffix as well as a secondary endpoint were set for account [" + account + "]");
         }
 
         if (hasEndpoint == false && hasSecondaryEndpoint) {
-            throw new SettingsException("A primary endpoint is required when setting a secondary endpoint");
+            throw new SettingsException("A primary endpoint is required when setting a secondary endpoint for account [" + account + "]");
         }
 
         if (hasEndpointSuffix) {
@@ -318,6 +322,10 @@ private static <T> T getValue(Settings settings, String groupName, Setting<T> se
     private static final String BLOB_ENDPOINT_NAME = "BlobEndpoint";
     private static final String BLOB_SECONDARY_ENDPOINT_NAME = "BlobSecondaryEndpoint";
 
+    public boolean hasCredentials() {
+        return hasCredentials;
+    }
+
     record StorageEndpoint(String primaryURI, @Nullable String secondaryURI) {}
 
     StorageEndpoint getStorageEndpoint() {
@@ -366,4 +374,8 @@ private String deriveURIFromSettings(boolean isPrimary) {
             throw new IllegalArgumentException(e);
         }
     }
+
+    public Set<String> credentialsUsageFeatures() {
+        return credentialsUsageFeatures;
+    }
 }
diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java
index e9b438339f361..79fa2afe5d49e 100644
--- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java
+++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java
@@ -92,9 +92,10 @@ public void testCanNotCreateAClientWithSecondaryLocationWithoutAProperEndpoint()
 
         LocationMode locationMode = LocationMode.SECONDARY_ONLY;
         RequestRetryOptions requestRetryOptions = new RequestRetryOptions();
-        expectThrows(IllegalArgumentException.class, () -> {
-            azureClientProvider.createClient(storageSettings, locationMode, requestRetryOptions, null, EMPTY_CONSUMER);
-        });
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> azureClientProvider.createClient(storageSettings, locationMode, requestRetryOptions, null, EMPTY_CONSUMER)
+        );
     }
 
     private static String encodeKey(final String value) {
diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
index 6f43f253db4c8..3f9fba35d9f4a 100644
--- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
+++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
@@ -186,23 +186,20 @@ public void testReinitClientWrongSettings() throws IOException {
         secureSettings1.setString("azure.client.azure1.account", "myaccount1");
         secureSettings1.setString("azure.client.azure1.key", encodeKey("mykey11"));
         final Settings settings1 = Settings.builder().setSecureSettings(secureSettings1).build();
+
         final MockSecureSettings secureSettings2 = new MockSecureSettings();
-        secureSettings2.setString("azure.client.azure1.account", "myaccount1");
-        // missing key
+        secureSettings2.setString("azure.client.azure1.account", "myaccount3");
+        secureSettings2.setString("azure.client.azure1.key", encodeKey("mykey33"));
+        secureSettings2.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33"));
         final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build();
-        final MockSecureSettings secureSettings3 = new MockSecureSettings();
-        secureSettings3.setString("azure.client.azure1.account", "myaccount3");
-        secureSettings3.setString("azure.client.azure1.key", encodeKey("mykey33"));
-        secureSettings3.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33"));
-        final Settings settings3 = Settings.builder().setSecureSettings(secureSettings3).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService.get();
             AzureBlobServiceClient client11 = azureStorageService.client("azure1", LocationMode.PRIMARY_ONLY);
             assertThat(client11.getSyncClient().getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
-            final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2));
-            assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set."));
-            final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3));
-            assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set."));
+            assertThat(
+                expectThrows(SettingsException.class, () -> plugin.reload(settings2)).getMessage(),
+                is("Both a secret as well as a shared access token were set for account [myaccount3]")
+            );
             // existing client untouched
             assertThat(client11.getSyncClient().getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
         }
@@ -499,7 +496,7 @@ public void testEndpointSettingValidation() {
                         .build()
                 )
             );
-            assertEquals("A primary endpoint is required when setting a secondary endpoint", e.getMessage());
+            assertEquals("A primary endpoint is required when setting a secondary endpoint for account [myaccount1]", e.getMessage());
         }
 
         {
@@ -513,7 +510,7 @@ public void testEndpointSettingValidation() {
                         .build()
                 )
             );
-            assertEquals("Both an endpoint suffix as well as a secondary endpoint were set", e.getMessage());
+            assertEquals("Both an endpoint suffix as well as a secondary endpoint were set for account [myaccount1]", e.getMessage());
         }
     }
 
diff --git a/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java b/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
index b9d5705aff91e..515b1aa2b6d6a 100644
--- a/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
+++ b/modules/repository-azure/src/yamlRestTest/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
@@ -13,22 +13,56 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestTrustStore;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import java.util.Map;
+import java.util.function.Predicate;
+
 public class RepositoryAzureClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.azure.fixture", "true"));
+    private static final boolean USE_HTTPS_FIXTURE = USE_FIXTURE && ESTestCase.inFipsJvm() == false;
+    // TODO when https://github.com/elastic/elasticsearch/issues/111532 addressed, use a HTTPS fixture in FIPS mode too
+
     private static final String AZURE_TEST_ACCOUNT = System.getProperty("test.azure.account");
     private static final String AZURE_TEST_CONTAINER = System.getProperty("test.azure.container");
     private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key");
     private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token");
+    private static final String AZURE_TEST_TENANT_ID = System.getProperty("test.azure.tenant_id");
+    private static final String AZURE_TEST_CLIENT_ID = System.getProperty("test.azure.client_id");
 
-    private static AzureHttpFixture fixture = new AzureHttpFixture(USE_FIXTURE, AZURE_TEST_ACCOUNT, AZURE_TEST_CONTAINER);
+    private static final AzureHttpFixture fixture = new AzureHttpFixture(
+        USE_HTTPS_FIXTURE ? AzureHttpFixture.Protocol.HTTPS : USE_FIXTURE ? AzureHttpFixture.Protocol.HTTP : AzureHttpFixture.Protocol.NONE,
+        AZURE_TEST_ACCOUNT,
+        AZURE_TEST_CONTAINER,
+        AZURE_TEST_TENANT_ID,
+        AZURE_TEST_CLIENT_ID,
+        decideAuthHeaderPredicate()
+    );
+
+    private static Predicate<String> decideAuthHeaderPredicate() {
+        if (Strings.hasText(AZURE_TEST_KEY) || Strings.hasText(AZURE_TEST_SASTOKEN)) {
+            return AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_TEST_ACCOUNT);
+        } else if (Strings.hasText(AZURE_TEST_TENANT_ID) && Strings.hasText(AZURE_TEST_CLIENT_ID)) {
+            return AzureHttpFixture.WORK_IDENTITY_BEARER_TOKEN_PREDICATE;
+        } else if (Strings.hasText(AZURE_TEST_TENANT_ID) || Strings.hasText(AZURE_TEST_CLIENT_ID)) {
+            fail(null, "Both [test.azure.tenant_id] and [test.azure.client_id] must be set if either is set");
+        }
+        return AzureHttpFixture.MANAGED_IDENTITY_BEARER_TOKEN_PREDICATE;
+    }
+
+    private static TestTrustStore trustStore = new TestTrustStore(
+        () -> AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem")
+    );
 
     private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-azure")
@@ -45,14 +79,31 @@ public class RepositoryAzureClientYamlTestSuiteIT extends ESClientYamlSuiteTestC
         )
         .setting(
             "azure.client.integration_test.endpoint_suffix",
-            () -> "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + fixture.getAddress(),
+            () -> "ignored;DefaultEndpointsProtocol=https;BlobEndpoint=" + fixture.getAddress(),
             s -> USE_FIXTURE
         )
+        .systemProperty(
+            "tests.azure.credentials.disable_instance_discovery",
+            () -> "true",
+            s -> USE_HTTPS_FIXTURE && Strings.hasText(AZURE_TEST_CLIENT_ID) && Strings.hasText(AZURE_TEST_TENANT_ID)
+        )
+        .systemProperty("AZURE_POD_IDENTITY_AUTHORITY_HOST", fixture::getMetadataAddress, s -> USE_FIXTURE)
+        .systemProperty("AZURE_AUTHORITY_HOST", fixture::getOAuthTokenServiceAddress, s -> USE_HTTPS_FIXTURE)
+        .systemProperty("AZURE_CLIENT_ID", () -> AZURE_TEST_CLIENT_ID, s -> Strings.hasText(AZURE_TEST_CLIENT_ID))
+        .systemProperty("AZURE_TENANT_ID", () -> AZURE_TEST_TENANT_ID, s -> Strings.hasText(AZURE_TEST_TENANT_ID))
+        .configFile("storage-azure/azure-federated-token", Resource.fromString(fixture.getFederatedToken()))
+        .environment(
+            nodeSpec -> USE_HTTPS_FIXTURE && Strings.hasText(AZURE_TEST_CLIENT_ID) && Strings.hasText(AZURE_TEST_TENANT_ID)
+                ? Map.of("AZURE_FEDERATED_TOKEN_FILE", "${ES_PATH_CONF}/storage-azure/azure-federated-token")
+                : Map.of()
+        )
         .setting("thread_pool.repository_azure.max", () -> String.valueOf(randomIntBetween(1, 10)), s -> USE_FIXTURE)
+        .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_HTTPS_FIXTURE)
+        .systemProperty("javax.net.ssl.trustStoreType", () -> "jks", s -> USE_HTTPS_FIXTURE)
         .build();
 
-    @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
+    @ClassRule(order = 1)
+    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster);
 
     public RepositoryAzureClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
diff --git a/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml b/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml
index 299183f26d9dc..a4a7d0b22a0ed 100644
--- a/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml
+++ b/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml
@@ -235,6 +235,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.azure.count: 1 }
+  - gte: { repositories.azure.read_write: 1 }
+
 ---
 teardown:
 
diff --git a/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml b/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml
index 68d61be4983c5..e8c34a4b6a20b 100644
--- a/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml
+++ b/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml
@@ -232,6 +232,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.gcs.count: 1 }
+  - gte: { repositories.gcs.read_write: 1 }
+
 ---
 teardown:
 
diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java
index 640293ecb80b0..31fa47fb7b196 100644
--- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java
+++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java
@@ -36,8 +36,9 @@
 import java.util.Map;
 import java.util.Queue;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.repositories.RepositoriesMetrics.HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM;
+import static org.elasticsearch.repositories.RepositoriesMetrics.HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM;
 import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_EXCEPTIONS_HISTOGRAM;
 import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL;
 import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_EXCEPTIONS_TOTAL;
@@ -52,6 +53,7 @@
 import static org.elasticsearch.rest.RestStatus.TOO_MANY_REQUESTS;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint")
 // Need to set up a new cluster for each test because cluster settings use randomized authentication settings
@@ -98,6 +100,35 @@ private static BlobContainer getBlobContainer(String dataNodeName, String reposi
         return blobStoreRepository.blobStore().blobContainer(BlobPath.EMPTY.add(randomIdentifier()));
     }
 
+    public void testHttpRequestTimeCaptureInMilliseconds() throws IOException {
+        final String repository = createRepository(randomRepositoryName());
+        final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData);
+        final TestTelemetryPlugin plugin = getPlugin(dataNodeName);
+        final OperationPurpose purpose = randomFrom(OperationPurpose.values());
+        final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository);
+        final String blobName = randomIdentifier();
+
+        long before = System.nanoTime();
+        blobContainer.writeBlob(purpose, blobName, new BytesArray(randomBytes(between(10, 1000))), false);
+        long elapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - before);
+        assertThat(getLongHistogramValue(plugin, HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM, Operation.PUT_OBJECT), lessThanOrEqualTo(elapsed));
+
+        plugin.resetMeter();
+        before = System.nanoTime();
+        blobContainer.readBlob(purpose, blobName).close();
+        elapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - before);
+        assertThat(getLongHistogramValue(plugin, HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM, Operation.GET_OBJECT), lessThanOrEqualTo(elapsed));
+
+        plugin.resetMeter();
+        before = System.nanoTime();
+        blobContainer.deleteBlobsIgnoringIfNotExists(purpose, Iterators.single(blobName));
+        elapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - before);
+        assertThat(
+            getLongHistogramValue(plugin, HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM, Operation.DELETE_OBJECTS),
+            lessThanOrEqualTo(elapsed)
+        );
+    }
+
     public void testMetricsWithErrors() throws IOException {
         final String repository = createRepository(randomRepositoryName());
 
@@ -121,7 +152,7 @@ public void testMetricsWithErrors() throws IOException {
             assertThat(getLongCounterValue(plugin, METRIC_THROTTLES_TOTAL, Operation.PUT_OBJECT), equalTo(2L * batch));
             assertThat(getLongHistogramValue(plugin, METRIC_EXCEPTIONS_HISTOGRAM, Operation.PUT_OBJECT), equalTo(batch));
             assertThat(getLongHistogramValue(plugin, METRIC_THROTTLES_HISTOGRAM, Operation.PUT_OBJECT), equalTo(2L * batch));
-            assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM, Operation.PUT_OBJECT), equalTo(batch));
+            assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM, Operation.PUT_OBJECT), equalTo(batch));
         }
 
         // Get not found
@@ -141,7 +172,7 @@ public void testMetricsWithErrors() throws IOException {
             assertThat(getLongCounterValue(plugin, METRIC_THROTTLES_TOTAL, Operation.GET_OBJECT), equalTo(batch));
             assertThat(getLongHistogramValue(plugin, METRIC_EXCEPTIONS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch));
             assertThat(getLongHistogramValue(plugin, METRIC_THROTTLES_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch));
-            assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch));
+            assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch));
 
             // Make sure we don't hit the request range not satisfied counters
             assertThat(getLongCounterValue(plugin, METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL, Operation.GET_OBJECT), equalTo(0L));
@@ -164,7 +195,7 @@ public void testMetricsWithErrors() throws IOException {
             assertThat(getLongCounterValue(plugin, METRIC_THROTTLES_TOTAL, Operation.LIST_OBJECTS), equalTo(5L * batch));
             assertThat(getLongHistogramValue(plugin, METRIC_EXCEPTIONS_HISTOGRAM, Operation.LIST_OBJECTS), equalTo(batch));
             assertThat(getLongHistogramValue(plugin, METRIC_THROTTLES_HISTOGRAM, Operation.LIST_OBJECTS), equalTo(5L * batch));
-            assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM, Operation.LIST_OBJECTS), equalTo(batch));
+            assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM, Operation.LIST_OBJECTS), equalTo(batch));
         }
 
         // Delete to clean up
@@ -176,7 +207,7 @@ public void testMetricsWithErrors() throws IOException {
         assertThat(getLongCounterValue(plugin, METRIC_THROTTLES_TOTAL, Operation.DELETE_OBJECTS), equalTo(0L));
         assertThat(getLongHistogramValue(plugin, METRIC_EXCEPTIONS_HISTOGRAM, Operation.DELETE_OBJECTS), equalTo(0L));
         assertThat(getLongHistogramValue(plugin, METRIC_THROTTLES_HISTOGRAM, Operation.DELETE_OBJECTS), equalTo(0L));
-        assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM, Operation.DELETE_OBJECTS), equalTo(1L));
+        assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM, Operation.DELETE_OBJECTS), equalTo(1L));
     }
 
     public void testMetricsForRequestRangeNotSatisfied() {
@@ -208,7 +239,7 @@ public void testMetricsForRequestRangeNotSatisfied() {
             );
             assertThat(getLongCounterValue(plugin, METRIC_THROTTLES_TOTAL, Operation.GET_OBJECT), equalTo(2 * batch));
             assertThat(getLongHistogramValue(plugin, METRIC_THROTTLES_HISTOGRAM, Operation.GET_OBJECT), equalTo(2 * batch));
-            assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch));
+            assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch));
         }
     }
 
diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
index 1132111826563..1ab370ad203fc 100644
--- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -10,13 +10,20 @@
 import fixture.s3.S3HttpHandler;
 
 import com.amazonaws.http.AmazonHttpClient;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
+import com.amazonaws.services.s3.model.MultipartUpload;
 import com.sun.net.httpserver.Headers;
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.LogEvent;
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
@@ -54,6 +61,7 @@
 import org.elasticsearch.telemetry.TestTelemetryPlugin;
 import org.elasticsearch.test.BackgroundIndexer;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.MockLog;
 import org.elasticsearch.test.junit.annotations.TestIssueLogging;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -70,6 +78,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
@@ -81,6 +90,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasEntry;
@@ -451,6 +461,106 @@ private Map<S3BlobStore.StatsKey, Long> getServerMetrics() {
         return Collections.emptyMap();
     }
 
+    public void testMultipartUploadCleanup() {
+        final String repoName = randomRepositoryName();
+        createRepository(repoName, repositorySettings(repoName), true);
+
+        createIndex("test-idx-1");
+        for (int i = 0; i < 100; i++) {
+            prepareIndex("test-idx-1").setId(Integer.toString(i)).setSource("foo", "bar" + i).get();
+        }
+        client().admin().indices().prepareRefresh().get();
+
+        final String snapshotName = randomIdentifier();
+        CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repoName, snapshotName)
+            .setWaitForCompletion(true)
+            .get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+        assertThat(
+            createSnapshotResponse.getSnapshotInfo().successfulShards(),
+            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())
+        );
+
+        final var repository = asInstanceOf(
+            S3Repository.class,
+            internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class).repository(repoName)
+        );
+        final var blobStore = asInstanceOf(S3BlobStore.class, asInstanceOf(BlobStoreWrapper.class, repository.blobStore()).delegate());
+
+        try (var clientRef = blobStore.clientReference()) {
+            final var danglingBlobName = randomIdentifier();
+            final var initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(
+                blobStore.bucket(),
+                blobStore.blobContainer(repository.basePath().add("test-multipart-upload")).path().buildAsString() + danglingBlobName
+            );
+            initiateMultipartUploadRequest.putCustomQueryParameter(
+                S3BlobStore.CUSTOM_QUERY_PARAMETER_PURPOSE,
+                OperationPurpose.SNAPSHOT_DATA.getKey()
+            );
+            final var multipartUploadResult = clientRef.client().initiateMultipartUpload(initiateMultipartUploadRequest);
+
+            final var listMultipartUploadsRequest = new ListMultipartUploadsRequest(blobStore.bucket()).withPrefix(
+                repository.basePath().buildAsString()
+            );
+            listMultipartUploadsRequest.putCustomQueryParameter(
+                S3BlobStore.CUSTOM_QUERY_PARAMETER_PURPOSE,
+                OperationPurpose.SNAPSHOT_DATA.getKey()
+            );
+            assertEquals(
+                List.of(multipartUploadResult.getUploadId()),
+                clientRef.client()
+                    .listMultipartUploads(listMultipartUploadsRequest)
+                    .getMultipartUploads()
+                    .stream()
+                    .map(MultipartUpload::getUploadId)
+                    .toList()
+            );
+
+            final var seenCleanupLogLatch = new CountDownLatch(1);
+            MockLog.assertThatLogger(() -> {
+                assertAcked(clusterAdmin().prepareDeleteSnapshot(TEST_REQUEST_TIMEOUT, repoName, snapshotName));
+                safeAwait(seenCleanupLogLatch);
+            },
+                S3BlobContainer.class,
+                new MockLog.SeenEventExpectation(
+                    "found-dangling",
+                    S3BlobContainer.class.getCanonicalName(),
+                    Level.INFO,
+                    "found [1] possibly-dangling multipart uploads; will clean them up after finalizing the current snapshot deletions"
+                ),
+                new MockLog.SeenEventExpectation(
+                    "cleaned-dangling",
+                    S3BlobContainer.class.getCanonicalName(),
+                    Level.INFO,
+                    Strings.format(
+                        "cleaned up dangling multipart upload [%s] of blob [%s]*test-multipart-upload/%s]",
+                        multipartUploadResult.getUploadId(),
+                        repoName,
+                        danglingBlobName
+                    )
+                ) {
+                    @Override
+                    public void match(LogEvent event) {
+                        super.match(event);
+                        if (Regex.simpleMatch(message, event.getMessage().getFormattedMessage())) {
+                            seenCleanupLogLatch.countDown();
+                        }
+                    }
+                }
+            );
+
+            assertThat(
+                clientRef.client()
+                    .listMultipartUploads(listMultipartUploadsRequest)
+                    .getMultipartUploads()
+                    .stream()
+                    .map(MultipartUpload::getUploadId)
+                    .toList(),
+                empty()
+            );
+        }
+    }
+
     /**
      * S3RepositoryPlugin that allows to disable chunked encoding and to set a low threshold between single upload and multipart upload.
      */
@@ -592,6 +702,9 @@ public void maybeTrack(final String rawRequest, Headers requestHeaders) {
                 trackRequest("ListObjects");
                 metricsCount.computeIfAbsent(new S3BlobStore.StatsKey(S3BlobStore.Operation.LIST_OBJECTS, purpose), k -> new AtomicLong())
                     .incrementAndGet();
+            } else if (Regex.simpleMatch("GET /*/?uploads&*", request)) {
+                // TODO track ListMultipartUploads requests
+                logger.info("--> ListMultipartUploads not tracked [{}] with parsed purpose [{}]", request, purpose.getKey());
             } else if (Regex.simpleMatch("GET /*/*", request)) {
                 trackRequest("GetObject");
                 metricsCount.computeIfAbsent(new S3BlobStore.StatsKey(S3BlobStore.Operation.GET_OBJECT, purpose), k -> new AtomicLong())
diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
index b5fc37e859b9b..cf3e73df2aee2 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
@@ -28,13 +28,17 @@
 import com.amazonaws.services.s3.model.UploadPartResult;
 import com.amazonaws.util.ValidationUtils;
 
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.RefCountingListener;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.action.support.ThreadedActionListener;
+import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobContainer;
@@ -54,6 +58,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.ChunkedBlobOutputStream;
 import org.elasticsearch.repositories.s3.S3BlobStore.Operation;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -644,7 +649,7 @@ void run(BytesReference expected, BytesReference updated, ActionListener<Optiona
 
                 // Step 4: Read the current register value.
 
-                .<OptionalBytesReference>andThen((l, ignored) -> getRegister(purpose, rawKey, l))
+                .<OptionalBytesReference>andThen(l -> getRegister(purpose, rawKey, l))
 
                 // Step 5: Perform the compare-and-swap by completing our upload iff the witnessed value matches the expected value.
 
@@ -912,4 +917,94 @@ public void getRegister(OperationPurpose purpose, String key, ActionListener<Opt
             }
         });
     }
+
+    ActionListener<Void> getMultipartUploadCleanupListener(int maxUploads, RefCountingRunnable refs) {
+        try (var clientReference = blobStore.clientReference()) {
+            final var bucket = blobStore.bucket();
+            final var request = new ListMultipartUploadsRequest(bucket).withPrefix(keyPath).withMaxUploads(maxUploads);
+            request.putCustomQueryParameter(S3BlobStore.CUSTOM_QUERY_PARAMETER_PURPOSE, OperationPurpose.SNAPSHOT_DATA.getKey());
+            final var multipartUploadListing = SocketAccess.doPrivileged(() -> clientReference.client().listMultipartUploads(request));
+            final var multipartUploads = multipartUploadListing.getMultipartUploads();
+            if (multipartUploads.isEmpty()) {
+                logger.debug("found no multipart uploads to clean up");
+                return ActionListener.noop();
+            } else {
+                // the uploads are only _possibly_ dangling because it's also possible we're no longer then master and the new master has
+                // started some more shard snapshots
+                if (multipartUploadListing.isTruncated()) {
+                    logger.info("""
+                        found at least [{}] possibly-dangling multipart uploads; will clean up the first [{}] after finalizing \
+                        the current snapshot deletions, and will check for further possibly-dangling multipart uploads in future \
+                        snapshot deletions""", multipartUploads.size(), multipartUploads.size());
+                } else {
+                    logger.info("""
+                        found [{}] possibly-dangling multipart uploads; \
+                        will clean them up after finalizing the current snapshot deletions""", multipartUploads.size());
+                }
+                return newMultipartUploadCleanupListener(
+                    refs,
+                    multipartUploads.stream().map(u -> new AbortMultipartUploadRequest(bucket, u.getKey(), u.getUploadId())).toList()
+                );
+            }
+        } catch (Exception e) {
+            // Cleanup is a best-effort thing, we can't do anything better than log and carry on here.
+            logger.warn("failure while checking for possibly-dangling multipart uploads", e);
+            return ActionListener.noop();
+        }
+    }
+
+    private ActionListener<Void> newMultipartUploadCleanupListener(
+        RefCountingRunnable refs,
+        List<AbortMultipartUploadRequest> abortMultipartUploadRequests
+    ) {
+        return new ThreadedActionListener<>(blobStore.getSnapshotExecutor(), ActionListener.releaseAfter(new ActionListener<>() {
+            @Override
+            public void onResponse(Void unused) {
+                try (var clientReference = blobStore.clientReference()) {
+                    for (final var abortMultipartUploadRequest : abortMultipartUploadRequests) {
+                        abortMultipartUploadRequest.putCustomQueryParameter(
+                            S3BlobStore.CUSTOM_QUERY_PARAMETER_PURPOSE,
+                            OperationPurpose.SNAPSHOT_DATA.getKey()
+                        );
+                        try {
+                            SocketAccess.doPrivilegedVoid(() -> clientReference.client().abortMultipartUpload(abortMultipartUploadRequest));
+                            logger.info(
+                                "cleaned up dangling multipart upload [{}] of blob [{}][{}][{}]",
+                                abortMultipartUploadRequest.getUploadId(),
+                                blobStore.getRepositoryMetadata().name(),
+                                abortMultipartUploadRequest.getBucketName(),
+                                abortMultipartUploadRequest.getKey()
+                            );
+                        } catch (Exception e) {
+                            // Cleanup is a best-effort thing, we can't do anything better than log and carry on here. Note that any failure
+                            // is surprising, even a 404 means that something else aborted/completed the upload at a point where there
+                            // should be no other processes interacting with the repository.
+                            logger.warn(
+                                Strings.format(
+                                    "failed to clean up multipart upload [{}] of blob [{}][{}][{}]",
+                                    abortMultipartUploadRequest.getUploadId(),
+                                    blobStore.getRepositoryMetadata().name(),
+                                    abortMultipartUploadRequest.getBucketName(),
+                                    abortMultipartUploadRequest.getKey()
+                                ),
+                                e
+                            );
+                        }
+                    }
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.log(
+                    MasterService.isPublishFailureException(e)
+                        || (e instanceof RepositoryException repositoryException
+                            && repositoryException.getCause() instanceof Exception cause
+                            && MasterService.isPublishFailureException(cause)) ? Level.DEBUG : Level.WARN,
+                    "failed to start cleanup of dangling multipart uploads",
+                    e
+                );
+            }
+        }, refs.acquire()));
+    }
 }
diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
index 5af53364fb765..03605d50750f0 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
@@ -224,11 +224,13 @@ private void maybeRecordHttpRequestTime(Request<?> request) {
                 return;
             }
 
-            final long totalTimeInMicros = getTotalTimeInMicros(requestTimesIncludingRetries);
-            if (totalTimeInMicros == 0) {
+            final long totalTimeInNanos = getTotalTimeInNanos(requestTimesIncludingRetries);
+            if (totalTimeInNanos == 0) {
                 logger.warn("Expected HttpRequestTime to be tracked for request [{}] but found no count.", request);
             } else {
-                s3RepositoriesMetrics.common().httpRequestTimeInMicroHistogram().record(totalTimeInMicros, attributes);
+                s3RepositoriesMetrics.common()
+                    .httpRequestTimeInMillisHistogram()
+                    .record(TimeUnit.NANOSECONDS.toMillis(totalTimeInNanos), attributes);
             }
         }
 
@@ -271,18 +273,20 @@ private static long getCountForMetric(TimingInfo info, AWSRequestMetrics.Field f
         }
     }
 
-    private static long getTotalTimeInMicros(List<TimingInfo> requestTimesIncludingRetries) {
-        // Here we calculate the timing in Microseconds for the sum of the individual subMeasurements with the goal of deriving the TTFB
-        // (time to first byte). We calculate the time in micros for later use with an APM style counter (exposed as a long), rather than
-        // using the default double exposed by getTimeTakenMillisIfKnown().
-        long totalTimeInMicros = 0;
+    private static long getTotalTimeInNanos(List<TimingInfo> requestTimesIncludingRetries) {
+        // Here we calculate the timing in Nanoseconds for the sum of the individual subMeasurements with the goal of deriving the TTFB
+        // (time to first byte). We use high precision time here to tell from the case when request time metric is missing (0).
+        // The time is converted to milliseconds for later use with an APM style counter (exposed as a long), rather than using the
+        // default double exposed by getTimeTakenMillisIfKnown().
+        // We don't need sub-millisecond precision. So no need perform the data type castings.
+        long totalTimeInNanos = 0;
         for (TimingInfo timingInfo : requestTimesIncludingRetries) {
             var endTimeInNanos = timingInfo.getEndTimeNanoIfKnown();
             if (endTimeInNanos != null) {
-                totalTimeInMicros += TimeUnit.NANOSECONDS.toMicros(endTimeInNanos - timingInfo.getStartTimeNano());
+                totalTimeInNanos += endTimeInNanos - timingInfo.getStartTimeNano();
             }
         }
-        return totalTimeInMicros;
+        return totalTimeInNanos;
     }
 
     @Override
diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index 72b48c5903629..d75a3e8ad433e 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ReferenceDocs;
@@ -28,6 +29,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.monitor.jvm.JvmInfo;
@@ -35,15 +37,17 @@
 import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository;
-import org.elasticsearch.snapshots.SnapshotDeleteListener;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
+import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 
@@ -183,6 +187,16 @@ class S3Repository extends MeteredBlobStoreRepository {
         S3BlobStore.MAX_BULK_DELETES
     );
 
+    /**
+     * Maximum number of uploads to request for cleanup when doing a snapshot delete.
+     */
+    static final Setting<Integer> MAX_MULTIPART_UPLOAD_CLEANUP_SIZE = Setting.intSetting(
+        "max_multipart_upload_cleanup_size",
+        1000,
+        0,
+        Setting.Property.Dynamic
+    );
+
     private final S3Service service;
 
     private final String bucket;
@@ -305,7 +319,7 @@ public void finalizeSnapshot(final FinalizeSnapshotContext finalizeSnapshotConte
                 finalizeSnapshotContext.clusterMetadata(),
                 finalizeSnapshotContext.snapshotInfo(),
                 finalizeSnapshotContext.repositoryMetaVersion(),
-                delayedListener(ActionListener.runAfter(finalizeSnapshotContext, () -> metadataDone.onResponse(null))),
+                wrapWithWeakConsistencyProtection(ActionListener.runAfter(finalizeSnapshotContext, () -> metadataDone.onResponse(null))),
                 info -> metadataDone.addListener(new ActionListener<>() {
                     @Override
                     public void onResponse(Void unused) {
@@ -324,50 +338,19 @@ public void onFailure(Exception e) {
         super.finalizeSnapshot(wrappedFinalizeContext);
     }
 
-    @Override
-    protected SnapshotDeleteListener wrapWithWeakConsistencyProtection(SnapshotDeleteListener listener) {
-        return new SnapshotDeleteListener() {
-            @Override
-            public void onDone() {
-                listener.onDone();
-            }
-
-            @Override
-            public void onRepositoryDataWritten(RepositoryData repositoryData) {
-                logCooldownInfo();
-                final Scheduler.Cancellable existing = finalizationFuture.getAndSet(threadPool.schedule(() -> {
-                    final Scheduler.Cancellable cancellable = finalizationFuture.getAndSet(null);
-                    assert cancellable != null;
-                    listener.onRepositoryDataWritten(repositoryData);
-                }, coolDown, snapshotExecutor));
-                assert existing == null : "Already have an ongoing finalization " + finalizationFuture;
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                logCooldownInfo();
-                final Scheduler.Cancellable existing = finalizationFuture.getAndSet(threadPool.schedule(() -> {
-                    final Scheduler.Cancellable cancellable = finalizationFuture.getAndSet(null);
-                    assert cancellable != null;
-                    listener.onFailure(e);
-                }, coolDown, snapshotExecutor));
-                assert existing == null : "Already have an ongoing finalization " + finalizationFuture;
-            }
-        };
-    }
-
     /**
      * Wraps given listener such that it is executed with a delay of {@link #coolDown} on the snapshot thread-pool after being invoked.
      * See {@link #COOLDOWN_PERIOD} for details.
      */
-    private <T> ActionListener<T> delayedListener(ActionListener<T> listener) {
-        final ActionListener<T> wrappedListener = ActionListener.runBefore(listener, () -> {
+    @Override
+    protected ActionListener<RepositoryData> wrapWithWeakConsistencyProtection(ActionListener<RepositoryData> listener) {
+        final ActionListener<RepositoryData> wrappedListener = ActionListener.runBefore(listener, () -> {
             final Scheduler.Cancellable cancellable = finalizationFuture.getAndSet(null);
             assert cancellable != null;
         });
         return new ActionListener<>() {
             @Override
-            public void onResponse(T response) {
+            public void onResponse(RepositoryData response) {
                 logCooldownInfo();
                 final Scheduler.Cancellable existing = finalizationFuture.getAndSet(
                     threadPool.schedule(ActionRunnable.wrap(wrappedListener, l -> l.onResponse(response)), coolDown, snapshotExecutor)
@@ -459,4 +442,75 @@ public String getAnalysisFailureExtraDetail() {
             ReferenceDocs.S3_COMPATIBLE_REPOSITORIES
         );
     }
+
+    // only one multipart cleanup process running at once
+    private final AtomicBoolean multipartCleanupInProgress = new AtomicBoolean();
+
+    @Override
+    public void deleteSnapshots(
+        Collection<SnapshotId> snapshotIds,
+        long repositoryDataGeneration,
+        IndexVersion minimumNodeVersion,
+        ActionListener<RepositoryData> repositoryDataUpdateListener,
+        Runnable onCompletion
+    ) {
+        getMultipartUploadCleanupListener(
+            isReadOnly() ? 0 : MAX_MULTIPART_UPLOAD_CLEANUP_SIZE.get(getMetadata().settings()),
+            new ActionListener<>() {
+                @Override
+                public void onResponse(ActionListener<Void> multipartUploadCleanupListener) {
+                    S3Repository.super.deleteSnapshots(snapshotIds, repositoryDataGeneration, minimumNodeVersion, new ActionListener<>() {
+                        @Override
+                        public void onResponse(RepositoryData repositoryData) {
+                            multipartUploadCleanupListener.onResponse(null);
+                            repositoryDataUpdateListener.onResponse(repositoryData);
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            multipartUploadCleanupListener.onFailure(e);
+                            repositoryDataUpdateListener.onFailure(e);
+                        }
+                    }, onCompletion);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    logger.warn("failed to get multipart uploads for cleanup during snapshot delete", e);
+                    assert false : e; // getMultipartUploadCleanupListener doesn't throw and snapshotExecutor doesn't reject anything
+                    repositoryDataUpdateListener.onFailure(e);
+                }
+            }
+        );
+    }
+
+    /**
+     * Capture the current list of multipart uploads, and (asynchronously) return a listener which, if completed successfully, aborts those
+     * uploads. Called at the start of a snapshot delete operation, at which point there should be no ongoing uploads (except in the case of
+     * a master failover). We protect against the master failover case by waiting until the delete operation successfully updates the root
+     * index-N blob before aborting any uploads.
+     */
+    void getMultipartUploadCleanupListener(int maxUploads, ActionListener<ActionListener<Void>> listener) {
+        if (maxUploads == 0) {
+            listener.onResponse(ActionListener.noop());
+            return;
+        }
+
+        if (multipartCleanupInProgress.compareAndSet(false, true) == false) {
+            logger.info("multipart upload cleanup already in progress");
+            listener.onResponse(ActionListener.noop());
+            return;
+        }
+
+        try (var refs = new RefCountingRunnable(() -> multipartCleanupInProgress.set(false))) {
+            snapshotExecutor.execute(
+                ActionRunnable.supply(
+                    ActionListener.releaseAfter(listener, refs.acquire()),
+                    () -> blobContainer() instanceof S3BlobContainer s3BlobContainer
+                        ? s3BlobContainer.getMultipartUploadCleanupListener(maxUploads, refs)
+                        : ActionListener.noop()
+                )
+            );
+        }
+    }
 }
diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml
index 77870697f93ae..e88a0861ec01c 100644
--- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml
+++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml
@@ -345,6 +345,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 
diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml
index 4a62d6183470d..501af980e17e3 100644
--- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml
+++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml
@@ -256,6 +256,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 
diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
index e24ff1ad0e559..129f0ba5d7588 100644
--- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
+++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
@@ -256,6 +256,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 
diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
index 9c332cc7d9301..de334b4b3df96 100644
--- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
+++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
@@ -256,6 +256,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 
diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml
index 24c2b2b1741d6..09a8526017960 100644
--- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml
+++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml
@@ -257,6 +257,19 @@ setup:
         snapshot: missing
         wait_for_completion: true
 
+---
+"Usage stats":
+  - requires:
+      cluster_features:
+        - repositories.supports_usage_stats
+      reason: requires this feature
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { repositories.s3.count: 1 }
+  - gte: { repositories.s3.read_write: 1 }
+
 ---
 teardown:
 
diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java
index 6b4b0a52b643a..fa48fc8d2907c 100644
--- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java
+++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java
@@ -14,9 +14,10 @@
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
@@ -33,7 +34,7 @@ public TransportMainAction(
         ActionFilters actionFilters,
         ClusterService clusterService
     ) {
-        super(MainRestPlugin.MAIN_ACTION.name(), actionFilters, transportService.getTaskManager());
+        super(MainRestPlugin.MAIN_ACTION.name(), actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.nodeName = Node.NODE_NAME_SETTING.get(settings);
         this.clusterService = clusterService;
     }
diff --git a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java
index 7f310caf4ea05..16d22407c25fa 100644
--- a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java
+++ b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java
@@ -73,7 +73,7 @@ public GrokHelper(TimeValue interval, TimeValue maxExecutionTime) {
                 return MatcherWatchdog.newInstance(
                     interval.millis(),
                     maxExecutionTime.millis(),
-                    threadPool::relativeTimeInMillis,
+                    threadPool.relativeTimeInMillisSupplier(),
                     (delay, command) -> threadPool.schedule(command, TimeValue.timeValueMillis(delay), threadPool.generic())
                 );
             })::getOrCompute;
diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java
index c4c35b410af78..7432a510763e0 100644
--- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java
+++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.ActionTestUtils;
 import org.elasticsearch.action.support.CountDownActionListener;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.TransportAction;
@@ -34,7 +33,6 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
@@ -57,6 +55,7 @@
 import org.elasticsearch.http.HttpRouteStats;
 import org.elasticsearch.http.HttpRouteStatsTracker;
 import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -71,6 +70,7 @@
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskCancelledException;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLog;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -435,13 +435,22 @@ public static class TransportYieldsContinuationsAction extends TransportAction<R
 
             @Inject
             public TransportYieldsContinuationsAction(ActionFilters actionFilters, TransportService transportService) {
-                super(TYPE.name(), actionFilters, transportService.getTaskManager());
-                executor = transportService.getThreadPool().executor(ThreadPool.Names.GENERIC);
+                this(actionFilters, transportService, transportService.getThreadPool().executor(ThreadPool.Names.GENERIC));
+            }
+
+            TransportYieldsContinuationsAction(ActionFilters actionFilters, TransportService transportService, ExecutorService executor) {
+                super(TYPE.name(), actionFilters, transportService.getTaskManager(), executor);
+                this.executor = executor;
             }
 
             @Override
             protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
-                executor.execute(ActionRunnable.supply(listener, () -> new Response(request.failIndex, executor)));
+                var response = new Response(request.failIndex, executor);
+                try {
+                    listener.onResponse(response);
+                } catch (Exception e) {
+                    ESTestCase.fail(e);
+                }
             }
         }
 
@@ -585,18 +594,22 @@ public static class TransportInfiniteContinuationsAction extends TransportAction
 
             @Inject
             public TransportInfiniteContinuationsAction(ActionFilters actionFilters, TransportService transportService) {
-                super(TYPE.name(), actionFilters, transportService.getTaskManager());
-                this.executor = transportService.getThreadPool().executor(ThreadPool.Names.GENERIC);
+                this(actionFilters, transportService, transportService.getThreadPool().executor(ThreadPool.Names.GENERIC));
+            }
+
+            TransportInfiniteContinuationsAction(ActionFilters actionFilters, TransportService transportService, ExecutorService executor) {
+                super(TYPE.name(), actionFilters, transportService.getTaskManager(), executor);
+                this.executor = executor;
             }
 
             @Override
             protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
-                executor.execute(
-                    ActionRunnable.supply(
-                        ActionTestUtils.assertNoFailureListener(listener::onResponse),
-                        () -> new Response(randomFrom(executor, EsExecutors.DIRECT_EXECUTOR_SERVICE))
-                    )
-                );
+                var response = new Response(randomFrom(executor, EsExecutors.DIRECT_EXECUTOR_SERVICE));
+                try {
+                    listener.onResponse(response);
+                } catch (Exception e) {
+                    ESTestCase.fail(e);
+                }
             }
         }
 
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
index f48a3143fd016..a309877e9aa83 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
@@ -53,6 +53,7 @@
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.netty4.internal.HttpHeadersAuthenticatorUtils;
 import org.elasticsearch.http.netty4.internal.HttpValidator;
+import org.elasticsearch.rest.ChunkedZipResponse;
 import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.netty4.AcceptChannelHandler;
@@ -382,7 +383,16 @@ protected boolean isContentAlwaysEmpty(HttpResponse msg) {
                 })
                 .addLast("aggregator", aggregator);
             if (handlingSettings.compression()) {
-                ch.pipeline().addLast("encoder_compress", new HttpContentCompressor(handlingSettings.compressionLevel()));
+                ch.pipeline().addLast("encoder_compress", new HttpContentCompressor(handlingSettings.compressionLevel()) {
+                    @Override
+                    protected Result beginEncode(HttpResponse httpResponse, String acceptEncoding) throws Exception {
+                        if (ChunkedZipResponse.ZIP_CONTENT_TYPE.equals(httpResponse.headers().get("content-type"))) {
+                            return null;
+                        } else {
+                            return super.beginEncode(httpResponse, acceptEncoding);
+                        }
+                    }
+                });
             }
             ch.pipeline()
                 .addLast(
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
index d85bf32da263f..8080170bf1ee7 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
@@ -372,7 +372,7 @@ private void setupPipeline(Channel ch, boolean isRemoteClusterServerChannel) {
     protected InboundPipeline getInboundPipeline(Channel ch, boolean isRemoteClusterServerChannel) {
         return new InboundPipeline(
             getStatsTracker(),
-            threadPool::relativeTimeInMillis,
+            threadPool.relativeTimeInMillisSupplier(),
             new InboundDecoder(recycler),
             new InboundAggregator(getInflightBreaker(), getRequestHandlers()::getHandler, ignoreDeserializationErrors()),
             this::inboundMessage
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
index 3e74a74dbd49c..ce7704e6e040c 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.mocksocket.MockSocket;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportSettings;
 import org.junit.After;
@@ -52,7 +53,7 @@ public class Netty4SizeHeaderFrameDecoderTests extends ESTestCase {
 
     @Before
     public void startThreadPool() {
-        threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
         NetworkService networkService = new NetworkService(Collections.emptyList());
         PageCacheRecycler recycler = new MockPageCacheRecycler(Settings.EMPTY);
         nettyTransport = new Netty4Transport(
diff --git a/muted-tests.yml b/muted-tests.yml
index ed7556039d0ab..57cc7abb899e7 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -38,29 +38,18 @@ tests:
 - class: "org.elasticsearch.xpack.deprecation.DeprecationHttpIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108628"
   method: "testDeprecatedSettingsReturnWarnings"
-- class: "org.elasticsearch.xpack.inference.InferenceCrudIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109391"
-  method: "testDeleteEndpointWhileReferencedByPipeline"
 - class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109687"
   method: "test {p0=sql/translate/Translate SQL}"
-- class: "org.elasticsearch.action.admin.indices.rollover.RolloverIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/110034"
-  method: "testRolloverWithClosedWriteIndex"
 - class: org.elasticsearch.index.store.FsDirectoryFactoryTests
   method: testStoreDirectory
   issue: https://github.com/elastic/elasticsearch/issues/110210
 - class: org.elasticsearch.index.store.FsDirectoryFactoryTests
   method: testPreload
   issue: https://github.com/elastic/elasticsearch/issues/110211
-- class: "org.elasticsearch.rest.RestControllerIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/110225"
 - class: org.elasticsearch.upgrades.SecurityIndexRolesMetadataMigrationIT
   method: testMetadataMigratedAfterUpgrade
   issue: https://github.com/elastic/elasticsearch/issues/110232
-- class: org.elasticsearch.compute.lucene.ValueSourceReaderTypeConversionTests
-  method: testLoadAll
-  issue: https://github.com/elastic/elasticsearch/issues/110244
 - class: org.elasticsearch.backwards.SearchWithMinCompatibleSearchNodeIT
   method: testMinVersionAsNewVersion
   issue: https://github.com/elastic/elasticsearch/issues/95384
@@ -73,74 +62,118 @@ tests:
 - class: "org.elasticsearch.xpack.searchablesnapshots.FrozenSearchableSnapshotsIntegTests"
   issue: "https://github.com/elastic/elasticsearch/issues/110408"
   method: "testCreateAndRestorePartialSearchableSnapshot"
-- class: org.elasticsearch.xpack.security.LicenseDLSFLSRoleIT
-  method: testQueryDLSFLSRolesShowAsDisabled
-  issue: https://github.com/elastic/elasticsearch/issues/110729
 - class: org.elasticsearch.xpack.security.authz.store.NativePrivilegeStoreCacheTests
   method: testPopulationOfCacheWhenLoadingPrivilegesForAllApplications
   issue: https://github.com/elastic/elasticsearch/issues/110789
-- class: org.elasticsearch.xpack.security.ScrollHelperIntegTests
-  method: testFetchAllEntities
-  issue: https://github.com/elastic/elasticsearch/issues/110786
 - class: org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFileTests
   method: testCacheFileCreatedAsSparseFile
   issue: https://github.com/elastic/elasticsearch/issues/110801
-- class: org.elasticsearch.nativeaccess.PreallocateTests
-  method: testPreallocate
-  issue: https://github.com/elastic/elasticsearch/issues/110948
 - class: org.elasticsearch.nativeaccess.VectorSystemPropertyTests
   method: testSystemPropertyDisabled
   issue: https://github.com/elastic/elasticsearch/issues/110949
-- class: org.elasticsearch.compute.lucene.ValueSourceReaderTypeConversionTests
-  method: testLoadAllStatusAllInOnePage
-  issue: https://github.com/elastic/elasticsearch/issues/111048
-- class: org.elasticsearch.action.search.KnnSearchSingleNodeTests
-  method: testKnnSearchAction
-  issue: https://github.com/elastic/elasticsearch/issues/111072
-- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTests
-  method: testKnnQuantizedFlatVectorsFormat
-  issue: https://github.com/elastic/elasticsearch/issues/111044
 - class: org.elasticsearch.xpack.esql.spatial.SpatialPushDownGeoPointIT
   method: testPushedDownQueriesSingleValue
   issue: https://github.com/elastic/elasticsearch/issues/111084
-- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
-  method: test {stats.MaxOfIpGrouping}
-  issue: https://github.com/elastic/elasticsearch/issues/111107
-- class: org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderIT
-  method: testEnterpriseDownloaderTask
-  issue: https://github.com/elastic/elasticsearch/issues/111002
-- class: org.elasticsearch.xpack.esql.spatial.SpatialPushDownCartesianPointIT
-  method: testPushedDownQueriesSingleValue
-  issue: https://github.com/elastic/elasticsearch/issues/110982
 - class: org.elasticsearch.multi_node.GlobalCheckpointSyncActionIT
   issue: https://github.com/elastic/elasticsearch/issues/111124
-- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
-  method: test {stats.MinOfIpGrouping}
-  issue: https://github.com/elastic/elasticsearch/issues/111125
-- class: org.elasticsearch.xpack.security.authz.store.NativePrivilegeStoreCacheTests
-  method: testGetPrivilegesUsesCache
-  issue: https://github.com/elastic/elasticsearch/issues/110788
-- class: org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankTests
-  method: testRerankInferenceResultMismatch
-  issue: https://github.com/elastic/elasticsearch/issues/111133
 - class: org.elasticsearch.cluster.PrevalidateShardPathIT
   method: testCheckShards
   issue: https://github.com/elastic/elasticsearch/issues/111134
-- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
-  method: test {stats.MaxOfIpGrouping SYNC}
-  issue: https://github.com/elastic/elasticsearch/issues/111136
-- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
-  method: test {stats.MinOfIpGrouping SYNC}
-  issue: https://github.com/elastic/elasticsearch/issues/111137
-- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
-  method: test {stats.MinOfIpGrouping ASYNC}
-  issue: https://github.com/elastic/elasticsearch/issues/111138
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test021InstallPlugin
   issue: https://github.com/elastic/elasticsearch/issues/110343
+- class: org.elasticsearch.xpack.security.authc.oidc.OpenIdConnectAuthIT
+  method: testAuthenticateWithImplicitFlow
+  issue: https://github.com/elastic/elasticsearch/issues/111191
+- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT
+  issue: https://github.com/elastic/elasticsearch/issues/111319
+- class: org.elasticsearch.xpack.ml.integration.InferenceIngestInputConfigIT
+  method: testIngestWithInputFields
+  issue: https://github.com/elastic/elasticsearch/issues/111383
+- class: org.elasticsearch.xpack.ml.integration.InferenceIngestInputConfigIT
+  method: testIngestWithMultipleInputFields
+  issue: https://github.com/elastic/elasticsearch/issues/111384
+- class: org.elasticsearch.xpack.security.authc.oidc.OpenIdConnectAuthIT
+  method: testAuthenticateWithCodeFlowAndClientPost
+  issue: https://github.com/elastic/elasticsearch/issues/111396
 - class: org.elasticsearch.xpack.restart.FullClusterRestartIT
-  method: testDisableFieldNameField {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/111141
+  method: testSingleDoc {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/111434
+- class: org.elasticsearch.xpack.restart.FullClusterRestartIT
+  method: testDataStreams {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/111448
+- class: org.elasticsearch.search.SearchServiceTests
+  issue: https://github.com/elastic/elasticsearch/issues/111529
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=rollup/security_tests/Index-based access}
+  issue: https://github.com/elastic/elasticsearch/issues/111631
+- class: org.elasticsearch.tdigest.ComparisonTests
+  method: testSparseGaussianDistribution
+  issue: https://github.com/elastic/elasticsearch/issues/111721
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSnapshotRestore {cluster=OLD}
+  issue: https://github.com/elastic/elasticsearch/issues/111777
+- class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT
+  method: testSnapshotRestore {cluster=OLD}
+  issue: https://github.com/elastic/elasticsearch/issues/111775
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSnapshotRestore {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/111798
+- class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT
+  method: testSnapshotRestore {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/111799
+- class: org.elasticsearch.xpack.esql.qa.mixed.FieldExtractorIT
+  method: testScaledFloat
+  issue: https://github.com/elastic/elasticsearch/issues/112003
+- class: org.elasticsearch.xpack.inference.InferenceRestIT
+  method: test {p0=inference/80_random_rerank_retriever/Random rerank retriever predictably shuffles results}
+  issue: https://github.com/elastic/elasticsearch/issues/111999
+- class: org.elasticsearch.xpack.ml.integration.MlJobIT
+  method: testDeleteJobAfterMissingIndex
+  issue: https://github.com/elastic/elasticsearch/issues/112088
+- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
+  method: test {stats.ByTwoCalculatedSecondOverwrites SYNC}
+  issue: https://github.com/elastic/elasticsearch/issues/112117
+- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
+  method: test {stats.ByTwoCalculatedSecondOverwritesReferencingFirst SYNC}
+  issue: https://github.com/elastic/elasticsearch/issues/112118
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=transform/preview_transforms/Test preview transform latest}
+  issue: https://github.com/elastic/elasticsearch/issues/112144
+- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
+  issue: https://github.com/elastic/elasticsearch/issues/112147
+- class: org.elasticsearch.smoketest.WatcherYamlRestIT
+  method: test {p0=watcher/usage/10_basic/Test watcher usage stats output}
+  issue: https://github.com/elastic/elasticsearch/issues/112189
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=ml/inference_processor/Test create processor with missing mandatory fields}
+  issue: https://github.com/elastic/elasticsearch/issues/112191
+- class: org.elasticsearch.xpack.ml.integration.MlJobIT
+  method: testDeleteJobAsync
+  issue: https://github.com/elastic/elasticsearch/issues/112212
+- class: org.elasticsearch.search.retriever.rankdoc.RankDocsSortBuilderTests
+  method: testEqualsAndHashcode
+  issue: https://github.com/elastic/elasticsearch/issues/112312
+- class: org.elasticsearch.blobcache.shared.SharedBlobCacheServiceTests
+  method: testGetMultiThreaded
+  issue: https://github.com/elastic/elasticsearch/issues/112314
+- class: org.elasticsearch.search.retriever.RankDocRetrieverBuilderIT
+  method: testRankDocsRetrieverWithCollapse
+  issue: https://github.com/elastic/elasticsearch/issues/112254
+- class: org.elasticsearch.search.ccs.CCSUsageTelemetryIT
+  issue: https://github.com/elastic/elasticsearch/issues/112324
+- class: org.elasticsearch.datastreams.logsdb.qa.StandardVersusLogsIndexModeRandomDataChallengeRestIT
+  method: testMatchAllQuery
+  issue: https://github.com/elastic/elasticsearch/issues/112374
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/rest-api/watcher/put-watch/line_120}
+  issue: https://github.com/elastic/elasticsearch/issues/99517
+- class: org.elasticsearch.xpack.ml.integration.MlJobIT
+  method: testMultiIndexDelete
+  issue: https://github.com/elastic/elasticsearch/issues/112381
+- class: org.elasticsearch.xpack.searchablesnapshots.cache.shared.NodesCachesStatsIntegTests
+  method: testNodesCachesStats
+  issue: https://github.com/elastic/elasticsearch/issues/112384
 
 # Examples:
 #
@@ -171,4 +204,12 @@ tests:
 #    method: "test {union_types.MultiIndexIpStringStatsInline}"
 #    issue: "https://github.com/elastic/elasticsearch/..."
 #  Note that this mutes for the unit-test-like CsvTests only.
-#  Muting for the integration tests needs to be done for each IT class individually.
+#  Muting all the integration tests can be done using the class "org.elasticsearch.xpack.esql.**".
+#  Consider however, that some tests are named as "test {file.test SYNC}" and "ASYNC" in the integration tests.
+#  To mute all 3 tests safely everywhere use:
+#  - class: "org.elasticsearch.xpack.esql.**"
+#    method: "test {union_types.MultiIndexIpStringStatsInline}"
+#    issue: "https://github.com/elastic/elasticsearch/..."
+#  - class: "org.elasticsearch.xpack.esql.**"
+#    method: "test {union_types.MultiIndexIpStringStatsInline *}"
+#    issue: "https://github.com/elastic/elasticsearch/..."
diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties
index efe2ff3449216..9036682bf0f0c 100644
--- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties
+++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip
+distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java
index 894f4ebe4bc54..dc429538fec3b 100644
--- a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java
+++ b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java
@@ -35,10 +35,7 @@
 public class ExpertScriptPlugin extends Plugin implements ScriptPlugin {
 
     @Override
-    public ScriptEngine getScriptEngine(
-        Settings settings,
-        Collection<ScriptContext<?>> contexts
-    ) {
+    public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
         return new MyExpertScriptEngine();
     }
 
@@ -143,6 +140,9 @@ public ScoreScript newInstance(DocReader docReader)
                         public double execute(
                             ExplanationHolder explanation
                         ) {
+                            if(explanation != null) {
+                                explanation.set("An example optional custom description to explain details for this script's execution; we'll provide a default one if you leave this out.");
+                            }
                             return 0.0d;
                         }
                     };
@@ -166,6 +166,9 @@ public void setDocument(int docid) {
                     }
                     @Override
                     public double execute(ExplanationHolder explanation) {
+                        if(explanation != null) {
+                            explanation.set("An example optional custom description to explain details for this script's execution; we'll provide a default one if you leave this out.");
+                        }
                         if (postings.docID() != currentDocid) {
                             /*
                              * advance moved past the current doc, so this
diff --git a/plugins/examples/script-expert-scoring/src/yamlRestTest/resources/rest-api-spec/test/script_expert_scoring/20_score.yml b/plugins/examples/script-expert-scoring/src/yamlRestTest/resources/rest-api-spec/test/script_expert_scoring/20_score.yml
index 89194d162872d..7436768416e00 100644
--- a/plugins/examples/script-expert-scoring/src/yamlRestTest/resources/rest-api-spec/test/script_expert_scoring/20_score.yml
+++ b/plugins/examples/script-expert-scoring/src/yamlRestTest/resources/rest-api-spec/test/script_expert_scoring/20_score.yml
@@ -4,26 +4,27 @@
 setup:
   - do:
       indices.create:
-          index:  test
+        index: test
 
   - do:
       index:
-        index:  test
-        id:     "1"
-        body:   { "important_field": "foo" }
+        index: test
+        id: "1"
+        body: { "important_field": "foo" }
   - do:
-        index:
-          index:  test
-          id:     "2"
-          body:   { "important_field": "foo foo foo" }
+      index:
+        index: test
+        id: "2"
+        body: { "important_field": "foo foo foo" }
   - do:
-        index:
-          index:  test
-          id:     "3"
-          body:   { "important_field": "foo foo" }
+      index:
+        index: test
+        id: "3"
+        body: { "important_field": "foo foo" }
 
   - do:
-      indices.refresh: {}
+      indices.refresh: { }
+
 ---
 "document scoring":
   - do:
@@ -46,6 +47,39 @@ setup:
                         term: "foo"
 
   - length: { hits.hits: 3 }
-  - match: {hits.hits.0._id: "2" }
-  - match: {hits.hits.1._id: "3" }
-  - match: {hits.hits.2._id: "1" }
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.2._id: "1" }
+
+---
+"document scoring with custom explanation":
+
+  - requires:
+      cluster_features: [ "gte_v8.15.1" ]
+      reason: "bug fixed where explanations were throwing npe prior to 8.16"
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          explain: true
+          query:
+            function_score:
+              query:
+                match:
+                  important_field: "foo"
+              functions:
+                - script_score:
+                    script:
+                      source: "pure_df"
+                      lang: "expert_scripts"
+                      params:
+                        field: "important_field"
+                        term: "foo"
+
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.2._id: "1" }
+  - match: { hits.hits.0._explanation.details.1.details.0.description: "An example optional custom description to explain details for this script's execution; we'll provide a default one if you leave this out." }
diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
index dac8e051f25f8..8d50a9f7e29a9 100644
--- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
+++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
@@ -584,7 +584,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
             );
         }
         if (fieldType.stored()) {
-            return new StringStoredFieldFieldLoader(fullPath(), leafName(), null) {
+            return new StringStoredFieldFieldLoader(fullPath(), leafName()) {
                 @Override
                 protected void write(XContentBuilder b, Object value) throws IOException {
                     b.value((String) value);
diff --git a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
index 49db5e3a1cd99..12c15a91be0e3 100644
--- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
+++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
@@ -310,8 +310,11 @@ protected ClientYamlTestExecutionContext createRestTestExecutionContext(
                 getClusterStateFeatures(adminSearchClient),
                 semanticNodeVersions
             );
-            final TestFeatureService combinedTestFeatureService = featureId -> testFeatureService.clusterHasFeature(featureId)
-                && searchTestFeatureService.clusterHasFeature(featureId);
+            final TestFeatureService combinedTestFeatureService = (featureId, any) -> {
+                boolean adminFeature = testFeatureService.clusterHasFeature(featureId, any);
+                boolean searchFeature = searchTestFeatureService.clusterHasFeature(featureId, any);
+                return any ? adminFeature || searchFeature : adminFeature && searchFeature;
+            };
             final Set<String> combinedOsSet = Stream.concat(osSet.stream(), Stream.of(searchOs)).collect(Collectors.toSet());
             final Set<String> combinedNodeVersions = Stream.concat(nodesVersions.stream(), searchNodeVersions.stream())
                 .collect(Collectors.toSet());
diff --git a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java
index e3639ffabf664..d3c8e64b85066 100644
--- a/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java
+++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java
@@ -294,8 +294,11 @@ protected ClientYamlTestExecutionContext createRestTestExecutionContext(
                 getClusterStateFeatures(adminSearchClient),
                 semanticNodeVersions
             );
-            final TestFeatureService combinedTestFeatureService = featureId -> testFeatureService.clusterHasFeature(featureId)
-                && searchTestFeatureService.clusterHasFeature(featureId);
+            final TestFeatureService combinedTestFeatureService = (featureId, any) -> {
+                boolean adminFeature = testFeatureService.clusterHasFeature(featureId, any);
+                boolean searchFeature = searchTestFeatureService.clusterHasFeature(featureId, any);
+                return any ? adminFeature || searchFeature : adminFeature && searchFeature;
+            };
 
             final Set<String> combinedOsSet = Stream.concat(osSet.stream(), Stream.of(searchOs)).collect(Collectors.toSet());
             final Set<String> combinedNodeVersions = Stream.concat(nodesVersions.stream(), searchNodeVersions.stream())
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
index 1a86947acab95..20c13ca92f5c1 100644
--- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -920,9 +920,7 @@ public void testEmptyShard() throws IOException {
         final String indexName = "test_empty_shard";
 
         if (isRunningAgainstOldCluster()) {
-            Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+            Settings.Builder settings = indexSettings(1, 1)
                 // if the node with the replica is the first to be restarted, while a replica is still recovering
                 // then delayed allocation will kick in. When the node comes back, the master will search for a copy
                 // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
@@ -1522,14 +1520,7 @@ public void testOperationBasedRecovery() throws Exception {
      */
     public void testTurnOffTranslogRetentionAfterUpgraded() throws Exception {
         if (isRunningAgainstOldCluster()) {
-            createIndex(
-                index,
-                Settings.builder()
-                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
-                    .build()
-            );
+            createIndex(index, indexSettings(1, 1).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build());
             ensureGreen(index);
             int numDocs = randomIntBetween(10, 100);
             for (int i = 0; i < numDocs; i++) {
@@ -1549,9 +1540,7 @@ public void testTurnOffTranslogRetentionAfterUpgraded() throws Exception {
     public void testResize() throws Exception {
         int numDocs;
         if (isRunningAgainstOldCluster()) {
-            final Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3)
-                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1);
+            final Settings.Builder settings = indexSettings(3, 1);
             if (minimumIndexVersion().before(IndexVersions.V_8_0_0) && randomBoolean()) {
                 settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
             }
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java
new file mode 100644
index 0000000000000..739b4e302bb54
--- /dev/null
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java
@@ -0,0 +1,227 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.FormatNames;
+import org.elasticsearch.test.MapMatcher;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.RestTestLegacyFeatures;
+import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.test.MapMatcher.assertMap;
+import static org.elasticsearch.test.MapMatcher.matchesMap;
+
+public class LogsIndexModeFullClusterRestartIT extends ParameterizedFullClusterRestartTestCase {
+
+    @ClassRule
+    public static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(getOldClusterTestVersion())
+        .module("constant-keyword")
+        .module("data-streams")
+        .module("mapper-extras")
+        .module("x-pack-aggregate-metric")
+        .module("x-pack-stack")
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .build();
+
+    public LogsIndexModeFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
+    @Override
+    protected ElasticsearchCluster getUpgradeCluster() {
+        return cluster;
+    }
+
+    private static final String BULK_INDEX_REQUEST = """
+        { "create": {} }
+        { "@timestamp": "%s", "host.name": "%s", "method": "%s", "ip.address": "%s", "message": "%s" }
+        """;
+
+    private static final String STANDARD_TEMPLATE = """
+        {
+          "index_patterns": [ "logs-*-*" ],
+          "data_stream": {},
+          "priority": 500,
+          "template": {
+            "mappings": {
+              "properties": {
+                "@timestamp" : {
+                  "type": "date"
+                },
+                "host.name": {
+                  "type": "keyword"
+                },
+                "method": {
+                  "type": "keyword"
+                },
+                "message": {
+                  "type": "text"
+                },
+                "ip.address": {
+                  "type": "ip"
+                }
+              }
+            }
+          }
+        }""";
+
+    private static final String LOGS_TEMPLATE = """
+        {
+          "index_patterns": [ "logs-*-*" ],
+          "data_stream": {},
+          "priority": 500,
+          "template": {
+            "settings": {
+              "index": {
+                "mode": "logsdb"
+              }
+            },
+            "mappings": {
+              "properties": {
+                "@timestamp" : {
+                  "type": "date"
+                },
+                "host.name": {
+                  "type": "keyword"
+                },
+                "method": {
+                  "type": "keyword"
+                },
+                "message": {
+                  "type": "text"
+                },
+                "ip.address": {
+                  "type": "ip"
+                }
+              }
+            }
+          }
+        }""";
+
+    public void testLogsIndexing() throws IOException {
+        assumeTrue("Test uses data streams", oldClusterHasFeature(RestTestLegacyFeatures.DATA_STREAMS_SUPPORTED));
+
+        if (isRunningAgainstOldCluster()) {
+            assertOK(client().performRequest(putTemplate(client(), "logs-template", STANDARD_TEMPLATE)));
+            assertOK(client().performRequest(createDataStream("logs-apache-production")));
+            final Response bulkIndexResponse = client().performRequest(bulkIndex("logs-apache-production", () -> {
+                final StringBuilder sb = new StringBuilder();
+                for (int i = 0; i < randomIntBetween(10, 20); i++) {
+                    sb.append(
+                        String.format(
+                            BULK_INDEX_REQUEST,
+                            DateFormatter.forPattern(FormatNames.DATE_TIME.getName()).format(Instant.now()),
+                            randomFrom("foo", "bar"),
+                            randomFrom("PUT", "POST", "GET"),
+                            InetAddresses.toAddrString(randomIp(randomBoolean())),
+                            randomAlphaOfLengthBetween(100, 200)
+                        )
+                    );
+                    sb.append("\n");
+                }
+                return sb.toString();
+            }));
+            assertOK(bulkIndexResponse);
+            assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
+        } else {
+            assertOK(client().performRequest(putTemplate(client(), "logs-template", LOGS_TEMPLATE)));
+            assertOK(client().performRequest(rolloverDataStream(client(), "logs-apache-production")));
+            final Response bulkIndexResponse = client().performRequest(bulkIndex("logs-apache-production", () -> {
+                final StringBuilder sb = new StringBuilder();
+                for (int i = 0; i < randomIntBetween(10, 20); i++) {
+                    sb.append(
+                        String.format(
+                            BULK_INDEX_REQUEST,
+                            DateFormatter.forPattern(FormatNames.DATE_TIME.getName()).format(Instant.now()),
+                            randomFrom("foo", "bar"),
+                            randomFrom("PUT", "POST", "GET"),
+                            InetAddresses.toAddrString(randomIp(randomBoolean())),
+                            randomAlphaOfLengthBetween(100, 200)
+                        )
+                    );
+                    sb.append("\n");
+                }
+                return sb.toString();
+            }));
+            assertOK(bulkIndexResponse);
+            assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
+
+            assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(
+                1,
+                Matchers.equalTo("logsdb"),
+                matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic"))
+            );
+        }
+    }
+
+    private void assertIndexMappingsAndSettings(int backingIndex, final Matcher<Object> indexModeMatcher, final MapMatcher mappingsMatcher)
+        throws IOException {
+        assertThat(
+            getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"),
+            indexModeMatcher
+        );
+        assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher);
+    }
+
+    private static Request createDataStream(final String dataStreamName) {
+        return new Request("PUT", "/_data_stream/" + dataStreamName);
+    }
+
+    private static Request bulkIndex(final String dataStreamName, final Supplier<String> bulkIndexRequestSupplier) {
+        final Request request = new Request("POST", dataStreamName + "/_bulk");
+        request.setJsonEntity(bulkIndexRequestSupplier.get());
+        request.addParameter("refresh", "true");
+        return request;
+    }
+
+    private static Request putTemplate(final RestClient client, final String templateName, final String mappings) throws IOException {
+        final Request request = new Request("PUT", "/_index_template/" + templateName);
+        request.setJsonEntity(mappings);
+        return request;
+    }
+
+    private static Request rolloverDataStream(final RestClient client, final String dataStreamName) throws IOException {
+        return new Request("POST", "/" + dataStreamName + "/_rollover");
+    }
+
+    @SuppressWarnings("unchecked")
+    private static String getWriteBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) throws IOException {
+        final Request request = new Request("GET", "_data_stream/" + dataStreamName);
+        final List<Object> dataStreams = (List<Object>) entityAsMap(client.performRequest(request)).get("data_streams");
+        final Map<String, Object> dataStream = (Map<String, Object>) dataStreams.get(0);
+        final List<Map<String, String>> backingIndices = (List<Map<String, String>>) dataStream.get("indices");
+        return backingIndices.get(backingIndex).get("index_name");
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<String, Object> getSettings(final RestClient client, final String indexName) throws IOException {
+        final Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings");
+        return ((Map<String, Map<String, Object>>) entityAsMap(client.performRequest(request)).get(indexName)).get("settings");
+    }
+}
diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java
index 6c924fe8e429a..825a866cdf2f8 100644
--- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java
+++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -75,10 +74,7 @@ public void testIndexVersionPropagation() throws Exception {
         logger.info("cluster discovered: {}", nodes.toString());
         final List<String> bwcNamesList = nodes.getBWCNodes().stream().map(MixedClusterTestNode::nodeName).collect(Collectors.toList());
         final String bwcNames = bwcNamesList.stream().collect(Collectors.joining(","));
-        Settings.Builder settings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
-            .put("index.routing.allocation.include._name", bwcNames);
+        Settings.Builder settings = indexSettings(1, 2).put("index.routing.allocation.include._name", bwcNames);
         final String index = "indexversionprop";
         final int minUpdates = 5;
         final int maxUpdates = 10;
@@ -165,10 +161,7 @@ public void testSeqNoCheckpoints() throws Exception {
         logger.info("cluster discovered: {}", nodes.toString());
         final List<String> bwcNamesList = nodes.getBWCNodes().stream().map(MixedClusterTestNode::nodeName).collect(Collectors.toList());
         final String bwcNames = bwcNamesList.stream().collect(Collectors.joining(","));
-        Settings.Builder settings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
-            .put("index.routing.allocation.include._name", bwcNames);
+        Settings.Builder settings = indexSettings(1, 2).put("index.routing.allocation.include._name", bwcNames);
 
         final String index = "test";
         createIndex(index, settings.build());
@@ -251,10 +244,7 @@ public void testUpdateSnapshotStatus() throws Exception {
         String bwcNames = nodes.getBWCNodes().stream().map(MixedClusterTestNode::nodeName).collect(Collectors.joining(","));
 
         // Allocating shards on the BWC nodes to makes sure that taking snapshot happens on those nodes.
-        Settings.Builder settings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(5, 10))
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-            .put("index.routing.allocation.include._name", bwcNames);
+        Settings.Builder settings = indexSettings(between(5, 10), 1).put("index.routing.allocation.include._name", bwcNames);
 
         final String index = "test-snapshot-index";
         createIndex(index, settings.build());
@@ -315,14 +305,7 @@ public void testSyncedFlushTransition() throws Exception {
         int numOfReplicas = randomIntBetween(0, nodes.getNewNodes().size() - 1);
         int totalShards = numShards * (numOfReplicas + 1);
         final String index = "test_synced_flush";
-        createIndex(
-            index,
-            Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), numShards)
-                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
-                .put("index.routing.allocation.include._name", newNodes)
-                .build()
-        );
+        createIndex(index, indexSettings(numShards, numOfReplicas).put("index.routing.allocation.include._name", newNodes).build());
         ensureGreen(index);
         indexDocs(index, randomIntBetween(0, 100), between(1, 100));
         try (
@@ -394,14 +377,7 @@ public void testFlushTransition() throws Exception {
         int numOfReplicas = randomIntBetween(0, nodes.getNewNodes().size() - 1);
         int totalShards = numShards * (numOfReplicas + 1);
         final String index = "test_flush";
-        createIndex(
-            index,
-            Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), numShards)
-                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
-                .put("index.routing.allocation.include._name", newNodes)
-                .build()
-        );
+        createIndex(index, indexSettings(numShards, numOfReplicas).put("index.routing.allocation.include._name", newNodes).build());
         ensureGreen(index);
         indexDocs(index, randomIntBetween(0, 100), between(1, 100));
         try (
diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java
index a33fc01d8446a..f2ca41e5ef8bc 100644
--- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java
+++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java
@@ -10,7 +10,6 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Strings;
@@ -43,9 +42,7 @@ private int indexDocs(int numDocs, int id) throws Exception {
     }
 
     public void testSingleValuedString() throws Exception {
-        final Settings.Builder settings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 2)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0);
+        final Settings.Builder settings = indexSettings(2, 0);
         createIndex(index, settings.build());
         // We want to trigger the usage oif cuckoo filters that happen only when there are
         // more than 10k distinct values in one shard.
diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java
index 461b731e518fb..808ebb764768f 100644
--- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java
+++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java
@@ -12,8 +12,6 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -50,13 +48,7 @@ public void prepareTestData() throws IOException {
         allNodes.addAll(nodes.getNewNodes());
 
         if (client().performRequest(new Request("HEAD", "/" + index)).getStatusLine().getStatusCode() == 404) {
-            createIndex(
-                index,
-                Settings.builder()
-                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), numShards)
-                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)
-                    .build()
-            );
+            createIndex(index, indexSettings(numShards, numReplicas).build());
             for (int i = 0; i < numDocs; i++) {
                 Request request = new Request("PUT", index + "/_doc/" + i);
                 request.setJsonEntity("{\"test\": \"test_" + randomAlphaOfLength(2) + "\"}");
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java
index 1925b1e8f36ab..2ce9eef29d903 100644
--- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java
+++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/ConfigurationTests.java
@@ -20,7 +20,6 @@
 
 import static java.nio.file.attribute.PosixFilePermissions.fromString;
 import static org.elasticsearch.packaging.util.FileUtils.append;
-import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assume.assumeFalse;
 
 public class ConfigurationTests extends PackagingTestCase {
@@ -50,13 +49,15 @@ public void test20HostnameSubstitution() throws Exception {
             // security auto-config requires that the archive owner and the node process user be the same
             Platforms.onWindows(() -> sh.chown(confPath, installation.getOwner()));
             assertWhileRunning(() -> {
-                final String nameResponse = ServerUtils.makeRequest(
-                    Request.Get("https://localhost:9200/_cat/nodes?h=name"),
-                    "test_superuser",
-                    "test_superuser_password",
-                    ServerUtils.getCaCert(confPath)
-                ).strip();
-                assertThat(nameResponse, equalTo("mytesthost"));
+                assertBusy(() -> {
+                    final String nameResponse = ServerUtils.makeRequest(
+                        Request.Get("https://localhost:9200/_cat/nodes?h=name"),
+                        "test_superuser",
+                        "test_superuser_password",
+                        ServerUtils.getCaCert(confPath)
+                    ).strip();
+                    assertEquals("mytesthost", nameResponse);
+                });
             });
             Platforms.onWindows(() -> sh.chown(confPath));
         });
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
index f9723f30cc371..18668b842b2d3 100644
--- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
+++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
@@ -1231,8 +1231,7 @@ public void test500Readiness() throws Exception {
         assertBusy(() -> assertTrue(readinessProbe(9399)));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99508")
-    public void test600Interrupt() {
+    public void test600Interrupt() throws Exception {
         waitForElasticsearch(installation, "elastic", PASSWORD);
         final Result containerLogs = getContainerLogs();
 
@@ -1242,10 +1241,12 @@ public void test600Interrupt() {
         final int maxPid = infos.stream().map(i -> i.pid()).max(Integer::compareTo).get();
 
         sh.run("bash -c 'kill -int " + maxPid + "'"); // send ctrl+c to all java processes
-        final Result containerLogsAfter = getContainerLogs();
 
-        assertThat("Container logs should contain stopping ...", containerLogsAfter.stdout(), containsString("stopping ..."));
-        assertThat("No errors stdout", containerLogsAfter.stdout(), not(containsString("java.security.AccessControlException:")));
-        assertThat("No errors stderr", containerLogsAfter.stderr(), not(containsString("java.security.AccessControlException:")));
+        assertBusy(() -> {
+            final Result containerLogsAfter = getContainerLogs();
+            assertThat("Container logs should contain stopping ...", containerLogsAfter.stdout(), containsString("stopping ..."));
+            assertThat("No errors stdout", containerLogsAfter.stdout(), not(containsString("java.security.AccessControlException:")));
+            assertThat("No errors stderr", containerLogsAfter.stderr(), not(containsString("java.security.AccessControlException:")));
+        });
     }
 }
diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/MemoryLockingTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/MemoryLockingTests.java
new file mode 100644
index 0000000000000..82a17c54b6d69
--- /dev/null
+++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/MemoryLockingTests.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.ServerUtils;
+import org.elasticsearch.packaging.util.docker.DockerRun;
+
+import java.util.Map;
+
+import static org.elasticsearch.packaging.util.docker.Docker.runContainer;
+import static org.elasticsearch.packaging.util.docker.DockerRun.builder;
+
+public class MemoryLockingTests extends PackagingTestCase {
+
+    public void test10Install() throws Exception {
+        install();
+    }
+
+    public void test20MemoryLockingEnabled() throws Exception {
+        configureAndRun(
+            Map.of(
+                "bootstrap.memory_lock",
+                "true",
+                "xpack.security.enabled",
+                "false",
+                "xpack.security.http.ssl.enabled",
+                "false",
+                "xpack.security.enrollment.enabled",
+                "false",
+                "discovery.type",
+                "single-node"
+            )
+        );
+        // TODO: very locking worked. logs? check memory of process? at least we know the process started successfully
+        stopElasticsearch();
+    }
+
+    public void configureAndRun(Map<String, String> settings) throws Exception {
+        if (distribution().isDocker()) {
+            DockerRun builder = builder();
+            settings.forEach(builder::envVar);
+            runContainer(distribution(), builder);
+        } else {
+
+            for (var setting : settings.entrySet()) {
+                ServerUtils.addSettingToExistingConfiguration(installation.config, setting.getKey(), setting.getValue());
+            }
+            ServerUtils.removeSettingFromExistingConfiguration(installation.config, "cluster.initial_master_nodes");
+        }
+
+        startElasticsearch();
+    }
+}
diff --git a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
index 43d5ea842f9ef..fe2236adc4904 100644
--- a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
+++ b/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
@@ -60,9 +60,7 @@ public class RecoveryIT extends AbstractRollingTestCase {
     public void testHistoryUUIDIsGenerated() throws Exception {
         final String index = "index_history_uuid";
         if (CLUSTER_TYPE == ClusterType.OLD) {
-            Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+            Settings.Builder settings = indexSettings(1, 1)
                 // if the node with the replica is the first to be restarted, while a replica is still recovering
                 // then delayed allocation will kick in. When the node comes back, the master will search for a copy
                 // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
@@ -128,9 +126,7 @@ public void testRecoveryWithConcurrentIndexing() throws Exception {
 
         switch (CLUSTER_TYPE) {
             case OLD -> {
-                Settings.Builder settings = Settings.builder()
-                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
+                Settings.Builder settings = indexSettings(1, 2)
                     // if the node with the replica is the first to be restarted, while a replica is still recovering
                     // then delayed allocation will kick in. When the node comes back, the master will search for a copy
                     // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
@@ -217,9 +213,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
         final String index = "relocation_with_concurrent_indexing";
         switch (CLUSTER_TYPE) {
             case OLD -> {
-                Settings.Builder settings = Settings.builder()
-                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
+                Settings.Builder settings = indexSettings(1, 2)
                     // if the node with the replica is the first to be restarted, while a replica is still recovering
                     // then delayed allocation will kick in. When the node comes back, the master will search for a copy
                     // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
@@ -296,9 +290,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
     public void testRecovery() throws Exception {
         final String index = "test_recovery";
         if (CLUSTER_TYPE == ClusterType.OLD) {
-            Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+            Settings.Builder settings = indexSettings(1, 1)
                 // if the node with the replica is the first to be restarted, while a replica is still recovering
                 // then delayed allocation will kick in. When the node comes back, the master will search for a copy
                 // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
@@ -413,9 +405,7 @@ public void testRecoveryClosedIndex() throws Exception {
         if (CLUSTER_TYPE == ClusterType.OLD) {
             createIndex(
                 indexName,
-                Settings.builder()
-                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                indexSettings(1, 1)
                     // if the node with the replica is the first to be restarted, while a replica is still recovering
                     // then delayed allocation will kick in. When the node comes back, the master will search for a copy
                     // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
@@ -453,13 +443,7 @@ public void testCloseIndexDuringRollingUpgrade() throws Exception {
         final String indexName = String.join("_", "index", CLUSTER_TYPE.toString(), Integer.toString(id)).toLowerCase(Locale.ROOT);
 
         if (indexExists(indexName) == false) {
-            createIndex(
-                indexName,
-                Settings.builder()
-                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                    .build()
-            );
+            createIndex(indexName, indexSettings(1, 0).build());
             ensureGreen(indexName);
             closeIndex(indexName);
         }
@@ -482,10 +466,7 @@ public void testClosedIndexNoopRecovery() throws Exception {
         if (CLUSTER_TYPE == ClusterType.OLD) {
             createIndex(
                 indexName,
-                Settings.builder()
-                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                    .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")
+                indexSettings(1, 0).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")
                     .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "24h")
                     .put("index.routing.allocation.include._name", CLUSTER_NAME + "-0")
                     .build()
@@ -578,9 +559,7 @@ private void assertClosedIndex(final String index, final boolean checkRoutingTab
     public void testUpdateDoc() throws Exception {
         final String index = "test_update_doc";
         if (CLUSTER_TYPE == ClusterType.OLD) {
-            Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2);
+            Settings.Builder settings = indexSettings(1, 2);
             createIndex(index, settings.build());
             indexDocs(index, 0, 100);
         }
@@ -648,9 +627,7 @@ private void assertNoopRecoveries(String indexName, Predicate<String> targetNode
     public void testOperationBasedRecovery() throws Exception {
         final String index = "test_operation_based_recovery";
         if (CLUSTER_TYPE == ClusterType.OLD) {
-            final Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2);
+            final Settings.Builder settings = indexSettings(1, 2);
             if (minimumIndexVersion().before(IndexVersions.V_8_0_0) && randomBoolean()) {
                 settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
             }
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java
new file mode 100644
index 0000000000000..1549789bcc44a
--- /dev/null
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java
@@ -0,0 +1,248 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.FormatNames;
+import org.elasticsearch.test.MapMatcher;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.test.MapMatcher.assertMap;
+import static org.elasticsearch.test.MapMatcher.matchesMap;
+
+public class LogsIndexModeRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
+
+    @ClassRule()
+    public static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .module("constant-keyword")
+        .module("data-streams")
+        .module("mapper-extras")
+        .module("x-pack-aggregate-metric")
+        .module("x-pack-stack")
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("cluster.logsdb.enabled", "true")
+        .setting("stack.templates.enabled", "false")
+        .build();
+
+    public LogsIndexModeRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    private static final String BULK_INDEX_REQUEST = """
+        { "create": {} }
+        { "@timestamp": "%s", "host.name": "%s", "method": "%s", "ip.address": "%s", "message": "%s" }
+        """;
+
+    private static final String STANDARD_TEMPLATE = """
+        {
+          "index_patterns": [ "logs-*-*" ],
+          "data_stream": {},
+          "priority": 500,
+          "template": {
+            "mappings": {
+              "properties": {
+                "@timestamp" : {
+                  "type": "date"
+                },
+                "host.name": {
+                  "type": "keyword"
+                },
+                "method": {
+                  "type": "keyword"
+                },
+                "message": {
+                  "type": "text"
+                },
+                "ip.address": {
+                  "type": "ip"
+                }
+              }
+            }
+          }
+        }""";
+
+    private static final String LOGS_TEMPLATE = """
+        {
+          "index_patterns": [ "logs-*-*" ],
+          "data_stream": {},
+          "priority": 500,
+          "template": {
+            "settings": {
+              "index": {
+                "mode": "logsdb"
+              }
+            },
+            "mappings": {
+              "properties": {
+                "@timestamp" : {
+                  "type": "date"
+                },
+                "host.name": {
+                  "type": "keyword"
+                },
+                "method": {
+                  "type": "keyword"
+                },
+                "message": {
+                  "type": "text"
+                },
+                "ip.address": {
+                  "type": "ip"
+                }
+              }
+            }
+          }
+        }""";
+
+    public void testLogsIndexing() throws IOException {
+        if (isOldCluster()) {
+            assertOK(client().performRequest(putTemplate(client(), "logs-template", STANDARD_TEMPLATE)));
+            assertOK(client().performRequest(createDataStream("logs-apache-production")));
+            final Response bulkIndexResponse = client().performRequest(bulkIndex("logs-apache-production", () -> {
+                final StringBuilder sb = new StringBuilder();
+                for (int i = 0; i < randomIntBetween(10, 20); i++) {
+                    sb.append(
+                        String.format(
+                            BULK_INDEX_REQUEST,
+                            DateFormatter.forPattern(FormatNames.DATE_TIME.getName()).format(Instant.now()),
+                            randomFrom("foo", "bar"),
+                            randomFrom("PUT", "POST", "GET"),
+                            InetAddresses.toAddrString(randomIp(randomBoolean())),
+                            randomIntBetween(20, 50)
+                        )
+                    );
+                    sb.append("\n");
+                }
+                return sb.toString();
+            }));
+            assertOK(bulkIndexResponse);
+            assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
+        } else if (isMixedCluster()) {
+            assertOK(client().performRequest(rolloverDataStream(client(), "logs-apache-production")));
+            final Response bulkIndexResponse = client().performRequest(bulkIndex("logs-apache-production", () -> {
+                final StringBuilder sb = new StringBuilder();
+                for (int i = 0; i < randomIntBetween(10, 20); i++) {
+                    sb.append(
+                        String.format(
+                            BULK_INDEX_REQUEST,
+                            DateFormatter.forPattern(FormatNames.DATE_TIME.getName()).format(Instant.now()),
+                            randomFrom("foo", "bar"),
+                            randomFrom("PUT", "POST", "GET"),
+                            InetAddresses.toAddrString(randomIp(randomBoolean())),
+                            randomIntBetween(20, 50)
+                        )
+                    );
+                    sb.append("\n");
+                }
+                return sb.toString();
+            }));
+            assertOK(bulkIndexResponse);
+            assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
+        } else if (isUpgradedCluster()) {
+            assertOK(client().performRequest(putTemplate(client(), "logs-template", LOGS_TEMPLATE)));
+            assertOK(client().performRequest(rolloverDataStream(client(), "logs-apache-production")));
+            final Response bulkIndexResponse = client().performRequest(bulkIndex("logs-apache-production", () -> {
+                final StringBuilder sb = new StringBuilder();
+                for (int i = 0; i < randomIntBetween(10, 20); i++) {
+                    sb.append(
+                        String.format(
+                            BULK_INDEX_REQUEST,
+                            DateFormatter.forPattern(FormatNames.DATE_TIME.getName()).format(Instant.now()),
+                            randomFrom("foo", "bar"),
+                            randomFrom("PUT", "POST", "GET"),
+                            InetAddresses.toAddrString(randomIp(randomBoolean())),
+                            randomIntBetween(20, 50)
+                        )
+                    );
+                    sb.append("\n");
+                }
+                return sb.toString();
+            }));
+            assertOK(bulkIndexResponse);
+            assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
+
+            assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(1, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(2, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(
+                3,
+                Matchers.equalTo("logsdb"),
+                matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic"))
+            );
+        }
+    }
+
+    private void assertIndexMappingsAndSettings(int backingIndex, final Matcher<Object> indexModeMatcher, final MapMatcher mappingsMatcher)
+        throws IOException {
+        assertThat(
+            getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"),
+            indexModeMatcher
+        );
+        assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher);
+    }
+
+    private static Request createDataStream(final String dataStreamName) {
+        return new Request("PUT", "/_data_stream/" + dataStreamName);
+    }
+
+    private static Request bulkIndex(final String dataStreamName, final Supplier<String> bulkIndexRequestSupplier) {
+        final Request request = new Request("POST", dataStreamName + "/_bulk");
+        request.setJsonEntity(bulkIndexRequestSupplier.get());
+        request.addParameter("refresh", "true");
+        return request;
+    }
+
+    private static Request putTemplate(final RestClient client, final String templateName, final String mappings) throws IOException {
+        final Request request = new Request("PUT", "/_index_template/" + templateName);
+        request.setJsonEntity(mappings);
+        return request;
+    }
+
+    private static Request rolloverDataStream(final RestClient client, final String dataStreamName) throws IOException {
+        return new Request("POST", "/" + dataStreamName + "/_rollover");
+    }
+
+    @SuppressWarnings("unchecked")
+    private static String getWriteBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) throws IOException {
+        final Request request = new Request("GET", "_data_stream/" + dataStreamName);
+        final List<Object> dataStreams = (List<Object>) entityAsMap(client.performRequest(request)).get("data_streams");
+        final Map<String, Object> dataStream = (Map<String, Object>) dataStreams.get(0);
+        final List<Map<String, String>> backingIndices = (List<Map<String, String>>) dataStream.get("indices");
+        return backingIndices.get(backingIndex).get("index_name");
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<String, Object> getSettings(final RestClient client, final String indexName) throws IOException {
+        final Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings");
+        return ((Map<String, Map<String, Object>>) entityAsMap(client.performRequest(request)).get(indexName)).get("settings");
+    }
+}
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java
index 593630546845d..2ee668c0e3fe1 100644
--- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java
@@ -59,10 +59,7 @@ public void testSnapshotBasedRecovery() throws Exception {
         final String repositoryName = "snapshot_based_recovery_repo";
         final int numDocs = 200;
         if (isOldCluster()) {
-            Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
+            Settings.Builder settings = indexSettings(1, 0).put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
                 .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
             createIndex(indexName, settings.build());
             ensureGreen(indexName);
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/capabilities.json b/rest-api-spec/src/main/resources/rest-api-spec/api/capabilities.json
index a96be0d63834e..f0537eee575d2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/capabilities.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/capabilities.json
@@ -41,6 +41,11 @@
       "capabilities": {
         "type": "string",
         "description": "Comma-separated list of arbitrary API capabilities to check"
+      },
+      "local_only": {
+        "type": "boolean",
+        "description": "True if only the node being called should be considered",
+        "visibility": "private"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json
index f8f3e238661c7..3df9232c3a5c2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json
@@ -26,6 +26,14 @@
       ]
     },
     "params":{
+      "timeout":{
+        "type":"time",
+        "description":"Specify timeout for acknowledging the cluster state update"
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json
index 26f015f6028c6..dc9da50e0417b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json
@@ -37,6 +37,10 @@
         ],
         "default":"open",
         "description":"Whether wildcard expressions should get expanded to open or closed indices (default: open)"
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json
index e8aa0f00afa01..6f05af1485f98 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_lifecycle.json
@@ -41,6 +41,10 @@
       "include_defaults":{
         "type":"boolean",
         "description":"Return all relevant default configurations for the data stream (default: false)"
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json
index ec878f1fc9f62..59cd8521f275e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json
@@ -47,6 +47,10 @@
       "include_defaults":{
         "type":"boolean",
         "description":"Return all relevant default configurations for the data stream (default: false)"
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.migrate_to_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.migrate_to_data_stream.json
index 4254ae79a23e8..879f086cc0675 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.migrate_to_data_stream.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.migrate_to_data_stream.json
@@ -26,6 +26,14 @@
       ]
     },
     "params":{
+      "timeout":{
+        "type":"time",
+        "description":"Specify timeout for acknowledging the cluster state update"
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.promote_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.promote_data_stream.json
index 5b51a900235ca..8c4c747fa8c16 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.promote_data_stream.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.promote_data_stream.json
@@ -26,6 +26,10 @@
       ]
     },
     "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
index f9afdbfb85f9c..8e80cd15d9ea2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
@@ -65,6 +65,10 @@
         ],
         "default":"open",
         "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout for connection to master node"
       }
     }
   }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc
index 4b9399d052ce6..0ddac662e73ef 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc
@@ -107,6 +107,10 @@ A `skip` section, on the other hand, defines certain conditions that, if met, wi
 - `features`: Only kept for a transition period, please use <<requires_test_runner_features, `test_runner_features`>>
   in the `requires` section instead.
 
+Note that `skip` with `capabilities` or `cluster_features` will skip the test if *any* node in the cluster
+has the feature or capability. `requires` will only run the test if *all* of the nodes in the cluster
+have the feature or capability.
+
 `requires` and `skip` sections must specify at least one of the options mentioned above.
 Unless only `test_runner_features` or legacy test runner `features` are specified, a `reason` must be given.
 
@@ -134,7 +138,7 @@ other test runners to skip tests if they do not support the capabilities API yet
               path: /_api
               parameters: [param1, param2]
               capabilities: [cap1, cap2]
-          test_runner_feature: [capabilities]
+          test_runner_features: [capabilities]
           reason: Capability required to run test
      - do:
        ... test definitions ...
@@ -142,13 +146,6 @@ other test runners to skip tests if they do not support the capabilities API yet
 
 The `capabilities` field is an array containing one or several capabilities checks.
 
-*NOTE: If planning to `skip` on capabilities, keep in mind this might lead to unexpected results in _mixed cluster_
-tests!* A test is only skipped if *all* nodes support the requested capabilities, in _mixed clusters_ this might not be
-the case: such a cluster can consist of a mix of nodes where some support respective capabilities and others don't,
-additionally there might even be nodes that do not support the capabilities API at all.
-In such cases the capabilities check will *not* succeed, hence the test is *not* skipped and might randomly hit one
-of the nodes that actually support what you intended to skip on. This might then break your assumptions and fail the test.
-
 Capabilities are declared as part of an implementation of `RestHandler`.
 Override the `supportedQueryParameters` and/or the `supportedCapabilities` methods:
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/bulk/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/bulk/10_basic.yml
index f4f6245603aab..a2dfe3784d5ae 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/bulk/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/bulk/10_basic.yml
@@ -229,3 +229,23 @@
   - match: { items.0.index.error.type: illegal_argument_exception }
   - match: { items.0.index.error.reason: "no write index is defined for alias [test_index]. The write index may be explicitly disabled using is_write_index=false or the alias points to multiple indices without one being designated as a write index" }
 
+---
+"Took is not orders of magnitude off":
+  - requires:
+      cluster_features: ["gte_v8.15.1"]
+      reason: "Bug reporting wrong took time introduced in 8.15.0, fixed in 8.15.1"
+  - do:
+      bulk:
+        body:
+          - index:
+              _index: took_test
+          - f: 1
+          - index:
+              _index: took_test
+          - f: 2
+          - index:
+              _index: took_test
+          - f: 3
+  - match: { errors: false }
+  - gte: { took: 0 }
+  - lte: { took: 60000 } # Making sure we have a reasonable upper bound and that we're not for example returning nanoseconds
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml
index 94c19a4d69e17..5881ec83ebe85 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml
@@ -1,25 +1,28 @@
 ---
 "Metrics object indexing":
   - requires:
-      test_runner_features: allowed_warnings_regex
+      test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
       cluster_features: ["gte_v8.3.0"]
       reason: added in 8.3.0
 
   - do:
-      indices.put_template:
+      allowed_warnings:
+        - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation"
+      indices.put_index_template:
         name: test
         body:
           index_patterns: test-*
-          mappings:
-            dynamic_templates:
-              - no_subobjects:
-                  match: metrics
-                  mapping:
-                    type: object
-                    subobjects: false
-                    properties:
-                      host.name:
-                        type: keyword
+          template:
+            mappings:
+              dynamic_templates:
+                - no_subobjects:
+                    match: metrics
+                    mapping:
+                      type: object
+                      subobjects: false
+                      properties:
+                        host.name:
+                          type: keyword
 
   - do:
       allowed_warnings_regex:
@@ -65,20 +68,23 @@
 ---
 "Root without subobjects":
   - requires:
-      test_runner_features: allowed_warnings_regex
+      test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
       cluster_features: ["gte_v8.3.0"]
       reason: added in 8.3.0
 
   - do:
-      indices.put_template:
+      allowed_warnings:
+        - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation"
+      indices.put_index_template:
         name: test
         body:
           index_patterns: test-*
-          mappings:
-            subobjects: false
-            properties:
-              host.name:
-                type: keyword
+          template:
+            mappings:
+              subobjects: false
+              properties:
+                host.name:
+                  type: keyword
 
   - do:
       allowed_warnings_regex:
@@ -124,27 +130,30 @@
 ---
 "Metrics object indexing with synthetic source":
   - requires:
-      test_runner_features: allowed_warnings_regex
+      test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
       cluster_features: ["gte_v8.4.0"]
       reason: added in 8.4.0
 
   - do:
-      indices.put_template:
+      allowed_warnings:
+        - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation"
+      indices.put_index_template:
         name: test
         body:
           index_patterns: test-*
-          mappings:
-            _source:
-              mode: synthetic
-            dynamic_templates:
-              - no_subobjects:
-                  match: metrics
-                  mapping:
-                    type: object
-                    subobjects: false
-                    properties:
-                      host.name:
-                        type: keyword
+          template:
+            mappings:
+              _source:
+                mode: synthetic
+              dynamic_templates:
+                - no_subobjects:
+                    match: metrics
+                    mapping:
+                      type: object
+                      subobjects: false
+                      properties:
+                        host.name:
+                          type: keyword
 
   - do:
       allowed_warnings_regex:
@@ -191,22 +200,25 @@
 ---
 "Root without subobjects with synthetic source":
   - requires:
-      test_runner_features: allowed_warnings_regex
+      test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
       cluster_features: ["gte_v8.4.0"]
       reason: added in 8.4.0
 
   - do:
-      indices.put_template:
+      allowed_warnings:
+        - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation"
+      indices.put_index_template:
         name: test
         body:
           index_patterns: test-*
-          mappings:
-            _source:
-              mode: synthetic
-            subobjects: false
-            properties:
-              host.name:
-                type: keyword
+          template:
+            mappings:
+              _source:
+                mode: synthetic
+              subobjects: false
+              properties:
+                host.name:
+                  type: keyword
 
   - do:
       allowed_warnings_regex:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml
new file mode 100644
index 0000000000000..414c24cfffd7d
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml
@@ -0,0 +1,262 @@
+---
+"Metrics object indexing":
+  - requires:
+      test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
+      cluster_features: ["mapper.subobjects_auto"]
+      reason: requires supporting subobjects auto setting
+
+  - do:
+      allowed_warnings:
+        - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation"
+      indices.put_index_template:
+        name: test
+        body:
+          index_patterns: test-*
+          template:
+            mappings:
+              dynamic_templates:
+                - no_subobjects:
+                    match: metrics
+                    mapping:
+                      type: object
+                      subobjects: auto
+                      properties:
+                        host.name:
+                          type: keyword
+
+  - do:
+      allowed_warnings_regex:
+        - "index \\[test-1\\] matches multiple legacy templates \\[global, test\\], composable templates will only match a single template"
+      index:
+        index: test-1
+        id: 1
+        refresh: true
+        body:
+          { metrics.host.name: localhost, metrics.host.id: 1, metrics.time: 10, metrics.time.max: 100, metrics.time.min: 1 }
+
+  - do:
+      field_caps:
+        index: test-1
+        fields: metrics*
+  - match: {fields.metrics\.host\.id.long.searchable: true}
+  - match: {fields.metrics\.host\.id.long.aggregatable: true}
+  - match: {fields.metrics\.host\.name.keyword.searchable: true}
+  - match: {fields.metrics\.host\.name.keyword.aggregatable: true}
+  - match: {fields.metrics\.time.long.searchable: true}
+  - match: {fields.metrics\.time.long.aggregatable: true}
+  - match: {fields.metrics\.time\.max.long.searchable: true}
+  - match: {fields.metrics\.time\.max.long.aggregatable: true}
+  - match: {fields.metrics\.time\.min.long.searchable: true}
+  - match: {fields.metrics\.time\.min.long.aggregatable: true}
+
+  - do:
+      get:
+        index: test-1
+        id:    1
+  - match: {_index: "test-1"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        metrics.host.name: localhost
+        metrics.host.id: 1
+        metrics.time: 10
+        metrics.time.max: 100
+        metrics.time.min: 1
+
+---
+"Root with metrics":
+  - requires:
+      test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
+      cluster_features: ["mapper.subobjects_auto"]
+      reason: requires supporting subobjects auto setting
+
+  - do:
+      allowed_warnings:
+        - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation"
+      indices.put_index_template:
+        name: test
+        body:
+          index_patterns: test-*
+          template:
+            mappings:
+              subobjects: auto
+              properties:
+                host.name:
+                  type: keyword
+
+  - do:
+      allowed_warnings_regex:
+        - "index \\[test-1\\] matches multiple legacy templates \\[global, test\\], composable templates will only match a single template"
+      index:
+        index: test-1
+        id: 1
+        refresh: true
+        body:
+          { host.name: localhost, host.id: 1, time: 10, time.max: 100, time.min: 1 }
+
+  - do:
+      field_caps:
+        index: test-1
+        fields: [host*, time*]
+  - match: {fields.host\.name.keyword.searchable: true}
+  - match: {fields.host\.name.keyword.aggregatable: true}
+  - match: {fields.host\.id.long.searchable: true}
+  - match: {fields.host\.id.long.aggregatable: true}
+  - match: {fields.time.long.searchable: true}
+  - match: {fields.time.long.aggregatable: true}
+  - match: {fields.time\.max.long.searchable: true}
+  - match: {fields.time\.max.long.aggregatable: true}
+  - match: {fields.time\.min.long.searchable: true}
+  - match: {fields.time\.min.long.aggregatable: true}
+
+  - do:
+      get:
+        index: test-1
+        id:    1
+  - match: {_index: "test-1"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        host.name: localhost
+        host.id: 1
+        time: 10
+        time.max: 100
+        time.min: 1
+
+---
+"Metrics object indexing with synthetic source":
+  - requires:
+      test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
+      cluster_features: ["mapper.subobjects_auto"]
+      reason: added in 8.4.0
+
+  - do:
+      allowed_warnings:
+        - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation"
+      indices.put_index_template:
+        name: test
+        body:
+          index_patterns: test-*
+          template:
+            mappings:
+              _source:
+                mode: synthetic
+              dynamic_templates:
+                - no_subobjects:
+                    match: metrics
+                    mapping:
+                      type: object
+                      subobjects: auto
+                      properties:
+                        host.name:
+                          type: keyword
+
+  - do:
+      allowed_warnings_regex:
+        - "index \\[test-1\\] matches multiple legacy templates \\[global, test\\], composable templates will only match a single template"
+      index:
+        index: test-1
+        id: 1
+        refresh: true
+        body:
+          { metrics.host.name: localhost, metrics.host.id: 1, metrics.time: 10, metrics.time.max: 100, metrics.time.min: 1 }
+
+  - do:
+      field_caps:
+        index: test-1
+        fields: metrics*
+  - match: {fields.metrics\.host\.id.long.searchable: true}
+  - match: {fields.metrics\.host\.id.long.aggregatable: true}
+  - match: {fields.metrics\.host\.name.keyword.searchable: true}
+  - match: {fields.metrics\.host\.name.keyword.aggregatable: true}
+  - match: {fields.metrics\.time.long.searchable: true}
+  - match: {fields.metrics\.time.long.aggregatable: true}
+  - match: {fields.metrics\.time\.max.long.searchable: true}
+  - match: {fields.metrics\.time\.max.long.aggregatable: true}
+  - match: {fields.metrics\.time\.min.long.searchable: true}
+  - match: {fields.metrics\.time\.min.long.aggregatable: true}
+
+  - do:
+      get:
+        index: test-1
+        id:    1
+  - match: {_index: "test-1"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        metrics:
+          host.name: localhost
+          host.id: 1
+          time: 10
+          time.max: 100
+          time.min: 1
+
+---
+"Root without subobjects with synthetic source":
+  - requires:
+      test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ]
+      cluster_features: ["mapper.subobjects_auto"]
+      reason: added in 8.4.0
+
+  - do:
+      allowed_warnings:
+        - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation"
+      indices.put_index_template:
+        name: test
+        body:
+          index_patterns: test-*
+          template:
+            mappings:
+              _source:
+                mode: synthetic
+              subobjects: auto
+              properties:
+                host.name:
+                  type: keyword
+
+  - do:
+      allowed_warnings_regex:
+        - "index \\[test-1\\] matches multiple legacy templates \\[global, test\\], composable templates will only match a single template"
+      index:
+        index: test-1
+        id: 1
+        refresh: true
+        body:
+          { host.name: localhost, host.id: 1, time: 10, time.max: 100, time.min: 1 }
+
+  - do:
+      field_caps:
+        index: test-1
+        fields: [host*, time*]
+  - match: {fields.host\.name.keyword.searchable: true}
+  - match: {fields.host\.name.keyword.aggregatable: true}
+  - match: {fields.host\.id.long.searchable: true}
+  - match: {fields.host\.id.long.aggregatable: true}
+  - match: {fields.time.long.searchable: true}
+  - match: {fields.time.long.aggregatable: true}
+  - match: {fields.time\.max.long.searchable: true}
+  - match: {fields.time\.max.long.aggregatable: true}
+  - match: {fields.time\.min.long.searchable: true}
+  - match: {fields.time\.min.long.aggregatable: true}
+
+  - do:
+      get:
+        index: test-1
+        id:    1
+  - match: {_index: "test-1"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        host.name: localhost
+        host.id: 1
+        time: 10
+        time.max: 100
+        time.min: 1
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
index 22deb7012c4ed..fa08efe402b43 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml
@@ -1204,3 +1204,187 @@ nested object with stored array:
   - match: { hits.hits.1._source.nested_array_stored.0.b.1.c: 100 }
   - match: { hits.hits.1._source.nested_array_stored.1.b.0.c: 20 }
   - match: { hits.hits.1._source.nested_array_stored.1.b.1.c: 200 }
+
+---
+empty nested object sorted as a first document:
+  - requires:
+      cluster_features: ["mapper.track_ignored_source"]
+      reason: requires tracking ignored source
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              sort.field: "name"
+              sort.order: "asc"
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              name:
+                type: keyword
+              nested:
+                type: nested
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "name": "B", "nested": { "a": "b" } }'
+          - '{ "create": { } }'
+          - '{ "name": "A" }'
+
+  - match: { errors: false }
+
+  - do:
+      search:
+        index: test
+        sort: name
+
+  - match:  { hits.total.value: 2 }
+  - match: { hits.hits.0._source.name: A }
+  - match: { hits.hits.1._source.name: B }
+  - match: { hits.hits.1._source.nested.a: "b" }
+
+---
+subobjects auto:
+  - requires:
+      cluster_features: ["mapper.subobjects_auto"]
+      reason: requires tracking ignored source and supporting subobjects auto setting
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            subobjects: auto
+            properties:
+              id:
+                type: integer
+              regular:
+                properties:
+                  span:
+                    properties:
+                      id:
+                        type: keyword
+                  trace:
+                    properties:
+                      id:
+                        type: keyword
+              stored:
+                store_array_source: true
+                properties:
+                  span:
+                    properties:
+                      id:
+                        type: keyword
+                  trace:
+                    properties:
+                      id:
+                        type: keyword
+              nested:
+                type: nested
+              auto_obj:
+                type: object
+                subobjects: auto
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{  "id": 1, "foo": 10, "foo.bar": 100, "regular": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }'
+          - '{ "create": { } }'
+          - '{  "id": 2, "foo": 20, "foo.bar": 200, "stored": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }'
+          - '{ "create": { } }'
+          - '{  "id": 3, "foo": 30, "foo.bar": 300, "nested": [ { "a": 10, "b": 20 }, { "a": 100, "b": 200 } ] }'
+          - '{ "create": { } }'
+          - '{  "id": 4, "auto_obj": {  "foo": 40, "foo.bar": 400 } }'
+
+  - match: { errors: false }
+
+  - do:
+      search:
+        index: test
+        sort: id
+
+  - match: { hits.hits.0._source.id: 1  }
+  - match: { hits.hits.0._source.foo: 10  }
+  - match: { hits.hits.0._source.foo\.bar: 100  }
+  - match: { hits.hits.0._source.regular.span.id: "1" }
+  - match: { hits.hits.0._source.regular.trace.id: [ "a", "b" ] }
+  - match: { hits.hits.1._source.id: 2  }
+  - match: { hits.hits.1._source.foo: 20 }
+  - match: { hits.hits.1._source.foo\.bar: 200 }
+  - match: { hits.hits.1._source.stored.0.trace.id: a  }
+  - match: { hits.hits.1._source.stored.0.span.id: "1"  }
+  - match: { hits.hits.1._source.stored.1.trace.id: b }
+  - match: { hits.hits.1._source.stored.1.span.id: "1" }
+  - match: { hits.hits.2._source.id: 3  }
+  - match: { hits.hits.2._source.foo: 30 }
+  - match: { hits.hits.2._source.foo\.bar: 300 }
+  - match: { hits.hits.2._source.nested.0.a: 10 }
+  - match: { hits.hits.2._source.nested.0.b: 20 }
+  - match: { hits.hits.2._source.nested.1.a: 100 }
+  - match: { hits.hits.2._source.nested.1.b: 200 }
+  - match: { hits.hits.3._source.id: 4  }
+  - match: { hits.hits.3._source.auto_obj.foo: 40 }
+  - match: { hits.hits.3._source.auto_obj.foo\.bar: 400 }
+
+---
+# 112156
+stored field under object with store_array_source:
+  - requires:
+      cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix"]
+      reason: requires bug fix to be implemented
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              sort.field: "name"
+              sort.order: "asc"
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              name:
+                type: keyword
+              obj:
+                store_array_source: true
+                properties:
+                  foo:
+                    type: keyword
+                    store: true
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "name": "B", "obj": null }'
+          - '{ "create": { } }'
+          - '{ "name": "A", "obj": [ { "foo": "hello_from_the_other_side" } ] }'
+
+  - match: { errors: false }
+
+  - do:
+      search:
+        index: test
+        sort: name
+
+  - match:  { hits.total.value: 2 }
+  - match: { hits.hits.0._source.name: A }
+  - match: { hits.hits.0._source.obj: [ { "foo": "hello_from_the_other_side" } ] }
+  - match: { hits.hits.1._source.name: B }
+  - match: { hits.hits.1._source.obj: null }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml
index 45bcf64f98945..3d82539944a97 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml
@@ -449,6 +449,115 @@
         index: test-generic
   - match: { test-generic.mappings.properties.parent.properties.child\.grandchild.type: "keyword" }
 
+
+---
+"Composable index templates that include subobjects: auto at root":
+  - requires:
+      cluster_features: ["mapper.subobjects_auto"]
+      reason: "https://github.com/elastic/elasticsearch/issues/96768 fixed at 8.11.0"
+      test_runner_features: "allowed_warnings"
+
+  - do:
+      cluster.put_component_template:
+        name: test-subobjects
+        body:
+          template:
+            mappings:
+              subobjects: auto
+              properties:
+                message:
+                  enabled: false
+
+  - do:
+      cluster.put_component_template:
+        name: test-field
+        body:
+          template:
+            mappings:
+              properties:
+                parent.subfield:
+                  type: keyword
+
+  - do:
+      allowed_warnings:
+        - "index template [test-composable-template] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: test-composable-template
+        body:
+          index_patterns:
+            - test-*
+          composed_of:
+            - test-subobjects
+            - test-field
+  - is_true: acknowledged
+
+  - do:
+      indices.create:
+        index: test-generic
+
+  - do:
+      indices.get_mapping:
+        index: test-generic
+  - match: { test-generic.mappings.properties.parent\.subfield.type: "keyword" }
+  - match: { test-generic.mappings.properties.message.type: "object" }
+
+---
+"Composable index templates that include subobjects: auto on arbitrary field":
+  - requires:
+      cluster_features: ["mapper.subobjects_auto"]
+      reason: "https://github.com/elastic/elasticsearch/issues/96768 fixed at 8.11.0"
+      test_runner_features: "allowed_warnings"
+
+  - do:
+      cluster.put_component_template:
+        name: test-subobjects
+        body:
+          template:
+            mappings:
+              properties:
+                parent:
+                  type: object
+                  subobjects: auto
+                  properties:
+                    message:
+                      enabled: false
+
+  - do:
+      cluster.put_component_template:
+        name: test-subfield
+        body:
+          template:
+            mappings:
+              properties:
+                parent:
+                  properties:
+                    child.grandchild:
+                      type: keyword
+
+  - do:
+      allowed_warnings:
+        - "index template [test-composable-template] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: test-composable-template
+        body:
+          index_patterns:
+            - test-*
+          composed_of:
+            - test-subobjects
+            - test-subfield
+  - is_true: acknowledged
+
+  - do:
+      indices.create:
+        index: test-generic
+
+  - do:
+      indices.get_mapping:
+        index: test-generic
+  - match: { test-generic.mappings.properties.parent.properties.child\.grandchild.type: "keyword" }
+  - match: { test-generic.mappings.properties.parent.properties.message.type: "object" }
+
+
 ---
 "Composition of component templates with different legal field mappings":
   - skip:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/40_routing_partition_size.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/40_routing_partition_size.yml
index 80a8ccf0d1063..11ffbe1d8464d 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/40_routing_partition_size.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/40_routing_partition_size.yml
@@ -16,22 +16,22 @@ more than 1:
   - do:
       index:
         index:   source
-        id:      1
-        routing: 1
+        id:      "1"
+        routing: "1"
         body:    { "foo": "hello world" }
 
   - do:
       index:
         index:   source
-        id:      2
-        routing: 2
+        id:      "2"
+        routing: "2"
         body:    { "foo": "hello world 2" }
 
   - do:
       index:
         index:   source
-        id:      3
-        routing: 3
+        id:      "3"
+        routing: "3"
         body:    { "foo": "hello world 3" }
 
   # make it read-only
@@ -66,8 +66,8 @@ more than 1:
   - do:
       get:
         index:   target
-        routing: 1
-        id:      1
+        routing: "1"
+        id:      "1"
 
   - match: { _index:   target }
   - match: { _id:      "1"     }
@@ -76,8 +76,8 @@ more than 1:
   - do:
       get:
         index:   target
-        routing: 2
-        id:      2
+        routing: "2"
+        id:      "2"
 
   - match: { _index:   target }
   - match: { _id:      "2"     }
@@ -86,8 +86,8 @@ more than 1:
   - do:
       get:
         index:   target
-        routing: 3
-        id:      3
+        routing: "3"
+        id:      "3"
 
   - match: { _index:   target }
   - match: { _id:      "3"     }
@@ -117,22 +117,22 @@ exactly 1:
   - do:
       index:
         index:   source
-        id:      1
-        routing: 1
+        id:      "1"
+        routing: "1"
         body:    { "foo": "hello world" }
 
   - do:
       index:
         index:   source
-        id:      2
-        routing: 2
+        id:      "2"
+        routing: "2"
         body:    { "foo": "hello world 2" }
 
   - do:
       index:
         index:   source
-        id:      3
-        routing: 3
+        id:      "3"
+        routing: "3"
         body:    { "foo": "hello world 3" }
 
   # make it read-only
@@ -167,8 +167,8 @@ exactly 1:
   - do:
       get:
         index:   target
-        routing: 1
-        id:      1
+        routing: "1"
+        id:      "1"
 
   - match: { _index:   target }
   - match: { _id:      "1"     }
@@ -177,8 +177,8 @@ exactly 1:
   - do:
       get:
         index:   target
-        routing: 2
-        id:      2
+        routing: "2"
+        id:      "2"
 
   - match: { _index:   target }
   - match: { _id:      "2"     }
@@ -187,8 +187,8 @@ exactly 1:
   - do:
       get:
         index:   target
-        routing: 3
-        id:      3
+        routing: "3"
+        id:      "3"
 
   - match: { _index:   target }
   - match: { _id:      "3"     }
@@ -221,22 +221,22 @@ nested:
   - do:
       index:
         index:   source
-        id:      1
-        routing: 1
+        id:      "1"
+        routing: "1"
         body:    { "foo": "hello world", "n": [{"foo": "goodbye world"}, {"foo": "more words"}] }
 
   - do:
       index:
         index:   source
-        id:      2
-        routing: 2
+        id:      "2"
+        routing: "2"
         body:    { "foo": "hello world 2" }
 
   - do:
       index:
         index:   source
-        id:      3
-        routing: 3
+        id:      "3"
+        routing: "3"
         body:    { "foo": "hello world 3" }
 
   # make it read-only
@@ -271,8 +271,8 @@ nested:
   - do:
       get:
         index:   target
-        routing: 1
-        id:      1
+        routing: "1"
+        id:      "1"
 
   - match: { _index:   target }
   - match: { _id:      "1"     }
@@ -281,8 +281,8 @@ nested:
   - do:
       get:
         index:   target
-        routing: 2
-        id:      2
+        routing: "2"
+        id:      "2"
 
   - match: { _index:   target }
   - match: { _id:      "2"     }
@@ -291,8 +291,8 @@ nested:
   - do:
       get:
         index:   target
-        routing: 3
-        id:      3
+        routing: "3"
+        id:      "3"
 
   - match: { _index:   target }
   - match: { _id:      "3"     }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/50_routing_required.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/50_routing_required.yml
index 38bf9d72ef8ff..4c8d7736631c9 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/50_routing_required.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/50_routing_required.yml
@@ -15,22 +15,22 @@ routing required:
   - do:
       index:
         index:   source
-        id:      1
-        routing: 1
+        id:      "1"
+        routing: "1"
         body:    { "foo": "hello world" }
 
   - do:
       index:
         index:   source
-        id:      2
-        routing: 2
+        id:      "2"
+        routing: "2"
         body:    { "foo": "hello world 2" }
 
   - do:
       index:
         index:   source
-        id:      3
-        routing: 3
+        id:      "3"
+        routing: "3"
         body:    { "foo": "hello world 3" }
 
   # make it read-only
@@ -65,8 +65,8 @@ routing required:
   - do:
       get:
         index:   target
-        routing: 1
-        id:      1
+        routing: "1"
+        id:      "1"
 
   - match: { _index:   target }
   - match: { _id:      "1"     }
@@ -75,8 +75,8 @@ routing required:
   - do:
       get:
         index:   target
-        routing: 2
-        id:      2
+        routing: "2"
+        id:      "2"
 
   - match: { _index:   target }
   - match: { _id:      "2"     }
@@ -85,8 +85,8 @@ routing required:
   - do:
       get:
         index:   target
-        routing: 3
-        id:      3
+        routing: "3"
+        id:      "3"
 
   - match: { _index:   target }
   - match: { _id:      "3"     }
@@ -122,22 +122,22 @@ nested:
   - do:
       index:
         index:   source
-        id:      1
-        routing: 1
+        id:      "1"
+        routing: "1"
         body:    { "foo": "hello world", "n": [{"foo": "goodbye world"}, {"foo": "more words"}] }
 
   - do:
       index:
         index:   source
-        id:      2
-        routing: 2
+        id:      "2"
+        routing: "2"
         body:    { "foo": "hello world 2" }
 
   - do:
       index:
         index:   source
-        id:      3
-        routing: 3
+        id:      "3"
+        routing: "3"
         body:    { "foo": "hello world 3" }
 
   # make it read-only
@@ -172,8 +172,8 @@ nested:
   - do:
       get:
         index:   target
-        routing: 1
-        id:      1
+        routing: "1"
+        id:      "1"
 
   - match: { _index:   target }
   - match: { _id:      "1"     }
@@ -182,8 +182,8 @@ nested:
   - do:
       get:
         index:   target
-        routing: 2
-        id:      2
+        routing: "2"
+        id:      "2"
 
   - match: { _index:   target }
   - match: { _id:      "2"     }
@@ -192,8 +192,8 @@ nested:
   - do:
       get:
         index:   target
-        routing: 3
-        id:      3
+        routing: "3"
+        id:      "3"
 
   - match: { _index:   target }
   - match: { _id:      "3"     }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_mapping.yml
deleted file mode 100644
index af4b1a1999669..0000000000000
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_mapping.yml
+++ /dev/null
@@ -1,94 +0,0 @@
----
-stored _source mode is not supported:
-  - requires:
-      test_runner_features: [capabilities]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [logsdb_index_mode]
-      reason: "Support for 'logsdb' index mode capability required"
-
-  - skip:
-      known_issues:
-        - cluster_feature: "gte_v8.15.0"
-          fixed_by: "gte_v8.16.0"
-      reason: "Development of logs index mode spans 8.15 and 8.16"
-
-  - do:
-      catch: bad_request
-      indices.create:
-        index: test-stored-source
-        body:
-          settings:
-            index:
-              mode: logsdb
-          mappings:
-            _source:
-              mode: stored
-            properties:
-              "@timestamp":
-                type: date
-              host.name:
-                type: keyword
-
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.root_cause.0.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logs] only support synthetic source" }
-
----
-disabled _source is not supported:
-  - requires:
-      test_runner_features: [capabilities]
-      capabilities:
-        - method: PUT
-          path: /{index}
-          capabilities: [logsdb_index_mode]
-      reason: "Support for 'logsdb' index mode capability required"
-
-  - skip:
-      known_issues:
-        - cluster_feature: "gte_v8.15.0"
-          fixed_by: "gte_v8.16.0"
-      reason: "Development of logs index mode spans 8.15 and 8.16"
-
-  - do:
-      catch: bad_request
-      indices.create:
-        index: test-disabled-source
-        body:
-          settings:
-            index:
-              mode: logsdb
-          mappings:
-            _source:
-              enabled: false
-            properties:
-              "@timestamp":
-                type: date
-              host.name:
-                type: keyword
-
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.root_cause.0.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
-
-  - do:
-      catch: bad_request
-      indices.create:
-        index: test-disabled-source
-        body:
-          settings:
-            index:
-              mode: logsdb
-          mappings:
-            _source:
-              mode: disabled
-            properties:
-              "@timestamp":
-                type: date
-              host.name:
-                type: keyword
-
-  - match: { error.type: "mapper_parsing_exception" }
-  - match: { error.root_cause.0.type: "mapper_parsing_exception" }
-  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logs] only support synthetic source" }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml
new file mode 100644
index 0000000000000..d209c839d904b
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml
@@ -0,0 +1,94 @@
+---
+stored _source mode is not supported:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [logsdb_index_mode]
+      reason: "Support for 'logsdb' index mode capability required"
+
+  - skip:
+      known_issues:
+        - cluster_feature: "gte_v8.15.0"
+          fixed_by: "gte_v8.16.0"
+      reason: "Development of logs index mode spans 8.15 and 8.16"
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test-stored-source
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            _source:
+              mode: stored
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.root_cause.0.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+
+---
+disabled _source is not supported:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [logsdb_index_mode]
+      reason: "Support for 'logsdb' index mode capability required"
+
+  - skip:
+      known_issues:
+        - cluster_feature: "gte_v8.15.0"
+          fixed_by: "gte_v8.16.0"
+      reason: "Development of logs index mode spans 8.15 and 8.16"
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test-disabled-source
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            _source:
+              enabled: false
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.root_cause.0.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test-disabled-source
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            _source:
+              mode: disabled
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+
+  - match: { error.type: "mapper_parsing_exception" }
+  - match: { error.root_cause.0.type: "mapper_parsing_exception" }
+  - match: { error.reason: "Failed to parse mapping: Indices with with index mode [logsdb] only support synthetic source" }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml
index 2935c0c1c41b5..ff17a92ed0fcc 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml
@@ -46,6 +46,94 @@ keyword:
       docs.1._source:
         kwd: bar
 
+---
+keyword with normalizer:
+  - requires:
+      cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ]
+      reason: support for normalizer on keyword fields
+  - do:
+      indices.create:
+        index: test-keyword-with-normalizer
+        body:
+          settings:
+            analysis:
+              normalizer:
+                lowercase:
+                  type: custom
+                  filter:
+                    - lowercase
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              keyword:
+                type: keyword
+                normalizer: lowercase
+              keyword_with_ignore_above:
+                type: keyword
+                normalizer: lowercase
+                ignore_above: 10
+              keyword_without_doc_values:
+                type: keyword
+                normalizer: lowercase
+                doc_values: false
+
+  - do:
+      index:
+        index: test-keyword-with-normalizer
+        id: 1
+        body:
+          keyword: "the Quick Brown Fox jumps over the lazy Dog"
+          keyword_with_ignore_above: "the Quick Brown Fox jumps over the lazy Dog"
+          keyword_without_doc_values: "the Quick Brown Fox jumps over the lazy Dog"
+
+  - do:
+      index:
+        index: test-keyword-with-normalizer
+        id: 2
+        body:
+          keyword: "The five BOXING wizards jump Quickly"
+          keyword_with_ignore_above: "The five BOXING wizards jump Quickly"
+          keyword_without_doc_values: "The five BOXING wizards jump Quickly"
+
+  - do:
+      index:
+        index: test-keyword-with-normalizer
+        id: 3
+        body:
+          keyword: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ]
+          keyword_with_ignore_above: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ]
+          keyword_without_doc_values: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ]
+
+  - do:
+      mget:
+        index: test-keyword-with-normalizer
+        body:
+          ids:    [ 1, 2, 3 ]
+  - match: { docs.0._index: "test-keyword-with-normalizer" }
+  - match: { docs.0._id: "1" }
+  - match:
+      docs.0._source:
+        keyword: "the Quick Brown Fox jumps over the lazy Dog"
+        keyword_with_ignore_above: "the Quick Brown Fox jumps over the lazy Dog"
+        keyword_without_doc_values: "the Quick Brown Fox jumps over the lazy Dog"
+
+  - match: { docs.1._index: "test-keyword-with-normalizer" }
+  - match: { docs.1._id: "2" }
+  - match:
+      docs.1._source:
+        keyword: "The five BOXING wizards jump Quickly"
+        keyword_with_ignore_above: "The five BOXING wizards jump Quickly"
+        keyword_without_doc_values: "The five BOXING wizards jump Quickly"
+
+  - match: { docs.2._index: "test-keyword-with-normalizer" }
+  - match: { docs.2._id: "3" }
+  - match:
+      docs.2._source:
+        keyword: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ]
+        keyword_with_ignore_above: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ]
+        keyword_without_doc_values: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ]
+
 ---
 stored text:
   - requires:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
index 35089cc4c85a7..9d6e8da8c1e1e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml
@@ -509,6 +509,102 @@
   - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 28672 }
 
 ---
+"Lucene segment level fields stats":
+
+  - requires:
+      cluster_features: ["mapper.segment_level_fields_stats"]
+      reason: "segment level fields stats"
+
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+            routing.rebalance.enable: none
+          mappings:
+            runtime:
+              a_source_field:
+                type: keyword
+            properties:
+              "@timestamp":
+                type: date
+              authors:
+                properties:
+                  age:
+                    type: long
+                  company:
+                    type: text
+                    fields:
+                      keyword:
+                        type: keyword
+                        ignore_above: 256
+                  name:
+                    properties:
+                      first_name:
+                        type: keyword
+                      full_name:
+                        type: text
+                      last_name:
+                        type: keyword
+              link:
+                type: alias
+                path: url
+              title:
+                type: text
+              url:
+                type: keyword
+  - do:
+      cluster.state: {}
+
+  - set:
+      routing_table.indices.index1.shards.0.0.node: node_id
+
+  - do:
+      nodes.stats: { metric: _all, level: "indices", human: true }
+
+  - do:
+      index:
+        index:  index1
+        body:   { "title": "foo", "@timestamp": "2023-10-15T14:12:12" }
+  - do:
+      indices.flush:
+        index: index1
+  - do:
+      nodes.stats: { metric: _all, level: "indices", human: true }
+
+  - gte: { nodes.$node_id.indices.mappings.total_count: 28 }
+  - lte: { nodes.$node_id.indices.mappings.total_count: 29 }
+  - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 28672 }
+  - lte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 29696 }
+  - match: { nodes.$node_id.indices.mappings.total_segments: 1 }
+  - gte: { nodes.$node_id.indices.mappings.total_segment_fields: 28 }
+  - lte: { nodes.$node_id.indices.mappings.total_segment_fields: 29 }
+  - gte: { nodes.$node_id.indices.mappings.average_fields_per_segment: 28 }
+  - lte: { nodes.$node_id.indices.mappings.average_fields_per_segment: 29 }
+
+  - do:
+      index:
+        index:  index1
+        body:   { "title": "bar", "@timestamp": "2023-11-15T14:12:12" }
+  - do:
+      indices.flush:
+        index: index1
+  - do:
+      nodes.stats: { metric: _all, level: "indices", human: true }
+
+  - gte: { nodes.$node_id.indices.mappings.total_count: 28 }
+  - lte: { nodes.$node_id.indices.mappings.total_count: 29 }
+  - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 28672 }
+  - lte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 29696 }
+  - match: { nodes.$node_id.indices.mappings.total_segments: 2 }
+  - gte: { nodes.$node_id.indices.mappings.total_segment_fields: 56 }
+  - lte: { nodes.$node_id.indices.mappings.total_segment_fields: 58 }
+  - gte: { nodes.$node_id.indices.mappings.average_fields_per_segment: 28 }
+  - lte: { nodes.$node_id.indices.mappings.average_fields_per_segment: 29 }
+---
+
 "indices mappings does not exist in shards level":
 
   - requires:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/100_knn_nested_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/100_knn_nested_search.yml
index 72c6abab22600..d627be2fb15c3 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/100_knn_nested_search.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/100_knn_nested_search.yml
@@ -411,3 +411,53 @@ setup:
 
   - match: {hits.total.value: 1}
   - match: {hits.hits.0._id: "2"}
+---
+"nested Knn search with required similarity appropriately filters inner_hits":
+  - requires:
+      cluster_features: "gte_v8.16.0"
+      reason: 'bugfix for 8.16'
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            nested:
+              path: nested
+              inner_hits:
+                size: 3
+                _source: false
+                fields:
+                  - nested.paragraph_id
+              query:
+                knn:
+                  field: nested.vector
+                  query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+                  num_candidates: 3
+                  similarity: 10.5
+
+  - match: {hits.total.value: 1}
+  - match: {hits.hits.0._id: "2"}
+  - length: {hits.hits.0.inner_hits.nested.hits.hits: 1}
+  - match: {hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.paragraph_id.0: "0"}
+
+  - do:
+      search:
+        index: test
+        body:
+          knn:
+            field: nested.vector
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            num_candidates: 3
+            k: 3
+            similarity: 10.5
+            inner_hits:
+              size: 3
+              _source: false
+              fields:
+                - nested.paragraph_id
+
+  - match: {hits.total.value: 1}
+  - match: {hits.hits.0._id: "2"}
+  - length: {hits.hits.0.inner_hits.nested.hits.hits: 1}
+  - match: {hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.paragraph_id.0: "0"}
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml
index df5d451e3a2e1..b3d86a066550e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml
@@ -28,15 +28,14 @@ setup:
                   type: hnsw
                   m: 16
                   ef_construction: 200
-
   - do:
       index:
         index: test
         id: "1"
         body:
           name: cow.jpg
-          vector: [230.0, 300.33, -34.8988, 15.555, -200.0]
-          another_vector: [130.0, 115.0, -1.02, 15.555, -100.0]
+          vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ]
+          another_vector: [ 130.0, 115.0, -1.02, 15.555, -100.0 ]
 
   - do:
       index:
@@ -44,8 +43,8 @@ setup:
         id: "2"
         body:
           name: moose.jpg
-          vector: [-0.5, 100.0, -13, 14.8, -156.0]
-          another_vector: [-0.5, 50.0, -1, 1, 120]
+          vector: [ -0.5, 100.0, -13, 14.8, -156.0 ]
+          another_vector: [ -0.5, 50.0, -1, 1, 120 ]
 
   - do:
       index:
@@ -53,11 +52,11 @@ setup:
         id: "3"
         body:
           name: rabbit.jpg
-          vector: [0.5, 111.3, -13.0, 14.8, -156.0]
-          another_vector: [-0.5, 11.0, 0, 12, 111.0]
+          vector: [ 0.5, 111.3, -13.0, 14.8, -156.0 ]
+          another_vector: [ -0.5, 11.0, 0, 12, 111.0 ]
 
   - do:
-      indices.refresh: {}
+      indices.refresh: { }
 
 ---
 "kNN search only":
@@ -71,15 +70,15 @@ setup:
           fields: [ "name" ]
           knn:
             field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
             k: 2
             num_candidates: 3
 
-  - match: {hits.hits.0._id: "2"}
-  - match: {hits.hits.0.fields.name.0: "moose.jpg"}
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.0.fields.name.0: "moose.jpg" }
 
-  - match: {hits.hits.1._id: "3"}
-  - match: {hits.hits.1.fields.name.0: "rabbit.jpg"}
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.1.fields.name.0: "rabbit.jpg" }
 ---
 "kNN multi-field search only":
   - requires:
@@ -91,14 +90,14 @@ setup:
         body:
           fields: [ "name" ]
           knn:
-           - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3}
-           - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3}
+            - { field: vector, query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ], k: 2, num_candidates: 3 }
+            - { field: another_vector, query_vector: [ -0.5, 11.0, 0, 12, 111.0 ], k: 2, num_candidates: 3 }
 
-  - match: {hits.hits.0._id: "3"}
-  - match: {hits.hits.0.fields.name.0: "rabbit.jpg"}
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0.fields.name.0: "rabbit.jpg" }
 
-  - match: {hits.hits.1._id: "2"}
-  - match: {hits.hits.1.fields.name.0: "moose.jpg"}
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1.fields.name.0: "moose.jpg" }
 ---
 "kNN search plus query":
   - requires:
@@ -111,21 +110,21 @@ setup:
           fields: [ "name" ]
           knn:
             field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
             k: 2
             num_candidates: 3
           query:
             term:
               name: cow.jpg
 
-  - match: {hits.hits.0._id: "1"}
-  - match: {hits.hits.0.fields.name.0: "cow.jpg"}
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.0.fields.name.0: "cow.jpg" }
 
-  - match: {hits.hits.1._id: "2"}
-  - match: {hits.hits.1.fields.name.0: "moose.jpg"}
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1.fields.name.0: "moose.jpg" }
 
-  - match: {hits.hits.2._id: "3"}
-  - match: {hits.hits.2.fields.name.0: "rabbit.jpg"}
+  - match: { hits.hits.2._id: "3" }
+  - match: { hits.hits.2.fields.name.0: "rabbit.jpg" }
 ---
 "kNN multi-field search with query":
   - requires:
@@ -137,20 +136,20 @@ setup:
         body:
           fields: [ "name" ]
           knn:
-            - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3}
-            - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3}
+            - { field: vector, query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ], k: 2, num_candidates: 3 }
+            - { field: another_vector, query_vector: [ -0.5, 11.0, 0, 12, 111.0 ], k: 2, num_candidates: 3 }
           query:
             term:
               name: cow.jpg
 
-  - match: {hits.hits.0._id: "3"}
-  - match: {hits.hits.0.fields.name.0: "rabbit.jpg"}
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0.fields.name.0: "rabbit.jpg" }
 
-  - match: {hits.hits.1._id: "1"}
-  - match: {hits.hits.1.fields.name.0: "cow.jpg"}
+  - match: { hits.hits.1._id: "1" }
+  - match: { hits.hits.1.fields.name.0: "cow.jpg" }
 
-  - match: {hits.hits.2._id: "2"}
-  - match: {hits.hits.2.fields.name.0: "moose.jpg"}
+  - match: { hits.hits.2._id: "2" }
+  - match: { hits.hits.2.fields.name.0: "moose.jpg" }
 ---
 "kNN search with filter":
   - requires:
@@ -163,16 +162,16 @@ setup:
           fields: [ "name" ]
           knn:
             field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
             k: 2
             num_candidates: 3
             filter:
               term:
                 name: "rabbit.jpg"
 
-  - match: {hits.total.value: 1}
-  - match: {hits.hits.0._id: "3"}
-  - match: {hits.hits.0.fields.name.0: "rabbit.jpg"}
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0.fields.name.0: "rabbit.jpg" }
 
   - do:
       search:
@@ -181,7 +180,7 @@ setup:
           fields: [ "name" ]
           knn:
             field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
             k: 2
             num_candidates: 3
             filter:
@@ -190,7 +189,7 @@ setup:
               - term:
                   _id: 2
 
-  - match: {hits.total.value: 0}
+  - match: { hits.total.value: 0 }
 
 ---
 "kNN search with explicit search_type":
@@ -206,7 +205,7 @@ setup:
           fields: [ "name" ]
           knn:
             field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
             k: 2
             num_candidates: 3
 
@@ -216,7 +215,7 @@ setup:
 ---
 "kNN search in _knn_search endpoint":
   - skip:
-      features: ["allowed_warnings"]
+      features: [ "allowed_warnings" ]
   - do:
       allowed_warnings:
         - "The kNN search API has been replaced by the `knn` option in the search API."
@@ -226,22 +225,22 @@ setup:
           fields: [ "name" ]
           knn:
             field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
             k: 2
             num_candidates: 3
 
-  - match: {hits.hits.0._id: "2"}
-  - match: {hits.hits.0.fields.name.0: "moose.jpg"}
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.0.fields.name.0: "moose.jpg" }
 
-  - match: {hits.hits.1._id: "3"}
-  - match: {hits.hits.1.fields.name.0: "rabbit.jpg"}
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.1.fields.name.0: "rabbit.jpg" }
 
 ---
 "kNN search with filter in _knn_search endpoint":
   - requires:
       cluster_features: "gte_v8.2.0"
       reason: 'kNN with filtering added in 8.2'
-      test_runner_features: ["allowed_warnings"]
+      test_runner_features: [ "allowed_warnings" ]
   - do:
       allowed_warnings:
         - "The kNN search API has been replaced by the `knn` option in the search API."
@@ -251,16 +250,16 @@ setup:
           fields: [ "name" ]
           knn:
             field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
             k: 2
             num_candidates: 3
           filter:
             term:
               name: "rabbit.jpg"
 
-  - match: {hits.total.value: 1}
-  - match: {hits.hits.0._id: "3"}
-  - match: {hits.hits.0.fields.name.0: "rabbit.jpg"}
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0.fields.name.0: "rabbit.jpg" }
 
   - do:
       allowed_warnings:
@@ -271,7 +270,7 @@ setup:
           fields: [ "name" ]
           knn:
             field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
             k: 2
             num_candidates: 3
           filter:
@@ -280,7 +279,7 @@ setup:
             - term:
                 _id: 2
 
-  - match: {hits.total.value: 0}
+  - match: { hits.total.value: 0 }
 
 ---
 "Test nonexistent field is match none":
@@ -298,7 +297,7 @@ setup:
             k: 2
             num_candidates: 3
 
-  - length: {hits.hits: 0}
+  - length: { hits.hits: 0 }
 
   - do:
       indices.create:
@@ -347,12 +346,12 @@ setup:
             k: 3
             field: vector
             similarity: 11
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
 
-  - length: {hits.hits: 1}
+  - length: { hits.hits: 1 }
 
-  - match: {hits.hits.0._id: "2"}
-  - match: {hits.hits.0.fields.name.0: "moose.jpg"}
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.0.fields.name.0: "moose.jpg" }
 ---
 "Vector similarity with filter only":
   - requires:
@@ -368,13 +367,13 @@ setup:
             k: 3
             field: vector
             similarity: 11
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
-            filter: {"term": {"name": "moose.jpg"}}
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+            filter: { "term": { "name": "moose.jpg" } }
 
-  - length: {hits.hits: 1}
+  - length: { hits.hits: 1 }
 
-  - match: {hits.hits.0._id: "2"}
-  - match: {hits.hits.0.fields.name.0: "moose.jpg"}
+  - match: { hits.hits.0._id: "2" }
+  - match: { hits.hits.0.fields.name.0: "moose.jpg" }
 
   - do:
       search:
@@ -386,10 +385,10 @@ setup:
             k: 3
             field: vector
             similarity: 110
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
-            filter: {"term": {"name": "cow.jpg"}}
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+            filter: { "term": { "name": "cow.jpg" } }
 
-  - length: {hits.hits: 0}
+  - length: { hits.hits: 0 }
 ---
 "Knn search with mip":
   - requires:
@@ -421,7 +420,7 @@ setup:
         id: "1"
         body:
           name: cow.jpg
-          vector: [230.0, 300.33, -34.8988, 15.555, -200.0]
+          vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ]
 
   - do:
       index:
@@ -429,7 +428,7 @@ setup:
         id: "2"
         body:
           name: moose.jpg
-          vector: [-0.5, 100.0, -13, 14.8, -156.0]
+          vector: [ -0.5, 100.0, -13, 14.8, -156.0 ]
 
   - do:
       index:
@@ -437,10 +436,10 @@ setup:
         id: "3"
         body:
           name: rabbit.jpg
-          vector: [0.5, 111.3, -13.0, 14.8, -156.0]
+          vector: [ 0.5, 111.3, -13.0, 14.8, -156.0 ]
 
   - do:
-      indices.refresh: {}
+      indices.refresh: { }
 
   - do:
       search:
@@ -451,16 +450,16 @@ setup:
             num_candidates: 3
             k: 3
             field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
 
 
-  - length: {hits.hits: 3}
-  - match: {hits.hits.0._id: "1"}
-  - close_to: {hits.hits.0._score: {value: 58694.902, error: 0.01}}
-  - match: {hits.hits.1._id: "3"}
-  - close_to: {hits.hits.1._score: {value: 34702.79, error: 0.01}}
-  - match: {hits.hits.2._id: "2"}
-  - close_to: {hits.hits.2._score: {value: 33686.29, error: 0.01}}
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._id: "1" }
+  - close_to: { hits.hits.0._score: { value: 58694.902, error: 0.01 } }
+  - match: { hits.hits.1._id: "3" }
+  - close_to: { hits.hits.1._score: { value: 34702.79, error: 0.01 } }
+  - match: { hits.hits.2._id: "2" }
+  - close_to: { hits.hits.2._score: { value: 33686.29, error: 0.01 } }
 
   - do:
       search:
@@ -471,14 +470,14 @@ setup:
             num_candidates: 3
             k: 3
             field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
             filter: { "term": { "name": "moose.jpg" } }
 
 
 
-  - length: {hits.hits: 1}
-  - match: {hits.hits.0._id: "2"}
-  - close_to: {hits.hits.0._score: {value: 33686.29, error: 0.01}}
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._id: "2" }
+  - close_to: { hits.hits.0._score: { value: 33686.29, error: 0.01 } }
 ---
 "Knn search with _name":
   - requires:
@@ -493,7 +492,7 @@ setup:
           fields: [ "name" ]
           knn:
             field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
             k: 3
             num_candidates: 3
             _name: "my_knn_query"
@@ -504,15 +503,41 @@ setup:
                 _name: "my_query"
 
 
-  - match: {hits.hits.0._id: "1"}
-  - match: {hits.hits.0.fields.name.0: "cow.jpg"}
-  - match: {hits.hits.0.matched_queries.0: "my_knn_query"}
-  - match: {hits.hits.0.matched_queries.1: "my_query"}
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.0.fields.name.0: "cow.jpg" }
+  - match: { hits.hits.0.matched_queries.0: "my_knn_query" }
+  - match: { hits.hits.0.matched_queries.1: "my_query" }
+
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1.fields.name.0: "moose.jpg" }
+  - match: { hits.hits.1.matched_queries.0: "my_knn_query" }
+
+  - match: { hits.hits.2._id: "3" }
+  - match: { hits.hits.2.fields.name.0: "rabbit.jpg" }
+  - match: { hits.hits.2.matched_queries.0: "my_knn_query" }
 
-  - match: {hits.hits.1._id: "2"}
-  - match: {hits.hits.1.fields.name.0: "moose.jpg"}
-  - match: {hits.hits.1.matched_queries.0: "my_knn_query"}
+---
+"kNN search on empty index should return 0 results and not an error":
+  - requires:
+      cluster_features: "gte_v8.15.1"
+      reason: 'Error fixed in 8.15.1'
+  - do:
+      indices.create:
+        index: test_empty
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+  - do:
+      search:
+        index: test_empty
+        body:
+          fields: [ "name" ]
+          knn:
+            field: vector
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+            k: 2
+            num_candidates: 3
 
-  - match: {hits.hits.2._id: "3"}
-  - match: {hits.hits.2.fields.name.0: "rabbit.jpg"}
-  - match: {hits.hits.2.matched_queries.0: "my_knn_query"}
+  - match: { hits.total.value: 0 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml
index 82fb18a879346..99bd001bd95e2 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml
@@ -21,6 +21,10 @@ setup:
             - '{"text" : "Baby its cold there outside"}'
             - '{"index": {"_index": "test", "_id": "4"}}'
             - '{"text" : "Outside it is cold and wet"}'
+            - '{"index": {"_index": "test", "_id": "5"}}'
+            - '{"text" : "the big bad wolf"}'
+            - '{"index": {"_index": "test", "_id": "6"}}'
+            - '{"text" : "the big wolf"}'
 
 ---
 "Test ordered matching":
@@ -444,4 +448,31 @@ setup:
                         prefix: out
   - match: { hits.total.value: 3 }
 
+---
+"Test rewrite disjunctions":
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            intervals:
+              text:
+                all_of:
+                  intervals:
+                    - "match":
+                        "query": "the"
+                    - "any_of":
+                        "intervals":
+                          - "match":
+                              "query": "big"
+                          - "match":
+                              "query": "big bad"
+                    - "match":
+                        "query": "wolf"
+                  max_gaps: 0
+                  ordered: true
+
+  - match: { hits.total.value: 2   }
+  - match: { hits.hits.0._id:  "6" }
+  - match: { hits.hits.1._id:  "5" }
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml
index 703f2a0352fbd..c120bed2d369d 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml
@@ -1125,3 +1125,55 @@ fetch geo_point:
   - match: { hits.hits.0.fields.root\.keyword.0: 'parent' }
   - match: { hits.hits.0.fields.root\.subfield.0: 'child' }
   - match: { hits.hits.0.fields.root\.subfield\.keyword.0: 'child' }
+
+---
+"Test with subobjects: auto":
+  - requires:
+      cluster_features: "mapper.subobjects_auto"
+      reason: requires support for subobjects auto setting
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            subobjects: auto
+            properties:
+              message:
+                type: object
+                subobjects: auto
+                enabled: false
+
+  - do:
+      index:
+        index: test
+        refresh: true
+        body: >
+          {
+            "root": "parent",
+            "root.subfield": "child",
+            "message": {
+              "foo": 10,
+              "foo.bar": 20
+            }
+          }
+  - match: {result: "created"}
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            term:
+              root.subfield:
+                value: 'child'
+          fields:
+            - field: 'root*'
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.root.0: 'parent' }
+  - match: { hits.hits.0.fields.root\.keyword.0: 'parent' }
+  - match: { hits.hits.0.fields.root\.subfield.0: 'child' }
+  - match: { hits.hits.0.fields.root\.subfield\.keyword.0: 'child' }
+  - is_false: hits.hits.0.fields.message
+  - match: { hits.hits.0._source.message.foo: 10 }
+  - match: { hits.hits.0._source.message.foo\.bar: 20 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml
new file mode 100644
index 0000000000000..d3d0a3bb4df70
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml
@@ -0,0 +1,309 @@
+setup:
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: Lenient handling of updateable synonyms by default is introduced in 8.16.0
+---
+"Load index with an invalid synonym rule":
+  - do:
+      synonyms.put_synonym:
+        id: set1
+        body:
+          synonyms_set:
+            synonyms: "foo => bar, baz"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+            analysis:
+              filter:
+                my_synonym_filter:
+                  type: synonym
+                  synonyms_set: set1
+                  updateable: true
+                my_stop_filter:
+                  type: stop
+                  stopwords: ["baz"]
+              analyzer:
+                my_analyzer:
+                  type: custom
+                  tokenizer: whitespace
+                  filter: [ lowercase, my_stop_filter, my_synonym_filter ]
+          mappings:
+            properties:
+              my_field:
+                type: text
+                search_analyzer: my_analyzer
+
+  - match: { acknowledged: true }
+  - match: { shards_acknowledged: true }
+
+  - do:
+      indices.stats: { index: test_index }
+
+  - match: { indices.test_index.health: "green" }
+
+  - do:
+      indices.analyze:
+        index: test_index
+        body:
+          analyzer: my_analyzer
+          text: foo
+
+  - length: { tokens: 1 }
+  - match: { tokens.0.token: bar }
+
+---
+"Reload index with invalid synonym rule":
+  - do:
+      synonyms.put_synonym:
+        id: set1
+        body:
+          synonyms_set:
+            synonyms: "foo => bar"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+            analysis:
+              filter:
+                my_synonym_filter:
+                  type: synonym
+                  synonyms_set: set1
+                  updateable: true
+                my_stop_filter:
+                  type: stop
+                  stopwords: ["baz"]
+              analyzer:
+                my_analyzer:
+                  type: custom
+                  tokenizer: whitespace
+                  filter: [ lowercase, my_stop_filter, my_synonym_filter ]
+          mappings:
+            properties:
+              my_field:
+                type: text
+                search_analyzer: my_analyzer
+
+  - match: { acknowledged: true }
+  - match: { shards_acknowledged: true }
+
+  - do:
+      indices.stats: { index: test_index }
+
+  - match: { indices.test_index.health: "green" }
+
+  - do:
+      indices.analyze:
+        index: test_index
+        body:
+          analyzer: my_analyzer
+          text: foo
+
+  - length: { tokens: 1 }
+  - match: { tokens.0.token: bar }
+
+  - do:
+      synonyms.put_synonym:
+        id: set1
+        body:
+          synonyms_set:
+            synonyms: "foo => bar, baz"
+
+  - do:
+      indices.stats: { index: test_index }
+
+  - match: { indices.test_index.health: "green" }
+
+  - do:
+      indices.analyze:
+        index: test_index
+        body:
+          analyzer: my_analyzer
+          text: foo
+
+  - length: { tokens: 1 }
+  - match: { tokens.0.token: bar }
+
+  - do:
+      indices.close:
+        index: test_index
+
+  - match: { acknowledged: true }
+  - length: { indices: 1 }
+
+  - do:
+      indices.open:
+        index: test_index
+        wait_for_active_shards: all
+
+  - match: { acknowledged: true }
+  - match: { shards_acknowledged: true }
+
+  - do:
+      indices.stats: { index: test_index }
+
+  - match: { indices.test_index.health: "green" }
+
+  - do:
+      indices.analyze:
+        index: test_index
+        body:
+          analyzer: my_analyzer
+          text: foo
+
+  - length: { tokens: 1 }
+  - match: { tokens.0.token: bar }
+
+---
+"Load index with an invalid synonym rule with lenient set to false":
+  - do:
+      synonyms.put_synonym:
+        id: set1
+        body:
+          synonyms_set:
+            synonyms: "foo => bar, baz"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+            analysis:
+              filter:
+                my_synonym_filter:
+                  type: synonym
+                  synonyms_set: set1
+                  updateable: true
+                  lenient: false
+                my_stop_filter:
+                  type: stop
+                  stopwords: ["baz"]
+              analyzer:
+                my_analyzer:
+                  type: custom
+                  tokenizer: whitespace
+                  filter: [ lowercase, my_stop_filter, my_synonym_filter ]
+          mappings:
+            properties:
+              my_field:
+                type: text
+                search_analyzer: my_analyzer
+
+  - match: { acknowledged: true }
+  - match: { shards_acknowledged: false }
+
+  - do:
+      indices.stats: { index: test_index }
+
+  - length: { indices: 0 }
+
+---
+"Reload index with an invalid synonym rule with lenient set to false":
+  - do:
+      synonyms.put_synonym:
+        id: set1
+        body:
+          synonyms_set:
+            synonyms: "foo => bar"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+            analysis:
+              filter:
+                my_synonym_filter:
+                  type: synonym
+                  synonyms_set: set1
+                  updateable: true
+                  lenient: false
+                my_stop_filter:
+                  type: stop
+                  stopwords: [ "baz" ]
+              analyzer:
+                my_analyzer:
+                  type: custom
+                  tokenizer: whitespace
+                  filter: [ lowercase, my_stop_filter, my_synonym_filter ]
+          mappings:
+            properties:
+              my_field:
+                type: text
+                search_analyzer: my_analyzer
+
+  - match: { acknowledged: true }
+  - match: { shards_acknowledged: true }
+
+  - do:
+      indices.stats: { index: test_index }
+
+  - match: { indices.test_index.health: "green" }
+
+  - do:
+      indices.analyze:
+        index: test_index
+        body:
+          analyzer: my_analyzer
+          text: foo
+
+  - length: { tokens: 1 }
+  - match: { tokens.0.token: bar }
+
+  - do:
+      synonyms.put_synonym:
+        id: set1
+        body:
+          synonyms_set:
+            synonyms: "foo => bar, baz"
+
+  - do:
+      indices.stats: { index: test_index }
+
+  - match: { indices.test_index.health: "green" }
+
+  - do:
+      indices.analyze:
+        index: test_index
+        body:
+          analyzer: my_analyzer
+          text: foo
+
+  - length: { tokens: 1 }
+  - match: { tokens.0.token: bar }
+
+  - do:
+      indices.close:
+        index: test_index
+
+  - match: { acknowledged: true }
+  - length: { indices: 1 }
+
+  - do:
+      indices.open:
+        index: test_index
+        wait_for_active_shards: all
+
+  - match: { acknowledged: true }
+  - match: { shards_acknowledged: false }
+
+  - do:
+      indices.stats: { index: test_index }
+
+  - length: { indices: 0 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml
index f6d09a4540eb7..89ad933370e1c 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/80_synonyms_from_index.yml
@@ -187,3 +187,44 @@ setup:
                 search_analyzer: my_analyzer
   - match: { acknowledged: true }
   - match: { shards_acknowledged: false }
+
+---
+"Load empty synonyms set from index for an analyzer":
+  - do:
+      synonyms.put_synonym:
+        id: empty_set
+        body:
+          synonyms_set: []
+
+  - do:
+      indices.create:
+        index: empty_set_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+            analysis:
+              filter:
+                my_synonym_filter:
+                  type: synonym
+                  synonyms_set: empty_set
+                  updateable : true
+              analyzer:
+                my_analyzer:
+                  type: custom
+                  tokenizer: standard
+                  filter: [ lowercase, my_synonym_filter ]
+          mappings:
+            properties:
+              my_field:
+                type: text
+                search_analyzer: my_analyzer
+
+  - match: { acknowledged: true }
+  - match: { shards_acknowledged: true }
+
+  - do:
+      indices.stats: { index: empty_set_index }
+
+  - match: { indices.empty_set_index.health: "green" }
diff --git a/server/build.gradle b/server/build.gradle
index deadd0a330ef8..c3ac4d9704091 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -267,7 +267,7 @@ tasks.named("dependencyLicenses").configure {
 
 tasks.named("licenseHeaders").configure {
     // Ignore our vendored version of Google Guice
-    excludes << 'org/elasticsearch/common/inject/**/*'
+    excludes << 'org/elasticsearch/injection/guice/**/*'
     // Ignore temporary copies of impending 8.7 Lucene classes
     excludes << 'org/apache/lucene/search/RegExp87*'
     excludes << 'org/apache/lucene/search/RegexpQuery87*'
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
index e1804368c2cad..69768d6efe3bd 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.CollectionUtils;
@@ -33,6 +32,7 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.CancellableTask;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
index 1fda9c67a0beb..0c3dac0f99b6c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
@@ -161,7 +161,7 @@ public void testIndicesShardStats() throws ExecutionException, InterruptedExcept
         ClusterStatsResponse response = clusterAdmin().prepareClusterStats().get();
         assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN));
 
-        prepareCreate("test1").setSettings(Settings.builder().put("number_of_shards", 2).put("number_of_replicas", 1)).get();
+        prepareCreate("test1").setSettings(indexSettings(2, 1)).get();
 
         response = clusterAdmin().prepareClusterStats().get();
         assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.YELLOW));
@@ -179,7 +179,7 @@ public void testIndicesShardStats() throws ExecutionException, InterruptedExcept
         assertThat(response.indicesStats.getDocs().getCount(), Matchers.equalTo(1L));
         assertShardStats(response.getIndicesStats().getShards(), 1, 4, 2, 1.0);
 
-        prepareCreate("test2").setSettings(Settings.builder().put("number_of_shards", 3).put("number_of_replicas", 0)).get();
+        prepareCreate("test2").setSettings(indexSettings(3, 0)).get();
         ensureGreen();
         response = clusterAdmin().prepareClusterStats().get();
         assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java
index 60462863dd09a..2acb38e9bda1e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 26a430123ccd9..27f0cd408e7fb 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -306,10 +306,7 @@ public void testRestartIndexCreationAfterFullClusterRestart() throws Exception {
 
     public void testFailureToCreateIndexCleansUpIndicesService() {
         final int numReplicas = internalCluster().numDataNodes();
-        Settings settings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), numReplicas)
-            .build();
+        Settings settings = indexSettings(1, numReplicas).build();
         assertAcked(indicesAdmin().prepareCreate("test-idx-1").setSettings(settings).addAlias(new Alias("alias1").writeIndex(true)).get());
 
         ActionRequestBuilder<?, ?> builder = indicesAdmin().prepareCreate("test-idx-2")
@@ -328,10 +325,7 @@ public void testFailureToCreateIndexCleansUpIndicesService() {
      */
     public void testDefaultWaitForActiveShardsUsesIndexSetting() throws Exception {
         final int numReplicas = internalCluster().numDataNodes();
-        Settings settings = Settings.builder()
-            .put(SETTING_WAIT_FOR_ACTIVE_SHARDS.getKey(), Integer.toString(numReplicas))
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), numReplicas)
+        Settings settings = indexSettings(1, numReplicas).put(SETTING_WAIT_FOR_ACTIVE_SHARDS.getKey(), Integer.toString(numReplicas))
             .build();
         assertAcked(indicesAdmin().prepareCreate("test-idx-1").setSettings(settings).get());
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
index 27fd54c39cc95..22549a1562dcd 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
@@ -276,6 +276,7 @@ public void testSplitIndexPrimaryTerm() throws Exception {
                 .put(indexSettings())
                 .put("number_of_shards", numberOfShards)
                 .put("index.number_of_routing_shards", numberOfTargetShards)
+                .put("index.routing.rebalance.enable", EnableAllocationDecider.Rebalance.NONE)
         ).get();
         ensureGreen(TimeValue.timeValueSeconds(120)); // needs more than the default to allocate many shards
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java
index 3560b74189d1d..415cfff459a67 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexBlocksIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -68,7 +69,9 @@ public void testClusterBlockMessageHasIndexName() {
             ClusterBlockException e = expectThrows(ClusterBlockException.class, prepareIndex("test").setId("1").setSource("foo", "bar"));
             assertEquals(
                 "index [test] blocked by: [TOO_MANY_REQUESTS/12/disk usage exceeded flood-stage watermark, "
-                    + "index has read-only-allow-delete block];",
+                    + "index has read-only-allow-delete block; for more information, see "
+                    + ReferenceDocs.FLOOD_STAGE_WATERMARK
+                    + "];",
                 e.getMessage()
             );
         } finally {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index 4d52383bfc4e1..16f8f51cb8aae 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -211,9 +211,8 @@ public void testRolloverWithIndexSettingsWithoutPrefix() throws Exception {
         assertAcked(prepareCreate("test_index-2").addAlias(testAlias).get());
         indexDoc("test_index-2", "1", "field", "value");
         flush("test_index-2");
-        final Settings settings = Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).build();
         final RolloverResponse response = indicesAdmin().prepareRolloverIndex("test_alias")
-            .settings(settings)
+            .settings(indexSettings(1, 0).build())
             .alias(new Alias("extra_alias"))
             .get();
         assertThat(response.getOldIndex(), equalTo("test_index-2"));
@@ -665,7 +664,7 @@ public void testRolloverWithClosedWriteIndex() throws Exception {
         assertAcked(prepareCreate(openNonwriteIndex).addAlias(new Alias(aliasName)).get());
         assertAcked(prepareCreate(closedIndex).addAlias(new Alias(aliasName)).get());
         assertAcked(prepareCreate(writeIndexPrefix + "000001").addAlias(new Alias(aliasName).writeIndex(true)).get());
-
+        ensureGreen(openNonwriteIndex, closedIndex, writeIndexPrefix + "000001");
         index(closedIndex, null, "{\"foo\": \"bar\"}");
         index(aliasName, null, "{\"foo\": \"bar\"}");
         index(aliasName, null, "{\"foo\": \"bar\"}");
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java
index d531686bb5207..0ed585164750a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkAfterWriteFsyncFailureIT.java
@@ -10,7 +10,6 @@
 
 import org.apache.lucene.tests.mockfile.FilterFileChannel;
 import org.apache.lucene.tests.mockfile.FilterFileSystemProvider;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.PathUtilsForTesting;
@@ -59,13 +58,7 @@ public void testFsyncFailureDoesNotAdvanceLocalCheckpoints() {
         client().admin()
             .indices()
             .prepareCreate(indexName)
-            .setSettings(
-                Settings.builder()
-                    .put(INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)
-                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                    .build()
-            )
+            .setSettings(indexSettings(1, 0).put(INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1))
             .setMapping("key", "type=keyword", "val", "type=long")
             .get();
         ensureGreen(indexName);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java
index a9a5bb074c9ac..da2dfc50d7fe9 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java
@@ -10,12 +10,16 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.NoShardAvailableActionException;
+import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
@@ -54,11 +58,14 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.hamcrest.Matchers.arrayWithSize;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.everyItem;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.in;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.not;
 
 public class PointInTimeIT extends ESIntegTestCase {
@@ -84,7 +91,7 @@ public void testBasic() {
             prepareIndex("test").setId(id).setSource("value", i).get();
         }
         refresh("test");
-        BytesReference pitId = openPointInTime(new String[] { "test" }, TimeValue.timeValueMinutes(2));
+        BytesReference pitId = openPointInTime(new String[] { "test" }, TimeValue.timeValueMinutes(2)).getPointInTimeId();
         assertResponse(prepareSearch().setPointInTime(new PointInTimeBuilder(pitId)), resp1 -> {
             assertThat(resp1.pointInTimeId(), equalTo(pitId));
             assertHitCount(resp1, numDocs);
@@ -130,7 +137,7 @@ public void testMultipleIndices() {
             prepareIndex(index).setId(id).setSource("value", i).get();
         }
         refresh();
-        BytesReference pitId = openPointInTime(new String[] { "*" }, TimeValue.timeValueMinutes(2));
+        BytesReference pitId = openPointInTime(new String[] { "*" }, TimeValue.timeValueMinutes(2)).getPointInTimeId();
         try {
             int moreDocs = randomIntBetween(10, 50);
             assertNoFailuresAndResponse(prepareSearch().setPointInTime(new PointInTimeBuilder(pitId)), resp -> {
@@ -212,7 +219,7 @@ public void testRelocation() throws Exception {
             prepareIndex("test").setId(Integer.toString(i)).setSource("value", i).get();
         }
         refresh();
-        BytesReference pitId = openPointInTime(new String[] { "test" }, TimeValue.timeValueMinutes(2));
+        BytesReference pitId = openPointInTime(new String[] { "test" }, TimeValue.timeValueMinutes(2)).getPointInTimeId();
         try {
             assertNoFailuresAndResponse(prepareSearch().setPointInTime(new PointInTimeBuilder(pitId)), resp -> {
                 assertHitCount(resp, numDocs);
@@ -264,7 +271,7 @@ public void testPointInTimeNotFound() throws Exception {
             prepareIndex("index").setId(id).setSource("value", i).get();
         }
         refresh();
-        BytesReference pit = openPointInTime(new String[] { "index" }, TimeValue.timeValueSeconds(5));
+        BytesReference pit = openPointInTime(new String[] { "index" }, TimeValue.timeValueSeconds(5)).getPointInTimeId();
         assertNoFailuresAndResponse(prepareSearch().setPointInTime(new PointInTimeBuilder(pit)), resp1 -> {
             assertHitCount(resp1, index1);
             if (rarely()) {
@@ -305,7 +312,7 @@ public void testIndexNotFound() {
             prepareIndex("index-2").setId(id).setSource("value", i).get();
         }
         refresh();
-        BytesReference pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueMinutes(2));
+        BytesReference pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueMinutes(2)).getPointInTimeId();
         try {
             assertNoFailuresAndResponse(
                 prepareSearch().setPointInTime(new PointInTimeBuilder(pit)),
@@ -348,7 +355,7 @@ public void testCanMatch() throws Exception {
         assertAcked(prepareCreate("test").setSettings(settings).setMapping("""
             {"properties":{"created_date":{"type": "date", "format": "yyyy-MM-dd"}}}"""));
         ensureGreen("test");
-        BytesReference pitId = openPointInTime(new String[] { "test*" }, TimeValue.timeValueMinutes(2));
+        BytesReference pitId = openPointInTime(new String[] { "test*" }, TimeValue.timeValueMinutes(2)).getPointInTimeId();
         try {
             for (String node : internalCluster().nodesInclude("test")) {
                 for (IndexService indexService : internalCluster().getInstance(IndicesService.class, node)) {
@@ -415,7 +422,7 @@ public void testPartialResults() throws Exception {
             prepareIndex(randomFrom("test-2")).setId(Integer.toString(i)).setSource("value", i).get();
         }
         refresh();
-        BytesReference pitId = openPointInTime(new String[] { "test-*" }, TimeValue.timeValueMinutes(2));
+        BytesReference pitId = openPointInTime(new String[] { "test-*" }, TimeValue.timeValueMinutes(2)).getPointInTimeId();
         try {
             assertNoFailuresAndResponse(prepareSearch().setPointInTime(new PointInTimeBuilder(pitId)), resp -> {
                 assertHitCount(resp, numDocs1 + numDocs2);
@@ -447,7 +454,7 @@ public void testPITTiebreak() throws Exception {
             }
         }
         refresh("index-*");
-        BytesReference pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueHours(1));
+        BytesReference pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueHours(1)).getPointInTimeId();
         try {
             for (int size = 1; size <= numIndex; size++) {
                 SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC;
@@ -532,6 +539,176 @@ public void testOpenPITConcurrentShardRequests() throws Exception {
         }
     }
 
+    public void testMissingShardsWithPointInTime() throws Exception {
+        final Settings nodeAttributes = Settings.builder().put("node.attr.foo", "bar").build();
+        final String masterNode = internalCluster().startMasterOnlyNode(nodeAttributes);
+        List<String> dataNodes = internalCluster().startDataOnlyNodes(2, nodeAttributes);
+
+        final String index = "my_test_index";
+        // tried to have randomIntBetween(3, 10) but having more shards than 3 was taking forever and throwing timeouts
+        final int numShards = 3;
+        final int numReplicas = 0;
+        // create an index with numShards shards and 0 replicas
+        createIndex(
+            index,
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)
+                .put("index.routing.allocation.require.foo", "bar")
+                .build()
+        );
+
+        // index some documents
+        int numDocs = randomIntBetween(10, 50);
+        for (int i = 0; i < numDocs; i++) {
+            String id = Integer.toString(i);
+            prepareIndex(index).setId(id).setSource("value", i).get();
+        }
+        refresh(index);
+
+        // create a PIT when all shards are present
+        OpenPointInTimeResponse pointInTimeResponse = openPointInTime(new String[] { index }, TimeValue.timeValueMinutes(1));
+        try {
+            // ensure that the PIT created has all the shards there
+            assertThat(numShards, equalTo(pointInTimeResponse.getTotalShards()));
+            assertThat(numShards, equalTo(pointInTimeResponse.getSuccessfulShards()));
+            assertThat(0, equalTo(pointInTimeResponse.getFailedShards()));
+            assertThat(0, equalTo(pointInTimeResponse.getSkippedShards()));
+
+            // make a request using the above PIT
+            assertResponse(
+                prepareSearch().setQuery(new MatchAllQueryBuilder())
+                    .setPointInTime(new PointInTimeBuilder(pointInTimeResponse.getPointInTimeId())),
+                resp -> {
+                    // ensure that al docs are returned
+                    assertThat(resp.pointInTimeId(), equalTo(pointInTimeResponse.getPointInTimeId()));
+                    assertHitCount(resp, numDocs);
+                }
+            );
+
+            // pick up a random data node to shut down
+            final String randomDataNode = randomFrom(dataNodes);
+
+            // find which shards to relocate
+            final String nodeId = admin().cluster().prepareNodesInfo(randomDataNode).get().getNodes().get(0).getNode().getId();
+            Set<Integer> shardsToRelocate = new HashSet<>();
+            for (ShardStats stats : admin().indices().prepareStats(index).get().getShards()) {
+                if (nodeId.equals(stats.getShardRouting().currentNodeId())) {
+                    shardsToRelocate.add(stats.getShardRouting().shardId().id());
+                }
+            }
+
+            final int shardsRemoved = shardsToRelocate.size();
+
+            // shut down the random data node
+            internalCluster().stopNode(randomDataNode);
+
+            // ensure that the index is Red
+            ensureRed(index);
+
+            // verify that not all documents can now be retrieved
+            assertResponse(prepareSearch().setQuery(new MatchAllQueryBuilder()), resp -> {
+                assertThat(resp.getSuccessfulShards(), equalTo(numShards - shardsRemoved));
+                assertThat(resp.getFailedShards(), equalTo(shardsRemoved));
+                assertNotNull(resp.getHits().getTotalHits());
+                assertThat(resp.getHits().getTotalHits().value, lessThan((long) numDocs));
+            });
+
+            // create a PIT when some shards are missing
+            OpenPointInTimeResponse pointInTimeResponseOneNodeDown = openPointInTime(
+                new String[] { index },
+                TimeValue.timeValueMinutes(10),
+                true
+            );
+            try {
+                // assert that some shards are indeed missing from PIT
+                assertThat(pointInTimeResponseOneNodeDown.getTotalShards(), equalTo(numShards));
+                assertThat(pointInTimeResponseOneNodeDown.getSuccessfulShards(), equalTo(numShards - shardsRemoved));
+                assertThat(pointInTimeResponseOneNodeDown.getFailedShards(), equalTo(shardsRemoved));
+                assertThat(pointInTimeResponseOneNodeDown.getSkippedShards(), equalTo(0));
+
+                // ensure that the response now contains fewer documents than the total number of indexed documents
+                assertResponse(
+                    prepareSearch().setQuery(new MatchAllQueryBuilder())
+                        .setPointInTime(new PointInTimeBuilder(pointInTimeResponseOneNodeDown.getPointInTimeId())),
+                    resp -> {
+                        assertThat(resp.getSuccessfulShards(), equalTo(numShards - shardsRemoved));
+                        assertThat(resp.getFailedShards(), equalTo(shardsRemoved));
+                        assertThat(resp.pointInTimeId(), equalTo(pointInTimeResponseOneNodeDown.getPointInTimeId()));
+                        assertNotNull(resp.getHits().getTotalHits());
+                        assertThat(resp.getHits().getTotalHits().value, lessThan((long) numDocs));
+                    }
+                );
+
+                // add another node to the cluster and re-allocate the shards
+                final String newNodeName = internalCluster().startDataOnlyNode(nodeAttributes);
+                try {
+                    for (int shardId : shardsToRelocate) {
+                        ClusterRerouteUtils.reroute(client(), new AllocateEmptyPrimaryAllocationCommand(index, shardId, newNodeName, true));
+                    }
+                    ensureGreen(TimeValue.timeValueMinutes(2), index);
+
+                    // index some more documents
+                    for (int i = numDocs; i < numDocs * 2; i++) {
+                        String id = Integer.toString(i);
+                        prepareIndex(index).setId(id).setSource("value", i).get();
+                    }
+                    refresh(index);
+
+                    // ensure that we now see at least numDocs results from the updated index
+                    assertResponse(prepareSearch().setQuery(new MatchAllQueryBuilder()), resp -> {
+                        assertThat(resp.getSuccessfulShards(), equalTo(numShards));
+                        assertThat(resp.getFailedShards(), equalTo(0));
+                        assertNotNull(resp.getHits().getTotalHits());
+                        assertThat(resp.getHits().getTotalHits().value, greaterThan((long) numDocs));
+                    });
+
+                    // ensure that when using the previously created PIT, we'd see the same number of documents as before regardless of the
+                    // newly indexed documents
+                    assertResponse(
+                        prepareSearch().setQuery(new MatchAllQueryBuilder())
+                            .setPointInTime(new PointInTimeBuilder(pointInTimeResponseOneNodeDown.getPointInTimeId())),
+                        resp -> {
+                            assertThat(resp.pointInTimeId(), equalTo(pointInTimeResponseOneNodeDown.getPointInTimeId()));
+                            assertThat(resp.getTotalShards(), equalTo(numShards));
+                            assertThat(resp.getSuccessfulShards(), equalTo(numShards - shardsRemoved));
+                            assertThat(resp.getFailedShards(), equalTo(shardsRemoved));
+                            assertThat(resp.getShardFailures().length, equalTo(shardsRemoved));
+                            for (var failure : resp.getShardFailures()) {
+                                assertTrue(shardsToRelocate.contains(failure.shardId()));
+                                assertThat(failure.getCause(), instanceOf(NoShardAvailableActionException.class));
+                            }
+                            assertNotNull(resp.getHits().getTotalHits());
+                            // we expect less documents as the newly indexed ones should not be part of the PIT
+                            assertThat(resp.getHits().getTotalHits().value, lessThan((long) numDocs));
+                        }
+                    );
+
+                    Exception exc = expectThrows(
+                        Exception.class,
+                        () -> prepareSearch().setQuery(new MatchAllQueryBuilder())
+                            .setPointInTime(new PointInTimeBuilder(pointInTimeResponseOneNodeDown.getPointInTimeId()))
+                            .setAllowPartialSearchResults(false)
+                            .get()
+                    );
+                    assertThat(exc.getCause().getMessage(), containsString("missing shards"));
+
+                } finally {
+                    internalCluster().stopNode(newNodeName);
+                }
+            } finally {
+                closePointInTime(pointInTimeResponseOneNodeDown.getPointInTimeId());
+            }
+
+        } finally {
+            closePointInTime(pointInTimeResponse.getPointInTimeId());
+            internalCluster().stopNode(masterNode);
+            for (String dataNode : dataNodes) {
+                internalCluster().stopNode(dataNode);
+            }
+        }
+    }
+
     @SuppressWarnings({ "rawtypes", "unchecked" })
     private void assertPagination(PointInTimeBuilder pit, int expectedNumDocs, int size, SortBuilder<?>... sorts) throws Exception {
         Set<String> seen = new HashSet<>();
@@ -590,10 +767,14 @@ private void assertPagination(PointInTimeBuilder pit, int expectedNumDocs, int s
         assertThat(seen.size(), equalTo(expectedNumDocs));
     }
 
-    private BytesReference openPointInTime(String[] indices, TimeValue keepAlive) {
-        OpenPointInTimeRequest request = new OpenPointInTimeRequest(indices).keepAlive(keepAlive);
-        final OpenPointInTimeResponse response = client().execute(TransportOpenPointInTimeAction.TYPE, request).actionGet();
-        return response.getPointInTimeId();
+    private OpenPointInTimeResponse openPointInTime(String[] indices, TimeValue keepAlive) {
+        return openPointInTime(indices, keepAlive, false);
+    }
+
+    private OpenPointInTimeResponse openPointInTime(String[] indices, TimeValue keepAlive, boolean allowPartialSearchResults) {
+        OpenPointInTimeRequest request = new OpenPointInTimeRequest(indices).keepAlive(keepAlive)
+            .allowPartialSearchResults(allowPartialSearchResults);
+        return client().execute(TransportOpenPointInTimeAction.TYPE, request).actionGet();
     }
 
     private void closePointInTime(BytesReference readerId) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
index e5dca62a97494..88d934973fc49 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
@@ -9,7 +9,9 @@
 package org.elasticsearch.action.search;
 
 import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.core.Strings;
@@ -23,7 +25,6 @@
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.test.junit.annotations.TestIssueLogging;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -39,10 +40,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThan;
 
-@TestIssueLogging(
-    issueUrl = "https://github.com/elastic/elasticsearch/issues/109830",
-    value = "org.elasticsearch.action.search:TRACE," + "org.elasticsearch.search.SearchService:TRACE"
-)
 public class SearchProgressActionListenerIT extends ESSingleNodeTestCase {
     private List<SearchShard> shards;
 
@@ -197,7 +194,11 @@ private static List<SearchShard> createRandomIndices(Client client) {
             client.prepareIndex(indexName).setSource("number", i, "foo", "bar").get();
         }
         client.admin().indices().prepareRefresh("index-*").get();
-        ClusterSearchShardsResponse resp = client.admin().cluster().prepareSearchShards("index-*").get();
+        ClusterSearchShardsResponse resp = safeExecute(
+            client,
+            TransportClusterSearchShardsAction.TYPE,
+            new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "index-*")
+        );
         return Arrays.stream(resp.getGroups()).map(e -> new SearchShard(null, e.getShardId())).sorted().toList();
     }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/TransportMasterNodeActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/TransportMasterNodeActionIT.java
index 7211585d766f4..e568b51e43b2e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/TransportMasterNodeActionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/master/TransportMasterNodeActionIT.java
@@ -26,12 +26,12 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionBypassCircuitBreakerOnReplicaIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionBypassCircuitBreakerOnReplicaIT.java
new file mode 100644
index 0000000000000..f9f047232e8c6
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionBypassCircuitBreakerOnReplicaIT.java
@@ -0,0 +1,192 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.support.replication;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.BiFunction;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class TransportReplicationActionBypassCircuitBreakerOnReplicaIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(TestPlugin.class);
+    }
+
+    public static class Request extends ReplicationRequest<Request> {
+        public Request(ShardId shardId) {
+            super(shardId);
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        public String toString() {
+            return "test-request";
+        }
+    }
+
+    public static class Response extends ReplicationResponse {
+        public Response() {}
+
+        public Response(StreamInput in) throws IOException {
+            super(in);
+        }
+    }
+
+    public static class TestAction extends TransportReplicationAction<Request, Request, Response> {
+        private static final String ACTION_NAME = "internal:test-replication-action";
+        private static final ActionType<Response> TYPE = new ActionType<>(ACTION_NAME);
+
+        @Inject
+        public TestAction(
+            Settings settings,
+            TransportService transportService,
+            ClusterService clusterService,
+            IndicesService indicesService,
+            ThreadPool threadPool,
+            ShardStateAction shardStateAction,
+            ActionFilters actionFilters
+        ) {
+            super(
+                settings,
+                ACTION_NAME,
+                transportService,
+                clusterService,
+                indicesService,
+                threadPool,
+                shardStateAction,
+                actionFilters,
+                Request::new,
+                Request::new,
+                threadPool.executor(ThreadPool.Names.GENERIC),
+                SyncGlobalCheckpointAfterOperation.DoNotSync,
+                PrimaryActionExecution.RejectOnOverload,
+                ReplicaActionExecution.BypassCircuitBreaker
+            );
+        }
+
+        @Override
+        protected Response newResponseInstance(StreamInput in) throws IOException {
+            return new Response(in);
+        }
+
+        @Override
+        protected void shardOperationOnPrimary(
+            Request shardRequest,
+            IndexShard primary,
+            ActionListener<PrimaryResult<Request, Response>> listener
+        ) {
+            listener.onResponse(new PrimaryResult<>(shardRequest, new Response()));
+        }
+
+        @Override
+        protected void shardOperationOnReplica(Request shardRequest, IndexShard replica, ActionListener<ReplicaResult> listener) {
+            listener.onResponse(new ReplicaResult());
+        }
+    }
+
+    public static class TestPlugin extends Plugin implements ActionPlugin {
+
+        public TestPlugin() {}
+
+        @Override
+        public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+            return List.of(new ActionHandler<>(TestAction.TYPE, TestAction.class));
+        }
+    }
+
+    private enum PrimaryOrReplica implements BiFunction<String, String, String> {
+        PRIMARY {
+            @Override
+            public String apply(String primaryName, String replicaName) {
+                return primaryName;
+            }
+        },
+        REPLICA {
+            @Override
+            public String apply(String primaryName, String replicaName) {
+                return replicaName;
+            }
+        }
+    }
+
+    public void testActionCompletesWhenReplicaCircuitBreakersAreAtCapacity() {
+        maxOutCircuitBreakersAndExecuteAction(PrimaryOrReplica.REPLICA);
+    }
+
+    public void testActionFailsWhenPrimaryCircuitBreakersAreAtCapacity() {
+        AssertionError assertionError = assertThrows(
+            AssertionError.class,
+            () -> maxOutCircuitBreakersAndExecuteAction(PrimaryOrReplica.PRIMARY)
+        );
+        assertNotNull(
+            "Not caused by CircuitBreakingException " + ExceptionsHelper.stackTrace(assertionError),
+            ExceptionsHelper.unwrap(assertionError, CircuitBreakingException.class)
+        );
+    }
+
+    private void maxOutCircuitBreakersAndExecuteAction(PrimaryOrReplica nodeToMaxOutCircuitBreakers) {
+        internalCluster().startMasterOnlyNodes(2);
+        String primary = internalCluster().startDataOnlyNode();
+        assertAcked(
+            prepareCreate("test").setSettings(
+                Settings.builder()
+                    .put(indexSettings())
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            )
+        );
+
+        String replica = internalCluster().startDataOnlyNode();
+        String coordinator = internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
+        ensureGreen("test");
+
+        try (
+            var ignored = fullyAllocateCircuitBreakerOnNode(
+                nodeToMaxOutCircuitBreakers.apply(primary, replica),
+                CircuitBreaker.IN_FLIGHT_REQUESTS
+            )
+        ) {
+            PlainActionFuture<Response> testActionResult = new PlainActionFuture<>();
+            client(coordinator).execute(TestAction.TYPE, new Request(new ShardId(resolveIndex("test"), 0)), testActionResult);
+            safeGet(testActionResult);
+        }
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java
index c4737468a766c..81f26243fb6fb 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
@@ -24,6 +23,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -107,7 +107,8 @@ public TestAction(
                 Request::new,
                 threadPool.executor(ThreadPool.Names.GENERIC),
                 SyncGlobalCheckpointAfterOperation.DoNotSync,
-                PrimaryActionExecution.RejectOnOverload
+                PrimaryActionExecution.RejectOnOverload,
+                ReplicaActionExecution.SubjectToCircuitBreaker
             );
         }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
index cf8decc5655ec..ca4bde8fd0f72 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
@@ -16,7 +16,9 @@
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.common.Strings;
@@ -1013,7 +1015,10 @@ public void testArtificialDocWithPreference() throws InterruptedException, IOExc
         indexRandom(true, prepareIndex("test").setId("1").setSource("field1", "random permutation"));
 
         // Get search shards
-        ClusterSearchShardsResponse searchShardsResponse = clusterAdmin().prepareSearchShards("test").get();
+        ClusterSearchShardsResponse searchShardsResponse = safeExecute(
+            TransportClusterSearchShardsAction.TYPE,
+            new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "test")
+        );
         List<Integer> shardIds = Arrays.stream(searchShardsResponse.getGroups()).map(s -> s.getShardId().id()).toList();
 
         // request termvectors of artificial document from each shard
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
index 7c13171ea76ad..dc93aaa814018 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -395,14 +394,7 @@ public void testMessageLogging() {
         assertThat(healthResponse.isTimedOut(), equalTo(false));
 
         final String indexName = "test_index";
-        indicesAdmin().prepareCreate(indexName)
-            .setWaitForActiveShards(ActiveShardCount.NONE)
-            .setSettings(
-                Settings.builder()
-                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 2)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-            )
-            .get();
+        indicesAdmin().prepareCreate(indexName).setWaitForActiveShards(ActiveShardCount.NONE).setSettings(indexSettings(2, 1)).get();
 
         try (var dryRunMockLog = MockLog.capture(TransportClusterRerouteAction.class)) {
             dryRunMockLog.addExpectation(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java
index 8b551e00caeeb..85a04ee6f1851 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java
@@ -13,6 +13,8 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
+import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
 import org.elasticsearch.action.admin.indices.refresh.TransportUnpromotableShardRefreshAction;
 import org.elasticsearch.action.search.ClosePointInTimeRequest;
 import org.elasticsearch.action.search.OpenPointInTimeRequest;
@@ -548,15 +550,15 @@ public void testSearchRouting() throws Exception {
             }
             // search-shards API
             for (int i = 0; i < 10; i++) {
-                final var search = clusterAdmin().prepareSearchShards(INDEX_NAME);
+                final var search = new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, INDEX_NAME);
                 switch (randomIntBetween(0, 2)) {
-                    case 0 -> search.setRouting(randomAlphaOfLength(10));
-                    case 1 -> search.setRouting(randomSearchPreference(routingTableWatcher.numShards, internalCluster().getNodeNames()));
+                    case 0 -> search.routing(randomAlphaOfLength(10));
+                    case 1 -> search.routing(randomSearchPreference(routingTableWatcher.numShards, internalCluster().getNodeNames()));
                     default -> {
                         // do nothing
                     }
                 }
-                ClusterSearchShardsGroup[] groups = search.get().getGroups();
+                ClusterSearchShardsGroup[] groups = safeExecute(client(), TransportClusterSearchShardsAction.TYPE, search).getGroups();
                 for (ClusterSearchShardsGroup group : groups) {
                     for (ShardRouting shr : group.getShards()) {
                         String profileKey = "[" + shr.currentNodeId() + "][" + INDEX_NAME + "][" + shr.id() + "]";
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
index fd5e54631fd7a..7464f83cb2814 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
@@ -106,7 +106,7 @@ public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
         }
         updateClusterSettings(settings);
         // Create an index with 10 shards so we can check allocation for it
-        assertAcked(prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 10).put("number_of_replicas", 0)));
+        assertAcked(prepareCreate("test").setSettings(indexSettings(10, 0)));
         ensureGreen("test");
 
         assertBusy(() -> {
@@ -184,7 +184,7 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception {
         updateClusterSettings(builder);
 
         // Create an index with 6 shards so we can check allocation for it
-        prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 6).put("number_of_replicas", 0)).get();
+        prepareCreate("test").setSettings(indexSettings(6, 0)).get();
         ensureGreen("test");
 
         {
@@ -269,7 +269,7 @@ public void testOnlyMovesEnoughShardsToDropBelowHighWatermark() throws Exception
             .map(RoutingNode::nodeId)
             .toList();
 
-        assertAcked(prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 6).put("number_of_replicas", 0)));
+        assertAcked(prepareCreate("test").setSettings(indexSettings(6, 0)));
 
         ensureGreen("test");
 
@@ -355,10 +355,10 @@ public void testDoesNotExceedLowWatermarkWhenRebalancing() throws Exception {
 
         assertAcked(
             prepareCreate("test").setSettings(
-                Settings.builder()
-                    .put("number_of_shards", 6)
-                    .put("number_of_replicas", 0)
-                    .put(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getConcreteSettingForNamespace("_id").getKey(), nodeIds.get(2))
+                indexSettings(6, 0).put(
+                    IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getConcreteSettingForNamespace("_id").getKey(),
+                    nodeIds.get(2)
+                )
             )
         );
         ensureGreen("test");
@@ -422,7 +422,7 @@ public void testMovesShardsOffSpecificDataPathAboveWatermark() throws Exception
             .map(RoutingNode::nodeId)
             .toList();
 
-        assertAcked(prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 6).put("number_of_replicas", 0)));
+        assertAcked(prepareCreate("test").setSettings(indexSettings(6, 0)));
 
         ensureGreen("test");
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java
index d52498043366a..895bd6932fdb9 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java
@@ -7,9 +7,13 @@
  */
 package org.elasticsearch.cluster.shards;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
+import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -38,10 +42,10 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         };
     }
 
-    public void testSingleShardAllocation() throws Exception {
+    public void testSingleShardAllocation() {
         indicesAdmin().prepareCreate("test").setSettings(indexSettings(1, 0).put("index.routing.allocation.include.tag", "A")).get();
         ensureGreen();
-        ClusterSearchShardsResponse response = clusterAdmin().prepareSearchShards("test").get();
+        ClusterSearchShardsResponse response = safeExecute(new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "test"));
         assertThat(response.getGroups().length, equalTo(1));
         assertThat(response.getGroups()[0].getShardId().getIndexName(), equalTo("test"));
         assertThat(response.getGroups()[0].getShardId().getId(), equalTo(0));
@@ -49,7 +53,7 @@ public void testSingleShardAllocation() throws Exception {
         assertThat(response.getNodes().length, equalTo(1));
         assertThat(response.getGroups()[0].getShards()[0].currentNodeId(), equalTo(response.getNodes()[0].getId()));
 
-        response = clusterAdmin().prepareSearchShards("test").setRouting("A").get();
+        response = safeExecute(new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "test").routing("A"));
         assertThat(response.getGroups().length, equalTo(1));
         assertThat(response.getGroups()[0].getShardId().getIndexName(), equalTo("test"));
         assertThat(response.getGroups()[0].getShardId().getId(), equalTo(0));
@@ -59,25 +63,25 @@ public void testSingleShardAllocation() throws Exception {
 
     }
 
-    public void testMultipleShardsSingleNodeAllocation() throws Exception {
+    public void testMultipleShardsSingleNodeAllocation() {
         indicesAdmin().prepareCreate("test").setSettings(indexSettings(4, 0).put("index.routing.allocation.include.tag", "A")).get();
         ensureGreen();
 
-        ClusterSearchShardsResponse response = clusterAdmin().prepareSearchShards("test").get();
+        ClusterSearchShardsResponse response = safeExecute(new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "test"));
         assertThat(response.getGroups().length, equalTo(4));
         assertThat(response.getGroups()[0].getShardId().getIndexName(), equalTo("test"));
         assertThat(response.getNodes().length, equalTo(1));
         assertThat(response.getGroups()[0].getShards()[0].currentNodeId(), equalTo(response.getNodes()[0].getId()));
 
-        response = clusterAdmin().prepareSearchShards("test").setRouting("ABC").get();
+        response = safeExecute(new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "test").routing("ABC"));
         assertThat(response.getGroups().length, equalTo(1));
 
-        response = clusterAdmin().prepareSearchShards("test").setPreference("_shards:2").get();
+        response = safeExecute(new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "test").preference("_shards:2"));
         assertThat(response.getGroups().length, equalTo(1));
         assertThat(response.getGroups()[0].getShardId().getId(), equalTo(2));
     }
 
-    public void testMultipleIndicesAllocation() throws Exception {
+    public void testMultipleIndicesAllocation() {
         createIndex("test1", 4, 1);
         createIndex("test2", 4, 1);
         indicesAdmin().prepareAliases()
@@ -86,7 +90,7 @@ public void testMultipleIndicesAllocation() throws Exception {
             .get();
         clusterAdmin().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().get();
 
-        ClusterSearchShardsResponse response = clusterAdmin().prepareSearchShards("routing_alias").get();
+        ClusterSearchShardsResponse response = safeExecute(new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "routing_alias"));
         assertThat(response.getGroups().length, equalTo(2));
         assertThat(response.getGroups()[0].getShards().length, equalTo(2));
         assertThat(response.getGroups()[1].getShards().length, equalTo(2));
@@ -128,7 +132,7 @@ public void testClusterSearchShardsWithBlocks() {
         )) {
             try {
                 enableIndexBlock("test-blocks", blockSetting);
-                ClusterSearchShardsResponse response = clusterAdmin().prepareSearchShards("test-blocks").get();
+                ClusterSearchShardsResponse response = safeExecute(new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "test-blocks"));
                 assertThat(response.getGroups().length, equalTo(numShards.numPrimaries));
             } finally {
                 disableIndexBlock("test-blocks", blockSetting);
@@ -138,9 +142,28 @@ public void testClusterSearchShardsWithBlocks() {
         // Request is blocked
         try {
             enableIndexBlock("test-blocks", SETTING_BLOCKS_METADATA);
-            assertBlocked(clusterAdmin().prepareSearchShards("test-blocks"));
+            assertBlocked(
+                null,
+                asInstanceOf(
+                    ClusterBlockException.class,
+                    ExceptionsHelper.unwrapCause(
+                        safeAwaitFailure(
+                            ClusterSearchShardsResponse.class,
+                            l -> client().execute(
+                                TransportClusterSearchShardsAction.TYPE,
+                                new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, "test-blocks"),
+                                l
+                            )
+                        )
+                    )
+                )
+            );
         } finally {
             disableIndexBlock("test-blocks", SETTING_BLOCKS_METADATA);
         }
     }
+
+    private static ClusterSearchShardsResponse safeExecute(ClusterSearchShardsRequest request) {
+        return safeExecute(TransportClusterSearchShardsAction.TYPE, request);
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java
index 3202f5513e9ac..31dd002a6af7d 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.index.IndexVersion;
@@ -152,7 +153,9 @@ public void testIncreaseReplicasOverLimit() {
                 + firstShardCount
                 + "]/["
                 + dataNodes * shardsPerNode
-                + "] maximum normal shards open;";
+                + "] maximum normal shards open; for more information, see "
+                + ReferenceDocs.MAX_SHARDS_PER_NODE
+                + ";";
             assertEquals(expectedError, e.getMessage());
         }
         Metadata clusterState = clusterAdmin().prepareState().get().getState().metadata();
@@ -211,7 +214,9 @@ public void testChangingMultipleIndicesOverLimit() {
                 + totalShardsBefore
                 + "]/["
                 + dataNodes * shardsPerNode
-                + "] maximum normal shards open;";
+                + "] maximum normal shards open; for more information, see "
+                + ReferenceDocs.MAX_SHARDS_PER_NODE
+                + ";";
             assertEquals(expectedError, e.getMessage());
         }
         Metadata clusterState = clusterAdmin().prepareState().get().getState().metadata();
@@ -403,7 +408,9 @@ private void verifyException(int dataNodes, ShardCounts counts, IllegalArgumentE
             + currentShards
             + "]/["
             + maxShards
-            + "] maximum normal shards open;";
+            + "] maximum normal shards open; for more information, see "
+            + ReferenceDocs.MAX_SHARDS_PER_NODE
+            + ";";
         assertEquals(expectedError, e.getMessage());
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java
index 71c6ef956c4d4..cad5c8f524bc7 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java
@@ -17,9 +17,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.breaker.CircuitBreakingException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.disruption.NetworkDisruption;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
@@ -33,9 +30,6 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
 
-import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING;
-import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING;
-
 /**
  * Tests for discovery during disruptions.
  */
@@ -138,13 +132,7 @@ public void testElectMasterWithLatestVersion() throws Exception {
         internalCluster().setDisruptionScheme(isolatePreferredMaster);
         isolatePreferredMaster.startDisrupting();
 
-        client(randomFrom(nonPreferredNodes)).admin()
-            .indices()
-            .prepareCreate("test")
-            .setSettings(
-                Settings.builder().put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1).put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            )
-            .get();
+        client(randomFrom(nonPreferredNodes)).admin().indices().prepareCreate("test").setSettings(indexSettings(1, 0)).get();
 
         internalCluster().clearDisruptionScheme(false);
         internalCluster().setDisruptionScheme(isolateAllNodes);
@@ -259,46 +247,31 @@ public void testJoinWaitsForCircuitBreaker() throws InterruptedException {
         logger.info("--> master [{}], victim [{}]", masterName, victimName);
 
         // fill up the circuit breaker to breaking point
-        final var circuitBreaker = internalCluster().getInstance(CircuitBreakerService.class, victimName)
-            .getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
-        long allocationSize = 1;
-        while (true) {
-            try {
-                circuitBreaker.addEstimateBytesAndMaybeBreak(allocationSize, "test");
-            } catch (CircuitBreakingException e) {
-                circuitBreaker.addWithoutBreaking(allocationSize);
-                break;
-            }
-            allocationSize <<= 1;
-            assert 0 <= allocationSize;
-        }
-
-        // drop the victim from the cluster with a network disruption
-        final var masterTransportService = MockTransportService.getInstance(masterName);
-        masterTransportService.addFailToSendNoConnectRule(internalCluster().getInstance(TransportService.class, victimName));
-        logger.info("--> waiting for victim's departure");
-        ensureStableCluster(2, masterName);
-
-        // verify that the victim sends no joins while the circuit breaker is breaking
-        final var victimTransportService = MockTransportService.getInstance(victimName);
-        victimTransportService.addSendBehavior((connection, requestId, action, request, options) -> {
-            assertNotEquals(action, JoinHelper.JOIN_ACTION_NAME);
-            connection.sendRequest(requestId, action, request, options);
-        });
-
-        // fix the network disruption
-        logger.info("--> removing network disruption");
-        masterTransportService.clearAllRules();
-        ensureStableCluster(2, masterName);
-
-        // permit joins again
-        victimTransportService.addSendBehavior(null);
-
-        // release the breaker
-        logger.info("--> releasing allocations from circuit breaker");
-        while (0 < allocationSize) {
-            circuitBreaker.addWithoutBreaking(-allocationSize);
-            allocationSize >>= 1;
+        try (var ignored = fullyAllocateCircuitBreakerOnNode(victimName, CircuitBreaker.IN_FLIGHT_REQUESTS)) {
+
+            // drop the victim from the cluster with a network disruption
+            final var masterTransportService = MockTransportService.getInstance(masterName);
+            masterTransportService.addFailToSendNoConnectRule(internalCluster().getInstance(TransportService.class, victimName));
+            logger.info("--> waiting for victim's departure");
+            ensureStableCluster(2, masterName);
+
+            // verify that the victim sends no joins while the circuit breaker is breaking
+            final var victimTransportService = MockTransportService.getInstance(victimName);
+            victimTransportService.addSendBehavior((connection, requestId, action, request, options) -> {
+                assertNotEquals(action, JoinHelper.JOIN_ACTION_NAME);
+                connection.sendRequest(requestId, action, request, options);
+            });
+
+            // fix the network disruption
+            logger.info("--> removing network disruption");
+            masterTransportService.clearAllRules();
+            ensureStableCluster(2, masterName);
+
+            // permit joins again
+            victimTransportService.addSendBehavior(null);
+
+            // release the breaker
+            logger.info("--> releasing allocations from circuit breaker");
         }
 
         logger.info("--> waiting for cluster to heal");
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
index 3baabe4cc888e..26573644790fa 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
@@ -441,13 +441,9 @@ public void testReuseInFileBasedPeerRecovery() throws Exception {
             .indices()
             .prepareCreate("test")
             .setSettings(
-                Settings.builder()
-                    .put("number_of_shards", 1)
-                    .put("number_of_replicas", 1)
-
+                indexSettings(1, 1)
                     // disable merges to keep segments the same
                     .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
-
                     // expire retention leases quickly
                     .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms")
             )
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java
index a96801b707808..76c501df1fd29 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -849,13 +848,7 @@ public void testAvoidWrappingSearcherInMultiGet() {
         SearcherWrapperPlugin.enabled = true;
         assertAcked(
             prepareCreate("test").setMapping("f", "type=keyword")
-                .setSettings(
-                    Settings.builder()
-                        .put("index.refresh_interval", "-1")
-                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                        .put("index.routing.rebalance.enable", "none")
-                )
+                .setSettings(indexSettings(1, 0).put("index.refresh_interval", "-1").put("index.routing.rebalance.enable", "none"))
         );
         // start tracking translog locations in the live version map
         {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java
index 224f0dd4dc822..a77c01e199942 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/get/GetFromTranslogActionIT.java
@@ -13,9 +13,7 @@
 import org.elasticsearch.action.get.TransportGetFromTranslogAction;
 import org.elasticsearch.action.get.TransportGetFromTranslogAction.Response;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -34,11 +32,8 @@ public void testGetFromTranslog() throws Exception {
         assertAcked(
             prepareCreate(INDEX).setMapping("field1", "type=keyword,store=true")
                 .setSettings(
-                    Settings.builder()
-                        .put("index.refresh_interval", -1)
-                        // A GetFromTranslogAction runs only Stateless where there is only one active indexing shard.
-                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    // A GetFromTranslogAction runs only Stateless where there is only one active indexing shard.
+                    indexSettings(1, 0).put("index.refresh_interval", -1)
                 )
                 .addAlias(new Alias(ALIAS).writeIndex(randomFrom(true, false, null)))
         );
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java
index b09a0284eba05..6da2802ce7cc8 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/get/ShardMultiGetFomTranslogActionIT.java
@@ -15,9 +15,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -42,11 +40,8 @@ public class ShardMultiGetFomTranslogActionIT extends ESIntegTestCase {
     public void testShardMultiGetFromTranslog() throws Exception {
         assertAcked(
             prepareCreate(INDEX).setSettings(
-                Settings.builder()
-                    .put("index.refresh_interval", -1)
-                    // A ShardMultiGetFromTranslogAction runs only Stateless where there is only one active indexing shard.
-                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                // A ShardMultiGetFromTranslogAction runs only Stateless where there is only one active indexing shard.
+                indexSettings(1, 0).put("index.refresh_interval", -1)
             ).addAlias(new Alias(ALIAS).writeIndex(randomFrom(true, false, null)))
         );
         ensureGreen();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java
index de783a28bce1d..89107c54ee820 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java
@@ -7,10 +7,10 @@
  */
 package org.elasticsearch.index;
 
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.AbstractModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/BackgroundRetentionLeaseSyncActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/BackgroundRetentionLeaseSyncActionIT.java
new file mode 100644
index 0000000000000..0bab5be245ecf
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/BackgroundRetentionLeaseSyncActionIT.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.seqno;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.stream.Stream;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class BackgroundRetentionLeaseSyncActionIT extends ESIntegTestCase {
+
+    public void testActionCompletesWhenReplicaCircuitBreakersAreAtCapacity() throws Exception {
+        internalCluster().startMasterOnlyNodes(1);
+        String primary = internalCluster().startDataOnlyNode();
+        assertAcked(
+            prepareCreate("test").setSettings(
+                Settings.builder()
+                    .put(indexSettings())
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            )
+        );
+
+        String replica = internalCluster().startDataOnlyNode();
+        ensureGreen("test");
+
+        try (var ignored = fullyAllocateCircuitBreakerOnNode(replica, CircuitBreaker.IN_FLIGHT_REQUESTS)) {
+            final ClusterState state = internalCluster().clusterService().state();
+            final Index testIndex = resolveIndex("test");
+            final ShardId testIndexShardZero = new ShardId(testIndex, 0);
+            final String testLeaseId = "test-lease/123";
+            RetentionLeases newLeases = addTestLeaseToRetentionLeases(primary, testIndex, testLeaseId);
+            internalCluster().getInstance(RetentionLeaseSyncer.class, primary)
+                .backgroundSync(
+                    testIndexShardZero,
+                    state.routingTable().shardRoutingTable(testIndexShardZero).primaryShard().allocationId().getId(),
+                    state.term(),
+                    newLeases
+                );
+
+            // Wait for test lease to appear on replica
+            IndicesService replicaIndicesService = internalCluster().getInstance(IndicesService.class, replica);
+            assertBusy(() -> {
+                RetentionLeases retentionLeases = replicaIndicesService.indexService(testIndex).getShard(0).getRetentionLeases();
+                assertTrue(retentionLeases.contains(testLeaseId));
+            });
+        }
+    }
+
+    private static RetentionLeases addTestLeaseToRetentionLeases(String primaryNodeName, Index index, String leaseId) {
+        IndicesService primaryIndicesService = internalCluster().getInstance(IndicesService.class, primaryNodeName);
+        RetentionLeases currentLeases = primaryIndicesService.indexService(index).getShard(0).getRetentionLeases();
+        RetentionLease newLease = new RetentionLease(leaseId, 0, System.currentTimeMillis(), "test source");
+        return new RetentionLeases(
+            currentLeases.primaryTerm(),
+            currentLeases.version() + 1,
+            Stream.concat(currentLeases.leases().stream(), Stream.of(newLease)).toList()
+        );
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java
new file mode 100644
index 0000000000000..2d8f455792172
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.seqno;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.index.IndexingPressure;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class RetentionLeaseSyncActionIT extends ESIntegTestCase {
+
+    public void testActionCompletesWhenReplicaCircuitBreakersAreAtCapacity() {
+        internalCluster().startMasterOnlyNodes(1);
+        String primary = internalCluster().startDataOnlyNode();
+        assertAcked(
+            prepareCreate("test").setSettings(
+                Settings.builder()
+                    .put(indexSettings())
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            )
+        );
+
+        String replica = internalCluster().startDataOnlyNode();
+        ensureGreen("test");
+
+        try (var ignored = fullyAllocateCircuitBreakerOnNode(replica, CircuitBreaker.IN_FLIGHT_REQUESTS)) {
+            assertThatRetentionLeaseSyncCompletesSuccessfully(primary);
+        }
+    }
+
+    public void testActionCompletesWhenPrimaryIndexingPressureIsAtCapacity() {
+        internalCluster().startMasterOnlyNodes(1);
+        String primary = internalCluster().startDataOnlyNode();
+        assertAcked(
+            prepareCreate("test").setSettings(
+                Settings.builder()
+                    .put(indexSettings())
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            )
+        );
+
+        String replica = internalCluster().startDataOnlyNode();
+        ensureGreen("test");
+
+        try (Releasable ignored = fullyAllocatePrimaryIndexingCapacityOnNode(primary)) {
+            assertThatRetentionLeaseSyncCompletesSuccessfully(primary);
+        }
+    }
+
+    private static void assertThatRetentionLeaseSyncCompletesSuccessfully(String primaryNodeName) {
+        RetentionLeaseSyncer instance = internalCluster().getInstance(RetentionLeaseSyncer.class, primaryNodeName);
+        PlainActionFuture<ReplicationResponse> retentionLeaseSyncResult = new PlainActionFuture<>();
+        ClusterState state = internalCluster().clusterService().state();
+        ShardId testIndexShardZero = new ShardId(resolveIndex("test"), 0);
+        ShardRouting primaryShard = state.routingTable().shardRoutingTable(testIndexShardZero).primaryShard();
+        instance.sync(
+            testIndexShardZero,
+            primaryShard.allocationId().getId(),
+            state.term(),
+            RetentionLeases.EMPTY,
+            retentionLeaseSyncResult
+        );
+        safeGet(retentionLeaseSyncResult);
+    }
+
+    /**
+     * Fully allocate primary indexing capacity on a node
+     *
+     * @param targetNode The name of the node on which to allocate
+     * @return A {@link Releasable} which will release the capacity when closed
+     */
+    private static Releasable fullyAllocatePrimaryIndexingCapacityOnNode(String targetNode) {
+        return internalCluster().getInstance(IndexingPressure.class, targetNode)
+            .markPrimaryOperationStarted(
+                1,
+                IndexingPressure.MAX_INDEXING_BYTES.get(internalCluster().getInstance(Settings.class, targetNode)).getBytes() + 1,
+                true
+            );
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/HierarchyCircuitBreakerTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/HierarchyCircuitBreakerTelemetryIT.java
new file mode 100644
index 0000000000000..ff2117ea93bb9
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/memory/breaker/HierarchyCircuitBreakerTelemetryIT.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices.memory.breaker;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.indices.breaker.CircuitBreakerMetrics;
+import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matchers;
+import org.junit.After;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.CIRCUIT_BREAKER_TYPE_ATTRIBUTE;
+import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING;
+import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING;
+import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING;
+import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING;
+import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING;
+import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING;
+import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, supportsDedicatedMasters = true)
+public class HierarchyCircuitBreakerTelemetryIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(TestTelemetryPlugin.class);
+    }
+
+    public void testCircuitBreakerTripCountMetric() {
+        final Settings circuitBreakerSettings = Settings.builder()
+            .put(FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), 100, ByteSizeUnit.BYTES)
+            .put(FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0)
+            .put(REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), 100, ByteSizeUnit.BYTES)
+            .put(REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0)
+            .put(IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), 100, ByteSizeUnit.BYTES)
+            .put(IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0)
+            .put(TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), 150, ByteSizeUnit.BYTES)
+            .put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), false)
+            .build();
+        String dataNodeName = null;
+        String masterNodeName = null;
+        try {
+            // NOTE: we start with empty circuitBreakerSettings to allow cluster formation
+            masterNodeName = internalCluster().startMasterOnlyNode(Settings.EMPTY);
+            dataNodeName = internalCluster().startDataOnlyNode(Settings.EMPTY);
+            assertTrue(clusterAdmin().prepareUpdateSettings().setPersistentSettings(circuitBreakerSettings).get().isAcknowledged());
+            assertTrue(
+                client().admin()
+                    .indices()
+                    .prepareCreate("test")
+                    .setSettings(
+                        Settings.builder()
+                            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                            .build()
+                    )
+                    .get()
+                    .isAcknowledged()
+            );
+            assertEquals(
+                RestStatus.OK.getStatus(),
+                client().prepareIndex("test").setWaitForActiveShards(1).setSource("field", "value").get().status().getStatus()
+            );
+        } catch (CircuitBreakingException cbex) {
+            final List<Measurement> dataNodeMeasurements = getMeasurements(dataNodeName);
+            final List<Measurement> masterNodeMeasurements = getMeasurements(masterNodeName);
+            final List<Measurement> allMeasurements = Stream.concat(dataNodeMeasurements.stream(), masterNodeMeasurements.stream())
+                .toList();
+            assertThat(allMeasurements, Matchers.not(Matchers.empty()));
+            final Measurement measurement = allMeasurements.get(0);
+            assertThat(1L, Matchers.equalTo(measurement.getLong()));
+            assertThat(1L, Matchers.equalTo(measurement.value()));
+            assertThat(Map.of(CIRCUIT_BREAKER_TYPE_ATTRIBUTE, "inflight_requests"), Matchers.equalTo(measurement.attributes()));
+            assertThat(true, Matchers.equalTo(measurement.isLong()));
+            return;
+        }
+        fail("Expected exception not thrown");
+    }
+
+    @After
+    public void resetClusterSetting() {
+        final var circuitBreakerSettings = Settings.builder()
+            .putNull(FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey())
+            .putNull(FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey())
+            .putNull(REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey())
+            .putNull(REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey())
+            .putNull(IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey())
+            .putNull(IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey())
+            .putNull(TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey())
+            .putNull(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey());
+        updateClusterSettings(circuitBreakerSettings);
+    }
+
+    private List<Measurement> getMeasurements(String nodeName) {
+        final TestTelemetryPlugin telemetryPlugin = internalCluster().getInstance(PluginsService.class, nodeName)
+            .filterPlugins(TestTelemetryPlugin.class)
+            .toList()
+            .get(0);
+        return Measurement.combine(
+            Stream.of(telemetryPlugin.getLongCounterMeasurement(CircuitBreakerMetrics.ES_BREAKER_TRIP_COUNT_TOTAL).stream())
+                .flatMap(Function.identity())
+                .toList()
+        );
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
index d56e4a372c17c..fbbeec4b4e9ba 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
@@ -914,10 +914,7 @@ private IndicesStatsResponse createAndPopulateIndex(String name, int nodeCount,
             prepareCreate(
                 name,
                 nodeCount,
-                Settings.builder()
-                    .put("number_of_shards", shardCount)
-                    .put("number_of_replicas", replicaCount)
-                    .put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0)
+                indexSettings(shardCount, replicaCount).put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0)
             )
         );
         ensureGreen();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
index 5201e4ab3d812..a7a2af57ef810 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
@@ -227,10 +227,7 @@ public void testCloseOpenAliasMultipleIndices() {
 
     public void testOpenWaitingForActiveShardsFailed() throws Exception {
         Client client = client();
-        Settings settings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            .build();
+        Settings settings = indexSettings(1, 0).build();
         assertAcked(client.admin().indices().prepareCreate("test").setSettings(settings).get());
         assertAcked(client.admin().indices().prepareClose("test").get());
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java
index 542a4cd2c4c92..db26d630fefea 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java
@@ -350,7 +350,7 @@ public void beforeIndexFoldersDeleted(Index index, IndexSettings indexSettings,
 
                 @Override
                 public void beforeShardFoldersDeleted(ShardId shardId, IndexSettings indexSettings, Path[] shardPaths) {
-                    deletedShards.computeIfAbsent(shardId.getIndex(), i -> new ArrayList<>()).add(shardId);
+                    deletedShards.computeIfAbsent(shardId.getIndex(), i -> Collections.synchronizedList(new ArrayList<>())).add(shardId);
                 }
             });
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverIT.java
deleted file mode 100644
index 7797371a2823b..0000000000000
--- a/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverIT.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.plugins.internal;
-
-import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.engine.EngineFactory;
-import org.elasticsearch.index.engine.InternalEngine;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.plugins.EnginePlugin;
-import org.elasticsearch.plugins.IngestPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.xcontent.FilterXContentParserWrapper;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentType;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.Optional;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.elasticsearch.xcontent.XContentFactory.cborBuilder;
-import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
-import static org.hamcrest.Matchers.equalTo;
-
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
-public class DocumentSizeObserverIT extends ESIntegTestCase {
-
-    private static String TEST_INDEX_NAME = "test-index-name";
-
-    @Override
-    protected boolean addMockInternalEngine() {
-        return false;
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return List.of(TestDocumentParsingProviderPlugin.class, TestEnginePlugin.class);
-    }
-
-    // the assertions are done in plugin which is static and will be created by ES server.
-    // hence a static flag to make sure it is indeed used
-    public static boolean hasWrappedParser;
-    public static AtomicLong COUNTER = new AtomicLong();
-
-    public void testDocumentIsReportedUponBulk() throws Exception {
-        hasWrappedParser = false;
-        client().index(
-            new IndexRequest(TEST_INDEX_NAME).id("1").source(jsonBuilder().startObject().field("test", "I am sam i am").endObject())
-        ).actionGet();
-        assertTrue(hasWrappedParser);
-        assertDocumentReported();
-
-        hasWrappedParser = false;
-        // the format of the request does not matter
-        client().index(
-            new IndexRequest(TEST_INDEX_NAME).id("2").source(cborBuilder().startObject().field("test", "I am sam i am").endObject())
-        ).actionGet();
-        assertTrue(hasWrappedParser);
-        assertDocumentReported();
-
-        hasWrappedParser = false;
-        // white spaces does not matter
-        client().index(new IndexRequest(TEST_INDEX_NAME).id("3").source("""
-            {
-            "test":
-
-            "I am sam i am"
-            }
-            """, XContentType.JSON)).actionGet();
-        assertTrue(hasWrappedParser);
-        assertDocumentReported();
-    }
-
-    private void assertDocumentReported() throws Exception {
-        assertBusy(() -> assertThat(COUNTER.get(), equalTo(5L)));
-        COUNTER.set(0);
-    }
-
-    public static class TestEnginePlugin extends Plugin implements EnginePlugin {
-        DocumentParsingProvider documentParsingProvider;
-
-        @Override
-        public Collection<?> createComponents(PluginServices services) {
-            documentParsingProvider = services.documentParsingProvider();
-            return super.createComponents(services);
-        }
-
-        @Override
-        public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
-            return Optional.of(config -> new InternalEngine(config) {
-                @Override
-                public IndexResult index(Index index) throws IOException {
-                    IndexResult result = super.index(index);
-
-                    DocumentSizeReporter documentParsingReporter = documentParsingProvider.newDocumentSizeReporter(
-                        shardId.getIndexName(),
-                        config().getMapperService(),
-                        DocumentSizeAccumulator.EMPTY_INSTANCE
-                    );
-                    ParsedDocument parsedDocument = index.parsedDoc();
-                    documentParsingReporter.onIndexingCompleted(parsedDocument);
-
-                    return result;
-                }
-            });
-        }
-    }
-
-    public static class TestDocumentParsingProviderPlugin extends Plugin implements DocumentParsingProviderPlugin, IngestPlugin {
-
-        public TestDocumentParsingProviderPlugin() {}
-
-        @Override
-        public DocumentParsingProvider getDocumentParsingProvider() {
-            return new DocumentParsingProvider() {
-                @Override
-                public <T> DocumentSizeObserver newDocumentSizeObserver(DocWriteRequest<T> request) {
-                    return new TestDocumentSizeObserver(0L);
-                }
-
-                @Override
-                public DocumentSizeReporter newDocumentSizeReporter(
-                    String indexName,
-                    MapperService mapperService,
-                    DocumentSizeAccumulator documentSizeAccumulator
-                ) {
-                    return new TestDocumentSizeReporter(indexName);
-                }
-            };
-        }
-    }
-
-    public static class TestDocumentSizeReporter implements DocumentSizeReporter {
-
-        private final String indexName;
-
-        public TestDocumentSizeReporter(String indexName) {
-            this.indexName = indexName;
-        }
-
-        @Override
-        public void onIndexingCompleted(ParsedDocument parsedDocument) {
-            COUNTER.addAndGet(parsedDocument.getDocumentSizeObserver().normalisedBytesParsed());
-            assertThat(indexName, equalTo(TEST_INDEX_NAME));
-        }
-    }
-
-    public static class TestDocumentSizeObserver implements DocumentSizeObserver {
-        long counter = 0;
-
-        public TestDocumentSizeObserver(long counter) {
-            this.counter = counter;
-        }
-
-        @Override
-        public XContentParser wrapParser(XContentParser xContentParser) {
-            hasWrappedParser = true;
-            return new FilterXContentParserWrapper(xContentParser) {
-
-                @Override
-                public Token nextToken() throws IOException {
-                    counter++;
-                    return super.nextToken();
-                }
-            };
-        }
-
-        @Override
-        public long normalisedBytesParsed() {
-            return counter;
-        }
-
-    }
-}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorIT.java
new file mode 100644
index 0000000000000..16fb618e97dfc
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorIT.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.internal;
+
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.engine.EngineFactory;
+import org.elasticsearch.index.engine.InternalEngine;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.plugins.EnginePlugin;
+import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.FilterXContentParserWrapper;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.elasticsearch.xcontent.XContentFactory.cborBuilder;
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class XContentMeteringParserDecoratorIT extends ESIntegTestCase {
+
+    private static String TEST_INDEX_NAME = "test-index-name";
+
+    @Override
+    protected boolean addMockInternalEngine() {
+        return false;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(TestDocumentParsingProviderPlugin.class, TestEnginePlugin.class);
+    }
+
+    // the assertions are done in plugin which is static and will be created by ES server.
+    // hence a static flag to make sure it is indeed used
+    public static boolean hasWrappedParser;
+    public static AtomicLong COUNTER = new AtomicLong();
+
+    public void testDocumentIsReportedUponBulk() throws Exception {
+        hasWrappedParser = false;
+        client().index(
+            new IndexRequest(TEST_INDEX_NAME).id("1").source(jsonBuilder().startObject().field("test", "I am sam i am").endObject())
+        ).actionGet();
+        assertTrue(hasWrappedParser);
+        assertDocumentReported();
+
+        hasWrappedParser = false;
+        // the format of the request does not matter
+        client().index(
+            new IndexRequest(TEST_INDEX_NAME).id("2").source(cborBuilder().startObject().field("test", "I am sam i am").endObject())
+        ).actionGet();
+        assertTrue(hasWrappedParser);
+        assertDocumentReported();
+
+        hasWrappedParser = false;
+        // white spaces does not matter
+        client().index(new IndexRequest(TEST_INDEX_NAME).id("3").source("""
+            {
+            "test":
+
+            "I am sam i am"
+            }
+            """, XContentType.JSON)).actionGet();
+        assertTrue(hasWrappedParser);
+        assertDocumentReported();
+    }
+
+    private void assertDocumentReported() throws Exception {
+        assertBusy(() -> assertThat(COUNTER.get(), equalTo(5L)));
+        COUNTER.set(0);
+    }
+
+    public static class TestEnginePlugin extends Plugin implements EnginePlugin {
+        DocumentParsingProvider documentParsingProvider;
+
+        @Override
+        public Collection<?> createComponents(PluginServices services) {
+            documentParsingProvider = services.documentParsingProvider();
+            return super.createComponents(services);
+        }
+
+        @Override
+        public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
+            return Optional.of(config -> new InternalEngine(config) {
+                @Override
+                public IndexResult index(Index index) throws IOException {
+                    IndexResult result = super.index(index);
+
+                    DocumentSizeReporter documentParsingReporter = documentParsingProvider.newDocumentSizeReporter(
+                        shardId.getIndexName(),
+                        config().getMapperService(),
+                        DocumentSizeAccumulator.EMPTY_INSTANCE
+                    );
+                    ParsedDocument parsedDocument = index.parsedDoc();
+                    documentParsingReporter.onIndexingCompleted(parsedDocument);
+
+                    return result;
+                }
+            });
+        }
+    }
+
+    public static class TestDocumentParsingProviderPlugin extends Plugin implements DocumentParsingProviderPlugin, IngestPlugin {
+
+        public TestDocumentParsingProviderPlugin() {}
+
+        @Override
+        public DocumentParsingProvider getDocumentParsingProvider() {
+            return new DocumentParsingProvider() {
+                @Override
+                public <T> XContentMeteringParserDecorator newMeteringParserDecorator(DocWriteRequest<T> request) {
+                    return new TestXContentMeteringParserDecorator(0L);
+                }
+
+                @Override
+                public DocumentSizeReporter newDocumentSizeReporter(
+                    String indexName,
+                    MapperService mapperService,
+                    DocumentSizeAccumulator documentSizeAccumulator
+                ) {
+                    return new TestDocumentSizeReporter(indexName);
+                }
+            };
+        }
+    }
+
+    public static class TestDocumentSizeReporter implements DocumentSizeReporter {
+
+        private final String indexName;
+
+        public TestDocumentSizeReporter(String indexName) {
+            this.indexName = indexName;
+        }
+
+        @Override
+        public void onIndexingCompleted(ParsedDocument parsedDocument) {
+            long delta = parsedDocument.getNormalizedSize().ingestedBytes();
+            if (delta > 0) {
+                COUNTER.addAndGet(delta);
+            }
+            assertThat(indexName, equalTo(TEST_INDEX_NAME));
+        }
+    }
+
+    public static class TestXContentMeteringParserDecorator implements XContentMeteringParserDecorator {
+        long counter = 0;
+
+        public TestXContentMeteringParserDecorator(long counter) {
+            this.counter = counter;
+        }
+
+        @Override
+        public XContentParser decorate(XContentParser xContentParser) {
+            hasWrappedParser = true;
+            return new FilterXContentParserWrapper(xContentParser) {
+
+                @Override
+                public Token nextToken() throws IOException {
+                    counter++;
+                    return super.nextToken();
+                }
+            };
+        }
+
+        @Override
+        public ParsedDocument.DocumentSize meteredDocumentSize() {
+            return new ParsedDocument.DocumentSize(counter, counter);
+        }
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java
new file mode 100644
index 0000000000000..4665dc486a904
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories.blobstore;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.action.support.ActionTestUtils;
+import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshotsIntegritySuppressor;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
+import org.elasticsearch.snapshots.SnapshotState;
+import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class BlobStoreCorruptionIT extends AbstractSnapshotIntegTestCase {
+
+    private static final Logger logger = LogManager.getLogger(BlobStoreCorruptionIT.class);
+
+    @Before
+    public void suppressConsistencyCheck() {
+        disableRepoConsistencyCheck("testing corruption detection involves breaking the repo");
+    }
+
+    public void testCorruptionDetection() throws Exception {
+        final var repositoryName = randomIdentifier();
+        final var indexName = randomIdentifier();
+        final var snapshotName = randomIdentifier();
+        final var repositoryRootPath = randomRepoPath();
+
+        createRepository(repositoryName, FsRepository.TYPE, repositoryRootPath);
+        createIndexWithRandomDocs(indexName, between(1, 100));
+        flushAndRefresh(indexName);
+        createSnapshot(repositoryName, snapshotName, List.of(indexName));
+
+        final var corruptedFile = BlobStoreCorruptionUtils.corruptRandomFile(repositoryRootPath);
+        final var corruptedFileType = RepositoryFileType.getRepositoryFileType(repositoryRootPath, corruptedFile);
+        final var corruptionDetectors = new ArrayList<CheckedConsumer<ActionListener<Exception>, ?>>();
+
+        // detect corruption by listing the snapshots
+        if (corruptedFileType == RepositoryFileType.SNAPSHOT_INFO) {
+            corruptionDetectors.add(exceptionListener -> {
+                logger.info("--> listing snapshots");
+                client().admin()
+                    .cluster()
+                    .prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repositoryName)
+                    .execute(ActionTestUtils.assertNoSuccessListener(exceptionListener::onResponse));
+            });
+        }
+
+        // detect corruption by taking another snapshot
+        if (corruptedFileType == RepositoryFileType.SHARD_GENERATION) {
+            corruptionDetectors.add(exceptionListener -> {
+                logger.info("--> taking another snapshot");
+                client().admin()
+                    .cluster()
+                    .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, randomIdentifier())
+                    .setWaitForCompletion(true)
+                    .execute(exceptionListener.map(createSnapshotResponse -> {
+                        assertNotEquals(SnapshotState.SUCCESS, createSnapshotResponse.getSnapshotInfo().state());
+                        return new ElasticsearchException("create-snapshot failed as expected");
+                    }));
+            });
+        }
+
+        // detect corruption by restoring the snapshot
+        switch (corruptedFileType) {
+            case SNAPSHOT_INFO, GLOBAL_METADATA, INDEX_METADATA -> corruptionDetectors.add(exceptionListener -> {
+                logger.info("--> restoring snapshot");
+                client().admin()
+                    .cluster()
+                    .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, snapshotName)
+                    .setRestoreGlobalState(corruptedFileType == RepositoryFileType.GLOBAL_METADATA || randomBoolean())
+                    .setWaitForCompletion(true)
+                    .execute(ActionTestUtils.assertNoSuccessListener(exceptionListener::onResponse));
+            });
+            case SHARD_SNAPSHOT_INFO, SHARD_DATA -> corruptionDetectors.add(exceptionListener -> {
+                logger.info("--> restoring snapshot and checking for failed shards");
+                SubscribableListener
+                    // if shard-level data is corrupted then the overall restore succeeds but the shard recoveries fail
+                    .<AcknowledgedResponse>newForked(l -> client().admin().indices().prepareDelete(indexName).execute(l))
+                    .andThenAccept(ElasticsearchAssertions::assertAcked)
+
+                    .<RestoreSnapshotResponse>andThen(
+                        l -> client().admin()
+                            .cluster()
+                            .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, snapshotName)
+                            .setRestoreGlobalState(randomBoolean())
+                            .setWaitForCompletion(true)
+                            .execute(l)
+                    )
+
+                    .addListener(exceptionListener.map(restoreSnapshotResponse -> {
+                        assertNotEquals(0, restoreSnapshotResponse.getRestoreInfo().failedShards());
+                        return new ElasticsearchException("post-restore recoveries failed as expected");
+                    }));
+            });
+        }
+
+        try (var ignored = new BlobStoreIndexShardSnapshotsIntegritySuppressor()) {
+            final var exception = safeAwait(randomFrom(corruptionDetectors));
+            logger.info(Strings.format("--> corrupted [%s] and caught exception", corruptedFile), exception);
+        }
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/ChunkedZipResponseIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/ChunkedZipResponseIT.java
new file mode 100644
index 0000000000000..be6a7387d8a0b
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/ChunkedZipResponseIT.java
@@ -0,0 +1,506 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.rest;
+
+import org.apache.http.ConnectionClosedException;
+import org.apache.http.HttpResponse;
+import org.apache.http.MalformedChunkCodingException;
+import org.apache.http.nio.ContentDecoder;
+import org.apache.http.nio.IOControl;
+import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
+import org.apache.http.protocol.HttpContext;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.support.ActionTestUtils;
+import org.elasticsearch.action.support.RefCountingRunnable;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.bytes.ReleasableBytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.recycler.Recycler;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.common.util.PageCacheRecycler;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.ThrottledIterator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import static org.elasticsearch.rest.ChunkedZipResponse.ZIP_CONTENT_TYPE;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.startsWith;
+
+@ESIntegTestCase.ClusterScope(numDataNodes = 1)
+public class ChunkedZipResponseIT extends ESIntegTestCase {
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopyNoNullElements(super.nodePlugins(), RandomZipResponsePlugin.class);
+    }
+
+    public static class RandomZipResponsePlugin extends Plugin implements ActionPlugin {
+
+        public static final String ROUTE = "/_random_zip_response";
+        public static final String RESPONSE_FILENAME = "test-response";
+
+        public static final String INFINITE_ROUTE = "/_infinite_zip_response";
+        public static final String GET_NEXT_PART_COUNT_DOWN_PARAM = "getNextPartCountDown";
+
+        public final AtomicReference<Response> responseRef = new AtomicReference<>();
+
+        public record EntryPart(List<BytesReference> chunks) {
+            public EntryPart {
+                Objects.requireNonNull(chunks);
+            }
+        }
+
+        public record EntryBody(List<EntryPart> parts) {
+            public EntryBody {
+                Objects.requireNonNull(parts);
+            }
+        }
+
+        public record Response(Map<String, EntryBody> entries, CountDownLatch completedLatch) {}
+
+        @Override
+        public Collection<RestHandler> getRestHandlers(
+            Settings settings,
+            NamedWriteableRegistry namedWriteableRegistry,
+            RestController restController,
+            ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings,
+            SettingsFilter settingsFilter,
+            IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster,
+            Predicate<NodeFeature> clusterSupportsFeature
+        ) {
+            return List.of(new RestHandler() {
+                @Override
+                public List<Route> routes() {
+                    return List.of(new Route(RestRequest.Method.GET, ROUTE));
+                }
+
+                @Override
+                public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) {
+                    final var response = new Response(new HashMap<>(), new CountDownLatch(1));
+                    final var maxSize = between(1, ByteSizeUnit.MB.toIntBytes(1));
+                    final var entryCount = between(0, ByteSizeUnit.MB.toIntBytes(10) / maxSize); // limit total size to 10MiB
+                    for (int i = 0; i < entryCount; i++) {
+                        response.entries().put(randomIdentifier(), randomContent(between(1, 10), maxSize));
+                    }
+                    assertTrue(responseRef.compareAndSet(null, response));
+                    handleZipRestRequest(
+                        channel,
+                        client.threadPool(),
+                        response.completedLatch(),
+                        () -> {},
+                        response.entries().entrySet().iterator()
+                    );
+                }
+            }, new RestHandler() {
+
+                @Override
+                public List<Route> routes() {
+                    return List.of(new Route(RestRequest.Method.GET, INFINITE_ROUTE));
+                }
+
+                @Override
+                public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) {
+                    final var response = new Response(null, new CountDownLatch(1));
+                    assertTrue(responseRef.compareAndSet(null, response));
+                    final var getNextPartCountDown = request.paramAsInt(GET_NEXT_PART_COUNT_DOWN_PARAM, -1);
+                    final Runnable onGetNextPart;
+                    final Supplier<EntryBody> entryBodySupplier;
+                    if (getNextPartCountDown <= 0) {
+                        onGetNextPart = () -> {};
+                        entryBodySupplier = () -> randomContent(between(1, 10), ByteSizeUnit.MB.toIntBytes(1));
+                    } else {
+                        final AtomicInteger remaining = new AtomicInteger(getNextPartCountDown);
+                        entryBodySupplier = () -> randomContent(between(2, 10), ByteSizeUnit.KB.toIntBytes(1));
+                        if (randomBoolean()) {
+                            onGetNextPart = () -> {
+                                final var newRemaining = remaining.decrementAndGet();
+                                assertThat(newRemaining, greaterThanOrEqualTo(0));
+                                if (newRemaining <= 0) {
+                                    throw new ElasticsearchException("simulated failure");
+                                }
+                            };
+                        } else {
+                            onGetNextPart = () -> {
+                                if (remaining.decrementAndGet() == 0) {
+                                    request.getHttpChannel().close();
+                                }
+                            };
+                        }
+                    }
+                    handleZipRestRequest(channel, client.threadPool(), response.completedLatch(), onGetNextPart, new Iterator<>() {
+
+                        private long id;
+
+                        // carry on yielding content even after the channel closes
+                        private final Semaphore trailingContentPermits = new Semaphore(between(0, 20));
+
+                        @Override
+                        public boolean hasNext() {
+                            return request.getHttpChannel().isOpen() || trailingContentPermits.tryAcquire();
+                        }
+
+                        @Override
+                        public Map.Entry<String, EntryBody> next() {
+                            return new Map.Entry<>() {
+                                private final String key = Long.toString(id++);
+                                private final EntryBody content = entryBodySupplier.get();
+
+                                @Override
+                                public String getKey() {
+                                    return key;
+                                }
+
+                                @Override
+                                public EntryBody getValue() {
+                                    return content;
+                                }
+
+                                @Override
+                                public EntryBody setValue(EntryBody value) {
+                                    return fail(null, "must not setValue");
+                                }
+                            };
+                        }
+                    });
+                }
+            });
+        }
+
+        private static EntryBody randomContent(int partCount, int maxSize) {
+            if (randomBoolean()) {
+                return null;
+            }
+
+            final var maxPartSize = maxSize / partCount;
+            return new EntryBody(randomList(partCount, partCount, () -> {
+                final var chunkCount = between(1, 10);
+                return randomEntryPart(chunkCount, maxPartSize / chunkCount);
+            }));
+        }
+
+        private static EntryPart randomEntryPart(int chunkCount, int maxChunkSize) {
+            final var chunks = randomList(chunkCount, chunkCount, () -> randomBytesReference(between(0, maxChunkSize)));
+            Collections.shuffle(chunks, random());
+            return new EntryPart(chunks);
+        }
+
+        private static void handleZipRestRequest(
+            RestChannel channel,
+            ThreadPool threadPool,
+            CountDownLatch completionLatch,
+            Runnable onGetNextPart,
+            Iterator<Map.Entry<String, EntryBody>> entryIterator
+        ) {
+            try (var refs = new RefCountingRunnable(completionLatch::countDown)) {
+                final var chunkedZipResponse = new ChunkedZipResponse(RESPONSE_FILENAME, channel, refs.acquire());
+                ThrottledIterator.run(
+                    entryIterator,
+                    (ref, entry) -> randomFrom(EsExecutors.DIRECT_EXECUTOR_SERVICE, threadPool.generic()).execute(
+                        ActionRunnable.supply(
+                            chunkedZipResponse.newEntryListener(entry.getKey(), Releasables.wrap(ref, refs.acquire())),
+                            () -> entry.getValue() == null && randomBoolean() // randomBoolean() to allow some null entries to fail with NPE
+                                ? null
+                                : new TestBytesReferenceBodyPart(
+                                    entry.getKey(),
+                                    threadPool,
+                                    entry.getValue().parts().iterator(),
+                                    refs,
+                                    onGetNextPart
+                                )
+                        )
+                    ),
+                    between(1, 10),
+                    () -> {},
+                    Releasables.wrap(refs.acquire(), chunkedZipResponse)::close
+                );
+            }
+        }
+    }
+
+    private static class TestBytesReferenceBodyPart implements ChunkedRestResponseBodyPart {
+
+        private final String name;
+        private final ThreadPool threadPool;
+        private final Iterator<BytesReference> chunksIterator;
+        private final Iterator<RandomZipResponsePlugin.EntryPart> partsIterator;
+        private final RefCountingRunnable refs;
+        private final Runnable onGetNextPart;
+
+        TestBytesReferenceBodyPart(
+            String name,
+            ThreadPool threadPool,
+            Iterator<RandomZipResponsePlugin.EntryPart> partsIterator,
+            RefCountingRunnable refs,
+            Runnable onGetNextPart
+        ) {
+            this.onGetNextPart = onGetNextPart;
+            assert partsIterator.hasNext();
+            this.name = name;
+            this.threadPool = threadPool;
+            this.partsIterator = partsIterator;
+            this.chunksIterator = partsIterator.next().chunks().iterator();
+            this.refs = refs;
+        }
+
+        @Override
+        public boolean isPartComplete() {
+            return chunksIterator.hasNext() == false;
+        }
+
+        @Override
+        public boolean isLastPart() {
+            return partsIterator.hasNext() == false;
+        }
+
+        @Override
+        public void getNextPart(ActionListener<ChunkedRestResponseBodyPart> listener) {
+            threadPool.generic().execute(ActionRunnable.supply(listener, () -> {
+                onGetNextPart.run();
+                return new TestBytesReferenceBodyPart(name, threadPool, partsIterator, refs, onGetNextPart);
+            }));
+        }
+
+        @Override
+        public ReleasableBytesReference encodeChunk(int sizeHint, Recycler<BytesRef> recycler) {
+            assert chunksIterator.hasNext();
+            return new ReleasableBytesReference(chunksIterator.next(), refs.acquire());
+        }
+
+        @Override
+        public String getResponseContentTypeString() {
+            return "application/binary";
+        }
+    }
+
+    public void testRandomZipResponse() throws IOException {
+        final var request = new Request("GET", RandomZipResponsePlugin.ROUTE);
+        if (randomBoolean()) {
+            request.setOptions(
+                RequestOptions.DEFAULT.toBuilder()
+                    .addHeader("accept-encoding", String.join(", ", randomSubsetOf(List.of("deflate", "gzip", "zstd", "br"))))
+            );
+        }
+        final var response = getRestClient().performRequest(request);
+        assertEquals(ZIP_CONTENT_TYPE, response.getHeader("Content-Type"));
+        assertNull(response.getHeader("content-encoding")); // zip file is already compressed
+        assertEquals(
+            "attachment; filename=\"" + RandomZipResponsePlugin.RESPONSE_FILENAME + ".zip\"",
+            response.getHeader("Content-Disposition")
+        );
+        final var pathPrefix = RandomZipResponsePlugin.RESPONSE_FILENAME + "/";
+
+        final var actualEntries = new HashMap<String, BytesReference>();
+        final var copyBuffer = new byte[PageCacheRecycler.BYTE_PAGE_SIZE];
+
+        try (var zipStream = new ZipInputStream(response.getEntity().getContent())) {
+            ZipEntry zipEntry;
+            while ((zipEntry = zipStream.getNextEntry()) != null) {
+                assertThat(zipEntry.getName(), startsWith(pathPrefix));
+                final var name = zipEntry.getName().substring(pathPrefix.length());
+                try (var bytesStream = new BytesStreamOutput()) {
+                    while (true) {
+                        final var readLength = zipStream.read(copyBuffer, 0, copyBuffer.length);
+                        if (readLength < 0) {
+                            break;
+                        }
+                        bytesStream.write(copyBuffer, 0, readLength);
+                    }
+                    actualEntries.put(name, bytesStream.bytes());
+                }
+            }
+        } finally {
+            assertEquals(getExpectedEntries(), actualEntries);
+        }
+    }
+
+    public void testAbort() throws IOException {
+        final var request = new Request("GET", RandomZipResponsePlugin.INFINITE_ROUTE);
+        final var responseStarted = new CountDownLatch(1);
+        final var bodyConsumed = new CountDownLatch(1);
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().setHttpAsyncResponseConsumerFactory(() -> new HttpAsyncResponseConsumer<>() {
+
+            final ByteBuffer readBuffer = ByteBuffer.allocate(ByteSizeUnit.KB.toIntBytes(4));
+            int bytesToConsume = ByteSizeUnit.MB.toIntBytes(1);
+
+            @Override
+            public void responseReceived(HttpResponse response) {
+                assertEquals("application/zip", response.getHeaders("Content-Type")[0].getValue());
+                final var contentDispositionHeader = response.getHeaders("Content-Disposition")[0].getElements()[0];
+                assertEquals("attachment", contentDispositionHeader.getName());
+                assertEquals(
+                    RandomZipResponsePlugin.RESPONSE_FILENAME + ".zip",
+                    contentDispositionHeader.getParameterByName("filename").getValue()
+                );
+                responseStarted.countDown();
+            }
+
+            @Override
+            public void consumeContent(ContentDecoder decoder, IOControl ioControl) throws IOException {
+                readBuffer.clear();
+                final var bytesRead = decoder.read(readBuffer);
+                if (bytesRead > 0) {
+                    bytesToConsume -= bytesRead;
+                }
+
+                if (bytesToConsume <= 0) {
+                    bodyConsumed.countDown();
+                    ioControl.shutdown();
+                }
+            }
+
+            @Override
+            public void responseCompleted(HttpContext context) {}
+
+            @Override
+            public void failed(Exception ex) {}
+
+            @Override
+            public Exception getException() {
+                return null;
+            }
+
+            @Override
+            public HttpResponse getResult() {
+                return null;
+            }
+
+            @Override
+            public boolean isDone() {
+                return false;
+            }
+
+            @Override
+            public void close() {}
+
+            @Override
+            public boolean cancel() {
+                return false;
+            }
+        }));
+
+        try {
+            try (var restClient = createRestClient(internalCluster().getRandomNodeName())) {
+                // one-node REST client to avoid retries
+                expectThrows(ConnectionClosedException.class, () -> restClient.performRequest(request));
+            }
+            safeAwait(responseStarted);
+            safeAwait(bodyConsumed);
+        } finally {
+            assertNull(getExpectedEntries()); // mainly just checking that all refs are released
+        }
+    }
+
+    public void testGetNextPartFailure() throws IOException {
+        final var request = new Request("GET", RandomZipResponsePlugin.INFINITE_ROUTE);
+        request.addParameter(RandomZipResponsePlugin.GET_NEXT_PART_COUNT_DOWN_PARAM, Integer.toString(between(1, 100)));
+
+        try (var restClient = createRestClient(internalCluster().getRandomNodeName())) {
+            // one-node REST client to avoid retries
+            assertThat(
+                safeAwaitFailure(
+                    Response.class,
+                    l -> restClient.performRequestAsync(request, ActionTestUtils.wrapAsRestResponseListener(l))
+                ),
+                anyOf(instanceOf(ConnectionClosedException.class), instanceOf(MalformedChunkCodingException.class))
+            );
+        } finally {
+            assertNull(getExpectedEntries()); // mainly just checking that all refs are released
+        }
+    }
+
+    private static Map<String, BytesReference> getExpectedEntries() {
+        final List<Map<String, BytesReference>> nodeResponses = StreamSupport
+            // concatenate all the chunks in all the entries
+            .stream(internalCluster().getInstances(PluginsService.class).spliterator(), false)
+            .flatMap(p -> p.filterPlugins(RandomZipResponsePlugin.class))
+            .flatMap(p -> {
+                final var maybeResponse = p.responseRef.getAndSet(null);
+                if (maybeResponse == null) {
+                    return Stream.of();
+                } else {
+                    safeAwait(maybeResponse.completedLatch()); // ensures that all refs have been released
+                    if (maybeResponse.entries() == null) {
+                        return Stream.of((Map<String, BytesReference>) null);
+                    } else {
+                        final var expectedEntries = Maps.<String, BytesReference>newMapWithExpectedSize(maybeResponse.entries().size());
+                        maybeResponse.entries().forEach((entryName, entryBody) -> {
+                            if (entryBody != null) {
+                                try (var bytesStreamOutput = new BytesStreamOutput()) {
+                                    for (final var part : entryBody.parts()) {
+                                        for (final var chunk : part.chunks()) {
+                                            chunk.writeTo(bytesStreamOutput);
+                                        }
+                                    }
+                                    expectedEntries.put(entryName, bytesStreamOutput.bytes());
+                                } catch (IOException e) {
+                                    throw new AssertionError(e);
+                                }
+                            }
+                        });
+                        return Stream.of(expectedEntries);
+                    }
+                }
+            })
+            .toList();
+        assertThat(nodeResponses, hasSize(1));
+        return nodeResponses.get(0);
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java
index 7ad464fee92ba..2e05dbfad6085 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/RestControllerIT.java
@@ -20,9 +20,12 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.plugins.TelemetryPlugin;
 import org.elasticsearch.telemetry.Measurement;
 import org.elasticsearch.telemetry.TestTelemetryPlugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -34,10 +37,10 @@
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
-import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numClientNodes = 1, numDataNodes = 0)
@@ -54,7 +57,7 @@ public void testHeadersEmittedWithChunkedResponses() throws IOException {
         assertEquals(ChunkedResponseWithHeadersPlugin.HEADER_VALUE, response.getHeader(ChunkedResponseWithHeadersPlugin.HEADER_NAME));
     }
 
-    public void testMetricsEmittedOnSuccess() throws IOException {
+    public void testMetricsEmittedOnSuccess() throws Exception {
         final var client = getRestClient();
         final var request = new Request("GET", TestEchoStatusCodePlugin.ROUTE);
         request.addParameter("status_code", "200");
@@ -70,7 +73,7 @@ public void testMetricsEmittedOnSuccess() throws IOException {
         });
     }
 
-    public void testMetricsEmittedOnRestError() throws IOException {
+    public void testMetricsEmittedOnRestError() throws Exception {
         final var client = getRestClient();
         final var request = new Request("GET", TestEchoStatusCodePlugin.ROUTE);
         request.addParameter("status_code", "503");
@@ -85,7 +88,7 @@ public void testMetricsEmittedOnRestError() throws IOException {
         });
     }
 
-    public void testMetricsEmittedOnWrongMethod() throws IOException {
+    public void testMetricsEmittedOnWrongMethod() throws Exception {
         final var client = getRestClient();
         final var request = new Request("DELETE", TestEchoStatusCodePlugin.ROUTE);
         final var response = expectThrows(ResponseException.class, () -> client.performRequest(request));
@@ -97,31 +100,39 @@ public void testMetricsEmittedOnWrongMethod() throws IOException {
         });
     }
 
-    private static void assertMeasurement(Consumer<Measurement> measurementConsumer) {
-        var measurements = new ArrayList<Measurement>();
-        for (PluginsService pluginsService : internalCluster().getInstances(PluginsService.class)) {
-            final TestTelemetryPlugin telemetryPlugin = pluginsService.filterPlugins(TestTelemetryPlugin.class).findFirst().orElseThrow();
-            telemetryPlugin.collect();
-
-            final var metrics = telemetryPlugin.getLongCounterMeasurement(RestController.METRIC_REQUESTS_TOTAL);
-            measurements.addAll(metrics);
-        }
-        assertThat(measurements, hasSize(1));
-        measurementConsumer.accept(measurements.get(0));
+    private void assertMeasurement(Consumer<Measurement> measurementConsumer) throws Exception {
+        assertBusy(() -> {
+            var measurements = new ArrayList<Measurement>();
+            for (var nodeName : internalCluster().getNodeNames()) {
+                PluginsService pluginsService = internalCluster().getInstance(PluginsService.class, nodeName);
+                var telemetryPlugins = pluginsService.filterPlugins(TelemetryPlugin.class).toList();
+
+                assertThat(telemetryPlugins, hasSize(1));
+                assertThat(telemetryPlugins.get(0), instanceOf(TestTelemetryPlugin.class));
+                var telemetryPlugin = (TestTelemetryPlugin) telemetryPlugins.get(0);
+
+                telemetryPlugin.collect();
+
+                final var metrics = telemetryPlugin.getLongCounterMeasurement(RestController.METRIC_REQUESTS_TOTAL);
+                logger.info("collecting [{}] metrics from [{}]", metrics.size(), nodeName);
+                measurements.addAll(metrics);
+            }
+            assertThat(measurements, hasSize(1));
+            measurementConsumer.accept(measurements.get(0));
+        });
     }
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Stream.concat(
-            super.nodePlugins().stream(),
-            Stream.of(ChunkedResponseWithHeadersPlugin.class, TestEchoStatusCodePlugin.class, TestTelemetryPlugin.class)
-        ).toList();
+        return List.of(ChunkedResponseWithHeadersPlugin.class, TestEchoStatusCodePlugin.class, TestTelemetryPlugin.class);
     }
 
     public static class TestEchoStatusCodePlugin extends Plugin implements ActionPlugin {
         static final String ROUTE = "/_test/echo_status_code";
         static final String NAME = "test_echo_status_code";
 
+        private static final Logger logger = LogManager.getLogger(TestEchoStatusCodePlugin.class);
+
         @Override
         public Collection<RestHandler> getRestHandlers(
             Settings settings,
@@ -148,7 +159,8 @@ public List<Route> routes() {
                 @Override
                 protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
                     var statusCode = request.param("status_code");
-                    client.getLocalNodeId();
+                    logger.info("received echo request for {}", statusCode);
+
                     var restStatus = RestStatus.fromCode(Integer.parseInt(statusCode));
                     return channel -> {
                         final var response = RestResponse.chunked(
@@ -161,6 +173,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
                             null
                         );
                         channel.sendResponse(response);
+                        logger.info("sent response");
                     };
                 }
             });
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/rest/StreamingXContentResponseIT.java b/server/src/internalClusterTest/java/org/elasticsearch/rest/StreamingXContentResponseIT.java
new file mode 100644
index 0000000000000..ae91caea888db
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/rest/StreamingXContentResponseIT.java
@@ -0,0 +1,300 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.rest;
+
+import org.apache.http.ConnectionClosedException;
+import org.apache.http.HttpResponse;
+import org.apache.http.nio.ContentDecoder;
+import org.apache.http.nio.IOControl;
+import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
+import org.apache.http.protocol.HttpContext;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.support.RefCountingRunnable;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.ThrottledIterator;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import static org.hamcrest.Matchers.hasSize;
+
+@ESIntegTestCase.ClusterScope(numDataNodes = 1)
+public class StreamingXContentResponseIT extends ESIntegTestCase {
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopyNoNullElements(super.nodePlugins(), RandomXContentResponsePlugin.class);
+    }
+
+    public static class RandomXContentResponsePlugin extends Plugin implements ActionPlugin {
+
+        public static final String ROUTE = "/_random_xcontent_response";
+
+        public static final String INFINITE_ROUTE = "/_random_infinite_xcontent_response";
+
+        public final AtomicReference<Response> responseRef = new AtomicReference<>();
+
+        public record Response(Map<String, String> fragments, CountDownLatch completedLatch) {}
+
+        @Override
+        public Collection<RestHandler> getRestHandlers(
+            Settings settings,
+            NamedWriteableRegistry namedWriteableRegistry,
+            RestController restController,
+            ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings,
+            SettingsFilter settingsFilter,
+            IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster,
+            Predicate<NodeFeature> clusterSupportsFeature
+        ) {
+            return List.of(
+                // handler that returns a normal (finite) response
+                new RestHandler() {
+                    @Override
+                    public List<Route> routes() {
+                        return List.of(new Route(RestRequest.Method.GET, ROUTE));
+                    }
+
+                    @Override
+                    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws IOException {
+                        final var response = new Response(new HashMap<>(), new CountDownLatch(1));
+                        final var entryCount = between(0, 10000);
+                        for (int i = 0; i < entryCount; i++) {
+                            response.fragments().put(randomIdentifier(), randomIdentifier());
+                        }
+                        assertTrue(responseRef.compareAndSet(null, response));
+                        handleStreamingXContentRestRequest(
+                            channel,
+                            client.threadPool(),
+                            response.completedLatch(),
+                            response.fragments().entrySet().iterator()
+                        );
+                    }
+                },
+
+                // handler that just keeps on yielding chunks until aborted
+                new RestHandler() {
+                    @Override
+                    public List<Route> routes() {
+                        return List.of(new Route(RestRequest.Method.GET, INFINITE_ROUTE));
+                    }
+
+                    @Override
+                    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws IOException {
+                        final var response = new Response(new HashMap<>(), new CountDownLatch(1));
+                        assertTrue(responseRef.compareAndSet(null, new Response(null, response.completedLatch())));
+                        handleStreamingXContentRestRequest(channel, client.threadPool(), response.completedLatch(), new Iterator<>() {
+
+                            private long id;
+
+                            // carry on yielding content even after the channel closes
+                            private final Semaphore trailingContentPermits = new Semaphore(between(0, 20));
+
+                            @Override
+                            public boolean hasNext() {
+                                return request.getHttpChannel().isOpen() || trailingContentPermits.tryAcquire();
+                            }
+
+                            @Override
+                            public Map.Entry<String, String> next() {
+                                return new Map.Entry<>() {
+                                    private final String key = Long.toString(id++);
+                                    private final String content = randomIdentifier();
+
+                                    @Override
+                                    public String getKey() {
+                                        return key;
+                                    }
+
+                                    @Override
+                                    public String getValue() {
+                                        return content;
+                                    }
+
+                                    @Override
+                                    public String setValue(String value) {
+                                        return fail(null, "must not setValue");
+                                    }
+                                };
+                            }
+                        });
+                    }
+                }
+            );
+        }
+
+        private static void handleStreamingXContentRestRequest(
+            RestChannel channel,
+            ThreadPool threadPool,
+            CountDownLatch completionLatch,
+            Iterator<Map.Entry<String, String>> fragmentIterator
+        ) throws IOException {
+            try (var refs = new RefCountingRunnable(completionLatch::countDown)) {
+                final var streamingXContentResponse = new StreamingXContentResponse(channel, channel.request(), refs.acquire());
+                streamingXContentResponse.writeFragment(p -> ChunkedToXContentHelper.startObject(), refs.acquire());
+                final var finalRef = refs.acquire();
+                ThrottledIterator.run(
+                    fragmentIterator,
+                    (ref, fragment) -> randomFrom(EsExecutors.DIRECT_EXECUTOR_SERVICE, threadPool.generic()).execute(
+                        ActionRunnable.run(ActionListener.releaseAfter(refs.acquireListener(), ref), () -> {
+                            Thread.yield();
+                            streamingXContentResponse.writeFragment(
+                                p -> ChunkedToXContentHelper.field(fragment.getKey(), fragment.getValue()),
+                                refs.acquire()
+                            );
+                        })
+                    ),
+                    between(1, 10),
+                    () -> {},
+                    () -> {
+                        try (streamingXContentResponse; finalRef) {
+                            streamingXContentResponse.writeFragment(p -> ChunkedToXContentHelper.endObject(), refs.acquire());
+                        }
+                    }
+                );
+            }
+        }
+    }
+
+    public void testRandomStreamingXContentResponse() throws IOException {
+        final var request = new Request("GET", RandomXContentResponsePlugin.ROUTE);
+        final var response = getRestClient().performRequest(request);
+        final var actualEntries = XContentHelper.convertToMap(JsonXContent.jsonXContent, response.getEntity().getContent(), false);
+        assertEquals(getExpectedEntries(), actualEntries);
+    }
+
+    public void testAbort() throws IOException {
+        final var request = new Request("GET", RandomXContentResponsePlugin.INFINITE_ROUTE);
+        final var responseStarted = new CountDownLatch(1);
+        final var bodyConsumed = new CountDownLatch(1);
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().setHttpAsyncResponseConsumerFactory(() -> new HttpAsyncResponseConsumer<>() {
+
+            final ByteBuffer readBuffer = ByteBuffer.allocate(ByteSizeUnit.KB.toIntBytes(4));
+            int bytesToConsume = ByteSizeUnit.MB.toIntBytes(1);
+
+            @Override
+            public void responseReceived(HttpResponse response) {
+                responseStarted.countDown();
+            }
+
+            @Override
+            public void consumeContent(ContentDecoder decoder, IOControl ioControl) throws IOException {
+                readBuffer.clear();
+                final var bytesRead = decoder.read(readBuffer);
+                if (bytesRead > 0) {
+                    bytesToConsume -= bytesRead;
+                }
+
+                if (bytesToConsume <= 0) {
+                    bodyConsumed.countDown();
+                    ioControl.shutdown();
+                }
+            }
+
+            @Override
+            public void responseCompleted(HttpContext context) {}
+
+            @Override
+            public void failed(Exception ex) {}
+
+            @Override
+            public Exception getException() {
+                return null;
+            }
+
+            @Override
+            public HttpResponse getResult() {
+                return null;
+            }
+
+            @Override
+            public boolean isDone() {
+                return false;
+            }
+
+            @Override
+            public void close() {}
+
+            @Override
+            public boolean cancel() {
+                return false;
+            }
+        }));
+
+        try {
+            try (var restClient = createRestClient(internalCluster().getRandomNodeName())) {
+                // one-node REST client to avoid retries
+                expectThrows(ConnectionClosedException.class, () -> restClient.performRequest(request));
+            }
+            safeAwait(responseStarted);
+            safeAwait(bodyConsumed);
+        } finally {
+            assertNull(getExpectedEntries()); // mainly just checking that all refs are released
+        }
+    }
+
+    private static Map<String, String> getExpectedEntries() {
+        final List<Map<String, String>> nodeResponses = StreamSupport
+            // concatenate all the chunks in all the entries
+            .stream(internalCluster().getInstances(PluginsService.class).spliterator(), false)
+            .flatMap(p -> p.filterPlugins(RandomXContentResponsePlugin.class))
+            .flatMap(p -> {
+                final var response = p.responseRef.getAndSet(null);
+                if (response == null) {
+                    return Stream.of();
+                } else {
+                    safeAwait(response.completedLatch()); // ensures that all refs have been released
+                    return Stream.of(response.fragments());
+                }
+            })
+            .toList();
+        assertThat(nodeResponses, hasSize(1));
+        return nodeResponses.get(0);
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java
index 619e7c9d9edec..fe798f9233cbb 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java
@@ -7,6 +7,14 @@
  */
 package org.elasticsearch.script;
 
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
@@ -20,6 +28,7 @@
 import java.util.Map;
 import java.util.function.Function;
 
+import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.putJsonStoredScript;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 
 public class StoredScriptsIT extends ESIntegTestCase {
@@ -41,34 +50,66 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testBasics() {
-        assertAcked(clusterAdmin().preparePutStoredScript().setId("foobar").setContent(new BytesArray(Strings.format("""
+        putJsonStoredScript("foobar", Strings.format("""
             {"script": {"lang": "%s", "source": "1"} }
-            """, LANG)), XContentType.JSON));
-        String script = clusterAdmin().prepareGetStoredScript("foobar").get().getSource().getSource();
+            """, LANG));
+        String script = safeExecute(GetStoredScriptAction.INSTANCE, new GetStoredScriptRequest(TEST_REQUEST_TIMEOUT, "foobar")).getSource()
+            .getSource();
         assertNotNull(script);
         assertEquals("1", script);
 
-        assertAcked(clusterAdmin().prepareDeleteStoredScript("foobar"));
-        StoredScriptSource source = clusterAdmin().prepareGetStoredScript("foobar").get().getSource();
+        assertAcked(
+            safeExecute(
+                TransportDeleteStoredScriptAction.TYPE,
+                new DeleteStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "foobar")
+            )
+        );
+        StoredScriptSource source = safeExecute(GetStoredScriptAction.INSTANCE, new GetStoredScriptRequest(TEST_REQUEST_TIMEOUT, "foobar"))
+            .getSource();
         assertNull(source);
 
-        IllegalArgumentException e = expectThrows(
-            IllegalArgumentException.class,
-            clusterAdmin().preparePutStoredScript().setId("id#").setContent(new BytesArray(Strings.format("""
-                {"script": {"lang": "%s", "source": "1"} }
-                """, LANG)), XContentType.JSON)
+        assertEquals(
+            "Validation Failed: 1: id cannot contain '#' for stored script;",
+            asInstanceOf(
+                IllegalArgumentException.class,
+                ExceptionsHelper.unwrapCause(
+                    safeAwaitFailure(
+                        AcknowledgedResponse.class,
+                        l -> client().execute(
+                            TransportPutStoredScriptAction.TYPE,
+                            new PutStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).id("id#")
+                                .content(new BytesArray(Strings.format("""
+                                    {"script": {"lang": "%s", "source": "1"} }
+                                    """, LANG)), XContentType.JSON),
+                            l
+                        )
+                    )
+                )
+            ).getMessage()
         );
-        assertEquals("Validation Failed: 1: id cannot contain '#' for stored script;", e.getMessage());
     }
 
     public void testMaxScriptSize() {
-        IllegalArgumentException e = expectThrows(
-            IllegalArgumentException.class,
-            clusterAdmin().preparePutStoredScript().setId("foobar").setContent(new BytesArray(Strings.format("""
-                {"script": { "lang": "%s", "source":"0123456789abcdef"} }\
-                """, LANG)), XContentType.JSON)
+        assertEquals(
+            "exceeded max allowed stored script size in bytes [64] with size [65] for script [foobar]",
+            asInstanceOf(
+                IllegalArgumentException.class,
+                ExceptionsHelper.unwrapCause(
+                    safeAwaitFailure(
+                        AcknowledgedResponse.class,
+                        l -> client().execute(
+                            TransportPutStoredScriptAction.TYPE,
+                            new PutStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).id("foobar")
+                                .content(new BytesArray(Strings.format("""
+                                    {"script": { "lang": "%s", "source":"0123456789abcdef"} }\
+                                    """, LANG)), XContentType.JSON),
+                            l
+                        )
+
+                    )
+                )
+            ).getMessage()
         );
-        assertEquals("exceeded max allowed stored script size in bytes [64] with size [65] for script [foobar]", e.getMessage());
     }
 
     public static class CustomScriptPlugin extends MockScriptPlugin {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
index efb283f047bb2..b83b74ca8b639 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
@@ -1549,8 +1549,7 @@ public void testDSTEndTransition() throws Exception {
      */
     public void testScriptCaching() throws Exception {
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=date")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=date").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         String date = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(date(1, 1));
         String date2 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format(date(2, 1));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
index 0b92372652597..9df1fae2431f0 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -616,7 +615,7 @@ public void testNoRangesInQuery() {
     public void testScriptCaching() throws Exception {
         assertAcked(
             prepareCreate("cache_test_idx").setMapping("date", "type=date")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
index 668b9d79c49a8..237f296f25751 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -942,7 +941,7 @@ public void testOtherDocCount() {
     public void testScriptCaching() throws Exception {
         assertAcked(
             prepareCreate("cache_test_idx").setMapping("d", "type=float")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
index 5894837e257bf..d117f593348d6 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
@@ -1115,7 +1114,7 @@ public void testDecimalIntervalAndOffset() throws Exception {
     public void testScriptCaching() throws Exception {
         assertAcked(
             prepareCreate("cache_test_idx").setMapping("d", "type=float")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
index f0c5cbf9c76bb..cbb3850422800 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -899,8 +898,7 @@ public void testOtherDocCount() {
      */
     public void testScriptCaching() throws Exception {
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=long").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java
index 71402d3e9c1d8..f8ace23057ea6 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RandomSamplerIT.java
@@ -25,7 +25,6 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThan;
 
 @ESIntegTestCase.SuiteScopeTestCase
@@ -93,6 +92,7 @@ public void testRandomSamplerConsistentSeed() {
         double[] sampleMonotonicValue = new double[1];
         double[] sampleNumericValue = new double[1];
         long[] sampledDocCount = new long[1];
+        double tolerance = 1e-14;
         // initialize the values
         assertResponse(
             prepareSearch("idx").setPreference("shard:0")
@@ -123,9 +123,12 @@ public void testRandomSamplerConsistentSeed() {
                     ),
                 response -> {
                     InternalRandomSampler sampler = response.getAggregations().get("sampler");
-                    assertThat(((Avg) sampler.getAggregations().get("mean_monotonic")).getValue(), equalTo(sampleMonotonicValue[0]));
-                    assertThat(((Avg) sampler.getAggregations().get("mean_numeric")).getValue(), equalTo(sampleNumericValue[0]));
-                    assertThat(sampler.getDocCount(), equalTo(sampledDocCount[0]));
+                    double monotonicValue = ((Avg) sampler.getAggregations().get("mean_monotonic")).getValue();
+                    double numericValue = ((Avg) sampler.getAggregations().get("mean_numeric")).getValue();
+                    long docCount = sampler.getDocCount();
+                    assertEquals(monotonicValue, sampleMonotonicValue[0], tolerance);
+                    assertEquals(numericValue, sampleNumericValue[0], tolerance);
+                    assertEquals(docCount, sampledDocCount[0]);
                 }
             );
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
index 10e3649e9f161..6a60969e632ee 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
@@ -896,7 +895,7 @@ public void testEmptyAggregation() throws Exception {
     public void testScriptCaching() throws Exception {
         assertAcked(
             prepareCreate("cache_test_idx").setMapping("i", "type=integer")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
index 35a117ac8922b..5fa010e4b091e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -548,7 +547,7 @@ public void testReduceFromSeveralShards() throws IOException, ExecutionException
     public void testScriptCaching() throws Exception {
         assertAcked(
             prepareCreate("cache_test_idx").setMapping("s", "type=long", "t", "type=text")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java
index c45cabf425b14..69b84c6b98286 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsIT.java
@@ -10,7 +10,6 @@
 
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
@@ -46,9 +45,7 @@ private void indexDocs(int numDocs) {
     }
 
     public void testSingleValuedString() {
-        final Settings.Builder settings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 2)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0);
+        final Settings.Builder settings = indexSettings(2, 0);
         createIndex(index, settings.build());
         // We want to trigger the usage of cuckoo filters that happen only when there are
         // more than 10k distinct values in one shard.
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java
index 662744ddfe77e..fa9a9ef2a7f41 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -1198,7 +1197,7 @@ public void testOtherDocCount() {
     public void testScriptCaching() throws Exception {
         assertAcked(
             prepareCreate("cache_test_idx").setMapping("d", "type=keyword")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+                .setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java
index 750868defde97..fde18fb283a6e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java
@@ -7,7 +7,6 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -905,8 +904,7 @@ private void checkUpperLowerBounds(ExtendedStats stats, double sigma) {
      */
     public void testScriptCaching() throws Exception {
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=long").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
index 13d66a5cf3949..a4da7c4e893be 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -568,8 +567,7 @@ public void testOrderByEmptyAggregation() throws Exception {
      */
     public void testScriptCaching() throws Exception {
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=long").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
index cd69fb8241ef2..43e4aecb07f7f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
@@ -541,8 +540,7 @@ public void testOrderByEmptyAggregation() throws Exception {
      */
     public void testScriptCaching() throws Exception {
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=long").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java
index 6c80931914ac6..4ded290f93961 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -494,8 +493,7 @@ public void testOrderByEmptyAggregation() throws Exception {
      */
     public void testScriptCaching() throws Exception {
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=long").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
 
         indexRandom(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
index eeee745b32f92..042c8c1fb0e35 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
@@ -10,8 +10,6 @@
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.plugins.Plugin;
@@ -27,7 +25,6 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.xcontent.XContentType;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -42,6 +39,7 @@
 import java.util.function.Consumer;
 import java.util.function.Function;
 
+import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.putJsonStoredScript;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.global;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
@@ -300,21 +298,21 @@ public void setupSuiteScopeCluster() throws Exception {
         // When using the MockScriptPlugin we can map Stored scripts to inline scripts:
         // the id of the stored script is used in test method while the source of the stored script
         // must match a predefined script from CustomScriptPlugin.pluginScripts() method
-        assertAcked(clusterAdmin().preparePutStoredScript().setId("initScript_stored").setContent(new BytesArray(Strings.format("""
+        putJsonStoredScript("initScript_stored", Strings.format("""
             {"script": {"lang": "%s", "source": "vars.multiplier = 3"} }
-            """, MockScriptPlugin.NAME)), XContentType.JSON));
+            """, MockScriptPlugin.NAME));
 
-        assertAcked(clusterAdmin().preparePutStoredScript().setId("mapScript_stored").setContent(new BytesArray(Strings.format("""
+        putJsonStoredScript("mapScript_stored", Strings.format("""
             {"script": {"lang": "%s", "source": "state.list.add(vars.multiplier)"} }
-            """, MockScriptPlugin.NAME)), XContentType.JSON));
+            """, MockScriptPlugin.NAME));
 
-        assertAcked(clusterAdmin().preparePutStoredScript().setId("combineScript_stored").setContent(new BytesArray(Strings.format("""
+        putJsonStoredScript("combineScript_stored", Strings.format("""
             {"script": {"lang": "%s", "source": "sum state values as a new aggregation"} }
-            """, MockScriptPlugin.NAME)), XContentType.JSON));
+            """, MockScriptPlugin.NAME));
 
-        assertAcked(clusterAdmin().preparePutStoredScript().setId("reduceScript_stored").setContent(new BytesArray(Strings.format("""
+        putJsonStoredScript("reduceScript_stored", Strings.format("""
             {"script": {"lang": "%s", "source": "sum all states (lists) values as a new aggregation"} }
-            """, MockScriptPlugin.NAME)), XContentType.JSON));
+            """, MockScriptPlugin.NAME));
 
         indexRandom(true, builders);
         ensureSearchable();
@@ -1139,8 +1137,7 @@ public void testScriptCaching() throws Exception {
         Script ndRandom = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "return Math.random()", Collections.emptyMap());
 
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=long").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
index 84e0bee396c9d..78adca3377f0b 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -234,8 +233,7 @@ private void assertShardExecutionState(SearchResponse response, int expectedFail
      */
     public void testScriptCaching() throws Exception {
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=long").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index d50c101dbd5d1..fd173b8f48a12 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -208,8 +207,7 @@ public void testOrderByEmptyAggregation() throws Exception {
      */
     public void testScriptCaching() throws Exception {
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=long").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
index 9c737cb734f16..9c11b6cd14d54 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
@@ -485,8 +484,7 @@ public void testOrderByEmptyAggregation() throws Exception {
      */
     public void testScriptCaching() throws Exception {
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=long").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
index 1c101324cd5fc..c67a237b2fc17 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
@@ -457,8 +456,7 @@ public void testOrderByEmptyAggregation() throws Exception {
      */
     public void testScriptCaching() throws Exception {
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=long").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
index fc753b0844c46..42f04ff54c82a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
@@ -1082,9 +1082,7 @@ public void testScriptCaching() throws Exception {
         try {
             assertAcked(
                 prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                    .setSettings(
-                        Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)
-                    )
+                    .setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
             );
             indexRandom(
                 true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index c3feff6f3eaaa..445ad8e0b9b11 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -7,7 +7,6 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -215,8 +214,7 @@ public void testMultiValuedScriptWithParams() throws Exception {
      */
     public void testScriptCaching() throws Exception {
         assertAcked(
-            prepareCreate("cache_test_idx").setMapping("d", "type=long")
-                .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1))
+            prepareCreate("cache_test_idx").setMapping("d", "type=long").setSettings(indexSettings(1, 1).put("requests.cache.enable", true))
         );
         indexRandom(
             true,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
index dc612d6bad5ce..bc8142a629f7c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
@@ -24,7 +23,6 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
-import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -35,12 +33,12 @@
 import java.util.Map;
 import java.util.function.Function;
 
+import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.putJsonStoredScript;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.dateRange;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
 import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.bucketScript;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.equalTo;
@@ -514,14 +512,11 @@ public void testInlineScriptReturnNull() {
     }
 
     public void testStoredScript() {
-        assertAcked(
-            clusterAdmin().preparePutStoredScript()
-                .setId("my_script")
-                // Script source is not interpreted but it references a pre-defined script from CustomScriptPlugin
-                .setContent(
-                    new BytesArray("{ \"script\": {\"lang\": \"" + CustomScriptPlugin.NAME + "\", \"source\": \"my_script\" } }"),
-                    XContentType.JSON
-                )
+
+        putJsonStoredScript(
+            "my_script",
+            // Script source is not interpreted but it references a pre-defined script from CustomScriptPlugin
+            "{ \"script\": {\"lang\": \"" + CustomScriptPlugin.NAME + "\", \"source\": \"my_script\" } }"
         );
 
         assertNoFailuresAndResponse(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java
new file mode 100644
index 0000000000000..bb18b8f1b702d
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java
@@ -0,0 +1,716 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.ccs;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot;
+import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.Result;
+import org.elasticsearch.action.search.ClosePointInTimeRequest;
+import org.elasticsearch.action.search.OpenPointInTimeRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.TransportClosePointInTimeAction;
+import org.elasticsearch.action.search.TransportOpenPointInTimeAction;
+import org.elasticsearch.action.search.TransportSearchAction;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.util.FeatureFlag;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.builder.PointInTimeBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.query.SlowRunningQueryBuilder;
+import org.elasticsearch.search.query.ThrowingQueryBuilder;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.test.AbstractMultiClustersTestCase;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.usage.UsageService;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE;
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.MRT_FEATURE;
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.WILDCARD_FEATURE;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
+import static org.hamcrest.Matchers.equalTo;
+
+public class CCSUsageTelemetryIT extends AbstractMultiClustersTestCase {
+    private static final Logger LOGGER = LogManager.getLogger(CCSUsageTelemetryIT.class);
+    private static final String REMOTE1 = "cluster-a";
+    private static final String REMOTE2 = "cluster-b";
+    private static final FeatureFlag CCS_TELEMETRY_FEATURE_FLAG = new FeatureFlag("ccs_telemetry");
+
+    @Override
+    protected boolean reuseClusters() {
+        return false;
+    }
+
+    @Override
+    protected Collection<String> remoteClusterAlias() {
+        return List.of(REMOTE1, REMOTE2);
+    }
+
+    @Rule
+    public SkipUnavailableRule skipOverride = new SkipUnavailableRule(REMOTE1, REMOTE2);
+
+    @BeforeClass
+    protected static void skipIfTelemetryDisabled() {
+        assumeTrue("Skipping test as CCS_TELEMETRY_FEATURE_FLAG is disabled", CCS_TELEMETRY_FEATURE_FLAG.isEnabled());
+    }
+
+    @Override
+    protected Map<String, Boolean> skipUnavailableForRemoteClusters() {
+        var map = skipOverride.getMap();
+        LOGGER.info("Using skip_unavailable map: [{}]", map);
+        return map;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
+        return CollectionUtils.appendToCopy(super.nodePlugins(clusterAlias), CrossClusterSearchIT.TestQueryBuilderPlugin.class);
+    }
+
+    private SearchRequest makeSearchRequest(String... indices) {
+        SearchRequest searchRequest = new SearchRequest(indices);
+        searchRequest.allowPartialSearchResults(false);
+        searchRequest.setBatchedReduceSize(randomIntBetween(3, 20));
+        searchRequest.setCcsMinimizeRoundtrips(randomBoolean());
+        if (randomBoolean()) {
+            searchRequest.setPreFilterShardSize(1);
+        }
+        searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(10));
+        return searchRequest;
+    }
+
+    /**
+    * Run search request and get telemetry from it
+    */
+    private CCSTelemetrySnapshot getTelemetryFromSearch(SearchRequest searchRequest) throws ExecutionException, InterruptedException {
+        // We want to send search to a specific node (we don't care which one) so that we could
+        // collect the CCS telemetry from it later
+        String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName();
+        // We don't care here too much about the response, we just want to trigger the telemetry collection.
+        // So we check it's not null and leave the rest to other tests.
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+        return getTelemetrySnapshot(nodeName);
+    }
+
+    private CCSTelemetrySnapshot getTelemetryFromFailedSearch(SearchRequest searchRequest) throws Exception {
+        // We want to send search to a specific node (we don't care which one) so that we could
+        // collect the CCS telemetry from it later
+        String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName();
+        PlainActionFuture<SearchResponse> queryFuture = new PlainActionFuture<>();
+        cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest, queryFuture);
+        assertBusy(() -> assertTrue(queryFuture.isDone()));
+
+        // We expect failure, but we don't care too much which failure it is in this test
+        ExecutionException ee = expectThrows(ExecutionException.class, queryFuture::get);
+        assertNotNull(ee.getCause());
+
+        return getTelemetrySnapshot(nodeName);
+    }
+
+    /**
+     * Create search request for indices and get telemetry from it
+     */
+    private CCSTelemetrySnapshot getTelemetryFromSearch(String... indices) throws ExecutionException, InterruptedException {
+        return getTelemetryFromSearch(makeSearchRequest(indices));
+    }
+
+    /**
+     * Search on all remotes
+     */
+    public void testAllRemotesSearch() throws ExecutionException, InterruptedException {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex);
+        boolean minimizeRoundtrips = TransportSearchAction.shouldMinimizeRoundtrips(searchRequest);
+
+        String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName();
+        assertResponse(
+            cluster(LOCAL_CLUSTER).client(nodeName)
+                .filterWithHeader(Map.of(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER, "kibana"))
+                .search(searchRequest),
+            Assert::assertNotNull
+        );
+        CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName);
+
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(1L));
+        assertThat(telemetry.getFailureReasons().size(), equalTo(0));
+        assertThat(telemetry.getTook().count(), equalTo(1L));
+        assertThat(telemetry.getTookMrtTrue().count(), equalTo(minimizeRoundtrips ? 1L : 0L));
+        assertThat(telemetry.getTookMrtFalse().count(), equalTo(minimizeRoundtrips ? 0L : 1L));
+        assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L));
+        assertThat(telemetry.getClientCounts().size(), equalTo(1));
+        assertThat(telemetry.getClientCounts().get("kibana"), equalTo(1L));
+        if (minimizeRoundtrips) {
+            assertThat(telemetry.getFeatureCounts().get(MRT_FEATURE), equalTo(1L));
+        } else {
+            assertThat(telemetry.getFeatureCounts().get(MRT_FEATURE), equalTo(null));
+        }
+        assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(null));
+
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(3));
+        for (String clusterAlias : remoteClusterAlias()) {
+            var clusterTelemetry = perCluster.get(clusterAlias);
+            assertThat(clusterTelemetry.getCount(), equalTo(1L));
+            assertThat(clusterTelemetry.getSkippedCount(), equalTo(0L));
+            assertThat(clusterTelemetry.getTook().count(), equalTo(1L));
+        }
+
+        // another search
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+        telemetry = getTelemetrySnapshot(nodeName);
+        assertThat(telemetry.getTotalCount(), equalTo(2L));
+        assertThat(telemetry.getSuccessCount(), equalTo(2L));
+        assertThat(telemetry.getFailureReasons().size(), equalTo(0));
+        assertThat(telemetry.getTook().count(), equalTo(2L));
+        assertThat(telemetry.getTookMrtTrue().count(), equalTo(minimizeRoundtrips ? 2L : 0L));
+        assertThat(telemetry.getTookMrtFalse().count(), equalTo(minimizeRoundtrips ? 0L : 2L));
+        assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L));
+        assertThat(telemetry.getClientCounts().size(), equalTo(1));
+        assertThat(telemetry.getClientCounts().get("kibana"), equalTo(1L));
+        perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(3));
+        for (String clusterAlias : remoteClusterAlias()) {
+            var clusterTelemetry = perCluster.get(clusterAlias);
+            assertThat(clusterTelemetry.getCount(), equalTo(2L));
+            assertThat(clusterTelemetry.getSkippedCount(), equalTo(0L));
+            assertThat(clusterTelemetry.getTook().count(), equalTo(2L));
+        }
+    }
+
+    /**
+     * Search on a specific remote
+     */
+    public void testOneRemoteSearch() throws ExecutionException, InterruptedException {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        // Make request to cluster a
+        SearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex);
+        String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName();
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+        CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName);
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(2));
+        assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE2), equalTo(null));
+        assertThat(telemetry.getClientCounts().size(), equalTo(0));
+
+        // Make request to cluster b
+        searchRequest = makeSearchRequest(localIndex, REMOTE2 + ":" + remoteIndex);
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+        telemetry = getTelemetrySnapshot(nodeName);
+        assertThat(telemetry.getTotalCount(), equalTo(2L));
+        assertThat(telemetry.getSuccessCount(), equalTo(2L));
+        perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(3));
+        assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE2).getCount(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE2).getTook().count(), equalTo(1L));
+    }
+
+    /**
+     * Local search should not produce any telemetry at all
+     */
+    public void testLocalOnlySearch() throws ExecutionException, InterruptedException {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+
+        CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(localIndex);
+        assertThat(telemetry.getTotalCount(), equalTo(0L));
+    }
+
+    /**
+    * Search on remotes only, without local index
+    */
+    public void testRemoteOnlySearch() throws ExecutionException, InterruptedException {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        CCSTelemetrySnapshot telemetry = getTelemetryFromSearch("*:" + remoteIndex);
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(1L));
+        assertThat(telemetry.getFailureReasons().size(), equalTo(0));
+        assertThat(telemetry.getTook().count(), equalTo(1L));
+        assertThat(perCluster.size(), equalTo(2));
+        assertThat(telemetry.getClientCounts().size(), equalTo(0));
+        assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(0L));
+        assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE2).getCount(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE2).getSkippedCount(), equalTo(0L));
+        assertThat(perCluster.get(REMOTE2).getTook().count(), equalTo(1L));
+    }
+
+    /**
+     * Count wildcard searches. Only wildcards in index names (not in cluster names) are counted.
+     */
+    public void testWildcardSearch() throws ExecutionException, InterruptedException {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex);
+        String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName();
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+        CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName);
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(null));
+
+        searchRequest = makeSearchRequest("*", REMOTE1 + ":" + remoteIndex);
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+        telemetry = getTelemetrySnapshot(nodeName);
+        assertThat(telemetry.getTotalCount(), equalTo(2L));
+        assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(1L));
+
+        searchRequest = makeSearchRequest(localIndex, REMOTE2 + ":*");
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+        telemetry = getTelemetrySnapshot(nodeName);
+        assertThat(telemetry.getTotalCount(), equalTo(3L));
+        assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(2L));
+
+        // Wildcards in cluster name do not count
+        searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex);
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+        telemetry = getTelemetrySnapshot(nodeName);
+        assertThat(telemetry.getTotalCount(), equalTo(4L));
+        assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(2L));
+
+        // Wildcard in the middle of the index name counts
+        searchRequest = makeSearchRequest(localIndex, REMOTE2 + ":rem*");
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+        telemetry = getTelemetrySnapshot(nodeName);
+        assertThat(telemetry.getTotalCount(), equalTo(5L));
+        assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(3L));
+
+        // Wildcard only counted once per search
+        searchRequest = makeSearchRequest("*", REMOTE1 + ":rem*", REMOTE2 + ":remote*");
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+        telemetry = getTelemetrySnapshot(nodeName);
+        assertThat(telemetry.getTotalCount(), equalTo(6L));
+        assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(4L));
+    }
+
+    /**
+     * Test complete search failure
+     */
+    public void testFailedSearch() throws Exception {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex);
+        // shardId -1 means to throw the Exception on all shards, so should result in complete search failure
+        ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder(randomLong(), new IllegalStateException("index corrupted"), -1);
+        searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10));
+        searchRequest.allowPartialSearchResults(true);
+
+        CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest);
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        assertThat(telemetry.getTook().count(), equalTo(0L));
+        assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L));
+        assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L));
+        Map<String, Long> expectedFailures = Map.of(Result.UNKNOWN.getName(), 1L);
+        assertThat(telemetry.getFailureReasons(), equalTo(expectedFailures));
+    }
+
+    /**
+     * Search when all the remotes failed and skipped
+     */
+    public void testSkippedAllRemotesSearch() throws Exception {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex);
+        // throw Exception on all shards of remoteIndex, but not against localIndex
+        ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder(
+            randomLong(),
+            new IllegalStateException("index corrupted"),
+            remoteIndex
+        );
+        searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10));
+        searchRequest.allowPartialSearchResults(true);
+
+        String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName();
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+
+        CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName);
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(1L));
+        // Note that this counts how many searches had skipped remotes, not how many remotes are skipped
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L));
+        // Still count the remote that failed
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        assertThat(telemetry.getTook().count(), equalTo(1L));
+        // Each remote will have its skipped count bumped
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(3));
+        for (String remote : remoteClusterAlias()) {
+            assertThat(perCluster.get(remote).getCount(), equalTo(0L));
+            assertThat(perCluster.get(remote).getSkippedCount(), equalTo(1L));
+            assertThat(perCluster.get(remote).getTook().count(), equalTo(0L));
+        }
+    }
+
+    public void testSkippedOneRemoteSearch() throws Exception {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        // Remote1 will fail, Remote2 will just do nothing but it counts as success
+        SearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex, REMOTE2 + ":" + "nosuchindex*");
+        // throw Exception on all shards of remoteIndex, but not against localIndex
+        ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder(
+            randomLong(),
+            new IllegalStateException("index corrupted"),
+            remoteIndex
+        );
+        searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10));
+        searchRequest.allowPartialSearchResults(true);
+
+        String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName();
+        assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull);
+
+        CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName);
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(1L));
+        // Note that this counts how many searches had skipped remotes, not how many remotes are skipped
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L));
+        // Still count the remote that failed
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        assertThat(telemetry.getTook().count(), equalTo(1L));
+        // Each remote will have its skipped count bumped
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(3));
+        // This one is skipped
+        assertThat(perCluster.get(REMOTE1).getCount(), equalTo(0L));
+        assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(0L));
+        // This one is OK
+        assertThat(perCluster.get(REMOTE2).getCount(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE2).getSkippedCount(), equalTo(0L));
+        assertThat(perCluster.get(REMOTE2).getTook().count(), equalTo(1L));
+    }
+
+    /**
+     * Test what happens if remote times out - it should be skipped
+     */
+    public void testRemoteTimesOut() throws Exception {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        SearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex);
+        // This works only with minimize_roundtrips enabled, since otherwise timed out shards will be counted as
+        // partial failure, and we disable partial results..
+        searchRequest.setCcsMinimizeRoundtrips(true);
+
+        TimeValue searchTimeout = new TimeValue(500, TimeUnit.MILLISECONDS);
+        // query builder that will sleep for the specified amount of time in the query phase
+        SlowRunningQueryBuilder slowRunningQueryBuilder = new SlowRunningQueryBuilder(searchTimeout.millis() * 5, remoteIndex);
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(slowRunningQueryBuilder).timeout(searchTimeout);
+        searchRequest.source(sourceBuilder);
+
+        CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(searchRequest);
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(1L));
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(1L));
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(2));
+        assertThat(perCluster.get(REMOTE1).getCount(), equalTo(0L));
+        assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(0L));
+        assertThat(perCluster.get(REMOTE2), equalTo(null));
+    }
+
+    /**
+    * Test what happens if remote times out and there's no local - it should be skipped
+    */
+    public void testRemoteOnlyTimesOut() throws Exception {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":" + remoteIndex);
+        // This works only with minimize_roundtrips enabled, since otherwise timed out shards will be counted as
+        // partial failure, and we disable partial results...
+        searchRequest.setCcsMinimizeRoundtrips(true);
+
+        TimeValue searchTimeout = new TimeValue(100, TimeUnit.MILLISECONDS);
+        // query builder that will sleep for the specified amount of time in the query phase
+        SlowRunningQueryBuilder slowRunningQueryBuilder = new SlowRunningQueryBuilder(searchTimeout.millis() * 5, remoteIndex);
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(slowRunningQueryBuilder).timeout(searchTimeout);
+        searchRequest.source(sourceBuilder);
+
+        CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(searchRequest);
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(1L));
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(1L));
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(1));
+        assertThat(perCluster.get(REMOTE1).getCount(), equalTo(0L));
+        assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(0L));
+        assertThat(perCluster.get(REMOTE2), equalTo(null));
+    }
+
+    @SkipOverride(aliases = { REMOTE1 })
+    public void testRemoteTimesOutFailure() throws Exception {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":" + remoteIndex);
+
+        TimeValue searchTimeout = new TimeValue(100, TimeUnit.MILLISECONDS);
+        // query builder that will sleep for the specified amount of time in the query phase
+        SlowRunningQueryBuilder slowRunningQueryBuilder = new SlowRunningQueryBuilder(searchTimeout.millis() * 5, remoteIndex);
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(slowRunningQueryBuilder).timeout(searchTimeout);
+        searchRequest.source(sourceBuilder);
+
+        CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest);
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        // Failure is not skipping
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L));
+        // Still count the remote that failed
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(1L));
+        assertThat(telemetry.getTook().count(), equalTo(0L));
+        Map<String, Long> expectedFailure = Map.of(Result.TIMEOUT.getName(), 1L);
+        assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure));
+        // No per-cluster data on total failure
+        assertThat(telemetry.getByRemoteCluster().size(), equalTo(0));
+    }
+
+    /**
+    * Search when all the remotes failed and not skipped
+    */
+    @SkipOverride(aliases = { REMOTE1, REMOTE2 })
+    public void testFailedAllRemotesSearch() throws Exception {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex);
+        // throw Exception on all shards of remoteIndex, but not against localIndex
+        ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder(
+            randomLong(),
+            new IllegalStateException("index corrupted"),
+            remoteIndex
+        );
+        searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10));
+
+        CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest);
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        // Failure is not skipping
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L));
+        // Still count the remote that failed
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        assertThat(telemetry.getTook().count(), equalTo(0L));
+        Map<String, Long> expectedFailure = Map.of(Result.REMOTES_UNAVAILABLE.getName(), 1L);
+        assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure));
+        // No per-cluster data on total failure
+        assertThat(telemetry.getByRemoteCluster().size(), equalTo(0));
+    }
+
+    /**
+     * Test that we're still counting remote search even if remote cluster has no such index
+     */
+    public void testRemoteHasNoIndex() throws Exception {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+
+        CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(localIndex, REMOTE1 + ":" + "no_such_index*");
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(1L));
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(2));
+        assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L));
+        assertThat(perCluster.get(REMOTE2), equalTo(null));
+    }
+
+    /**
+     * Test that we're still counting remote search even if remote cluster has no such index
+     */
+    @SkipOverride(aliases = { REMOTE1 })
+    public void testRemoteHasNoIndexFailure() throws Exception {
+        SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":no_such_index");
+        CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest);
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(0));
+        Map<String, Long> expectedFailure = Map.of(Result.NOT_FOUND.getName(), 1L);
+        assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure));
+    }
+
+    public void testPITSearch() throws ExecutionException, InterruptedException {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        OpenPointInTimeRequest openPITRequest = new OpenPointInTimeRequest(localIndex, "*:" + remoteIndex).keepAlive(
+            TimeValue.timeValueMinutes(5)
+        );
+        String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName();
+        var client = cluster(LOCAL_CLUSTER).client(nodeName);
+        BytesReference pitID = client.execute(TransportOpenPointInTimeAction.TYPE, openPITRequest).actionGet().getPointInTimeId();
+        SearchRequest searchRequest = new SearchRequest().source(
+            new SearchSourceBuilder().pointInTimeBuilder(new PointInTimeBuilder(pitID).setKeepAlive(TimeValue.timeValueMinutes(5)))
+                .sort("@timestamp")
+                .size(10)
+        );
+        searchRequest.setCcsMinimizeRoundtrips(randomBoolean());
+
+        assertResponse(client.search(searchRequest), Assert::assertNotNull);
+        // do it again
+        assertResponse(client.search(searchRequest), Assert::assertNotNull);
+        client.execute(TransportClosePointInTimeAction.TYPE, new ClosePointInTimeRequest(pitID)).actionGet();
+        CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName);
+
+        assertThat(telemetry.getTotalCount(), equalTo(2L));
+        assertThat(telemetry.getSuccessCount(), equalTo(2L));
+    }
+
+    private CCSTelemetrySnapshot getTelemetrySnapshot(String nodeName) {
+        var usage = cluster(LOCAL_CLUSTER).getInstance(UsageService.class, nodeName);
+        return usage.getCcsUsageHolder().getCCSTelemetrySnapshot();
+    }
+
+    private Map<String, Object> setupClusters() {
+        String localIndex = "demo";
+        int numShardsLocal = randomIntBetween(2, 10);
+        Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build();
+        assertAcked(
+            client(LOCAL_CLUSTER).admin()
+                .indices()
+                .prepareCreate(localIndex)
+                .setSettings(localSettings)
+                .setMapping("@timestamp", "type=date", "f", "type=text")
+        );
+        indexDocs(client(LOCAL_CLUSTER), localIndex);
+
+        String remoteIndex = "prod";
+        int numShardsRemote = randomIntBetween(2, 10);
+        for (String clusterAlias : remoteClusterAlias()) {
+            final InternalTestCluster remoteCluster = cluster(clusterAlias);
+            remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(1, 3));
+            assertAcked(
+                client(clusterAlias).admin()
+                    .indices()
+                    .prepareCreate(remoteIndex)
+                    .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1)))
+                    .setMapping("@timestamp", "type=date", "f", "type=text")
+            );
+            assertFalse(
+                client(clusterAlias).admin()
+                    .cluster()
+                    .prepareHealth(remoteIndex)
+                    .setWaitForYellowStatus()
+                    .setTimeout(TimeValue.timeValueSeconds(10))
+                    .get()
+                    .isTimedOut()
+            );
+            indexDocs(client(clusterAlias), remoteIndex);
+        }
+
+        Map<String, Object> clusterInfo = new HashMap<>();
+        clusterInfo.put("local.index", localIndex);
+        clusterInfo.put("remote.index", remoteIndex);
+        return clusterInfo;
+    }
+
+    private int indexDocs(Client client, String index) {
+        int numDocs = between(5, 20);
+        for (int i = 0; i < numDocs; i++) {
+            client.prepareIndex(index).setSource("f", "v", "@timestamp", randomNonNegativeLong()).get();
+        }
+        client.admin().indices().prepareRefresh(index).get();
+        return numDocs;
+    }
+
+    /**
+     * Annotation to mark specific cluster in a test as not to be skipped when unavailable
+     */
+    @Retention(RetentionPolicy.RUNTIME)
+    @Target(ElementType.METHOD)
+    @interface SkipOverride {
+        String[] aliases();
+    }
+
+    /**
+     * Test rule to process skip annotations
+     */
+    static class SkipUnavailableRule implements TestRule {
+        private final Map<String, Boolean> skipMap;
+
+        SkipUnavailableRule(String... clusterAliases) {
+            this.skipMap = Arrays.stream(clusterAliases).collect(Collectors.toMap(Function.identity(), alias -> true));
+        }
+
+        public Map<String, Boolean> getMap() {
+            return skipMap;
+        }
+
+        @Override
+        public Statement apply(Statement base, Description description) {
+            // Check for annotation named "SkipOverride" and set the overrides accordingly
+            var aliases = description.getAnnotation(SkipOverride.class);
+            if (aliases != null) {
+                for (String alias : aliases.aliases()) {
+                    skipMap.put(alias, false);
+                }
+            }
+            return base;
+        }
+
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
index 076158ee22037..cc272042d5384 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
@@ -89,6 +89,7 @@
 import static org.hamcrest.Matchers.array;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasKey;
@@ -711,6 +712,63 @@ public void testCancel() throws Exception {
         }
     }
 
+    public void testIndexMode() throws Exception {
+        Map<String, IndexMode> indexModes = new HashMap<>();
+        // metrics
+        {
+            final String metricsMapping = """
+                 {
+                     "properties": {
+                       "@timestamp": { "type": "date" },
+                       "hostname": { "type": "keyword", "time_series_dimension": true },
+                       "request_count" : { "type" : "long", "time_series_metric" : "counter" },
+                       "cluster": {"type": "keyword"}
+                     }
+                 }
+                """;
+            Settings settings = Settings.builder().put("mode", "time_series").putList("routing_path", List.of("hostname")).build();
+            int numIndices = between(1, 5);
+            for (int i = 0; i < numIndices; i++) {
+                assertAcked(indicesAdmin().prepareCreate("test_metrics_" + i).setSettings(settings).setMapping(metricsMapping).get());
+                indexModes.put("test_metrics_" + i, IndexMode.TIME_SERIES);
+                assertAcked(indicesAdmin().prepareCreate("test_old_metrics_" + i).setMapping(metricsMapping).get());
+                indexModes.put("test_old_metrics_" + i, IndexMode.STANDARD);
+            }
+        }
+        // logsdb
+        {
+            final String logsMapping = """
+                 {
+                     "properties": {
+                       "@timestamp": { "type": "date" },
+                       "hostname": { "type": "keyword"},
+                       "request_count" : { "type" : "long"},
+                       "cluster": {"type": "keyword"}
+                     }
+                 }
+                """;
+            Settings settings = Settings.builder().put("mode", "logsdb").build();
+            int numIndices = between(1, 5);
+            for (int i = 0; i < numIndices; i++) {
+                assertAcked(indicesAdmin().prepareCreate("test_logs_" + i).setSettings(settings).setMapping(logsMapping).get());
+                indexModes.put("test_logs_" + i, IndexMode.LOGSDB);
+                assertAcked(indicesAdmin().prepareCreate("test_old_logs_" + i).setMapping(logsMapping).get());
+                indexModes.put("test_old_logs_" + i, IndexMode.STANDARD);
+            }
+        }
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
+        request.setMergeResults(false);
+        request.indices("test_*");
+        request.fields(randomFrom("*", "@timestamp", "host*"));
+        var resp = client().fieldCaps(request).get();
+        assertThat(resp.getFailures(), empty());
+        Map<String, IndexMode> actualIndexModes = new HashMap<>();
+        for (var indexResp : resp.getIndexResponses()) {
+            actualIndexModes.put(indexResp.getIndexName(), indexResp.getIndexMode());
+        }
+        assertThat(actualIndexModes, equalTo(indexModes));
+    }
+
     private void assertIndices(FieldCapabilitiesResponse response, String... indices) {
         assertNotNull(response.getIndices());
         Arrays.sort(indices);
@@ -859,7 +917,7 @@ protected String contentType() {
 
         @Override
         public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
-            return new StringStoredFieldFieldLoader(fullPath(), leafName(), null) {
+            return new StringStoredFieldFieldLoader(fullPath(), leafName()) {
                 @Override
                 protected void write(XContentBuilder b, Object value) throws IOException {
                     BytesRef ref = (BytesRef) value;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
index ee60888d7a0a8..c59fc0f68c4d4 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
@@ -73,6 +73,11 @@ public boolean needs_score() {
                             return false;
                         }
 
+                        @Override
+                        public boolean needs_termStats() {
+                            return false;
+                        }
+
                         @Override
                         public ScoreScript newInstance(DocReader docReader) {
                             return new MyScript(params1, lookup, ((DocValuesDocReader) docReader).getLeafReaderContext());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
index 0acf9be574ffe..81bb143c47729 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
@@ -93,7 +93,7 @@ protected int numberOfShards() {
     protected void setupSuiteScopeCluster() throws Exception {
         assertAcked(
             indicesAdmin().prepareCreate("idx")
-                .setSettings(Map.of("number_of_shards", 1, "number_of_replicas", 0))
+                .setSettings(indexSettings(1, 0))
                 .setMapping(STRING_FIELD, "type=keyword", NUMBER_FIELD, "type=integer", TAG_FIELD, "type=keyword")
         );
         List<IndexRequestBuilder> builders = new ArrayList<>();
@@ -634,11 +634,7 @@ public void testNoProfile() {
      * documents and that is hard to express in yaml.
      */
     public void testFilterByFilter() throws InterruptedException, IOException {
-        assertAcked(
-            indicesAdmin().prepareCreate("dateidx")
-                .setSettings(Map.of("number_of_shards", 1, "number_of_replicas", 0))
-                .setMapping("date", "type=date")
-        );
+        assertAcked(indicesAdmin().prepareCreate("dateidx").setSettings(indexSettings(1, 0)).setMapping("date", "type=date"));
         List<IndexRequestBuilder> builders = new ArrayList<>();
         for (int i = 0; i < RangeAggregator.DOCS_PER_RANGE_TO_USE_FILTERS * 2; i++) {
             String date = Instant.ofEpochSecond(i).toString();
@@ -713,7 +709,7 @@ public void testDateHistogramFilterByFilterDisabled() throws InterruptedExceptio
         try {
             assertAcked(
                 indicesAdmin().prepareCreate("date_filter_by_filter_disabled")
-                    .setSettings(Map.of("number_of_shards", 1, "number_of_replicas", 0))
+                    .setSettings(indexSettings(1, 0))
                     .setMapping("date", "type=date", "keyword", "type=keyword")
             );
             List<IndexRequestBuilder> builders = new ArrayList<>();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java
index 0d6d17cbaeb1f..636a7b033a4ee 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/MockedRequestActionBasedRerankerIT.java
@@ -23,11 +23,11 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java
new file mode 100644
index 0000000000000..fa4cafc66c822
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java
@@ -0,0 +1,755 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.retriever;
+
+import org.apache.lucene.search.TotalHits;
+import org.apache.lucene.search.join.ScoreMode;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.MultiSearchRequest;
+import org.elasticsearch.action.search.MultiSearchResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.TransportMultiSearchAction;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.MockSearchService;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.builder.PointInTimeBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.collapse.CollapseBuilder;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.NestedSortBuilder;
+import org.elasticsearch.search.sort.ScoreSortBuilder;
+import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentType;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+import static org.hamcrest.Matchers.equalTo;
+
+public class RankDocRetrieverBuilderIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(MockSearchService.TestPlugin.class);
+    }
+
+    public record RetrieverSource(RetrieverBuilder retriever, SearchSourceBuilder source) {}
+
+    private static String INDEX = "test_index";
+    private static final String ID_FIELD = "_id";
+    private static final String DOC_FIELD = "doc";
+    private static final String TEXT_FIELD = "text";
+    private static final String VECTOR_FIELD = "vector";
+    private static final String TOPIC_FIELD = "topic";
+    private static final String LAST_30D_FIELD = "views.last30d";
+    private static final String ALL_TIME_FIELD = "views.all";
+
+    @Before
+    public void setup() throws Exception {
+        String mapping = """
+            {
+              "properties": {
+                "vector": {
+                  "type": "dense_vector",
+                  "dims": 3,
+                  "element_type": "float",
+                  "index": true,
+                  "similarity": "l2_norm",
+                  "index_options": {
+                    "type": "hnsw"
+                  }
+                },
+                "text": {
+                  "type": "text"
+                },
+                "doc": {
+                  "type": "keyword"
+                },
+                "topic": {
+                  "type": "keyword"
+                },
+                "views": {
+                    "type": "nested",
+                    "properties": {
+                        "last30d": {
+                            "type": "integer"
+                        },
+                        "all": {
+                            "type": "integer"
+                        }
+                    }
+                }
+              }
+            }
+            """;
+        createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).build());
+        admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get();
+        indexDoc(
+            INDEX,
+            "doc_1",
+            DOC_FIELD,
+            "doc_1",
+            TOPIC_FIELD,
+            "technology",
+            TEXT_FIELD,
+            "the quick brown fox jumps over the lazy dog",
+            LAST_30D_FIELD,
+            100
+        );
+        indexDoc(
+            INDEX,
+            "doc_2",
+            DOC_FIELD,
+            "doc_2",
+            TOPIC_FIELD,
+            "astronomy",
+            TEXT_FIELD,
+            "you know, for Search!",
+            VECTOR_FIELD,
+            new float[] { 1.0f, 2.0f, 3.0f },
+            LAST_30D_FIELD,
+            3
+        );
+        indexDoc(INDEX, "doc_3", DOC_FIELD, "doc_3", TOPIC_FIELD, "technology", VECTOR_FIELD, new float[] { 6.0f, 6.0f, 6.0f });
+        indexDoc(
+            INDEX,
+            "doc_4",
+            DOC_FIELD,
+            "doc_4",
+            TOPIC_FIELD,
+            "technology",
+            TEXT_FIELD,
+            "aardvark is a really awesome animal, but not very quick",
+            ALL_TIME_FIELD,
+            100,
+            LAST_30D_FIELD,
+            40
+        );
+        indexDoc(INDEX, "doc_5", DOC_FIELD, "doc_5", TOPIC_FIELD, "science", TEXT_FIELD, "irrelevant stuff");
+        indexDoc(
+            INDEX,
+            "doc_6",
+            DOC_FIELD,
+            "doc_6",
+            TEXT_FIELD,
+            "quick quick quick quick search",
+            VECTOR_FIELD,
+            new float[] { 10.0f, 30.0f, 100.0f },
+            LAST_30D_FIELD,
+            15
+        );
+        indexDoc(
+            INDEX,
+            "doc_7",
+            DOC_FIELD,
+            "doc_7",
+            TOPIC_FIELD,
+            "biology",
+            TEXT_FIELD,
+            "dog",
+            VECTOR_FIELD,
+            new float[] { 3.0f, 3.0f, 3.0f },
+            ALL_TIME_FIELD,
+            1000
+        );
+        refresh(INDEX);
+    }
+
+    public void testRankDocsRetrieverBasicWithPagination() {
+        final int rankWindowSize = 100;
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder();
+        // this one retrieves docs 1, 4, and 6
+        standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD))
+            .boost(10L);
+        StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder();
+        // this one retrieves docs 2 and 6 due to prefilter
+        standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L);
+        standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
+        // this one retrieves docs 7, 2, 3, and 6
+        KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(
+            VECTOR_FIELD,
+            new float[] { 3.0f, 3.0f, 3.0f },
+            null,
+            10,
+            100,
+            null
+        );
+        // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
+        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 4, 7, 3 and with pagination, we'd just omit the first result
+        source.retriever(
+            new CompoundRetrieverWithRankDocs(
+                rankWindowSize,
+                Arrays.asList(
+                    new RetrieverSource(standard0, null),
+                    new RetrieverSource(standard1, null),
+                    new RetrieverSource(knnRetrieverBuilder, null)
+                )
+            )
+        );
+        // include some pagination as well
+        source.from(1);
+        SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source);
+        ElasticsearchAssertions.assertResponse(req, resp -> {
+            assertNull(resp.pointInTimeId());
+            assertNotNull(resp.getHits().getTotalHits());
+            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
+            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
+            assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1"));
+            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_4"));
+            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7"));
+            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_3"));
+        });
+    }
+
+    public void testRankDocsRetrieverWithAggs() {
+        // same as above, but we only want to bring back the top result from each subsearch
+        // so that would be 1, 2, and 7
+        // and final rank would be (based on score): 2, 1, 7
+        // aggs should still account for the same docs as the testRankDocsRetriever test, i.e. all but doc_5
+        final int rankWindowSize = 1;
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder();
+        // this one retrieves docs 1, 4, and 6
+        standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD))
+            .boost(10L);
+        StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder();
+        // this one retrieves docs 2 and 6 due to prefilter
+        standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L);
+        standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
+        // this one retrieves docs 7, 2, 3, and 6
+        KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(
+            VECTOR_FIELD,
+            new float[] { 3.0f, 3.0f, 3.0f },
+            null,
+            10,
+            100,
+            null
+        );
+        source.retriever(
+            new CompoundRetrieverWithRankDocs(
+                rankWindowSize,
+                Arrays.asList(
+                    new RetrieverSource(standard0, null),
+                    new RetrieverSource(standard1, null),
+                    new RetrieverSource(knnRetrieverBuilder, null)
+                )
+            )
+        );
+        source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD));
+        SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source);
+        ElasticsearchAssertions.assertResponse(req, resp -> {
+            assertNull(resp.pointInTimeId());
+            assertNotNull(resp.getHits().getTotalHits());
+            assertThat(resp.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
+            assertNotNull(resp.getAggregations());
+            assertNotNull(resp.getAggregations().get("topic"));
+            Terms terms = resp.getAggregations().get("topic");
+            // doc_3 is not part of the final aggs computation as it is only retrieved through the knn retriever
+            // and is outside of the rank window
+            assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(2L));
+            assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L));
+            assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L));
+        });
+    }
+
+    public void testRankDocsRetrieverWithCollapse() {
+        final int rankWindowSize = 100;
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder();
+        // this one retrieves docs 1, 4, and 6
+        standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD))
+            .boost(10L);
+        StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder();
+        // this one retrieves docs 2 and 6 due to prefilter
+        standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L);
+        standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
+        // this one retrieves docs 7, 2, 3, and 6
+        KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(
+            VECTOR_FIELD,
+            new float[] { 3.0f, 3.0f, 3.0f },
+            null,
+            10,
+            100,
+            null
+        );
+        // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
+        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 4, 7, 3
+        // with collapsing on topic field we would have 6, 2, 1, 7
+        source.retriever(
+            new CompoundRetrieverWithRankDocs(
+                rankWindowSize,
+                Arrays.asList(
+                    new RetrieverSource(standard0, null),
+                    new RetrieverSource(standard1, null),
+                    new RetrieverSource(knnRetrieverBuilder, null)
+                )
+            )
+        );
+        source.collapse(
+            new CollapseBuilder(TOPIC_FIELD).setInnerHits(
+                new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10)
+            )
+        );
+        source.fetchField(TOPIC_FIELD);
+        SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source);
+        ElasticsearchAssertions.assertResponse(req, resp -> {
+            assertNull(resp.pointInTimeId());
+            assertNotNull(resp.getHits().getTotalHits());
+            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
+            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getHits().length, equalTo(4));
+            assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6"));
+            assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2"));
+            assertThat(resp.getHits().getAt(1).field(TOPIC_FIELD).getValue().toString(), equalTo("astronomy"));
+            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1"));
+            assertThat(resp.getHits().getAt(2).field(TOPIC_FIELD).getValue().toString(), equalTo("technology"));
+            assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getHits().length, equalTo(3));
+            assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(0).getId(), equalTo("doc_4"));
+            assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(1).getId(), equalTo("doc_3"));
+            assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(2).getId(), equalTo("doc_1"));
+            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7"));
+            assertThat(resp.getHits().getAt(3).field(TOPIC_FIELD).getValue().toString(), equalTo("biology"));
+        });
+    }
+
+    public void testRankDocsRetrieverWithCollapseAndAggs() {
+        // same as above, but we only want to bring back the top result from each subsearch
+        // so that would be 1, 2, and 7
+        // and final rank would be (based on score): 2, 1, 7
+        // aggs should still account for the same docs as the testRankDocsRetriever test, i.e. all but doc_5
+        final int rankWindowSize = 10;
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder();
+        // this one retrieves docs 1 and 6 as doc_4 is collapsed to doc_1
+        standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD))
+            .boost(10L);
+        standard0.collapseBuilder = new CollapseBuilder(TOPIC_FIELD).setInnerHits(
+            new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10)
+        );
+        StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder();
+        // this one retrieves docs 2 and 6 due to prefilter
+        standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L);
+        standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
+        // this one retrieves docs 7, 2, 3, and 6
+        KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(
+            VECTOR_FIELD,
+            new float[] { 3.0f, 3.0f, 3.0f },
+            null,
+            10,
+            100,
+            null
+        );
+        // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
+        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 4, 7, 3
+        source.retriever(
+            new CompoundRetrieverWithRankDocs(
+                rankWindowSize,
+                Arrays.asList(
+                    new RetrieverSource(standard0, null),
+                    new RetrieverSource(standard1, null),
+                    new RetrieverSource(knnRetrieverBuilder, null)
+                )
+            )
+        );
+        source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD));
+        SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source);
+        ElasticsearchAssertions.assertResponse(req, resp -> {
+            assertNull(resp.pointInTimeId());
+            assertNotNull(resp.getHits().getTotalHits());
+            assertThat(resp.getHits().getTotalHits().value, equalTo(5L));
+            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6"));
+            assertNotNull(resp.getAggregations());
+            assertNotNull(resp.getAggregations().get("topic"));
+            Terms terms = resp.getAggregations().get("topic");
+            // doc_3 is not part of the final aggs computation as it is only retrieved through the knn retriever
+            // and is outside of the rank window
+            assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(3L));
+            assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L));
+            assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L));
+        });
+    }
+
+    public void testRankDocsRetrieverWithNestedQuery() {
+        final int rankWindowSize = 100;
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder();
+        // this one retrieves docs 1, 4, and 6
+        standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(10L), ScoreMode.Avg)
+            .innerHit(new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10));
+        StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder();
+        // this one retrieves docs 2 and 6 due to prefilter
+        standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L);
+        standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
+        // this one retrieves docs 7, 2, 3, and 6
+        KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(
+            VECTOR_FIELD,
+            new float[] { 3.0f, 3.0f, 3.0f },
+            null,
+            10,
+            100,
+            null
+        );
+        // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
+        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 4, 3, 7
+        source.retriever(
+            new CompoundRetrieverWithRankDocs(
+                rankWindowSize,
+                Arrays.asList(
+                    new RetrieverSource(standard0, null),
+                    new RetrieverSource(standard1, null),
+                    new RetrieverSource(knnRetrieverBuilder, null)
+                )
+            )
+        );
+        source.fetchField(TOPIC_FIELD);
+        SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source);
+        ElasticsearchAssertions.assertResponse(req, resp -> {
+            assertNull(resp.pointInTimeId());
+            assertNotNull(resp.getHits().getTotalHits());
+            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
+            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6"));
+            assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2"));
+            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1"));
+            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7"));
+            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_4"));
+            assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_3"));
+        });
+    }
+
+    public void testRankDocsRetrieverMultipleCompoundRetrievers() {
+        final int rankWindowSize = 100;
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder();
+        // this one retrieves docs 1, 4, and 6
+        standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD))
+            .boost(10L);
+        StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder();
+        // this one retrieves docs 2 and 6 due to prefilter
+        standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L);
+        standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD));
+        // this one retrieves docs 7, 2, 3, and 6
+        KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(
+            VECTOR_FIELD,
+            new float[] { 3.0f, 3.0f, 3.0f },
+            null,
+            10,
+            100,
+            null
+        );
+        // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
+        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 4, 7, 3
+        CompoundRetrieverWithRankDocs compoundRetriever1 = new CompoundRetrieverWithRankDocs(
+            rankWindowSize,
+            Arrays.asList(
+                new RetrieverSource(standard0, null),
+                new RetrieverSource(standard1, null),
+                new RetrieverSource(knnRetrieverBuilder, null)
+            )
+        );
+        // simple standard retriever that would have the doc_4 as its first (and only) result
+        StandardRetrieverBuilder standard2 = new StandardRetrieverBuilder();
+        standard2.queryBuilder = QueryBuilders.queryStringQuery("aardvark").defaultField(TEXT_FIELD);
+
+        // combining the two retrievers would bring doc_4 at the top as it would be the only one present in both doc sets
+        // the rest of the docs would be sorted based on their ranks as they have the same score (1/2)
+        source.retriever(
+            new CompoundRetrieverWithRankDocs(
+                rankWindowSize,
+                Arrays.asList(new RetrieverSource(compoundRetriever1, null), new RetrieverSource(standard2, null))
+            )
+        );
+
+        SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source);
+        ElasticsearchAssertions.assertResponse(req, resp -> {
+            assertNull(resp.pointInTimeId());
+            assertNotNull(resp.getHits().getTotalHits());
+            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
+            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4"));
+            assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_6"));
+            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2"));
+            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_1"));
+            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7"));
+            assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_3"));
+        });
+    }
+
+    public void testRankDocsRetrieverDifferentNestedSorting() {
+        final int rankWindowSize = 100;
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder();
+        // this one retrieves docs 1, 4, 6, 2
+        standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(0), ScoreMode.Avg);
+        standard0.sortBuilders = List.of(
+            new FieldSortBuilder(LAST_30D_FIELD).setNestedSort(new NestedSortBuilder("views")).order(SortOrder.DESC)
+        );
+        StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder();
+        // this one retrieves docs 4, 7
+        standard1.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(ALL_TIME_FIELD).gt(0), ScoreMode.Avg);
+        standard1.sortBuilders = List.of(
+            new FieldSortBuilder(ALL_TIME_FIELD).setNestedSort(new NestedSortBuilder("views")).order(SortOrder.ASC)
+        );
+
+        source.retriever(
+            new CompoundRetrieverWithRankDocs(
+                rankWindowSize,
+                Arrays.asList(new RetrieverSource(standard0, null), new RetrieverSource(standard1, null))
+            )
+        );
+
+        SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source);
+        ElasticsearchAssertions.assertResponse(req, resp -> {
+            assertNull(resp.pointInTimeId());
+            assertNotNull(resp.getHits().getTotalHits());
+            assertThat(resp.getHits().getTotalHits().value, equalTo(5L));
+            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4"));
+            assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1"));
+            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_7"));
+            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_6"));
+            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_2"));
+        });
+    }
+
+    class CompoundRetrieverWithRankDocs extends RetrieverBuilder {
+
+        private final List<RetrieverSource> sources;
+        private final int rankWindowSize;
+
+        private CompoundRetrieverWithRankDocs(int rankWindowSize, List<RetrieverSource> sources) {
+            this.rankWindowSize = rankWindowSize;
+            this.sources = Collections.unmodifiableList(sources);
+        }
+
+        @Override
+        public boolean isCompound() {
+            return true;
+        }
+
+        @Override
+        public QueryBuilder topDocsQuery() {
+            throw new UnsupportedOperationException("should not be called");
+        }
+
+        @Override
+        public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException {
+            if (ctx.getPointInTimeBuilder() == null) {
+                throw new IllegalStateException("PIT is required");
+            }
+
+            // Rewrite prefilters
+            boolean hasChanged = false;
+            var newPreFilters = rewritePreFilters(ctx);
+            hasChanged |= newPreFilters != preFilterQueryBuilders;
+
+            // Rewrite retriever sources
+            List<RetrieverSource> newRetrievers = new ArrayList<>();
+            for (var entry : sources) {
+                RetrieverBuilder newRetriever = entry.retriever.rewrite(ctx);
+                if (newRetriever != entry.retriever) {
+                    newRetrievers.add(new RetrieverSource(newRetriever, null));
+                    hasChanged |= newRetriever != entry.retriever;
+                } else if (newRetriever == entry.retriever) {
+                    var sourceBuilder = entry.source != null
+                        ? entry.source
+                        : createSearchSourceBuilder(ctx.getPointInTimeBuilder(), newRetriever);
+                    var rewrittenSource = sourceBuilder.rewrite(ctx);
+                    newRetrievers.add(new RetrieverSource(newRetriever, rewrittenSource));
+                    hasChanged |= rewrittenSource != entry.source;
+                }
+            }
+            if (hasChanged) {
+                return new CompoundRetrieverWithRankDocs(rankWindowSize, newRetrievers);
+            }
+
+            // execute searches
+            final SetOnce<RankDoc[]> results = new SetOnce<>();
+            final MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
+            for (var entry : sources) {
+                SearchRequest searchRequest = new SearchRequest().source(entry.source);
+                // The can match phase can reorder shards, so we disable it to ensure the stable ordering
+                searchRequest.setPreFilterShardSize(Integer.MAX_VALUE);
+                multiSearchRequest.add(searchRequest);
+            }
+            ctx.registerAsyncAction((client, listener) -> {
+                client.execute(TransportMultiSearchAction.TYPE, multiSearchRequest, new ActionListener<>() {
+                    @Override
+                    public void onResponse(MultiSearchResponse items) {
+                        List<RankDoc[]> topDocs = new ArrayList<>();
+                        for (int i = 0; i < items.getResponses().length; i++) {
+                            var item = items.getResponses()[i];
+                            var rankDocs = getRankDocs(item.getResponse());
+                            sources.get(i).retriever().setRankDocs(rankDocs);
+                            topDocs.add(rankDocs);
+                        }
+                        results.set(combineResults(topDocs));
+                        listener.onResponse(null);
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        listener.onFailure(e);
+                    }
+                });
+            });
+
+            return new RankDocsRetrieverBuilder(
+                rankWindowSize,
+                newRetrievers.stream().map(s -> s.retriever).toList(),
+                results::get,
+                newPreFilters
+            );
+        }
+
+        @Override
+        public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
+            throw new UnsupportedOperationException("should not be called");
+        }
+
+        @Override
+        public String getName() {
+            return "compound_retriever";
+        }
+
+        @Override
+        protected void doToXContent(XContentBuilder builder, Params params) throws IOException {
+
+        }
+
+        @Override
+        protected boolean doEquals(Object o) {
+            return false;
+        }
+
+        @Override
+        protected int doHashCode() {
+            return 0;
+        }
+
+        private RankDoc[] getRankDocs(SearchResponse searchResponse) {
+            assert searchResponse != null;
+            int size = Math.min(rankWindowSize, searchResponse.getHits().getHits().length);
+            RankDoc[] docs = new RankDoc[size];
+            for (int i = 0; i < size; i++) {
+                var hit = searchResponse.getHits().getAt(i);
+                long sortValue = (long) hit.getRawSortValues()[hit.getRawSortValues().length - 1];
+                int doc = decodeDoc(sortValue);
+                int shardRequestIndex = decodeShardRequestIndex(sortValue);
+                docs[i] = new RankDoc(doc, hit.getScore(), shardRequestIndex);
+                docs[i].rank = i + 1;
+            }
+            return docs;
+        }
+
+        public static int decodeDoc(long value) {
+            return (int) value;
+        }
+
+        public static int decodeShardRequestIndex(long value) {
+            return (int) (value >> 32);
+        }
+
+        record RankDocAndHitRatio(RankDoc rankDoc, float hitRatio) {}
+
+        /**
+         * Combines the provided {@code rankResults} to return the final top documents.
+         */
+        public RankDoc[] combineResults(List<RankDoc[]> rankResults) {
+            int totalQueries = rankResults.size();
+            final float step = 1.0f / totalQueries;
+            Map<RankDoc.RankKey, RankDocAndHitRatio> docsToRankResults = Maps.newMapWithExpectedSize(rankWindowSize);
+            for (var rankResult : rankResults) {
+                for (RankDoc scoreDoc : rankResult) {
+                    docsToRankResults.compute(new RankDoc.RankKey(scoreDoc.doc, scoreDoc.shardIndex), (key, value) -> {
+                        if (value == null) {
+                            RankDoc res = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex);
+                            res.rank = scoreDoc.rank;
+                            return new RankDocAndHitRatio(res, step);
+                        } else {
+                            RankDoc res = new RankDoc(scoreDoc.doc, Math.max(scoreDoc.score, value.rankDoc.score), scoreDoc.shardIndex);
+                            res.rank = Math.min(scoreDoc.rank, value.rankDoc.rank);
+                            return new RankDocAndHitRatio(res, value.hitRatio + step);
+                        }
+                    });
+                }
+            }
+            // sort the results based on hit ratio, then doc, then rank, and final tiebreaker is based on smaller doc id
+            RankDocAndHitRatio[] sortedResults = docsToRankResults.values().toArray(RankDocAndHitRatio[]::new);
+            Arrays.sort(sortedResults, (RankDocAndHitRatio doc1, RankDocAndHitRatio doc2) -> {
+                if (doc1.hitRatio != doc2.hitRatio) {
+                    return doc1.hitRatio < doc2.hitRatio ? 1 : -1;
+                }
+                if (false == (Float.isNaN(doc1.rankDoc.score) || Float.isNaN(doc2.rankDoc.score))
+                    && (doc1.rankDoc.score != doc2.rankDoc.score)) {
+                    return doc1.rankDoc.score < doc2.rankDoc.score ? 1 : -1;
+                }
+                if (doc1.rankDoc.rank != doc2.rankDoc.rank) {
+                    return doc1.rankDoc.rank < doc2.rankDoc.rank ? -1 : 1;
+                }
+                return doc1.rankDoc.doc < doc2.rankDoc.doc ? -1 : 1;
+            });
+            // trim the results if needed, otherwise each shard will always return `rank_window_size` results.
+            // pagination and all else will happen on the coordinator when combining the shard responses
+            RankDoc[] topResults = new RankDoc[Math.min(rankWindowSize, sortedResults.length)];
+            for (int rank = 0; rank < topResults.length; ++rank) {
+                topResults[rank] = sortedResults[rank].rankDoc;
+                topResults[rank].rank = rank + 1;
+                topResults[rank].score = sortedResults[rank].hitRatio;
+            }
+            return topResults;
+        }
+    }
+
+    private SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) {
+        var sourceBuilder = new SearchSourceBuilder().pointInTimeBuilder(pit).trackTotalHits(false).size(100);
+        retrieverBuilder.extractToSearchSourceBuilder(sourceBuilder, false);
+
+        // Record the shard id in the sort result
+        List<SortBuilder<?>> sortBuilders = sourceBuilder.sorts() != null ? new ArrayList<>(sourceBuilder.sorts()) : new ArrayList<>();
+        if (sortBuilders.isEmpty()) {
+            sortBuilders.add(new ScoreSortBuilder());
+        }
+        sortBuilders.add(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME));
+        sourceBuilder.sort(sortBuilders);
+        return sourceBuilder;
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java
new file mode 100644
index 0000000000000..e618a1b75cc4d
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java
@@ -0,0 +1,275 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.retriever;
+
+import org.apache.lucene.search.TotalHits;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.MockSearchService;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(numDataNodes = 3)
+public class RetrieverRewriteIT extends ESIntegTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(MockSearchService.TestPlugin.class);
+    }
+
+    private static String INDEX_DOCS = "docs";
+    private static String INDEX_QUERIES = "queries";
+    private static final String ID_FIELD = "_id";
+    private static final String QUERY_FIELD = "query";
+
+    @Before
+    public void setup() throws Exception {
+        createIndex(INDEX_DOCS);
+        index(INDEX_DOCS, "doc_0", "{}");
+        index(INDEX_DOCS, "doc_1", "{}");
+        index(INDEX_DOCS, "doc_2", "{}");
+        refresh(INDEX_DOCS);
+
+        createIndex(INDEX_QUERIES);
+        index(INDEX_QUERIES, "query_0", "{ \"" + QUERY_FIELD + "\": \"doc_2\"}");
+        index(INDEX_QUERIES, "query_1", "{ \"" + QUERY_FIELD + "\": \"doc_1\"}");
+        index(INDEX_QUERIES, "query_2", "{ \"" + QUERY_FIELD + "\": \"doc_0\"}");
+        refresh(INDEX_QUERIES);
+    }
+
+    public void testRewrite() {
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        StandardRetrieverBuilder standard = new StandardRetrieverBuilder();
+        standard.queryBuilder = QueryBuilders.termQuery(ID_FIELD, "doc_0");
+        source.retriever(new AssertingRetrieverBuilder(standard));
+        SearchRequestBuilder req = client().prepareSearch(INDEX_DOCS, INDEX_QUERIES).setSource(source);
+        ElasticsearchAssertions.assertResponse(req, resp -> {
+            assertNull(resp.pointInTimeId());
+            assertNotNull(resp.getHits().getTotalHits());
+            assertThat(resp.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_0"));
+        });
+    }
+
+    public void testRewriteCompound() {
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        source.retriever(new AssertingCompoundRetrieverBuilder("query_0"));
+        SearchRequestBuilder req = client().prepareSearch(INDEX_DOCS, INDEX_QUERIES).setSource(source);
+        ElasticsearchAssertions.assertResponse(req, resp -> {
+            assertNull(resp.pointInTimeId());
+            assertNotNull(resp.getHits().getTotalHits());
+            assertThat(resp.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
+        });
+    }
+
+    public void testRewriteCompoundRetrieverShouldThrowForPartialResults() throws Exception {
+        final String testIndex = "test";
+        createIndex(testIndex, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 3).put(SETTING_NUMBER_OF_REPLICAS, 0).build());
+        for (int i = 0; i < 50; i++) {
+            index(testIndex, "doc_" + i, "{}");
+        }
+        refresh(testIndex);
+
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        source.retriever(new AssertingCompoundRetrieverBuilder("doc_0"));
+        final String randomDataNode = internalCluster().getNodeNameThat(
+            settings -> DiscoveryNode.hasRole(settings, DiscoveryNodeRole.DATA_ROLE)
+        );
+        try {
+            ensureGreen(testIndex);
+            if (false == internalCluster().stopNode(randomDataNode)) {
+                throw new IllegalStateException("node did not stop");
+            }
+            assertBusy(() -> {
+                ClusterHealthResponse healthResponse = clusterAdmin().prepareHealth(testIndex)
+                    .setWaitForStatus(ClusterHealthStatus.RED) // we are now known red because the primary shard is missing
+                    .setWaitForEvents(Priority.LANGUID) // ensures that the update has occurred
+                    .execute()
+                    .actionGet();
+                assertThat(healthResponse.getStatus(), equalTo(ClusterHealthStatus.RED));
+            });
+            SearchPhaseExecutionException ex = expectThrows(
+                SearchPhaseExecutionException.class,
+                client().prepareSearch(testIndex).setSource(source)::get
+            );
+            assertThat(
+                ex.getDetailedMessage(),
+                containsString("[open_point_in_time] action requires all shards to be available. Missing shards")
+            );
+        } finally {
+            internalCluster().restartNode(randomDataNode);
+        }
+    }
+
+    private static class AssertingRetrieverBuilder extends RetrieverBuilder {
+        private final RetrieverBuilder innerRetriever;
+
+        private AssertingRetrieverBuilder(RetrieverBuilder innerRetriever) {
+            this.innerRetriever = innerRetriever;
+        }
+
+        @Override
+        public QueryBuilder topDocsQuery() {
+            return null;
+        }
+
+        @Override
+        public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException {
+            assertNull(ctx.getPointInTimeBuilder());
+            assertNull(ctx.convertToInnerHitsRewriteContext());
+            assertNull(ctx.convertToCoordinatorRewriteContext());
+            assertNull(ctx.convertToIndexMetadataContext());
+            assertNull(ctx.convertToSearchExecutionContext());
+            assertNull(ctx.convertToDataRewriteContext());
+            var newRetriever = innerRetriever.rewrite(ctx);
+            if (newRetriever != innerRetriever) {
+                return new AssertingRetrieverBuilder(newRetriever);
+            }
+            return this;
+        }
+
+        @Override
+        public void extractToSearchSourceBuilder(SearchSourceBuilder sourceBuilder, boolean compoundUsed) {
+            assertNull(sourceBuilder.retriever());
+            innerRetriever.extractToSearchSourceBuilder(sourceBuilder, compoundUsed);
+        }
+
+        @Override
+        public String getName() {
+            return "asserting";
+        }
+
+        @Override
+        protected void doToXContent(XContentBuilder builder, Params params) throws IOException {}
+
+        @Override
+        protected boolean doEquals(Object o) {
+            return false;
+        }
+
+        @Override
+        protected int doHashCode() {
+            return innerRetriever.doHashCode();
+        }
+    }
+
+    private static class AssertingCompoundRetrieverBuilder extends RetrieverBuilder {
+        private final String id;
+        private final SetOnce<RetrieverBuilder> innerRetriever;
+
+        private AssertingCompoundRetrieverBuilder(String id) {
+            this.id = id;
+            this.innerRetriever = new SetOnce<>(null);
+        }
+
+        private AssertingCompoundRetrieverBuilder(String id, SetOnce<RetrieverBuilder> innerRetriever) {
+            this.id = id;
+            this.innerRetriever = innerRetriever;
+        }
+
+        @Override
+        public boolean isCompound() {
+            return true;
+        }
+
+        @Override
+        public QueryBuilder topDocsQuery() {
+            return null;
+        }
+
+        @Override
+        public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException {
+            assertNotNull(ctx.getPointInTimeBuilder());
+            assertNull(ctx.convertToInnerHitsRewriteContext());
+            assertNull(ctx.convertToCoordinatorRewriteContext());
+            assertNull(ctx.convertToIndexMetadataContext());
+            assertNull(ctx.convertToSearchExecutionContext());
+            assertNull(ctx.convertToDataRewriteContext());
+            if (innerRetriever.get() != null) {
+                return this;
+            }
+            SetOnce<RetrieverBuilder> innerRetriever = new SetOnce<>();
+            ctx.registerAsyncAction((client, actionListener) -> {
+                SearchSourceBuilder source = new SearchSourceBuilder().pointInTimeBuilder(ctx.getPointInTimeBuilder())
+                    .query(QueryBuilders.termQuery(ID_FIELD, id))
+                    .fetchField(QUERY_FIELD);
+                client.search(new SearchRequest().source(source), new ActionListener<>() {
+                    @Override
+                    public void onResponse(SearchResponse response) {
+                        String query = response.getHits().getAt(0).field(QUERY_FIELD).getValue();
+                        StandardRetrieverBuilder standard = new StandardRetrieverBuilder();
+                        standard.queryBuilder = QueryBuilders.termQuery(ID_FIELD, query);
+                        innerRetriever.set(standard);
+                        actionListener.onResponse(null);
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        actionListener.onFailure(e);
+                    }
+                });
+            });
+            return new AssertingCompoundRetrieverBuilder(id, innerRetriever);
+        }
+
+        @Override
+        public void extractToSearchSourceBuilder(SearchSourceBuilder sourceBuilder, boolean compoundUsed) {
+            assertNull(sourceBuilder.retriever());
+            innerRetriever.get().extractToSearchSourceBuilder(sourceBuilder, compoundUsed);
+        }
+
+        @Override
+        public String getName() {
+            return "asserting";
+        }
+
+        @Override
+        protected void doToXContent(XContentBuilder builder, Params params) throws IOException {
+            throw new AssertionError("not implemented");
+        }
+
+        @Override
+        protected boolean doEquals(Object o) {
+            return false;
+        }
+
+        @Override
+        protected int doHashCode() {
+            return id.hashCode();
+        }
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java
index 836bd26f08eee..71616abf0dcfa 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java
@@ -2209,7 +2209,7 @@ public void testDeleteIndexWithOutOfOrderFinalization() {
                     .anyMatch(e -> e.snapshot().getSnapshotId().getName().equals("snapshot-with-index-1") && e.state().completed())
             )
             // execute the index deletion _directly on the master_ so it happens before the snapshot finalization executes
-            .andThen((l, ignored) -> masterDeleteIndexService.deleteIndices(new DeleteIndexClusterStateUpdateRequest(l.map(r -> {
+            .andThen(l -> masterDeleteIndexService.deleteIndices(new DeleteIndexClusterStateUpdateRequest(l.map(r -> {
                 assertTrue(r.isAcknowledged());
                 return null;
             })).indices(new Index[] { internalCluster().clusterService().state().metadata().index(indexToDelete).getIndex() })
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java
index 1130ddaa74f38..477fd9737394e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java
@@ -8,26 +8,60 @@
 
 package org.elasticsearch.snapshots;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
+import org.elasticsearch.action.admin.cluster.repositories.delete.TransportDeleteRepositoryAction;
+import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
+import org.elasticsearch.action.admin.cluster.repositories.put.TransportPutRepositoryAction;
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequestBuilder;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.get.SnapshotSortKey;
+import org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.action.support.RefCountingListener;
+import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.SnapshotsInProgress;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.blobstore.fs.FsBlobStore;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Predicates;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.repositories.RepositoryMissingException;
+import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
+import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xcontent.json.JsonXContent;
 
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.empty;
@@ -745,4 +779,351 @@ private static GetSnapshotsRequestBuilder baseGetSnapshotsRequest(String[] repoN
         return clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repoNames)
             .setSnapshots("*", "-" + AbstractSnapshotIntegTestCase.OLD_VERSION_SNAPSHOT_PREFIX + "*");
     }
+
+    public void testAllFeatures() {
+        // A test that uses (potentially) as many of the features of the get-snapshots API at once as possible, to verify that they interact
+        // in the expected order etc.
+
+        // Create a few repositories and a few indices
+        final var repositories = randomList(1, 4, ESTestCase::randomIdentifier);
+        final var indices = randomList(1, 4, ESTestCase::randomIdentifier);
+        final var slmPolicies = randomList(1, 4, ESTestCase::randomIdentifier);
+
+        safeAwait(l -> {
+            try (var listeners = new RefCountingListener(l.map(v -> null))) {
+                for (final var repository : repositories) {
+                    client().execute(
+                        TransportPutRepositoryAction.TYPE,
+                        new PutRepositoryRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, repository).type(FsRepository.TYPE)
+                            .settings(Settings.builder().put("location", randomRepoPath()).build()),
+                        listeners.acquire(ElasticsearchAssertions::assertAcked)
+                    );
+                }
+
+                for (final var index : indices) {
+                    client().execute(
+                        TransportCreateIndexAction.TYPE,
+                        new CreateIndexRequest(index, indexSettings(1, 0).build()),
+                        listeners.acquire(ElasticsearchAssertions::assertAcked)
+                    );
+                }
+            }
+        });
+        ensureGreen();
+
+        // Create a few snapshots
+        final var snapshotInfos = Collections.synchronizedList(new ArrayList<SnapshotInfo>());
+        safeAwait(l -> {
+            try (var listeners = new RefCountingListener(l.map(v -> null))) {
+                for (int i = 0; i < 10; i++) {
+                    client().execute(
+                        TransportCreateSnapshotAction.TYPE,
+                        new CreateSnapshotRequest(
+                            TEST_REQUEST_TIMEOUT,
+                            // at least one snapshot per repository to satisfy consistency checks
+                            i < repositories.size() ? repositories.get(i) : randomFrom(repositories),
+                            randomIdentifier()
+                        ).indices(randomNonEmptySubsetOf(indices))
+                            .userMetadata(
+                                randomBoolean() ? Map.of() : Map.of(SnapshotsService.POLICY_ID_METADATA_FIELD, randomFrom(slmPolicies))
+                            )
+                            .waitForCompletion(true),
+                        listeners.acquire(
+                            createSnapshotResponse -> snapshotInfos.add(Objects.requireNonNull(createSnapshotResponse.getSnapshotInfo()))
+                        )
+                    );
+                }
+            }
+        });
+
+        if (randomBoolean()) {
+            // Sometimes also simulate bwc repository contents where some details are missing from the root blob
+            safeAwait(l -> {
+                try (var listeners = new RefCountingListener(l.map(v -> null))) {
+                    for (final var repositoryName : randomSubsetOf(repositories)) {
+                        removeDetailsForRandomSnapshots(repositoryName, listeners.acquire());
+                    }
+                }
+            });
+        }
+
+        Predicate<SnapshotInfo> snapshotInfoPredicate = Predicates.always();
+
+        // {repository} path parameter
+        final String[] requestedRepositories;
+        if (randomBoolean()) {
+            requestedRepositories = new String[] { randomFrom("_all", "*") };
+        } else {
+            final var selectedRepositories = Set.copyOf(randomNonEmptySubsetOf(repositories));
+            snapshotInfoPredicate = snapshotInfoPredicate.and(si -> selectedRepositories.contains(si.repository()));
+            requestedRepositories = selectedRepositories.toArray(new String[0]);
+        }
+
+        // {snapshot} path parameter
+        final String[] requestedSnapshots;
+        if (randomBoolean()) {
+            requestedSnapshots = randomBoolean() ? Strings.EMPTY_ARRAY : new String[] { randomFrom("_all", "*") };
+        } else {
+            final var selectedSnapshots = randomNonEmptySubsetOf(snapshotInfos).stream()
+                .map(si -> si.snapshotId().getName())
+                .collect(Collectors.toSet());
+            snapshotInfoPredicate = snapshotInfoPredicate.and(si -> selectedSnapshots.contains(si.snapshotId().getName()));
+            requestedSnapshots = selectedSnapshots.stream()
+                // if we have multiple repositories, add a trailing wildcard to each requested snapshot name, because if we specify exact
+                // names then there must be a snapshot with that name in every requested repository
+                .map(n -> repositories.size() == 1 && randomBoolean() ? n : n + "*")
+                .toArray(String[]::new);
+        }
+
+        // ?slm_policy_filter parameter
+        final String[] requestedSlmPolicies;
+        switch (between(0, 3)) {
+            default -> requestedSlmPolicies = Strings.EMPTY_ARRAY;
+            case 1 -> {
+                requestedSlmPolicies = new String[] { "*" };
+                snapshotInfoPredicate = snapshotInfoPredicate.and(
+                    si -> si.userMetadata().get(SnapshotsService.POLICY_ID_METADATA_FIELD) != null
+                );
+            }
+            case 2 -> {
+                requestedSlmPolicies = new String[] { "_none" };
+                snapshotInfoPredicate = snapshotInfoPredicate.and(
+                    si -> si.userMetadata().get(SnapshotsService.POLICY_ID_METADATA_FIELD) == null
+                );
+            }
+            case 3 -> {
+                final var selectedPolicies = Set.copyOf(randomNonEmptySubsetOf(slmPolicies));
+                requestedSlmPolicies = selectedPolicies.stream()
+                    .map(policy -> randomBoolean() ? policy : policy + "*")
+                    .toArray(String[]::new);
+                snapshotInfoPredicate = snapshotInfoPredicate.and(
+                    si -> si.userMetadata().get(SnapshotsService.POLICY_ID_METADATA_FIELD) instanceof String policy
+                        && selectedPolicies.contains(policy)
+                );
+            }
+        }
+
+        // ?sort and ?order parameters
+        final var sortKey = randomFrom(SnapshotSortKey.values());
+        final var order = randomFrom(SortOrder.values());
+        // NB we sometimes choose to sort by FAILED_SHARDS, but there are no failed shards in these snapshots. We're still testing the
+        // fallback sorting by snapshot ID in this case. We also have no multi-shard indices so there's no difference between sorting by
+        // INDICES and by SHARDS. The actual sorting behaviour for these cases is tested elsewhere, here we're just checking that sorting
+        // interacts correctly with the other parameters to the API.
+
+        // compute the ordered sequence of snapshots which match the repository/snapshot name filters and SLM policy filter
+        final var selectedSnapshots = snapshotInfos.stream()
+            .filter(snapshotInfoPredicate)
+            .sorted(sortKey.getSnapshotInfoComparator(order))
+            .toList();
+
+        final var getSnapshotsRequest = new GetSnapshotsRequest(TEST_REQUEST_TIMEOUT, requestedRepositories, requestedSnapshots).policies(
+            requestedSlmPolicies
+        )
+            // apply sorting params
+            .sort(sortKey)
+            .order(order);
+
+        // sometimes use ?from_sort_value to skip some items; note that snapshots skipped in this way are subtracted from
+        // GetSnapshotsResponse.totalCount whereas snapshots skipped by ?after and ?offset are not
+        final int skippedByFromSortValue;
+        if (randomBoolean()) {
+            final var startingSnapshot = randomFrom(snapshotInfos);
+            getSnapshotsRequest.fromSortValue(switch (sortKey) {
+                case START_TIME -> Long.toString(startingSnapshot.startTime());
+                case NAME -> startingSnapshot.snapshotId().getName();
+                case DURATION -> Long.toString(startingSnapshot.endTime() - startingSnapshot.startTime());
+                case INDICES, SHARDS -> Integer.toString(startingSnapshot.indices().size());
+                case FAILED_SHARDS -> "0";
+                case REPOSITORY -> startingSnapshot.repository();
+            });
+            final Predicate<SnapshotInfo> fromSortValuePredicate = snapshotInfo -> {
+                final var comparison = switch (sortKey) {
+                    case START_TIME -> Long.compare(snapshotInfo.startTime(), startingSnapshot.startTime());
+                    case NAME -> snapshotInfo.snapshotId().getName().compareTo(startingSnapshot.snapshotId().getName());
+                    case DURATION -> Long.compare(
+                        snapshotInfo.endTime() - snapshotInfo.startTime(),
+                        startingSnapshot.endTime() - startingSnapshot.startTime()
+                    );
+                    case INDICES, SHARDS -> Integer.compare(snapshotInfo.indices().size(), startingSnapshot.indices().size());
+                    case FAILED_SHARDS -> 0;
+                    case REPOSITORY -> snapshotInfo.repository().compareTo(startingSnapshot.repository());
+                };
+                return order == SortOrder.ASC ? comparison < 0 : comparison > 0;
+            };
+
+            int skipCount = 0;
+            for (final var snapshotInfo : selectedSnapshots) {
+                if (fromSortValuePredicate.test(snapshotInfo)) {
+                    skipCount += 1;
+                } else {
+                    break;
+                }
+            }
+            skippedByFromSortValue = skipCount;
+        } else {
+            skippedByFromSortValue = 0;
+        }
+
+        // ?offset parameter
+        if (randomBoolean()) {
+            getSnapshotsRequest.offset(between(0, selectedSnapshots.size() + 1));
+        }
+
+        // ?size parameter
+        if (randomBoolean()) {
+            getSnapshotsRequest.size(between(1, selectedSnapshots.size() + 1));
+        }
+
+        // compute the expected offset and size of the returned snapshots as indices in selectedSnapshots:
+        final var expectedOffset = Math.min(selectedSnapshots.size(), skippedByFromSortValue + getSnapshotsRequest.offset());
+        final var expectedSize = Math.min(
+            selectedSnapshots.size() - expectedOffset,
+            getSnapshotsRequest.size() == GetSnapshotsRequest.NO_LIMIT ? Integer.MAX_VALUE : getSnapshotsRequest.size()
+        );
+
+        // get the actual response
+        final GetSnapshotsResponse getSnapshotsResponse = safeAwait(
+            l -> client().execute(TransportGetSnapshotsAction.TYPE, getSnapshotsRequest, l)
+        );
+
+        // verify it returns the expected results
+        assertEquals(
+            selectedSnapshots.stream().skip(expectedOffset).limit(expectedSize).map(SnapshotInfo::snapshotId).toList(),
+            getSnapshotsResponse.getSnapshots().stream().map(SnapshotInfo::snapshotId).toList()
+        );
+        assertEquals(expectedSize, getSnapshotsResponse.getSnapshots().size());
+        assertEquals(selectedSnapshots.size() - skippedByFromSortValue, getSnapshotsResponse.totalCount());
+        assertEquals(selectedSnapshots.size() - expectedOffset - expectedSize, getSnapshotsResponse.remaining());
+        assertEquals(getSnapshotsResponse.remaining() > 0, getSnapshotsResponse.next() != null);
+
+        // now use ?after to page through the rest of the results
+        var nextRequestAfter = getSnapshotsResponse.next();
+        var nextExpectedOffset = expectedOffset + expectedSize;
+        var remaining = getSnapshotsResponse.remaining();
+        while (nextRequestAfter != null) {
+            final var nextSize = between(1, remaining);
+            final var nextRequest = new GetSnapshotsRequest(TEST_REQUEST_TIMEOUT, requestedRepositories, requestedSnapshots)
+                // same name/policy filters, same ?sort and ?order params, new ?size, but no ?offset or ?from_sort_value because of ?after
+                .policies(requestedSlmPolicies)
+                .sort(sortKey)
+                .order(order)
+                .size(nextSize)
+                .after(SnapshotSortKey.decodeAfterQueryParam(nextRequestAfter));
+            final GetSnapshotsResponse nextResponse = safeAwait(l -> client().execute(TransportGetSnapshotsAction.TYPE, nextRequest, l));
+
+            assertEquals(
+                selectedSnapshots.stream().skip(nextExpectedOffset).limit(nextSize).map(SnapshotInfo::snapshotId).toList(),
+                nextResponse.getSnapshots().stream().map(SnapshotInfo::snapshotId).toList()
+            );
+            assertEquals(nextSize, nextResponse.getSnapshots().size());
+            assertEquals(selectedSnapshots.size(), nextResponse.totalCount());
+            assertEquals(remaining - nextSize, nextResponse.remaining());
+            assertEquals(nextResponse.remaining() > 0, nextResponse.next() != null);
+
+            nextRequestAfter = nextResponse.next();
+            nextExpectedOffset += nextSize;
+            remaining -= nextSize;
+        }
+
+        assertEquals(0, remaining);
+    }
+
+    /**
+     * Older versions of Elasticsearch don't record in {@link RepositoryData} all the details needed for the get-snapshots API to pick out
+     * the right snapshots, so in this case the API must fall back to reading those details from each candidate {@link SnapshotInfo} blob.
+     * Simulate this situation by manipulating the {@link RepositoryData} blob directly to remove all the optional details from some subset
+     * of its snapshots.
+     */
+    private static void removeDetailsForRandomSnapshots(String repositoryName, ActionListener<Void> listener) {
+        final Set<SnapshotId> snapshotsWithoutDetails = ConcurrentCollections.newConcurrentSet();
+        final var masterRepositoriesService = internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class);
+        final var repository = asInstanceOf(FsRepository.class, masterRepositoriesService.repository(repositoryName));
+        final var repositoryMetadata = repository.getMetadata();
+        final var repositorySettings = repositoryMetadata.settings();
+        final var repositoryDataBlobPath = asInstanceOf(FsBlobStore.class, repository.blobStore()).path()
+            .resolve(BlobStoreRepository.INDEX_FILE_PREFIX + repositoryMetadata.generation());
+
+        SubscribableListener
+
+            // unregister the repository while we're mucking around with its internals
+            .<AcknowledgedResponse>newForked(
+                l -> client().execute(
+                    TransportDeleteRepositoryAction.TYPE,
+                    new DeleteRepositoryRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, repositoryName),
+                    l
+                )
+            )
+            .andThenAccept(ElasticsearchAssertions::assertAcked)
+
+            // rewrite the RepositoryData blob with some details removed
+            .andThenAccept(ignored -> {
+                // load the existing RepositoryData JSON blob as raw maps/lists/etc.
+                final var repositoryDataBytes = Files.readAllBytes(repositoryDataBlobPath);
+                final var repositoryDataMap = XContentHelper.convertToMap(
+                    JsonXContent.jsonXContent,
+                    repositoryDataBytes,
+                    0,
+                    repositoryDataBytes.length,
+                    true
+                );
+
+                // modify the contents
+                final var snapshotsList = asInstanceOf(List.class, repositoryDataMap.get("snapshots"));
+                for (final var snapshotObj : snapshotsList) {
+                    if (randomBoolean()) {
+                        continue;
+                    }
+                    final var snapshotMap = asInstanceOf(Map.class, snapshotObj);
+                    snapshotsWithoutDetails.add(
+                        new SnapshotId(
+                            asInstanceOf(String.class, snapshotMap.get("name")),
+                            asInstanceOf(String.class, snapshotMap.get("uuid"))
+                        )
+                    );
+
+                    // remove the optional details fields
+                    assertNotNull(snapshotMap.remove("start_time_millis"));
+                    assertNotNull(snapshotMap.remove("end_time_millis"));
+                    assertNotNull(snapshotMap.remove("slm_policy"));
+                }
+
+                // overwrite the RepositoryData JSON blob with its new contents
+                final var updatedRepositoryDataBytes = XContentTestUtils.convertToXContent(repositoryDataMap, XContentType.JSON);
+                try (var outputStream = Files.newOutputStream(repositoryDataBlobPath)) {
+                    BytesRef bytesRef;
+                    final var iterator = updatedRepositoryDataBytes.iterator();
+                    while ((bytesRef = iterator.next()) != null) {
+                        outputStream.write(bytesRef.bytes, bytesRef.offset, bytesRef.length);
+                    }
+                }
+            })
+
+            // re-register the repository
+            .<AcknowledgedResponse>andThen(
+                l -> client().execute(
+                    TransportPutRepositoryAction.TYPE,
+                    new PutRepositoryRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, repositoryName).type(FsRepository.TYPE)
+                        .settings(repositorySettings),
+                    l
+                )
+            )
+            .andThenAccept(ElasticsearchAssertions::assertAcked)
+
+            // verify that the details are indeed now missing
+            .<RepositoryData>andThen(
+                l -> masterRepositoriesService.repository(repositoryName).getRepositoryData(EsExecutors.DIRECT_EXECUTOR_SERVICE, l)
+            )
+            .andThenAccept(repositoryData -> {
+                for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) {
+                    assertEquals(
+                        repositoryName + "/" + snapshotId.toString() + ": " + repositoryData.getSnapshotDetails(snapshotId),
+                        snapshotsWithoutDetails.contains(snapshotId),
+                        repositoryData.hasMissingDetails(snapshotId)
+                    );
+                }
+            })
+
+            .addListener(listener);
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
index 057d7124f83d9..d4c0a4c80a3b5 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
@@ -298,7 +298,7 @@ public void testRepositoryConflict() throws Exception {
 
         logger.info("--> snapshot");
         final String index = "test-idx";
-        assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
+        assertAcked(prepareCreate(index, 1, indexSettings(1, 0)));
         for (int i = 0; i < 10; i++) {
             indexDoc(index, Integer.toString(i), "foo", "bar" + i);
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index a651537c77539..531e9f4f45afa 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -1788,9 +1788,7 @@ public void testSnapshotCanceledOnRemovedShard() throws Exception {
         final String index = "test-idx";
         final String snapshot = "test-snap";
 
-        assertAcked(
-            prepareCreate(index, 1, Settings.builder().put("number_of_shards", numPrimaries).put("number_of_replicas", numReplicas))
-        );
+        assertAcked(prepareCreate(index, 1, indexSettings(numPrimaries, numReplicas)));
 
         indexRandomDocs(index, 100);
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java
index b0c5e73de5859..8f2702099c102 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotCustomPluginStateIT.java
@@ -12,11 +12,14 @@
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
+import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.action.ingest.GetPipelineResponse;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.ingest.IngestTestPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -29,6 +32,7 @@
 import java.util.Collection;
 import java.util.Collections;
 
+import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.putJsonStoredScript;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertIndexTemplateExists;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertIndexTemplateMissing;
@@ -96,14 +100,7 @@ public void testIncludeGlobalState() throws Exception {
 
         if (testScript) {
             logger.info("-->  creating test script");
-            assertAcked(
-                clusterAdmin().preparePutStoredScript()
-                    .setId("foobar")
-                    .setContent(
-                        new BytesArray("{\"script\": { \"lang\": \"" + MockScriptEngine.NAME + "\", \"source\": \"1\"} }"),
-                        XContentType.JSON
-                    )
-            );
+            putJsonStoredScript("foobar", "{\"script\": { \"lang\": \"" + MockScriptEngine.NAME + "\", \"source\": \"1\"} }");
         }
 
         logger.info("--> snapshot without global state");
@@ -152,7 +149,12 @@ public void testIncludeGlobalState() throws Exception {
 
         if (testScript) {
             logger.info("-->  delete test script");
-            assertAcked(clusterAdmin().prepareDeleteStoredScript("foobar").get());
+            assertAcked(
+                safeExecute(
+                    TransportDeleteStoredScriptAction.TYPE,
+                    new DeleteStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "foobar")
+                )
+            );
         }
 
         logger.info("--> try restoring from snapshot without global state");
@@ -188,7 +190,10 @@ public void testIncludeGlobalState() throws Exception {
 
         if (testScript) {
             logger.info("--> check that script is restored");
-            GetStoredScriptResponse getStoredScriptResponse = clusterAdmin().prepareGetStoredScript("foobar").get();
+            GetStoredScriptResponse getStoredScriptResponse = safeExecute(
+                GetStoredScriptAction.INSTANCE,
+                new GetStoredScriptRequest(TEST_REQUEST_TIMEOUT, "foobar")
+            );
             assertNotNull(getStoredScriptResponse.getSource());
         }
 
@@ -217,7 +222,12 @@ public void testIncludeGlobalState() throws Exception {
         }
 
         if (testScript) {
-            assertAcked(clusterAdmin().prepareDeleteStoredScript("foobar").get());
+            assertAcked(
+                safeExecute(
+                    TransportDeleteStoredScriptAction.TYPE,
+                    new DeleteStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "foobar")
+                )
+            );
         }
 
         getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates().get();
@@ -236,7 +246,7 @@ public void testIncludeGlobalState() throws Exception {
         getIndexTemplatesResponse = indicesAdmin().prepareGetTemplates().get();
         assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
         assertFalse(clusterAdmin().prepareGetPipeline("barbaz").get().isFound());
-        assertNull(clusterAdmin().prepareGetStoredScript("foobar").get().getSource());
+        assertNull(safeExecute(GetStoredScriptAction.INSTANCE, new GetStoredScriptRequest(TEST_REQUEST_TIMEOUT, "foobar")).getSource());
         assertDocCount("test-idx", 100L);
     }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java
index a45471f273732..2d1e16dc64273 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java
@@ -483,7 +483,7 @@ private static void putShutdownForRemovalMetadata(ClusterService clusterService,
         SubscribableListener
 
             .<Void>newForked(l -> putShutdownMetadata(clusterService, shutdownMetadata, nodeName, l))
-            .<Void>andThen((l, ignored) -> flushMasterQueue(clusterService, l))
+            .<Void>andThen(l -> flushMasterQueue(clusterService, l))
             .addListener(listener);
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java
index b8b6dcb25b557..9c9076dff00e2 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -1257,7 +1256,7 @@ public void clusterStateProcessed(ClusterState initialState, ClusterState newSta
                         )
 
                         .<Void>andThen(
-                            (l, ignored) -> clusterService.submitUnbatchedStateUpdateTask(
+                            l -> clusterService.submitUnbatchedStateUpdateTask(
                                 "unmark [" + node + "] for removal",
                                 new ClusterStateUpdateTask() {
                                     @Override
@@ -1461,11 +1460,7 @@ private void createIndexAndContinue(Releasable releasable) {
                 docPermits = new Semaphore(between(1000, 3000));
                 logger.info("--> create index [{}] with max [{}] docs", indexName, docPermits.availablePermits());
                 indicesAdmin().prepareCreate(indexName)
-                    .setSettings(
-                        Settings.builder()
-                            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shardCount)
-                            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, cluster.numDataNodes() - 1))
-                    )
+                    .setSettings(indexSettings(shardCount, between(0, cluster.numDataNodes() - 1)))
                     .execute(mustSucceed(response -> {
                         assertTrue(response.isAcknowledged());
                         logger.info("--> finished create index [{}]", indexName);
diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java
index 1c07b5b4564ec..086bfece87172 100644
--- a/server/src/main/java/module-info.java
+++ b/server/src/main/java/module-info.java
@@ -190,14 +190,15 @@
     exports org.elasticsearch.common.file;
     exports org.elasticsearch.common.geo;
     exports org.elasticsearch.common.hash;
-    exports org.elasticsearch.common.inject;
-    exports org.elasticsearch.common.inject.binder;
-    exports org.elasticsearch.common.inject.internal;
-    exports org.elasticsearch.common.inject.matcher;
-    exports org.elasticsearch.common.inject.multibindings;
-    exports org.elasticsearch.common.inject.name;
-    exports org.elasticsearch.common.inject.spi;
-    exports org.elasticsearch.common.inject.util;
+    exports org.elasticsearch.injection.api;
+    exports org.elasticsearch.injection.guice;
+    exports org.elasticsearch.injection.guice.binder;
+    exports org.elasticsearch.injection.guice.internal;
+    exports org.elasticsearch.injection.guice.matcher;
+    exports org.elasticsearch.injection.guice.multibindings;
+    exports org.elasticsearch.injection.guice.name;
+    exports org.elasticsearch.injection.guice.spi;
+    exports org.elasticsearch.injection.guice.util;
     exports org.elasticsearch.common.io;
     exports org.elasticsearch.common.io.stream;
     exports org.elasticsearch.common.logging;
@@ -364,6 +365,7 @@
     exports org.elasticsearch.search.rank.rerank;
     exports org.elasticsearch.search.rescore;
     exports org.elasticsearch.search.retriever;
+    exports org.elasticsearch.search.retriever.rankdoc;
     exports org.elasticsearch.search.runtime;
     exports org.elasticsearch.search.searchafter;
     exports org.elasticsearch.search.slice;
@@ -428,6 +430,7 @@
             org.elasticsearch.cluster.metadata.MetadataFeatures,
             org.elasticsearch.rest.RestFeatures,
             org.elasticsearch.indices.IndicesFeatures,
+            org.elasticsearch.repositories.RepositoriesFeatures,
             org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures,
             org.elasticsearch.index.mapper.MapperFeatures,
             org.elasticsearch.ingest.IngestGeoIpFeatures,
diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java
index 2983a2d62de71..d7db8f4ec09dd 100644
--- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -1908,25 +1908,31 @@ private enum ElasticsearchExceptionHandle {
             FailureIndexNotSupportedException.class,
             FailureIndexNotSupportedException::new,
             178,
-            TransportVersions.ADD_FAILURE_STORE_INDICES_OPTIONS
+            TransportVersions.V_8_14_0
         ),
         NOT_PERSISTENT_TASK_NODE_EXCEPTION(
             NotPersistentTaskNodeException.class,
             NotPersistentTaskNodeException::new,
             179,
-            TransportVersions.ADD_PERSISTENT_TASK_EXCEPTIONS
+            TransportVersions.V_8_14_0
         ),
         PERSISTENT_TASK_NODE_NOT_ASSIGNED_EXCEPTION(
             PersistentTaskNodeNotAssignedException.class,
             PersistentTaskNodeNotAssignedException::new,
             180,
-            TransportVersions.ADD_PERSISTENT_TASK_EXCEPTIONS
+            TransportVersions.V_8_14_0
         ),
         RESOURCE_ALREADY_UPLOADED_EXCEPTION(
             ResourceAlreadyUploadedException.class,
             ResourceAlreadyUploadedException::new,
             181,
             TransportVersions.ADD_RESOURCE_ALREADY_UPLOADED_EXCEPTION
+        ),
+        INGEST_PIPELINE_EXCEPTION(
+            org.elasticsearch.ingest.IngestPipelineException.class,
+            org.elasticsearch.ingest.IngestPipelineException::new,
+            182,
+            TransportVersions.INGEST_PIPELINE_EXCEPTION_ADDED
         );
 
         final Class<? extends ElasticsearchException> exceptionClass;
diff --git a/server/src/main/java/org/elasticsearch/ExceptionsHelper.java b/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
index b67b59aeee076..3e109fb1600b9 100644
--- a/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
+++ b/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
@@ -97,6 +97,193 @@ public static String stackTrace(Throwable e) {
         return stackTraceStringWriter.toString();
     }
 
+    /**
+     * Constructs a limited and compressed stack trace string. Each exception printed as part of the full stack trace will have its printed
+     * stack frames capped at the given trace depth. Stack traces that are longer than the given trace depth will summarize the count of the
+     * remaining frames at the end of the trace. Each stack frame omits the module information and limits the package names to single
+     * characters per part.
+     * <br><br>
+     * An example result when using a trace depth of 2 and one nested cause:
+     * <pre><code>
+     * o.e.s.GenericException: some generic exception!
+     *   at o.e.s.SomeClass.method(SomeClass.java:100)
+     *   at o.e.s.SomeOtherClass.earlierMethod(SomeOtherClass.java:24)
+     *   ... 5 more
+     * Caused by: o.e.s.GenericException: some other generic exception!
+     *   at o.e.s.SomeClass.method(SomeClass.java:115)
+     *   at o.e.s.SomeOtherClass.earlierMethod(SomeOtherClass.java:16)
+     *   ... 12 more
+     * </code></pre>
+     *
+     * @param e Throwable object to construct a printed stack trace for
+     * @param traceDepth The maximum number of stack trace elements to display per exception referenced
+     * @return A string containing a limited and compressed stack trace.
+     */
+    public static String limitedStackTrace(Throwable e, int traceDepth) {
+        assert traceDepth >= 0 : "Cannot print stacktraces with negative trace depths";
+        StringWriter stackTraceStringWriter = new StringWriter();
+        PrintWriter printWriter = new PrintWriter(stackTraceStringWriter);
+        printLimitedStackTrace(e, printWriter, traceDepth);
+        return stackTraceStringWriter.toString();
+    }
+
+    /** Caption for labeling causative exception stack traces */
+    private static final String CAUSE_CAPTION = "Caused by: ";
+    /** Caption for labeling suppressed exception stack traces */
+    private static final String SUPPRESSED_CAPTION = "Suppressed: ";
+
+    private static void printLimitedStackTrace(Throwable e, PrintWriter s, int maxLines) {
+        // Guard against malicious overrides of Throwable.equals by
+        // using a Set with identity equality semantics.
+        Set<Throwable> dejaVu = Collections.newSetFromMap(new IdentityHashMap<>());
+        dejaVu.add(e);
+
+        // Print our stack trace
+        s.println(compressExceptionMessage(e));
+        StackTraceElement[] trace = e.getStackTrace();
+        int linesPrinted = 0;
+        for (StackTraceElement traceElement : trace) {
+            if (linesPrinted >= maxLines) {
+                break;
+            } else {
+                s.println(compressStackTraceElement(new StringBuilder("\tat "), traceElement));
+                linesPrinted++;
+            }
+        }
+        if (trace.length > linesPrinted) {
+            s.println("\t... " + (trace.length - linesPrinted) + " more");
+        }
+
+        // Print suppressed exceptions, if any
+        for (Throwable se : e.getSuppressed()) {
+            limitAndPrintEnclosedStackTrace(se, s, trace, SUPPRESSED_CAPTION, "\t", maxLines, dejaVu);
+        }
+
+        // Print cause, if any
+        Throwable ourCause = e.getCause();
+        if (ourCause != null) {
+            limitAndPrintEnclosedStackTrace(ourCause, s, trace, CAUSE_CAPTION, "", maxLines, dejaVu);
+        }
+    }
+
+    private static void limitAndPrintEnclosedStackTrace(
+        Throwable e,
+        PrintWriter s,
+        StackTraceElement[] enclosingTrace,
+        String caption,
+        String prefix,
+        int maxLines,
+        Set<Throwable> dejaVu
+    ) {
+        if (dejaVu.contains(e)) {
+            s.println(prefix + caption + "[CIRCULAR REFERENCE: " + compressExceptionMessage(e) + "]");
+        } else {
+            dejaVu.add(e);
+            // Compute number of frames in common between this and enclosing trace
+            StackTraceElement[] trace = e.getStackTrace();
+            int m = trace.length - 1;
+            int n = enclosingTrace.length - 1;
+            while (m >= 0 && n >= 0 && trace[m].equals(enclosingTrace[n])) {
+                m--;
+                n--;
+            }
+            int framesInCommon = trace.length - 1 - m;
+
+            // Instead of breaking out of the print loop below when it reaches the maximum
+            // print lines, we simply cap how many frames we plan on printing here.
+            int linesToPrint = m + 1;
+            if (linesToPrint > maxLines) {
+                // The print loop below is "<=" based instead of "<", so subtract
+                // one from the max lines to convert a count value to an array index
+                // value and avoid an off by one error.
+                m = maxLines - 1;
+                framesInCommon = trace.length - 1 - m;
+            }
+
+            // Print our stack trace
+            s.println(prefix + caption + compressExceptionMessage(e));
+            for (int i = 0; i <= m; i++) {
+                s.println(compressStackTraceElement(new StringBuilder(prefix).append("\tat "), trace[i]));
+            }
+            if (framesInCommon != 0) {
+                s.println(prefix + "\t... " + framesInCommon + " more");
+            }
+
+            // Print suppressed exceptions, if any
+            for (Throwable se : e.getSuppressed()) {
+                limitAndPrintEnclosedStackTrace(se, s, trace, SUPPRESSED_CAPTION, prefix + "\t", maxLines, dejaVu);
+            }
+
+            // Print cause, if any
+            Throwable ourCause = e.getCause();
+            if (ourCause != null) {
+                limitAndPrintEnclosedStackTrace(ourCause, s, trace, CAUSE_CAPTION, prefix, maxLines, dejaVu);
+            }
+        }
+    }
+
+    private static String compressExceptionMessage(Throwable e) {
+        StringBuilder msg = new StringBuilder();
+        compressPackages(msg, e.getClass().getName());
+        String message = e.getLocalizedMessage();
+        if (message != null) {
+            msg.append(": ").append(message);
+        }
+        return msg.toString();
+    }
+
+    private static StringBuilder compressStackTraceElement(StringBuilder s, final StackTraceElement stackTraceElement) {
+        String declaringClass = stackTraceElement.getClassName();
+        compressPackages(s, declaringClass);
+
+        String methodName = stackTraceElement.getMethodName();
+        s.append(".").append(methodName).append("(");
+
+        if (stackTraceElement.isNativeMethod()) {
+            s.append("Native Method)");
+        } else {
+            String fileName = stackTraceElement.getFileName();
+            int lineNumber = stackTraceElement.getLineNumber();
+            if (fileName != null && lineNumber >= 0) {
+                s.append(fileName).append(":").append(lineNumber).append(")");
+            } else if (fileName != null) {
+                s.append(fileName).append(")");
+            } else {
+                s.append("Unknown Source)");
+            }
+        }
+        return s;
+    }
+
+    // Visible for testing
+    static void compressPackages(StringBuilder s, String className) {
+        assert s != null : "s cannot be null";
+        assert className != null : "className cannot be null";
+        int finalDot = className.lastIndexOf('.');
+        if (finalDot < 0) {
+            s.append(className);
+            return;
+        }
+        int lastPackageName = className.lastIndexOf('.', finalDot - 1);
+        if (lastPackageName < 0) {
+            if (finalDot >= 1) {
+                s.append(className.charAt(0)).append('.');
+            }
+            s.append(className.substring(finalDot + 1));
+            return;
+        }
+        boolean firstChar = true;
+        char[] charArray = className.toCharArray();
+        for (int idx = 0; idx <= lastPackageName + 1; idx++) {
+            char c = charArray[idx];
+            if (firstChar && '.' != c) {
+                s.append(c).append('.');
+            }
+            firstChar = '.' == c;
+        }
+        s.append(className.substring(finalDot + 1));
+    }
+
     public static String formatStackTrace(final StackTraceElement[] stackTrace) {
         return Arrays.stream(stackTrace).skip(1).map(e -> "\tat " + e).collect(Collectors.joining("\n"));
     }
diff --git a/server/src/main/java/org/elasticsearch/ReleaseVersions.java b/server/src/main/java/org/elasticsearch/ReleaseVersions.java
index 7b5c8d1d42382..cacdca1c5b528 100644
--- a/server/src/main/java/org/elasticsearch/ReleaseVersions.java
+++ b/server/src/main/java/org/elasticsearch/ReleaseVersions.java
@@ -41,7 +41,7 @@ public class ReleaseVersions {
 
     private static final Pattern VERSION_LINE = Pattern.compile("(\\d+\\.\\d+\\.\\d+),(\\d+)");
 
-    public static IntFunction<String> generateVersionsLookup(Class<?> versionContainer) {
+    public static IntFunction<String> generateVersionsLookup(Class<?> versionContainer, int current) {
         if (USES_VERSIONS == false) return Integer::toString;
 
         try {
@@ -52,6 +52,9 @@ public static IntFunction<String> generateVersionsLookup(Class<?> versionContain
             }
 
             NavigableMap<Integer, List<Version>> versions = new TreeMap<>();
+            // add the current version id, which won't be in the csv
+            versions.put(current, List.of(Version.CURRENT));
+
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(versionsFile, StandardCharsets.UTF_8))) {
                 String line;
                 while ((line = reader.readLine()) != null) {
@@ -121,8 +124,8 @@ private static IntFunction<String> lookupFunction(NavigableMap<Integer, List<Ver
                     // too hard to guess what version this id might be for using the next version - just use it directly
                     upperBound = upperRange.getValue().get(0).toString();
                 } else {
-                    // likely a version created after the last release tagged version - ok
-                    upperBound = "snapshot[" + id + "]";
+                    // a newer version than all we know about? Can't map it...
+                    upperBound = "[" + id + "]";
                 }
             }
 
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index afd4aa4f11d0d..7581ee7649696 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -100,49 +100,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion V_8_12_1 = def(8_560_00_1);
     public static final TransportVersion V_8_13_0 = def(8_595_00_0);
     public static final TransportVersion V_8_13_4 = def(8_595_00_1);
-    // 8.14.0+
-    public static final TransportVersion RANDOM_AGG_SHARD_SEED = def(8_596_00_0);
-    public static final TransportVersion ESQL_TIMINGS = def(8_597_00_0);
-    public static final TransportVersion DATA_STREAM_AUTO_SHARDING_EVENT = def(8_598_00_0);
-    public static final TransportVersion ADD_FAILURE_STORE_INDICES_OPTIONS = def(8_599_00_0);
-    public static final TransportVersion ESQL_ENRICH_OPERATOR_STATUS = def(8_600_00_0);
-    public static final TransportVersion ESQL_SERIALIZE_ARRAY_VECTOR = def(8_601_00_0);
-    public static final TransportVersion ESQL_SERIALIZE_ARRAY_BLOCK = def(8_602_00_0);
-    public static final TransportVersion ADD_DATA_STREAM_GLOBAL_RETENTION = def(8_603_00_0);
-    public static final TransportVersion ALLOCATION_STATS = def(8_604_00_0);
-    public static final TransportVersion ESQL_EXTENDED_ENRICH_TYPES = def(8_605_00_0);
-    public static final TransportVersion KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING = def(8_606_00_0);
-    public static final TransportVersion ESQL_EXTENDED_ENRICH_INPUT_TYPE = def(8_607_00_0);
-    public static final TransportVersion ESQL_SERIALIZE_BIG_VECTOR = def(8_608_00_0);
-    public static final TransportVersion AGGS_EXCLUDED_DELETED_DOCS = def(8_609_00_0);
-    public static final TransportVersion ESQL_SERIALIZE_BIG_ARRAY = def(8_610_00_0);
-    public static final TransportVersion AUTO_SHARDING_ROLLOVER_CONDITION = def(8_611_00_0);
-    public static final TransportVersion KNN_QUERY_VECTOR_BUILDER = def(8_612_00_0);
-    public static final TransportVersion USE_DATA_STREAM_GLOBAL_RETENTION = def(8_613_00_0);
-    public static final TransportVersion ML_COMPLETION_INFERENCE_SERVICE_ADDED = def(8_614_00_0);
-    public static final TransportVersion ML_INFERENCE_EMBEDDING_BYTE_ADDED = def(8_615_00_0);
-    public static final TransportVersion ML_INFERENCE_L2_NORM_SIMILARITY_ADDED = def(8_616_00_0);
-    public static final TransportVersion SEARCH_NODE_LOAD_AUTOSCALING = def(8_617_00_0);
-    public static final TransportVersion ESQL_ES_SOURCE_OPTIONS = def(8_618_00_0);
-    public static final TransportVersion ADD_PERSISTENT_TASK_EXCEPTIONS = def(8_619_00_0);
-    public static final TransportVersion ESQL_REDUCER_NODE_FRAGMENT = def(8_620_00_0);
-    public static final TransportVersion FAILURE_STORE_ROLLOVER = def(8_621_00_0);
-    public static final TransportVersion CCR_STATS_API_TIMEOUT_PARAM = def(8_622_00_0);
-    public static final TransportVersion ESQL_ORDINAL_BLOCK = def(8_623_00_0);
-    public static final TransportVersion ML_INFERENCE_COHERE_RERANK = def(8_624_00_0);
-    public static final TransportVersion INDEXING_PRESSURE_DOCUMENT_REJECTIONS_COUNT = def(8_625_00_0);
-    public static final TransportVersion ALIAS_ACTION_RESULTS = def(8_626_00_0);
-    public static final TransportVersion HISTOGRAM_AGGS_KEY_SORTED = def(8_627_00_0);
-    public static final TransportVersion INFERENCE_FIELDS_METADATA = def(8_628_00_0);
-    public static final TransportVersion ML_INFERENCE_TIMEOUT_ADDED = def(8_629_00_0);
-    public static final TransportVersion MODIFY_DATA_STREAM_FAILURE_STORES = def(8_630_00_0);
-    public static final TransportVersion ML_INFERENCE_RERANK_NEW_RESPONSE_FORMAT = def(8_631_00_0);
-    public static final TransportVersion HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL = def(8_632_00_0);
-    public static final TransportVersion TRACK_FLUSH_TIME_EXCLUDING_WAITING_ON_LOCKS = def(8_633_00_0);
-    public static final TransportVersion ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS = def(8_634_00_0);
-    public static final TransportVersion ILM_SHRINK_ENABLE_WRITE = def(8_635_00_0);
-    public static final TransportVersion GEOIP_CACHE_STATS = def(8_636_00_0);
-    public static final TransportVersion SHUTDOWN_REQUEST_TIMEOUTS_FIX_8_14 = def(8_636_00_1);
+    public static final TransportVersion V_8_14_0 = def(8_636_00_1);
+    // 8.15.0+
     public static final TransportVersion WATERMARK_THRESHOLDS_STATS = def(8_637_00_0);
     public static final TransportVersion ENRICH_CACHE_ADDITIONAL_STATS = def(8_638_00_0);
     public static final TransportVersion ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED = def(8_639_00_0);
@@ -210,6 +169,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion VERSIONED_MASTER_NODE_REQUESTS = def(8_701_00_0);
     public static final TransportVersion ML_INFERENCE_AMAZON_BEDROCK_ADDED = def(8_702_00_0);
     public static final TransportVersion ENTERPRISE_GEOIP_DOWNLOADER_BACKPORT_8_15 = def(8_702_00_1);
+    public static final TransportVersion FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15 = def(8_702_00_2);
     public static final TransportVersion ML_INFERENCE_DONT_DELETE_WHEN_SEMANTIC_TEXT_EXISTS = def(8_703_00_0);
     public static final TransportVersion INFERENCE_ADAPTIVE_ALLOCATIONS = def(8_704_00_0);
     public static final TransportVersion INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN = def(8_705_00_0);
@@ -218,6 +178,31 @@ static TransportVersion def(int id) {
     public static final TransportVersion ENTERPRISE_GEOIP_DOWNLOADER = def(8_708_00_0);
     public static final TransportVersion NODES_STATS_ENUM_SET = def(8_709_00_0);
     public static final TransportVersion MASTER_NODE_METRICS = def(8_710_00_0);
+    public static final TransportVersion SEGMENT_LEVEL_FIELDS_STATS = def(8_711_00_0);
+    public static final TransportVersion ML_ADD_DETECTION_RULE_PARAMS = def(8_712_00_0);
+    public static final TransportVersion FIX_VECTOR_SIMILARITY_INNER_HITS = def(8_713_00_0);
+    public static final TransportVersion INDEX_REQUEST_UPDATE_BY_DOC_ORIGIN = def(8_714_00_0);
+    public static final TransportVersion ESQL_ATTRIBUTE_CACHED_SERIALIZATION = def(8_715_00_0);
+    public static final TransportVersion REGISTER_SLM_STATS = def(8_716_00_0);
+    public static final TransportVersion ESQL_NESTED_UNSUPPORTED = def(8_717_00_0);
+    public static final TransportVersion ESQL_SINGLE_VALUE_QUERY_SOURCE = def(8_718_00_0);
+    public static final TransportVersion ESQL_ORIGINAL_INDICES = def(8_719_00_0);
+    public static final TransportVersion ML_INFERENCE_EIS_INTEGRATION_ADDED = def(8_720_00_0);
+    public static final TransportVersion INGEST_PIPELINE_EXCEPTION_ADDED = def(8_721_00_0);
+    public static final TransportVersion ZDT_NANOS_SUPPORT_BROKEN = def(8_722_00_0);
+    public static final TransportVersion REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES = def(8_723_00_0);
+    public static final TransportVersion RANDOM_RERANKER_RETRIEVER = def(8_724_00_0);
+    public static final TransportVersion ESQL_PROFILE_SLEEPS = def(8_725_00_0);
+    public static final TransportVersion ZDT_NANOS_SUPPORT = def(8_726_00_0);
+    public static final TransportVersion LTR_SERVERLESS_RELEASE = def(8_727_00_0);
+    public static final TransportVersion ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT = def(8_728_00_0);
+    public static final TransportVersion RANK_DOCS_RETRIEVER = def(8_729_00_0);
+    public static final TransportVersion ESQL_ES_FIELD_CACHED_SERIALIZATION = def(8_730_00_0);
+    public static final TransportVersion ADD_MANAGE_ROLES_PRIVILEGE = def(8_731_00_0);
+    public static final TransportVersion REPOSITORIES_TELEMETRY = def(8_732_00_0);
+    public static final TransportVersion ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED = def(8_733_00_0);
+
+    public static final TransportVersion FIELD_CAPS_RESPONSE_INDEX_MODE = def(8_734_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
@@ -282,7 +267,7 @@ static TransportVersion def(int id) {
      * Reference to the minimum transport version that can be used with CCS.
      * This should be the transport version used by the previous minor release.
      */
-    public static final TransportVersion MINIMUM_CCS_VERSION = SHUTDOWN_REQUEST_TIMEOUTS_FIX_8_14;
+    public static final TransportVersion MINIMUM_CCS_VERSION = FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15;
 
     static final NavigableMap<Integer, TransportVersion> VERSION_IDS = getAllVersionIds(TransportVersions.class);
 
@@ -339,7 +324,7 @@ static Collection<TransportVersion> getAllVersions() {
         return VERSION_IDS.values();
     }
 
-    static final IntFunction<String> VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(TransportVersions.class);
+    static final IntFunction<String> VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(TransportVersions.class, LATEST_DEFINED.id());
 
     // no instance
     private TransportVersions() {}
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index fefe2ea486485..333669ca8079c 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -123,6 +123,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_7_17_21 = new Version(7_17_21_99);
     public static final Version V_7_17_22 = new Version(7_17_22_99);
     public static final Version V_7_17_23 = new Version(7_17_23_99);
+    public static final Version V_7_17_24 = new Version(7_17_24_99);
 
     public static final Version V_8_0_0 = new Version(8_00_00_99);
     public static final Version V_8_0_1 = new Version(8_00_01_99);
@@ -179,8 +180,8 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_14_1 = new Version(8_14_01_99);
     public static final Version V_8_14_2 = new Version(8_14_02_99);
     public static final Version V_8_14_3 = new Version(8_14_03_99);
-    public static final Version V_8_14_4 = new Version(8_14_04_99);
     public static final Version V_8_15_0 = new Version(8_15_00_99);
+    public static final Version V_8_15_1 = new Version(8_15_01_99);
     public static final Version V_8_16_0 = new Version(8_16_00_99);
     public static final Version CURRENT = V_8_16_0;
 
diff --git a/server/src/main/java/org/elasticsearch/action/ActionListener.java b/server/src/main/java/org/elasticsearch/action/ActionListener.java
index ec01d88cb5e6e..f3fa1dd2e105f 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionListener.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionListener.java
@@ -187,6 +187,13 @@ default <T> ActionListener<T> delegateFailureAndWrap(CheckedBiConsumer<ActionLis
         return new ActionListenerImplementations.ResponseWrappingActionListener<>(this, bc);
     }
 
+    /**
+     * Same as {@link #delegateFailureAndWrap(CheckedBiConsumer)} except that the response is ignored and not passed to the delegate.
+     */
+    default <T> ActionListener<T> delegateFailureIgnoreResponseAndWrap(CheckedConsumer<ActionListener<Response>, ? extends Exception> c) {
+        return new ActionListenerImplementations.ResponseDroppingActionListener<>(this, c);
+    }
+
     /**
      * Creates a listener which releases the given resource on completion (whether success or failure)
      */
diff --git a/server/src/main/java/org/elasticsearch/action/ActionListenerImplementations.java b/server/src/main/java/org/elasticsearch/action/ActionListenerImplementations.java
index bf4f2dcc2d8db..93f6ac6e49d41 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionListenerImplementations.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionListenerImplementations.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.action;
 
 import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.Releasable;
@@ -254,6 +255,33 @@ public String toString() {
         }
     }
 
+    /**
+     * The same as {@link ResponseWrappingActionListener} except that the response is dropped
+     */
+    static final class ResponseDroppingActionListener<T, R> extends DelegatingActionListener<T, R> {
+
+        private final CheckedConsumer<ActionListener<R>, ? extends Exception> consumer;
+
+        ResponseDroppingActionListener(ActionListener<R> delegate, CheckedConsumer<ActionListener<R>, ? extends Exception> consumer) {
+            super(delegate);
+            this.consumer = consumer;
+        }
+
+        @Override
+        public void onResponse(T ignored) {
+            try {
+                consumer.accept(delegate);
+            } catch (Exception e) {
+                onFailure(e);
+            }
+        }
+
+        @Override
+        public String toString() {
+            return super.toString() + "/" + consumer;
+        }
+    }
+
     static final class RunAfterActionListener<T> extends DelegatingActionListener<T, T> {
 
         private final Runnable runAfter;
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index a9c6894355cb6..37a33eab4e4e8 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -218,9 +218,6 @@
 import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.NamedRegistry;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.TypeLiteral;
-import org.elasticsearch.common.inject.multibindings.MapBinder;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -241,6 +238,9 @@
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetadata;
+import org.elasticsearch.injection.guice.AbstractModule;
+import org.elasticsearch.injection.guice.TypeLiteral;
+import org.elasticsearch.injection.guice.multibindings.MapBinder;
 import org.elasticsearch.persistent.CompletionPersistentTaskAction;
 import org.elasticsearch.persistent.RemovePersistentTaskAction;
 import org.elasticsearch.persistent.StartPersistentTaskAction;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
index 8e6f029c71013..c8d6a25c3ba6e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.SnapshotsInfoService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java
index 76a9b5a245a84..f296dbb9bb5cb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceAction.java
@@ -27,8 +27,8 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
index 228ad640f9fca..2f094b0fc6006 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
@@ -25,12 +25,12 @@
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.routing.allocation.NodeAllocationStats;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -78,7 +78,7 @@ public TransportGetAllocationStatsAction(
 
     @Override
     protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
-        if (clusterService.state().getMinTransportVersion().before(TransportVersions.ALLOCATION_STATS)) {
+        if (clusterService.state().getMinTransportVersion().before(TransportVersions.V_8_14_0)) {
             // The action is not available before ALLOCATION_STATS
             listener.onResponse(new Response(Map.of(), null));
             return;
@@ -108,6 +108,7 @@ public static class Request extends MasterNodeReadRequest<Request> {
 
         private final EnumSet<Metric> metrics;
 
+        @SuppressWarnings("this-escape")
         public Request(TimeValue masterNodeTimeout, TaskId parentTaskId, EnumSet<Metric> metrics) {
             super(masterNodeTimeout);
             setParentTask(parentTaskId);
@@ -123,7 +124,7 @@ public Request(StreamInput in) throws IOException {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            assert out.getTransportVersion().onOrAfter(TransportVersions.ALLOCATION_STATS);
+            assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0);
             super.writeTo(out);
             if (out.getTransportVersion().onOrAfter(TransportVersions.MASTER_NODE_METRICS)) {
                 out.writeEnumSet(metrics);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java
index fca7b5c44fd29..8337353e1f40d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java
@@ -28,8 +28,8 @@
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardAssignment;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java
index c540d535e60d4..12db5e664a784 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -37,6 +36,7 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java
index bbe292e817389..29aa93cd7d513 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java
@@ -29,10 +29,10 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoAction.java
index 95025c851fd94..ff73bc0b9c56a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper;
 import org.elasticsearch.cluster.coordination.Coordinator;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/CoordinationDiagnosticsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/CoordinationDiagnosticsAction.java
index 858a85b6bdfe8..bd41f5933e058 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/CoordinationDiagnosticsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/CoordinationDiagnosticsAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.coordination.CoordinationDiagnosticsService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryAction.java
index b38cee5e2a042..82a2ede90c945 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryAction.java
@@ -17,10 +17,10 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.coordination.MasterHistoryService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java
index 46e41d306cefe..11bdd41f458d3 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDeleteDesiredNodesAction.java
@@ -25,10 +25,10 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java
index e06918355e7a9..6909ce1f9366b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportGetDesiredNodesAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.cluster.metadata.DesiredNodes;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java
index ee8295381dd88..9ec8feeb5d405 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java
@@ -28,9 +28,9 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
index 8cca2c5bf6472..522bbb4eee629 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
@@ -31,12 +31,12 @@
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java
index 63c2be9050ab0..ae538e7c72334 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java
@@ -17,12 +17,12 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java
index 281e26a44f335..485f91f0fd4a2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java
@@ -19,9 +19,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesRequest.java
index c69d273727238..7adcff9f19ccb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/NodesCapabilitiesRequest.java
@@ -24,10 +24,15 @@ public class NodesCapabilitiesRequest extends BaseNodesRequest<NodesCapabilities
     private RestApiVersion restApiVersion = RestApiVersion.current();
 
     public NodesCapabilitiesRequest() {
-        // always send to all nodes
+        // send to all nodes
         super(Strings.EMPTY_ARRAY);
     }
 
+    public NodesCapabilitiesRequest(String nodeId) {
+        // only send to this node (the local node)
+        super(new String[] { nodeId });
+    }
+
     public NodesCapabilitiesRequest path(String path) {
         this.path = path;
         return this;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java
index 71aa95908d3b7..3691c07fe95e2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java
@@ -15,11 +15,11 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java
index d1b7a4f1b7e95..723f3f5720ff9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java
@@ -14,10 +14,10 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
index 719a96ecb4d57..811e2d8152067 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
@@ -17,11 +17,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.monitor.jvm.HotThreads;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
index ce962fb454a88..542dea59e7197 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
@@ -14,9 +14,9 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
index f906b7d659b7b..fc75608317011 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
@@ -19,11 +19,11 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.tasks.Task;
@@ -123,6 +123,7 @@ protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation(
             final List<Exception> exceptions = new ArrayList<>();
             // broadcast the new settings object (with the open embedded keystore) to all reloadable plugins
             pluginsService.filterPlugins(ReloadablePlugin.class).forEach(p -> {
+                logger.debug("Reloading plugin [" + p.getClass().getSimpleName() + "]");
                 try {
                     p.reload(settingsWithKeystore);
                 } catch (final Exception e) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java
index 901f8b1e83c69..052263a00811d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java
@@ -23,11 +23,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateShardPathAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateShardPathAction.java
index 40fff1b77a4fb..401a415756abe 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateShardPathAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateShardPathAction.java
@@ -16,13 +16,13 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
index a438983e855e9..1a53cec1bdbd7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
@@ -125,7 +125,7 @@ public NodeStats(StreamInput in) throws IOException {
         repositoriesStats = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)
             ? in.readOptionalWriteable(RepositoriesStats::new)
             : null;
-        nodeAllocationStats = in.getTransportVersion().onOrAfter(TransportVersions.ALLOCATION_STATS)
+        nodeAllocationStats = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
             ? in.readOptionalWriteable(NodeAllocationStats::new)
             : null;
     }
@@ -337,7 +337,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
             out.writeOptionalWriteable(repositoriesStats);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ALLOCATION_STATS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeOptionalWriteable(nodeAllocationStats);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index 3416b77fdd7fd..be2cd5fb187a3 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -22,9 +22,9 @@
 import org.elasticsearch.cluster.routing.allocation.NodeAllocationStats;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeService;
 import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.tasks.CancellableTask;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
index 4582a1cb26f82..1ff0a78575cac 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
index 85de3c65c798e..36d31f9ec59fc 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -24,12 +24,12 @@
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.RemovedTaskListener;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
index c4888b9900428..83b4b2fc45626 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
@@ -17,12 +17,12 @@
 import org.elasticsearch.action.support.ListenableActionFuture;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.RefCounted;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.RemovedTaskListener;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java
index 72bbe2683d157..1f8a049a678e5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java
@@ -15,9 +15,9 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.aggregations.support.AggregationUsageService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java
index 5f3fd654eeb84..71a5614cf5b25 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesAction.java
@@ -23,11 +23,11 @@
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.RemoteClusterServerInfo;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java
index c7d74fc414115..b72b573796b1e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java
index 237e241c8900f..e6c5b63b5c2cf 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java
@@ -24,11 +24,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.blobstore.DeleteResult;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryCleanupResult;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
index 301cc6a4255f7..0b3645cd47364 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
@@ -19,8 +19,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
index bed02ef2cbc19..df7e235c00a3f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.repositories.ResolvedRepositories;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
index eb7e26b30e874..29999f98b553c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
@@ -19,8 +19,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
index 353cc2994afc7..84e3ed77c64fc 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
index 7eea49861333e..3d90b4061f194 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
@@ -35,11 +35,11 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportResponseHandler;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java
index 302efb5867065..41d010a6a9f5b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
index e4093486da39c..de53ef70ff54a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
@@ -26,12 +26,12 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsUpdater;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java
index 91c302c8aa7be..1f53840c20c06 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -31,22 +32,16 @@ public final class ClusterSearchShardsRequest extends MasterNodeReadRequest<Clus
     private String preference;
     private IndicesOptions indicesOptions = IndicesOptions.lenientExpandOpen();
 
-    public ClusterSearchShardsRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
-    }
-
-    public ClusterSearchShardsRequest(String... indices) {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    public ClusterSearchShardsRequest(TimeValue masterNodeTimeout, String... indices) {
+        super(masterNodeTimeout);
         indices(indices);
     }
 
     public ClusterSearchShardsRequest(StreamInput in) throws IOException {
         super(in);
         indices = in.readStringArray();
-
         routing = in.readOptionalString();
         preference = in.readOptionalString();
-
         indicesOptions = IndicesOptions.readIndicesOptions(in);
     }
 
@@ -56,7 +51,6 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringArray(indices);
         out.writeOptionalString(routing);
         out.writeOptionalString(preference);
-
         indicesOptions.writeIndicesOptions(out);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java
deleted file mode 100644
index daae51f04f442..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.action.admin.cluster.shards;
-
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-
-public class ClusterSearchShardsRequestBuilder extends MasterNodeReadOperationRequestBuilder<
-    ClusterSearchShardsRequest,
-    ClusterSearchShardsResponse,
-    ClusterSearchShardsRequestBuilder> {
-
-    public ClusterSearchShardsRequestBuilder(ElasticsearchClient client) {
-        super(client, TransportClusterSearchShardsAction.TYPE, new ClusterSearchShardsRequest());
-    }
-
-    /**
-     * Sets the indices the search will be executed on.
-     */
-    public ClusterSearchShardsRequestBuilder setIndices(String... indices) {
-        request.indices(indices);
-        return this;
-    }
-
-    /**
-     * A comma separated list of routing values to control the shards the search will be executed on.
-     */
-    public ClusterSearchShardsRequestBuilder setRouting(String routing) {
-        request.routing(routing);
-        return this;
-    }
-
-    /**
-     * The routing values to control the shards that the search will be executed on.
-     */
-    public ClusterSearchShardsRequestBuilder setRouting(String... routing) {
-        request.routing(routing);
-        return this;
-    }
-
-    /**
-     * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * {@code _local} to prefer local shards or a custom value, which guarantees that the same order
-     * will be used across different requests.
-     */
-    public ClusterSearchShardsRequestBuilder setPreference(String preference) {
-        request.preference(preference);
-        return this;
-    }
-
-    /**
-     * Specifies what type of requested indices to ignore and how to deal indices wildcard expressions.
-     * For example indices that don't exist.
-     */
-    public ClusterSearchShardsRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) {
-        request().indicesOptions(indicesOptions);
-        return this;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
index b7164f81c71ac..e727809ec56c1 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
@@ -21,10 +21,10 @@
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java
index 130cada5f6742..99e2ee6ca6e15 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
index 2c460319e3d86..d17ad9674a19d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -82,6 +83,9 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
     @Nullable
     private Map<String, Object> userMetadata;
 
+    @Nullable
+    private String uuid = null;
+
     public CreateSnapshotRequest(TimeValue masterNodeTimeout) {
         super(masterNodeTimeout);
     }
@@ -96,6 +100,7 @@ public CreateSnapshotRequest(TimeValue masterNodeTimeout, String repository, Str
         this(masterNodeTimeout);
         this.snapshot = snapshot;
         this.repository = repository;
+        this.uuid = UUIDs.randomBase64UUID();
     }
 
     public CreateSnapshotRequest(StreamInput in) throws IOException {
@@ -112,6 +117,7 @@ public CreateSnapshotRequest(StreamInput in) throws IOException {
         waitForCompletion = in.readBoolean();
         partial = in.readBoolean();
         userMetadata = in.readGenericMap();
+        uuid = in.getTransportVersion().onOrAfter(TransportVersions.REGISTER_SLM_STATS) ? in.readOptionalString() : null;
     }
 
     @Override
@@ -129,6 +135,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(waitForCompletion);
         out.writeBoolean(partial);
         out.writeGenericMap(userMetadata);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.REGISTER_SLM_STATS)) {
+            out.writeOptionalString(uuid);
+        }
     }
 
     @Override
@@ -364,6 +373,35 @@ public CreateSnapshotRequest userMetadata(@Nullable Map<String, Object> userMeta
         return this;
     }
 
+    /**
+     * Set a uuid to identify snapshot.
+     * If no uuid is specified, one will be created within SnapshotService
+     */
+    public CreateSnapshotRequest uuid(String uuid) {
+        this.uuid = uuid;
+        return this;
+    }
+
+    /**
+     * Get the uuid, generating it if one does not yet exist.
+     * Because the uuid can be set, this method is NOT thread-safe.
+     * <p>
+     * The uuid was previously generated in SnapshotService.createSnapshot
+     * but was moved to the CreateSnapshotRequest constructor so that the caller could
+     * uniquely identify the snapshot. Unfortunately, in a mixed-version cluster,
+     * the CreateSnapshotRequest could be created on a node which does not yet
+     * generate the uuid in the constructor. In this case, the uuid
+     * must be generated when it is first accessed with this getter.
+     *
+     * @return the uuid that will be used for the snapshot
+     */
+    public String uuid() {
+        if (this.uuid == null) {
+            this.uuid = UUIDs.randomBase64UUID();
+        }
+        return this.uuid;
+    }
+
     /**
      * @return Which plugin states should be included in the snapshot
      */
@@ -469,12 +507,13 @@ public boolean equals(Object o) {
             && Objects.equals(indicesOptions, that.indicesOptions)
             && Arrays.equals(featureStates, that.featureStates)
             && Objects.equals(masterNodeTimeout(), that.masterNodeTimeout())
-            && Objects.equals(userMetadata, that.userMetadata);
+            && Objects.equals(userMetadata, that.userMetadata)
+            && Objects.equals(uuid, that.uuid);
     }
 
     @Override
     public int hashCode() {
-        int result = Objects.hash(snapshot, repository, indicesOptions, partial, includeGlobalState, waitForCompletion, userMetadata);
+        int result = Objects.hash(snapshot, repository, indicesOptions, partial, includeGlobalState, waitForCompletion, userMetadata, uuid);
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(featureStates);
         return result;
@@ -505,6 +544,8 @@ public String toString() {
             + masterNodeTimeout()
             + ", metadata="
             + userMetadata
+            + ", uuid="
+            + uuid
             + '}';
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
index 03810f027363f..9752d764bd6e6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
index 9522a7afaec8b..5c6259d133b44 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
@@ -19,8 +19,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java
index 5d8ab3daaf85c..bb3fbd401b3a9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java
@@ -18,9 +18,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java
index e7deabd341312..c12b12a18dd2d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
index 1a279e3488123..063fe49160d49 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.concurrent.AbstractThrottledTaskRunner;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@@ -32,6 +31,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.repositories.IndexId;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java
index 6933725476adf..6b31dd0c2d358 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java
@@ -20,9 +20,9 @@
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.IndexSnapshotsService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.RepositoryException;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index f0d47813dad77..f9ee2d84f8732 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -68,11 +68,6 @@ public RestoreSnapshotRequest(TimeValue masterNodeTimeout) {
         super(masterNodeTimeout);
     }
 
-    @Deprecated(forRemoval = true) // temporary compatibility shim
-    public RestoreSnapshotRequest(String repository, String snapshot) {
-        this(MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, repository, snapshot);
-    }
-
     /**
      * Constructs a new put repository request with the provided repository and snapshot names.
      *
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
index cc0d4cdfd9ee9..ba34b8cab1021 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
@@ -17,8 +17,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -49,7 +48,7 @@ public TransportRestoreSnapshotAction(
             RestoreSnapshotRequest::new,
             indexNameExpressionResolver,
             RestoreSnapshotResponse::new,
-            EsExecutors.DIRECT_EXECUTOR_SERVICE
+            threadPool.executor(ThreadPool.Names.SNAPSHOT_META)
         );
         this.restoreService = restoreService;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
index 82f32d2472b97..7c5d17ced1de5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
@@ -19,11 +19,11 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotShardsService;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index caedc3363e9a3..7bd0c929773a8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -24,12 +24,12 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
@@ -332,8 +332,7 @@ private void loadRepositoryData(
                     final SnapshotsInProgress.State state = switch (snapshotInfo.state()) {
                         case FAILED -> SnapshotsInProgress.State.FAILED;
                         case SUCCESS, PARTIAL ->
-                            // Translating both PARTIAL and SUCCESS to SUCCESS for now
-                            // TODO: add the differentiation on the metadata level in the next major release
+                            // Both of these means the snapshot has completed.
                             SnapshotsInProgress.State.SUCCESS;
                         default -> throw new IllegalArgumentException("Unexpected snapshot state " + snapshotInfo.state());
                     };
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
index 8d72715219e0f..0e5d7cda4a0d4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
@@ -28,11 +28,11 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.version.CompatibilityVersions;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java
new file mode 100644
index 0000000000000..fe1da86dd54c7
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java
@@ -0,0 +1,404 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.action.admin.cluster.stats.LongMetric.LongMetricValue;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.xcontent.ToXContentFragment;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Holds a snapshot of the CCS telemetry statistics from {@link CCSUsageTelemetry}.
+ * Used to hold the stats for a single node that's part of a {@link ClusterStatsNodeResponse}, as well as to
+ * accumulate stats for the entire cluster and return them as part of the {@link ClusterStatsResponse}.
+ * <br>
+ * Theory of operation:
+ * - The snapshot is created on each particular node with the stats for the node, and is sent to the coordinating node
+ * - Coordinating node creates an empty snapshot and merges all the node snapshots into it using add()
+ * <br>
+ * The snapshot contains {@link LongMetricValue}s for latencies, which currently contain full histograms (since you can't
+ * produce p90 from a set of node p90s, you need the full histogram for that). To avoid excessive copying (histogram weighs several KB),
+ * the snapshot is designed to be mutable, so that you can add multiple snapshots to it without copying the histograms all the time.
+ * It is not the intent to mutate the snapshot objects otherwise.
+ * <br>
+ */
+public final class CCSTelemetrySnapshot implements Writeable, ToXContentFragment {
+    public static final String CCS_TELEMETRY_FIELD_NAME = "_search";
+    private long totalCount;
+    private long successCount;
+    private final Map<String, Long> failureReasons;
+
+    /**
+     * Latency metrics, overall.
+     */
+    private final LongMetricValue took;
+    /**
+     * Latency metrics with minimize_roundtrips=true
+     */
+    private final LongMetricValue tookMrtTrue;
+    /**
+     * Latency metrics with minimize_roundtrips=false
+     */
+    private final LongMetricValue tookMrtFalse;
+    private long remotesPerSearchMax;
+    private double remotesPerSearchAvg;
+    private long skippedRemotes;
+
+    private final Map<String, Long> featureCounts;
+
+    private final Map<String, Long> clientCounts;
+    private final Map<String, PerClusterCCSTelemetry> byRemoteCluster;
+
+    /**
+    * Creates a new stats instance with the provided info.
+    */
+    public CCSTelemetrySnapshot(
+        long totalCount,
+        long successCount,
+        Map<String, Long> failureReasons,
+        LongMetricValue took,
+        LongMetricValue tookMrtTrue,
+        LongMetricValue tookMrtFalse,
+        long remotesPerSearchMax,
+        double remotesPerSearchAvg,
+        long skippedRemotes,
+        Map<String, Long> featureCounts,
+        Map<String, Long> clientCounts,
+        Map<String, PerClusterCCSTelemetry> byRemoteCluster
+    ) {
+        this.totalCount = totalCount;
+        this.successCount = successCount;
+        this.failureReasons = failureReasons;
+        this.took = took;
+        this.tookMrtTrue = tookMrtTrue;
+        this.tookMrtFalse = tookMrtFalse;
+        this.remotesPerSearchMax = remotesPerSearchMax;
+        this.remotesPerSearchAvg = remotesPerSearchAvg;
+        this.skippedRemotes = skippedRemotes;
+        this.featureCounts = featureCounts;
+        this.clientCounts = clientCounts;
+        this.byRemoteCluster = byRemoteCluster;
+    }
+
+    /**
+     * Creates a new empty stats instance, that will get additional stats added through {@link #add(CCSTelemetrySnapshot)}
+     */
+    public CCSTelemetrySnapshot() {
+        // Note this produces modifiable maps, so other snapshots can be merged into it
+        failureReasons = new HashMap<>();
+        featureCounts = new HashMap<>();
+        clientCounts = new HashMap<>();
+        byRemoteCluster = new HashMap<>();
+        took = new LongMetricValue();
+        tookMrtTrue = new LongMetricValue();
+        tookMrtFalse = new LongMetricValue();
+    }
+
+    public CCSTelemetrySnapshot(StreamInput in) throws IOException {
+        this.totalCount = in.readVLong();
+        this.successCount = in.readVLong();
+        this.failureReasons = in.readMap(StreamInput::readLong);
+        this.took = LongMetricValue.fromStream(in);
+        this.tookMrtTrue = LongMetricValue.fromStream(in);
+        this.tookMrtFalse = LongMetricValue.fromStream(in);
+        this.remotesPerSearchMax = in.readVLong();
+        this.remotesPerSearchAvg = in.readDouble();
+        this.skippedRemotes = in.readVLong();
+        this.featureCounts = in.readMap(StreamInput::readLong);
+        this.clientCounts = in.readMap(StreamInput::readLong);
+        this.byRemoteCluster = in.readMap(PerClusterCCSTelemetry::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(totalCount);
+        out.writeVLong(successCount);
+        out.writeMap(failureReasons, StreamOutput::writeLong);
+        took.writeTo(out);
+        tookMrtTrue.writeTo(out);
+        tookMrtFalse.writeTo(out);
+        out.writeVLong(remotesPerSearchMax);
+        out.writeDouble(remotesPerSearchAvg);
+        out.writeVLong(skippedRemotes);
+        out.writeMap(featureCounts, StreamOutput::writeLong);
+        out.writeMap(clientCounts, StreamOutput::writeLong);
+        out.writeMap(byRemoteCluster, StreamOutput::writeWriteable);
+    }
+
+    public long getTotalCount() {
+        return totalCount;
+    }
+
+    public long getSuccessCount() {
+        return successCount;
+    }
+
+    public Map<String, Long> getFailureReasons() {
+        return Collections.unmodifiableMap(failureReasons);
+    }
+
+    public LongMetricValue getTook() {
+        return took;
+    }
+
+    public LongMetricValue getTookMrtTrue() {
+        return tookMrtTrue;
+    }
+
+    public LongMetricValue getTookMrtFalse() {
+        return tookMrtFalse;
+    }
+
+    public long getRemotesPerSearchMax() {
+        return remotesPerSearchMax;
+    }
+
+    public double getRemotesPerSearchAvg() {
+        return remotesPerSearchAvg;
+    }
+
+    public long getSearchCountWithSkippedRemotes() {
+        return skippedRemotes;
+    }
+
+    public Map<String, Long> getFeatureCounts() {
+        return Collections.unmodifiableMap(featureCounts);
+    }
+
+    public Map<String, Long> getClientCounts() {
+        return Collections.unmodifiableMap(clientCounts);
+    }
+
+    public Map<String, PerClusterCCSTelemetry> getByRemoteCluster() {
+        return Collections.unmodifiableMap(byRemoteCluster);
+    }
+
+    public static class PerClusterCCSTelemetry implements Writeable, ToXContentFragment {
+        private long count;
+        private long skippedCount;
+        private final LongMetricValue took;
+
+        public PerClusterCCSTelemetry() {
+            took = new LongMetricValue();
+        }
+
+        public PerClusterCCSTelemetry(long count, long skippedCount, LongMetricValue took) {
+            this.took = took;
+            this.skippedCount = skippedCount;
+            this.count = count;
+        }
+
+        public PerClusterCCSTelemetry(PerClusterCCSTelemetry other) {
+            this.count = other.count;
+            this.skippedCount = other.skippedCount;
+            this.took = new LongMetricValue(other.took);
+        }
+
+        public PerClusterCCSTelemetry(StreamInput in) throws IOException {
+            this.count = in.readVLong();
+            this.skippedCount = in.readVLong();
+            this.took = LongMetricValue.fromStream(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVLong(count);
+            out.writeVLong(skippedCount);
+            took.writeTo(out);
+        }
+
+        public PerClusterCCSTelemetry add(PerClusterCCSTelemetry v) {
+            count += v.count;
+            skippedCount += v.skippedCount;
+            took.add(v.took);
+            return this;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field("total", count);
+            builder.field("skipped", skippedCount);
+            publishLatency(builder, "took", took);
+            builder.endObject();
+            return builder;
+        }
+
+        public long getCount() {
+            return count;
+        }
+
+        public long getSkippedCount() {
+            return skippedCount;
+        }
+
+        public LongMetricValue getTook() {
+            return took;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            PerClusterCCSTelemetry that = (PerClusterCCSTelemetry) o;
+            return count == that.count && skippedCount == that.skippedCount && Objects.equals(took, that.took);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(count, skippedCount, took);
+        }
+    }
+
+    /**
+     * Add the provided stats to the ones held by the current instance, effectively merging the two.
+     * @param stats the other stats object to add to this one
+     */
+    public void add(CCSTelemetrySnapshot stats) {
+        // This should be called in ClusterStatsResponse ctor only, so we don't need to worry about concurrency
+        if (stats.totalCount == 0) {
+            // Just ignore the empty stats.
+            // This could happen if the node is brand new or if the stats are not available, e.g. because it runs an old version.
+            return;
+        }
+        long oldCount = totalCount;
+        totalCount += stats.totalCount;
+        successCount += stats.successCount;
+        skippedRemotes += stats.skippedRemotes;
+        stats.failureReasons.forEach((k, v) -> failureReasons.merge(k, v, Long::sum));
+        stats.featureCounts.forEach((k, v) -> featureCounts.merge(k, v, Long::sum));
+        stats.clientCounts.forEach((k, v) -> clientCounts.merge(k, v, Long::sum));
+        took.add(stats.took);
+        tookMrtTrue.add(stats.tookMrtTrue);
+        tookMrtFalse.add(stats.tookMrtFalse);
+        remotesPerSearchMax = Math.max(remotesPerSearchMax, stats.remotesPerSearchMax);
+        if (totalCount > 0 && oldCount > 0) {
+            // Weighted average
+            remotesPerSearchAvg = (remotesPerSearchAvg * oldCount + stats.remotesPerSearchAvg * stats.totalCount) / totalCount;
+        } else {
+            // If we didn't have any old value, we just take the new one
+            remotesPerSearchAvg = stats.remotesPerSearchAvg;
+        }
+        // we copy the object here since we'll be modifying it later on subsequent adds
+        // TODO: this may be sub-optimal, as we'll be copying histograms when adding first snapshot to an empty container,
+        // which we could have avoided probably.
+        stats.byRemoteCluster.forEach((r, v) -> byRemoteCluster.merge(r, new PerClusterCCSTelemetry(v), PerClusterCCSTelemetry::add));
+    }
+
+    /**
+     * Publishes the latency statistics to the provided {@link XContentBuilder}.
+     * Example:
+     * "took": {
+     *      "max": 345032,
+     *      "avg": 1620,
+     *      "p90": 2570
+     * }
+     */
+    public static void publishLatency(XContentBuilder builder, String name, LongMetricValue took) throws IOException {
+        builder.startObject(name);
+        {
+            builder.field("max", took.max());
+            builder.field("avg", took.avg());
+            builder.field("p90", took.p90());
+        }
+        builder.endObject();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(CCS_TELEMETRY_FIELD_NAME);
+        {
+            builder.field("total", totalCount);
+            builder.field("success", successCount);
+            builder.field("skipped", skippedRemotes);
+            publishLatency(builder, "took", took);
+            publishLatency(builder, "took_mrt_true", tookMrtTrue);
+            publishLatency(builder, "took_mrt_false", tookMrtFalse);
+            builder.field("remotes_per_search_max", remotesPerSearchMax);
+            builder.field("remotes_per_search_avg", remotesPerSearchAvg);
+            builder.field("failure_reasons", failureReasons);
+            builder.field("features", featureCounts);
+            builder.field("clients", clientCounts);
+            builder.startObject("clusters");
+            {
+                for (var entry : byRemoteCluster.entrySet()) {
+                    String remoteName = entry.getKey();
+                    if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(remoteName)) {
+                        remoteName = SearchResponse.LOCAL_CLUSTER_NAME_REPRESENTATION;
+                    }
+                    builder.field(remoteName, entry.getValue());
+                }
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        CCSTelemetrySnapshot that = (CCSTelemetrySnapshot) o;
+        return totalCount == that.totalCount
+            && successCount == that.successCount
+            && skippedRemotes == that.skippedRemotes
+            && Objects.equals(failureReasons, that.failureReasons)
+            && Objects.equals(took, that.took)
+            && Objects.equals(tookMrtTrue, that.tookMrtTrue)
+            && Objects.equals(tookMrtFalse, that.tookMrtFalse)
+            && Objects.equals(remotesPerSearchMax, that.remotesPerSearchMax)
+            && Objects.equals(remotesPerSearchAvg, that.remotesPerSearchAvg)
+            && Objects.equals(featureCounts, that.featureCounts)
+            && Objects.equals(clientCounts, that.clientCounts)
+            && Objects.equals(byRemoteCluster, that.byRemoteCluster);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            totalCount,
+            successCount,
+            failureReasons,
+            took,
+            tookMrtTrue,
+            tookMrtFalse,
+            remotesPerSearchMax,
+            remotesPerSearchAvg,
+            skippedRemotes,
+            featureCounts,
+            clientCounts,
+            byRemoteCluster
+        );
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java
new file mode 100644
index 0000000000000..b2d75ac8f61f3
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.Result;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.action.search.TransportSearchAction;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.query.SearchTimeoutException;
+import org.elasticsearch.tasks.TaskCancelledException;
+import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.NoSeedNodeLeftException;
+import org.elasticsearch.transport.NoSuchRemoteClusterException;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.transport.RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
+
+/**
+ * This is a container for telemetry data from an individual cross-cluster search for _search or _async_search (or
+ * other search endpoints that use the {@link TransportSearchAction} such as _msearch).
+ */
+public class CCSUsage {
+    private final long took;
+    private final Result status;
+    private final Set<String> features;
+    private final int remotesCount;
+
+    private final String client;
+
+    private final Set<String> skippedRemotes;
+    private final Map<String, PerClusterUsage> perClusterUsage;
+
+    public static class Builder {
+        private long took;
+        private final Set<String> features;
+        private Result status = Result.SUCCESS;
+        private int remotesCount;
+        private String client;
+        private final Set<String> skippedRemotes;
+        private final Map<String, PerClusterUsage> perClusterUsage;
+
+        public Builder() {
+            features = new HashSet<>();
+            skippedRemotes = new HashSet<>();
+            perClusterUsage = new HashMap<>();
+        }
+
+        public Builder took(long took) {
+            this.took = took;
+            return this;
+        }
+
+        public Builder setFailure(Result failureType) {
+            this.status = failureType;
+            return this;
+        }
+
+        public Builder setFailure(Exception e) {
+            return setFailure(getFailureType(e));
+        }
+
+        public Builder setFeature(String feature) {
+            this.features.add(feature);
+            return this;
+        }
+
+        public Builder setClient(String client) {
+            this.client = client;
+            return this;
+        }
+
+        public Builder skippedRemote(String remote) {
+            this.skippedRemotes.add(remote);
+            return this;
+        }
+
+        public Builder perClusterUsage(String remote, TimeValue took) {
+            this.perClusterUsage.put(remote, new PerClusterUsage(took));
+            return this;
+        }
+
+        public CCSUsage build() {
+            return new CCSUsage(took, status, remotesCount, skippedRemotes, features, client, perClusterUsage);
+        }
+
+        public Builder setRemotesCount(int remotesCount) {
+            this.remotesCount = remotesCount;
+            return this;
+        }
+
+        public int getRemotesCount() {
+            return remotesCount;
+        }
+
+        /**
+         * Get failure type as {@link Result} from the search failure exception.
+         */
+        public static Result getFailureType(Exception e) {
+            var unwrapped = ExceptionsHelper.unwrapCause(e);
+            if (unwrapped instanceof Exception) {
+                e = (Exception) unwrapped;
+            }
+            if (isRemoteUnavailable(e)) {
+                return Result.REMOTES_UNAVAILABLE;
+            }
+            if (ExceptionsHelper.unwrap(e, ResourceNotFoundException.class) != null) {
+                return Result.NOT_FOUND;
+            }
+            if (e instanceof TaskCancelledException || (ExceptionsHelper.unwrap(e, TaskCancelledException.class) != null)) {
+                return Result.CANCELED;
+            }
+            if (ExceptionsHelper.unwrap(e, SearchTimeoutException.class) != null) {
+                return Result.TIMEOUT;
+            }
+            if (ExceptionsHelper.unwrap(e, ElasticsearchSecurityException.class) != null) {
+                return Result.SECURITY;
+            }
+            if (ExceptionsHelper.unwrapCorruption(e) != null) {
+                return Result.CORRUPTION;
+            }
+            // This is kind of last resort check - if we still don't know the reason but all shard failures are remote,
+            // we assume it's remote's fault somehow.
+            if (e instanceof SearchPhaseExecutionException spe) {
+                // If this is a failure that happened because of remote failures only
+                var groupedFails = ExceptionsHelper.groupBy(spe.shardFailures());
+                if (Arrays.stream(groupedFails).allMatch(Builder::isRemoteFailure)) {
+                    return Result.REMOTES_UNAVAILABLE;
+                }
+            }
+            // OK we don't know what happened
+            return Result.UNKNOWN;
+        }
+
+        /**
+         * Is this failure exception because remote was unavailable?
+         * See also: TransportResolveClusterAction#notConnectedError
+         */
+        static boolean isRemoteUnavailable(Exception e) {
+            if (ExceptionsHelper.unwrap(
+                e,
+                ConnectTransportException.class,
+                NoSuchRemoteClusterException.class,
+                NoSeedNodeLeftException.class
+            ) != null) {
+                return true;
+            }
+            Throwable ill = ExceptionsHelper.unwrap(e, IllegalStateException.class, IllegalArgumentException.class);
+            if (ill != null && (ill.getMessage().contains("Unable to open any connections") || ill.getMessage().contains("unknown host"))) {
+                return true;
+            }
+            // Ok doesn't look like any of the known remote exceptions
+            return false;
+        }
+
+        /**
+         * Is this failure coming from a remote cluster?
+         */
+        static boolean isRemoteFailure(ShardOperationFailedException failure) {
+            if (failure instanceof ShardSearchFailure shardFailure) {
+                SearchShardTarget shard = shardFailure.shard();
+                return shard != null && shard.getClusterAlias() != null && LOCAL_CLUSTER_GROUP_KEY.equals(shard.getClusterAlias()) == false;
+            }
+            return false;
+        }
+    }
+
+    private CCSUsage(
+        long took,
+        Result status,
+        int remotesCount,
+        Set<String> skippedRemotes,
+        Set<String> features,
+        String client,
+        Map<String, PerClusterUsage> perClusterUsage
+    ) {
+        this.status = status;
+        this.remotesCount = remotesCount;
+        this.features = features;
+        this.client = client;
+        this.took = took;
+        this.skippedRemotes = skippedRemotes;
+        this.perClusterUsage = perClusterUsage;
+    }
+
+    public Map<String, PerClusterUsage> getPerClusterUsage() {
+        return perClusterUsage;
+    }
+
+    public Result getStatus() {
+        return status;
+    }
+
+    public Set<String> getFeatures() {
+        return features;
+    }
+
+    public long getRemotesCount() {
+        return remotesCount;
+    }
+
+    public String getClient() {
+        return client;
+    }
+
+    public long getTook() {
+        return took;
+    }
+
+    public Set<String> getSkippedRemotes() {
+        return skippedRemotes;
+    }
+
+    public static class PerClusterUsage {
+
+        // if MRT=true, the took time on the remote cluster (if MRT=true), otherwise the overall took time
+        private long took;
+
+        public PerClusterUsage(TimeValue took) {
+            if (took != null) {
+                this.took = took.millis();
+            }
+        }
+
+        public long getTook() {
+            return took;
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java
new file mode 100644
index 0000000000000..60766bd4068e3
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.common.util.Maps;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.LongAdder;
+
+/**
+ * Service holding accumulated CCS search usage statistics. Individual cross-cluster searches will pass
+ * CCSUsage data here to have it collated and aggregated. Snapshots of the current CCS Telemetry Usage
+ * can be obtained by getting {@link CCSTelemetrySnapshot} objects.
+ * <br>
+ * Theory of operation:
+ * Each search creates a {@link CCSUsage.Builder}, which can be updated during the progress of the search request,
+ * and then it instantiates a {@link CCSUsage} object when the request is finished.
+ * That object is passed to {@link #updateUsage(CCSUsage)} on the request processing end (whether successful or not).
+ * The {@link #updateUsage(CCSUsage)} method will then update the internal counters and metrics.
+ * <br>
+ * When we need to return the current state of the telemetry, we can call {@link #getCCSTelemetrySnapshot()} which produces
+ * a snapshot of the current state of the telemetry as {@link CCSTelemetrySnapshot}. These snapshots are additive so
+ * when collecting the snapshots from multiple nodes, an empty snapshot is created and then all the node's snapshots are added
+ * to it to obtain the summary telemetry.
+ */
+public class CCSUsageTelemetry {
+
+    /**
+     * Result of the request execution.
+     * Either "success" or a failure reason.
+     */
+    public enum Result {
+        SUCCESS("success"),
+        REMOTES_UNAVAILABLE("remotes_unavailable"),
+        CANCELED("canceled"),
+        NOT_FOUND("not_found"),
+        TIMEOUT("timeout"),
+        CORRUPTION("corruption"),
+        SECURITY("security"),
+        // May be helpful if there's a lot of other reasons, and it may be hard to calculate the unknowns for some clients.
+        UNKNOWN("other");
+
+        private final String name;
+
+        Result(String name) {
+            this.name = name;
+        }
+
+        public String getName() {
+            return name;
+        }
+    }
+
+    // Not enum because we won't mind other places adding their own features
+    public static final String MRT_FEATURE = "mrt_on";
+    public static final String ASYNC_FEATURE = "async";
+    public static final String WILDCARD_FEATURE = "wildcards";
+
+    // The list of known Elastic clients. May be incomplete.
+    public static final Set<String> KNOWN_CLIENTS = Set.of(
+        "kibana",
+        "cloud",
+        "logstash",
+        "beats",
+        "fleet",
+        "ml",
+        "security",
+        "observability",
+        "enterprise-search",
+        "elasticsearch",
+        "connectors",
+        "connectors-cli"
+    );
+
+    private final LongAdder totalCount;
+    private final LongAdder successCount;
+    private final Map<Result, LongAdder> failureReasons;
+
+    /**
+     * Latency metrics overall
+     */
+    private final LongMetric took;
+    /**
+     * Latency metrics with minimize_roundtrips=true
+     */
+    private final LongMetric tookMrtTrue;
+    /**
+     * Latency metrics with minimize_roundtrips=false
+     */
+    private final LongMetric tookMrtFalse;
+    private final LongMetric remotesPerSearch;
+    private final LongAdder skippedRemotes;
+
+    private final Map<String, LongAdder> featureCounts;
+
+    private final Map<String, LongAdder> clientCounts;
+    private final Map<String, PerClusterCCSTelemetry> byRemoteCluster;
+
+    public CCSUsageTelemetry() {
+        this.byRemoteCluster = new ConcurrentHashMap<>();
+        totalCount = new LongAdder();
+        successCount = new LongAdder();
+        failureReasons = new ConcurrentHashMap<>();
+        took = new LongMetric();
+        tookMrtTrue = new LongMetric();
+        tookMrtFalse = new LongMetric();
+        remotesPerSearch = new LongMetric();
+        skippedRemotes = new LongAdder();
+        featureCounts = new ConcurrentHashMap<>();
+        clientCounts = new ConcurrentHashMap<>();
+    }
+
+    public void updateUsage(CCSUsage ccsUsage) {
+        assert ccsUsage.getRemotesCount() > 0 : "Expected at least one remote cluster in CCSUsage";
+        // TODO: fork this to a background thread?
+        doUpdate(ccsUsage);
+    }
+
+    // This is not synchronized, instead we ensure that every metric in the class is thread-safe.
+    private void doUpdate(CCSUsage ccsUsage) {
+        totalCount.increment();
+        long searchTook = ccsUsage.getTook();
+        if (isSuccess(ccsUsage)) {
+            successCount.increment();
+            took.record(searchTook);
+            if (isMRT(ccsUsage)) {
+                tookMrtTrue.record(searchTook);
+            } else {
+                tookMrtFalse.record(searchTook);
+            }
+            ccsUsage.getPerClusterUsage().forEach((r, u) -> byRemoteCluster.computeIfAbsent(r, PerClusterCCSTelemetry::new).update(u));
+        } else {
+            failureReasons.computeIfAbsent(ccsUsage.getStatus(), k -> new LongAdder()).increment();
+        }
+
+        remotesPerSearch.record(ccsUsage.getRemotesCount());
+        if (ccsUsage.getSkippedRemotes().isEmpty() == false) {
+            skippedRemotes.increment();
+            ccsUsage.getSkippedRemotes().forEach(remote -> byRemoteCluster.computeIfAbsent(remote, PerClusterCCSTelemetry::new).skipped());
+        }
+        ccsUsage.getFeatures().forEach(f -> featureCounts.computeIfAbsent(f, k -> new LongAdder()).increment());
+        String client = ccsUsage.getClient();
+        if (client != null && KNOWN_CLIENTS.contains(client)) {
+            // We count only known clients for now
+            clientCounts.computeIfAbsent(ccsUsage.getClient(), k -> new LongAdder()).increment();
+        }
+    }
+
+    private boolean isMRT(CCSUsage ccsUsage) {
+        return ccsUsage.getFeatures().contains(MRT_FEATURE);
+    }
+
+    private boolean isSuccess(CCSUsage ccsUsage) {
+        return ccsUsage.getStatus() == Result.SUCCESS;
+    }
+
+    public Map<String, PerClusterCCSTelemetry> getTelemetryByCluster() {
+        return byRemoteCluster;
+    }
+
+    /**
+     * Telemetry of each remote involved in cross cluster searches
+     */
+    public static class PerClusterCCSTelemetry {
+        private final String clusterAlias;
+        // The number of successful (not skipped) requests to this cluster.
+        private final LongAdder count;
+        private final LongAdder skippedCount;
+        // This is only over the successful requetss, skipped ones do not count here.
+        private final LongMetric took;
+
+        PerClusterCCSTelemetry(String clusterAlias) {
+            this.clusterAlias = clusterAlias;
+            this.count = new LongAdder();
+            took = new LongMetric();
+            this.skippedCount = new LongAdder();
+        }
+
+        void update(CCSUsage.PerClusterUsage remoteUsage) {
+            count.increment();
+            took.record(remoteUsage.getTook());
+        }
+
+        void skipped() {
+            skippedCount.increment();
+        }
+
+        public long getCount() {
+            return count.longValue();
+        }
+
+        @Override
+        public String toString() {
+            return "PerClusterCCSTelemetry{"
+                + "clusterAlias='"
+                + clusterAlias
+                + '\''
+                + ", count="
+                + count
+                + ", latency="
+                + took.toString()
+                + '}';
+        }
+
+        public long getSkippedCount() {
+            return skippedCount.longValue();
+        }
+
+        public CCSTelemetrySnapshot.PerClusterCCSTelemetry getSnapshot() {
+            return new CCSTelemetrySnapshot.PerClusterCCSTelemetry(count.longValue(), skippedCount.longValue(), took.getValue());
+        }
+
+    }
+
+    public CCSTelemetrySnapshot getCCSTelemetrySnapshot() {
+        Map<String, Long> reasonsMap = Maps.newMapWithExpectedSize(failureReasons.size());
+        failureReasons.forEach((k, v) -> reasonsMap.put(k.getName(), v.longValue()));
+
+        LongMetric.LongMetricValue remotes = remotesPerSearch.getValue();
+
+        // Maps returned here are unmodifiable, but the empty ctor produces modifiable maps
+        return new CCSTelemetrySnapshot(
+            totalCount.longValue(),
+            successCount.longValue(),
+            Collections.unmodifiableMap(reasonsMap),
+            took.getValue(),
+            tookMrtTrue.getValue(),
+            tookMrtFalse.getValue(),
+            remotes.max(),
+            remotes.avg(),
+            skippedRemotes.longValue(),
+            Collections.unmodifiableMap(Maps.transformValues(featureCounts, LongAdder::longValue)),
+            Collections.unmodifiableMap(Maps.transformValues(clientCounts, LongAdder::longValue)),
+            Collections.unmodifiableMap(Maps.transformValues(byRemoteCluster, PerClusterCCSTelemetry::getSnapshot))
+        );
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
index d74889b623589..b48295dc8b3eb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
@@ -20,29 +20,33 @@
 import org.elasticsearch.core.Nullable;
 
 import java.io.IOException;
+import java.util.Objects;
 
 public class ClusterStatsNodeResponse extends BaseNodeResponse {
 
     private final NodeInfo nodeInfo;
     private final NodeStats nodeStats;
     private final ShardStats[] shardsStats;
-    private ClusterHealthStatus clusterStatus;
+    private final ClusterHealthStatus clusterStatus;
     private final SearchUsageStats searchUsageStats;
+    private final RepositoryUsageStats repositoryUsageStats;
 
     public ClusterStatsNodeResponse(StreamInput in) throws IOException {
         super(in);
-        clusterStatus = null;
-        if (in.readBoolean()) {
-            clusterStatus = ClusterHealthStatus.readFrom(in);
-        }
+        this.clusterStatus = in.readOptionalWriteable(ClusterHealthStatus::readFrom);
         this.nodeInfo = new NodeInfo(in);
         this.nodeStats = new NodeStats(in);
-        shardsStats = in.readArray(ShardStats::new, ShardStats[]::new);
+        this.shardsStats = in.readArray(ShardStats::new, ShardStats[]::new);
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_6_0)) {
             searchUsageStats = new SearchUsageStats(in);
         } else {
             searchUsageStats = new SearchUsageStats();
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.REPOSITORIES_TELEMETRY)) {
+            repositoryUsageStats = RepositoryUsageStats.readFrom(in);
+        } else {
+            repositoryUsageStats = RepositoryUsageStats.EMPTY;
+        }
     }
 
     public ClusterStatsNodeResponse(
@@ -51,14 +55,16 @@ public ClusterStatsNodeResponse(
         NodeInfo nodeInfo,
         NodeStats nodeStats,
         ShardStats[] shardsStats,
-        SearchUsageStats searchUsageStats
+        SearchUsageStats searchUsageStats,
+        RepositoryUsageStats repositoryUsageStats
     ) {
         super(node);
         this.nodeInfo = nodeInfo;
         this.nodeStats = nodeStats;
         this.shardsStats = shardsStats;
         this.clusterStatus = clusterStatus;
-        this.searchUsageStats = searchUsageStats;
+        this.searchUsageStats = Objects.requireNonNull(searchUsageStats);
+        this.repositoryUsageStats = Objects.requireNonNull(repositoryUsageStats);
     }
 
     public NodeInfo nodeInfo() {
@@ -85,20 +91,22 @@ public SearchUsageStats searchUsageStats() {
         return searchUsageStats;
     }
 
+    public RepositoryUsageStats repositoryUsageStats() {
+        return repositoryUsageStats;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (clusterStatus == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            out.writeByte(clusterStatus.value());
-        }
+        out.writeOptionalWriteable(clusterStatus);
         nodeInfo.writeTo(out);
         nodeStats.writeTo(out);
         out.writeArray(shardsStats);
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_6_0)) {
             searchUsageStats.writeTo(out);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.REPOSITORIES_TELEMETRY)) {
+            repositoryUsageStats.writeTo(out);
+        } // else just drop these stats, ok for bwc
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
index 36e7b247befac..b6dd40e8c8b79 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
@@ -30,6 +30,7 @@ public class ClusterStatsResponse extends BaseNodesResponse<ClusterStatsNodeResp
     final ClusterStatsIndices indicesStats;
     final ClusterHealthStatus status;
     final ClusterSnapshotStats clusterSnapshotStats;
+    final RepositoryUsageStats repositoryUsageStats;
     final long timestamp;
     final String clusterUUID;
 
@@ -59,6 +60,14 @@ public ClusterStatsResponse(
         }
         this.status = status;
         this.clusterSnapshotStats = clusterSnapshotStats;
+
+        this.repositoryUsageStats = nodes.stream()
+            .map(ClusterStatsNodeResponse::repositoryUsageStats)
+            // only populated on snapshot nodes (i.e. master and data nodes)
+            .filter(r -> r.isEmpty() == false)
+            // stats should be the same on every node so just pick one of them
+            .findAny()
+            .orElse(RepositoryUsageStats.EMPTY);
     }
 
     public String getClusterUUID() {
@@ -113,6 +122,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field("snapshots");
         clusterSnapshotStats.toXContent(builder, params);
 
+        builder.field("repositories");
+        repositoryUsageStats.toXContent(builder, params);
+
         return builder;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java
new file mode 100644
index 0000000000000..f3bb936b108c0
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.HdrHistogram.ConcurrentHistogram;
+import org.HdrHistogram.Histogram;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+import java.util.zip.DataFormatException;
+
+/**
+ * Metric class that accepts longs and provides count, average, max and percentiles.
+ * Abstracts out the details of how exactly the values are stored and calculated.
+ * {@link LongMetricValue} is a snapshot of the current state of the metric.
+ */
+public class LongMetric {
+    private final Histogram values;
+    private static final int SIGNIFICANT_DIGITS = 2;
+
+    LongMetric() {
+        values = new ConcurrentHistogram(SIGNIFICANT_DIGITS);
+    }
+
+    void record(long v) {
+        values.recordValue(v);
+    }
+
+    LongMetricValue getValue() {
+        return new LongMetricValue(values);
+    }
+
+    /**
+     * Snapshot of {@link LongMetric} value that provides the current state of the metric.
+     * Can be added with another {@link LongMetricValue} object.
+     */
+    public static final class LongMetricValue implements Writeable {
+        // We have to carry the full histogram around since we might need to calculate aggregate percentiles
+        // after collecting individual stats from the nodes, and we can't do that without having the full histogram.
+        // This costs about 2K per metric, which was deemed acceptable.
+        private final Histogram values;
+
+        public LongMetricValue(Histogram values) {
+            // Copy here since we don't want the snapshot value to change if somebody updates the original one
+            this.values = values.copy();
+        }
+
+        public LongMetricValue(LongMetricValue v) {
+            this.values = v.values.copy();
+        }
+
+        LongMetricValue() {
+            this.values = new Histogram(SIGNIFICANT_DIGITS);
+        }
+
+        public void add(LongMetricValue v) {
+            this.values.add(v.values);
+        }
+
+        public static LongMetricValue fromStream(StreamInput in) throws IOException {
+            byte[] b = in.readByteArray();
+            ByteBuffer bb = ByteBuffer.wrap(b);
+            try {
+                // TODO: not sure what is the good value for minBarForHighestToLowestValueRatio here?
+                Histogram dh = Histogram.decodeFromCompressedByteBuffer(bb, 1);
+                return new LongMetricValue(dh);
+            } catch (DataFormatException e) {
+                throw new IOException(e);
+            }
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            ByteBuffer b = ByteBuffer.allocate(values.getNeededByteBufferCapacity());
+            values.encodeIntoCompressedByteBuffer(b);
+            int size = b.position();
+            out.writeVInt(size);
+            out.writeBytes(b.array(), 0, size);
+        }
+
+        public long count() {
+            return values.getTotalCount();
+        }
+
+        public long max() {
+            return values.getMaxValue();
+        }
+
+        public long avg() {
+            return (long) Math.ceil(values.getMean());
+        }
+
+        public long p90() {
+            return values.getValueAtPercentile(90);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == this) return true;
+            if (obj == null || obj.getClass() != this.getClass()) return false;
+            var that = (LongMetricValue) obj;
+            return this.values.equals(that.values);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(values);
+        }
+
+        @Override
+        public String toString() {
+            return "LongMetricValue[count=" + count() + ", " + "max=" + max() + ", " + "avg=" + avg() + "]";
+        }
+
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RepositoryUsageStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RepositoryUsageStats.java
new file mode 100644
index 0000000000000..771aa0fbef842
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RepositoryUsageStats.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Stats on repository feature usage exposed in cluster stats for telemetry.
+ *
+ * @param statsByType a count of the repositories using various named features, keyed by repository type and then by feature name.
+ */
+public record RepositoryUsageStats(Map<String, Map<String, Long>> statsByType) implements Writeable, ToXContentObject {
+
+    public static final RepositoryUsageStats EMPTY = new RepositoryUsageStats(Map.of());
+
+    public static RepositoryUsageStats readFrom(StreamInput in) throws IOException {
+        final var statsByType = in.readMap(i -> i.readMap(StreamInput::readVLong));
+        if (statsByType.isEmpty()) {
+            return EMPTY;
+        } else {
+            return new RepositoryUsageStats(statsByType);
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(statsByType, (o, m) -> o.writeMap(m, StreamOutput::writeVLong));
+    }
+
+    public boolean isEmpty() {
+        return statsByType.isEmpty();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        for (Map.Entry<String, Map<String, Long>> typeAndStats : statsByType.entrySet()) {
+            builder.startObject(typeAndStats.getKey());
+            for (Map.Entry<String, Long> statAndValue : typeAndStats.getValue().entrySet()) {
+                builder.field(statAndValue.getKey(), statAndValue.getValue());
+            }
+            builder.endObject();
+        }
+        return builder.endObject();
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 5d12cb5c0f657..1912de3cfa4d2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.CancellableSingleObjectCache;
@@ -40,7 +39,9 @@
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeService;
+import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -78,6 +79,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<
 
     private final NodeService nodeService;
     private final IndicesService indicesService;
+    private final RepositoriesService repositoriesService;
     private final SearchUsageHolder searchUsageHolder;
 
     private final MetadataStatsCache<MappingStats> mappingStatsCache;
@@ -90,6 +92,7 @@ public TransportClusterStatsAction(
         TransportService transportService,
         NodeService nodeService,
         IndicesService indicesService,
+        RepositoriesService repositoriesService,
         UsageService usageService,
         ActionFilters actionFilters
     ) {
@@ -103,6 +106,7 @@ public TransportClusterStatsAction(
         );
         this.nodeService = nodeService;
         this.indicesService = indicesService;
+        this.repositoriesService = repositoriesService;
         this.searchUsageHolder = usageService.getSearchUsageHolder();
         this.mappingStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), MappingStats::of);
         this.analysisStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), AnalysisStats::of);
@@ -237,12 +241,14 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
             }
         }
 
-        ClusterHealthStatus clusterStatus = null;
-        if (clusterService.state().nodes().isLocalNodeElectedMaster()) {
-            clusterStatus = new ClusterStateHealth(clusterService.state()).getStatus();
-        }
+        final ClusterState clusterState = clusterService.state();
+        final ClusterHealthStatus clusterStatus = clusterState.nodes().isLocalNodeElectedMaster()
+            ? new ClusterStateHealth(clusterState).getStatus()
+            : null;
+
+        final SearchUsageStats searchUsageStats = searchUsageHolder.getSearchUsageStats();
 
-        SearchUsageStats searchUsageStats = searchUsageHolder.getSearchUsageStats();
+        final RepositoryUsageStats repositoryUsageStats = repositoriesService.getUsageStats();
 
         return new ClusterStatsNodeResponse(
             nodeInfo.getNode(),
@@ -250,7 +256,8 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
             nodeInfo,
             nodeStats,
             shardsStats.toArray(new ShardStats[shardsStats.size()]),
-            searchUsageStats
+            searchUsageStats,
+            repositoryUsageStats
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
index 96dbaec7a4487..a893a77f660f3 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -26,13 +27,12 @@ public DeleteStoredScriptRequest(StreamInput in) throws IOException {
         id = in.readString();
     }
 
-    DeleteStoredScriptRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+    DeleteStoredScriptRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+        super(masterNodeTimeout, ackTimeout);
     }
 
-    public DeleteStoredScriptRequest(String id) {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-
+    public DeleteStoredScriptRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout, String id) {
+        super(masterNodeTimeout, ackTimeout);
         this.id = id;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java
deleted file mode 100644
index ce074e17ebb75..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.action.admin.cluster.storedscripts;
-
-import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-
-public class DeleteStoredScriptRequestBuilder extends AcknowledgedRequestBuilder<
-    DeleteStoredScriptRequest,
-    AcknowledgedResponse,
-    DeleteStoredScriptRequestBuilder> {
-
-    public DeleteStoredScriptRequestBuilder(ElasticsearchClient client) {
-        super(client, TransportDeleteStoredScriptAction.TYPE, new DeleteStoredScriptRequest());
-    }
-
-    public DeleteStoredScriptRequestBuilder setId(String id) {
-        request.id(id);
-
-        return this;
-    }
-
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java
index a0b6f9b26cb6c..3a3756553bd76 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java
@@ -10,12 +10,12 @@
 
 import org.elasticsearch.action.ActionType;
 
-public class GetStoredScriptAction extends ActionType<GetStoredScriptResponse> {
+public class GetStoredScriptAction {
 
-    public static final GetStoredScriptAction INSTANCE = new GetStoredScriptAction();
-    public static final String NAME = "cluster:admin/script/get";
+    public static final ActionType<GetStoredScriptResponse> INSTANCE = new ActionType<>("cluster:admin/script/get");
+    public static final String NAME = INSTANCE.name();
 
     private GetStoredScriptAction() {
-        super(NAME);
+        /* no instances */
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
index 1e8d865d9eb8c..f2f651b127573 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -21,13 +22,12 @@ public class GetStoredScriptRequest extends MasterNodeReadRequest<GetStoredScrip
 
     protected String id;
 
-    GetStoredScriptRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    GetStoredScriptRequest(TimeValue masterNodeTimeout) {
+        super(masterNodeTimeout);
     }
 
-    public GetStoredScriptRequest(String id) {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
-
+    public GetStoredScriptRequest(TimeValue masterNodeTimeout, String id) {
+        super(masterNodeTimeout);
         this.id = id;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestBuilder.java
deleted file mode 100644
index 3a3bda5be016a..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestBuilder.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.action.admin.cluster.storedscripts;
-
-import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-
-public class GetStoredScriptRequestBuilder extends MasterNodeReadOperationRequestBuilder<
-    GetStoredScriptRequest,
-    GetStoredScriptResponse,
-    GetStoredScriptRequestBuilder> {
-
-    public GetStoredScriptRequestBuilder(ElasticsearchClient client) {
-        super(client, GetStoredScriptAction.INSTANCE, new GetStoredScriptRequest());
-    }
-
-    public GetStoredScriptRequestBuilder setId(String id) {
-        request.id(id);
-        return this;
-    }
-
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
index 7d6ac9d319fa1..1ac151f0f4971 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.script.StoredScriptSource;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -41,12 +42,20 @@ public PutStoredScriptRequest(StreamInput in) throws IOException {
         source = new StoredScriptSource(in);
     }
 
-    public PutStoredScriptRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+    public PutStoredScriptRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+        super(masterNodeTimeout, ackTimeout);
     }
 
-    public PutStoredScriptRequest(String id, String context, BytesReference content, XContentType xContentType, StoredScriptSource source) {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+    public PutStoredScriptRequest(
+        TimeValue masterNodeTimeout,
+        TimeValue ackTimeout,
+        String id,
+        String context,
+        BytesReference content,
+        XContentType xContentType,
+        StoredScriptSource source
+    ) {
+        super(masterNodeTimeout, ackTimeout);
         this.id = id;
         this.context = context;
         this.content = content;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java
deleted file mode 100644
index 9e353382f84a9..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.action.admin.cluster.storedscripts;
-
-import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.xcontent.XContentType;
-
-public class PutStoredScriptRequestBuilder extends AcknowledgedRequestBuilder<
-    PutStoredScriptRequest,
-    AcknowledgedResponse,
-    PutStoredScriptRequestBuilder> {
-
-    public PutStoredScriptRequestBuilder(ElasticsearchClient client) {
-        super(client, TransportPutStoredScriptAction.TYPE, new PutStoredScriptRequest());
-    }
-
-    public PutStoredScriptRequestBuilder setId(String id) {
-        request.id(id);
-        return this;
-    }
-
-    public PutStoredScriptRequestBuilder setContext(String context) {
-        request.context(context);
-        return this;
-    }
-
-    /**
-     * Set the source of the script along with the content type of the source
-     */
-    public PutStoredScriptRequestBuilder setContent(BytesReference source, XContentType xContentType) {
-        request.content(source, xContentType);
-        return this;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java
index a5d1fd7e151c5..9362564e0855d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptContextAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptContextAction.java
index 47e1ddc4c9ac1..58a68c558f4c7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptContextAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptContextAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.ScriptContextInfo;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java
index ec18439653380..f8fe7fb9b85a1 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java
index f5c674df2e475..daf8411d0808f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
index 6a73cd0b91264..2ce5fa942c948 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java
index b9a16a4fa44bf..7c4d77be1364b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java
@@ -19,8 +19,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.PendingClusterTask;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java
index b4f483e6f8161..b8ad70ad8b6f8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesResponse.java
@@ -48,7 +48,7 @@ public class IndicesAliasesResponse extends AcknowledgedResponse {
     protected IndicesAliasesResponse(StreamInput in) throws IOException {
         super(in);
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ALIAS_ACTION_RESULTS)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             this.errors = in.readBoolean();
             this.actionResults = in.readCollectionAsImmutableList(AliasActionResult::new);
         } else {
@@ -91,7 +91,7 @@ public static IndicesAliasesResponse build(final List<AliasActionResult> actionR
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ALIAS_ACTION_RESULTS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeBoolean(errors);
             out.writeCollection(actionResults);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
index f9c255bb057d8..fbbe2b1b60a1d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
@@ -30,11 +30,11 @@
 import org.elasticsearch.cluster.metadata.MetadataIndexAliasesService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
index 4f7525c700fc2..77f2aca410e68 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -27,6 +26,7 @@
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
index b0ac4311f6b09..76a972f5b900b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.IOUtils;
@@ -43,6 +42,7 @@
 import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportReloadAnalyzersAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportReloadAnalyzersAction.java
index 3e13b55dbd9d4..5b9810eb466d7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportReloadAnalyzersAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportReloadAnalyzersAction.java
@@ -25,12 +25,12 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
index 428fd6e083116..51402bf8c80b5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
@@ -20,9 +20,9 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
index f87ea1e4cd6c7..9b6331a9d320d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
@@ -21,13 +21,13 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java
index 643f92ec3378f..cfeab14d05e32 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -29,6 +28,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -68,7 +68,8 @@ public TransportVerifyShardBeforeCloseAction(
             ShardRequest::new,
             threadPool.executor(ThreadPool.Names.MANAGEMENT),
             SyncGlobalCheckpointAfterOperation.DoNotSync,
-            PrimaryActionExecution.RejectOnOverload
+            PrimaryActionExecution.RejectOnOverload,
+            ReplicaActionExecution.SubjectToCircuitBreaker
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java
index e68263aab5330..fed8044477cc0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java
@@ -37,7 +37,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -46,6 +45,7 @@
 import org.elasticsearch.indices.SystemDataStreamDescriptor;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
index 8a46daa45e73b..948199fbe74f4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest;
-import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
@@ -43,8 +42,6 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
 
     private final Set<Alias> aliases = new HashSet<>();
 
-    private final Set<ClusterBlock> blocks = new HashSet<>();
-
     private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
 
     private boolean performReroute = true;
@@ -125,10 +122,6 @@ public Set<Alias> aliases() {
         return aliases;
     }
 
-    public Set<ClusterBlock> blocks() {
-        return blocks;
-    }
-
     public Index recoverFrom() {
         return recoverFrom;
     }
@@ -229,8 +222,6 @@ public String toString() {
             + settings
             + ", aliases="
             + aliases
-            + ", blocks="
-            + blocks
             + ", waitForActiveShards="
             + waitForActiveShards
             + ", systemDataStreamDescriptor="
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
index e62205938f6fd..438f3930dee7e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
@@ -23,12 +23,12 @@
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java
index 95b1410e16565..7b8d70693f40d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java
@@ -31,10 +31,10 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java
index b6e8693acc66d..ff2b32ac5cc6c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/TransportFindDanglingIndexAction.java
@@ -15,9 +15,9 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.gateway.DanglingIndicesState;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java
index d3957be682cfd..4c90ce9875266 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java
@@ -22,9 +22,9 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.gateway.LocalAllocateDangledIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java
index f6b809ea1ea49..40ca4e6de9435 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/TransportListDanglingIndicesAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.gateway.DanglingIndicesState;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
index 35034eebfaa93..a2cda7fc1ac33 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
@@ -21,9 +21,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageAction.java
index 8380edb4cb6ed..8676daf0a55d9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageAction.java
@@ -24,13 +24,13 @@
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
index 96b4a0191b10c..bc34e3fb99e04 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
index 69e1309b89aef..52662a79d7cbd 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
@@ -16,13 +16,13 @@
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportChannel;
@@ -60,7 +60,8 @@ public TransportShardFlushAction(
             ShardFlushRequest::new,
             threadPool.executor(ThreadPool.Names.FLUSH),
             SyncGlobalCheckpointAfterOperation.DoNotSync,
-            PrimaryActionExecution.RejectOnOverload
+            PrimaryActionExecution.RejectOnOverload,
+            ReplicaActionExecution.SubjectToCircuitBreaker
         );
         transportService.registerRequestHandler(
             PRE_SYNCED_FLUSH_ACTION_NAME,
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java
index df98e8f12f18e..c474a3d40cdc9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java
@@ -20,10 +20,10 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
index eb1f823dc0302..7de14400ec2b5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
@@ -19,11 +19,11 @@
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
index 4fd770d9a3576..849977a39f2a6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
index 27516e0ad5a7f..7b024737dc70a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -29,6 +28,7 @@
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.ToXContent;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java
index f55eef4ebbae6..7691dac2e1dc4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java
index 9c3b08ef49add..6a3977bcc8d4a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java
@@ -20,10 +20,10 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataMappingService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
index ca6e97c3e1334..737ee28bbf873 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
@@ -23,12 +23,12 @@
 import org.elasticsearch.cluster.metadata.MetadataMappingService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
index e8fc62d480bc4..bd89d124b3f76 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
@@ -21,9 +21,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java
index 72731bc636b13..f5b12dbe209ac 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java
@@ -21,9 +21,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java
index e93b3983ee85b..27234976f780b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockAction.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -26,6 +25,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -69,7 +69,8 @@ public TransportVerifyShardIndexBlockAction(
             ShardRequest::new,
             threadPool.executor(ThreadPool.Names.MANAGEMENT),
             SyncGlobalCheckpointAfterOperation.DoNotSync,
-            PrimaryActionExecution.RejectOnOverload
+            PrimaryActionExecution.RejectOnOverload,
+            ReplicaActionExecution.SubjectToCircuitBreaker
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java
index c74981d475389..cec8e26ded364 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java
@@ -18,12 +18,12 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
index 5d6f60216ae05..a5d0ec36a3bfd 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
index cc4edcf0efb81..5fa9ba6662c2e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
@@ -20,12 +20,12 @@
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -70,7 +70,8 @@ public TransportShardRefreshAction(
             ShardRefreshReplicaRequest::new,
             threadPool.executor(ThreadPool.Names.REFRESH),
             SyncGlobalCheckpointAfterOperation.DoNotSync,
-            PrimaryActionExecution.RejectOnOverload
+            PrimaryActionExecution.RejectOnOverload,
+            ReplicaActionExecution.SubjectToCircuitBreaker
         );
         // registers the unpromotable version of shard refresh action
         new TransportUnpromotableShardRefreshAction(clusterService, transportService, shardStateAction, actionFilters, indicesService);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java
index b4357c69c46ae..29eaeb273a8a3 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java
@@ -14,9 +14,9 @@
 import org.elasticsearch.action.support.broadcast.unpromotable.TransportBroadcastUnpromotableAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java
index 118f139045971..224ea63150420 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java
@@ -54,12 +54,14 @@ public ResolveClusterActionRequest(String[] names) {
         this(names, DEFAULT_INDICES_OPTIONS);
     }
 
+    @SuppressWarnings("this-escape")
     public ResolveClusterActionRequest(String[] names, IndicesOptions indicesOptions) {
         this.names = names;
         this.localIndicesRequested = localIndicesPresent(names);
         this.indicesOptions = indicesOptions;
     }
 
+    @SuppressWarnings("this-escape")
     public ResolveClusterActionRequest(StreamInput in) throws IOException {
         super(in);
         if (in.getTransportVersion().before(TransportVersions.V_8_13_0)) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java
index 1f23ee724e542..1e5036a9bfc63 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -36,6 +35,7 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.RemoteClusterAware;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java
index 8b171b0d12bf5..1ac7def758ab5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java
@@ -26,10 +26,10 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java
index 29be34bc3b2c2..ef72fdd93caeb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java
@@ -29,8 +29,8 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java
index 5c7518abdbbf8..9d34b9ab5f126 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
@@ -45,6 +44,7 @@
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.indices.SystemDataStreamDescriptor;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.SnapshotInProgressException;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.telemetry.TelemetryProvider;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java
index 93a11b8fe0855..acd26f2984c99 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/OptimalShardCountCondition.java
@@ -65,6 +65,6 @@ public static OptimalShardCountCondition fromXContent(XContentParser parser) thr
 
     @Override
     boolean includedInVersion(TransportVersion version) {
-        return version.onOrAfter(TransportVersions.AUTO_SHARDING_ROLLOVER_CONDITION);
+        return version.onOrAfter(TransportVersions.V_8_14_0);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index 6302b1c9ef9fb..db4fad99d4f48 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -116,7 +116,7 @@ public RolloverRequest(StreamInput in) throws IOException {
         } else {
             lazy = false;
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_ROLLOVER)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             indicesOptions = IndicesOptions.readIndicesOptions(in);
         }
     }
@@ -168,7 +168,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
             out.writeBoolean(lazy);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_ROLLOVER)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             indicesOptions.writeIndicesOptions(out);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index d76cfedd279b5..9df3be1994fdf 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -44,11 +44,11 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
index bd12cfdbc7962..429ebe365bbe1 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
@@ -36,7 +36,7 @@ public class IndicesSegmentResponse extends ChunkedBroadcastResponse {
 
     private volatile Map<String, IndexSegments> indicesSegments;
 
-    IndicesSegmentResponse(
+    public IndicesSegmentResponse(
         ShardSegments[] shards,
         int totalShards,
         int successfulShards,
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java
index 8cff400e6dde2..f5a79eec7ab33 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java
@@ -18,11 +18,11 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java
index e11c5a1a6103d..7396e1d383bd5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -25,6 +24,7 @@
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
index 36c0634fa9dba..25a22ae52ac61 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
@@ -23,12 +23,12 @@
 import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
index 4b1c5b3f58dd5..08399cdd3f9cb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThrottledIterator;
@@ -41,6 +40,7 @@
 import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards;
 import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
index 8a2eb18b5164f..0fe8ee9670f00 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
@@ -27,12 +27,12 @@
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.routing.IndexRouting;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java
index d463c01bfda81..db49c199e05b0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java
@@ -18,11 +18,11 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
index dcde4ef6047c1..4c042570704d9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.CommitStats;
@@ -27,6 +26,7 @@
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java
index 9ac10d782a605..3fcdd96feaf5f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java
@@ -23,10 +23,10 @@
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java
index fa40a901c705b..2c6a0a2945abb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java
@@ -23,10 +23,10 @@
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
index 8d3a83a929389..757930bc0505b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
@@ -20,8 +20,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java
index 5483097b140da..f0552cc3226f5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java
@@ -121,8 +121,6 @@ public static class Response extends ActionResponse implements ToXContentObject
         private final Map<String, ComponentTemplate> componentTemplates;
         @Nullable
         private final RolloverConfiguration rolloverConfiguration;
-        @Nullable
-        private final DataStreamGlobalRetention globalRetention;
 
         public Response(StreamInput in) throws IOException {
             super(in);
@@ -132,25 +130,39 @@ public Response(StreamInput in) throws IOException {
             } else {
                 rolloverConfiguration = null;
             }
-            if (in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) {
-                globalRetention = in.readOptionalWriteable(DataStreamGlobalRetention::read);
-            } else {
-                globalRetention = null;
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
+                && in.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) {
+                in.readOptionalWriteable(DataStreamGlobalRetention::read);
             }
         }
 
+        /**
+         * Please use {@link GetComponentTemplateAction.Response#Response(Map)}
+         */
+        @Deprecated
         public Response(Map<String, ComponentTemplate> componentTemplates, @Nullable DataStreamGlobalRetention globalRetention) {
-            this(componentTemplates, null, globalRetention);
+            this(componentTemplates, (RolloverConfiguration) null);
         }
 
+        /**
+         * Please use {@link GetComponentTemplateAction.Response#Response(Map, RolloverConfiguration)}
+         */
+        @Deprecated
         public Response(
             Map<String, ComponentTemplate> componentTemplates,
             @Nullable RolloverConfiguration rolloverConfiguration,
-            @Nullable DataStreamGlobalRetention globalRetention
+            @Nullable DataStreamGlobalRetention ignored
         ) {
+            this(componentTemplates, rolloverConfiguration);
+        }
+
+        public Response(Map<String, ComponentTemplate> componentTemplates) {
+            this(componentTemplates, (RolloverConfiguration) null);
+        }
+
+        public Response(Map<String, ComponentTemplate> componentTemplates, @Nullable RolloverConfiguration rolloverConfiguration) {
             this.componentTemplates = componentTemplates;
             this.rolloverConfiguration = rolloverConfiguration;
-            this.globalRetention = globalRetention;
         }
 
         public Map<String, ComponentTemplate> getComponentTemplates() {
@@ -161,8 +173,14 @@ public RolloverConfiguration getRolloverConfiguration() {
             return rolloverConfiguration;
         }
 
+        /**
+         * @return null
+         * @deprecated The global retention is not used anymore in the component template response
+         */
+        @Deprecated
+        @Nullable
         public DataStreamGlobalRetention getGlobalRetention() {
-            return globalRetention;
+            return null;
         }
 
         @Override
@@ -171,8 +189,9 @@ public void writeTo(StreamOutput out) throws IOException {
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
                 out.writeOptionalWriteable(rolloverConfiguration);
             }
-            if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) {
-                out.writeOptionalWriteable(globalRetention);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
+                && out.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) {
+                out.writeOptionalWriteable(null);
             }
         }
 
@@ -182,13 +201,12 @@ public boolean equals(Object o) {
             if (o == null || getClass() != o.getClass()) return false;
             Response that = (Response) o;
             return Objects.equals(componentTemplates, that.componentTemplates)
-                && Objects.equals(rolloverConfiguration, that.rolloverConfiguration)
-                && Objects.equals(globalRetention, that.globalRetention);
+                && Objects.equals(rolloverConfiguration, that.rolloverConfiguration);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(componentTemplates, rolloverConfiguration, globalRetention);
+            return Objects.hash(componentTemplates, rolloverConfiguration);
         }
 
         @Override
@@ -208,5 +226,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
 
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java
index 5cb35d23c8b7c..ba07c87e753e6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java
@@ -122,8 +122,6 @@ public static class Response extends ActionResponse implements ToXContentObject
         private final Map<String, ComposableIndexTemplate> indexTemplates;
         @Nullable
         private final RolloverConfiguration rolloverConfiguration;
-        @Nullable
-        private final DataStreamGlobalRetention globalRetention;
 
         public Response(StreamInput in) throws IOException {
             super(in);
@@ -133,33 +131,57 @@ public Response(StreamInput in) throws IOException {
             } else {
                 rolloverConfiguration = null;
             }
-            if (in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) {
-                globalRetention = in.readOptionalWriteable(DataStreamGlobalRetention::read);
-            } else {
-                globalRetention = null;
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
+                && in.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) {
+                in.readOptionalWriteable(DataStreamGlobalRetention::read);
             }
         }
 
+        /**
+         * Please use {@link GetComposableIndexTemplateAction.Response#Response(Map)}
+         */
         public Response(Map<String, ComposableIndexTemplate> indexTemplates, @Nullable DataStreamGlobalRetention globalRetention) {
-            this(indexTemplates, null, globalRetention);
+            this(indexTemplates, (RolloverConfiguration) null);
         }
 
+        /**
+         * Please use {@link GetComposableIndexTemplateAction.Response#Response(Map, RolloverConfiguration)}
+         */
+        @Deprecated
         public Response(
             Map<String, ComposableIndexTemplate> indexTemplates,
             @Nullable RolloverConfiguration rolloverConfiguration,
             @Nullable DataStreamGlobalRetention globalRetention
         ) {
+            this(indexTemplates, rolloverConfiguration);
+        }
+
+        public Response(Map<String, ComposableIndexTemplate> indexTemplates) {
+            this(indexTemplates, (RolloverConfiguration) null);
+        }
+
+        public Response(Map<String, ComposableIndexTemplate> indexTemplates, @Nullable RolloverConfiguration rolloverConfiguration) {
             this.indexTemplates = indexTemplates;
             this.rolloverConfiguration = rolloverConfiguration;
-            this.globalRetention = globalRetention;
         }
 
         public Map<String, ComposableIndexTemplate> indexTemplates() {
             return indexTemplates;
         }
 
+        /**
+         * @return null
+         * @deprecated global retention is not used in composable templates anymore
+         */
+        @Deprecated
+        @Nullable
         public DataStreamGlobalRetention getGlobalRetention() {
-            return globalRetention;
+            return null;
+        }
+
+        @Nullable
+        public RolloverConfiguration getRolloverConfiguration() {
+            return rolloverConfiguration;
         }
 
         @Override
@@ -168,8 +190,9 @@ public void writeTo(StreamOutput out) throws IOException {
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
                 out.writeOptionalWriteable(rolloverConfiguration);
             }
-            if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) {
-                out.writeOptionalWriteable(globalRetention);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
+                && out.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) {
+                out.writeOptionalWriteable(null);
             }
         }
 
@@ -178,14 +201,12 @@ public boolean equals(Object o) {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
             GetComposableIndexTemplateAction.Response that = (GetComposableIndexTemplateAction.Response) o;
-            return Objects.equals(indexTemplates, that.indexTemplates)
-                && Objects.equals(rolloverConfiguration, that.rolloverConfiguration)
-                && Objects.equals(globalRetention, that.globalRetention);
+            return Objects.equals(indexTemplates, that.indexTemplates) && Objects.equals(rolloverConfiguration, that.rolloverConfiguration);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(indexTemplates, rolloverConfiguration, globalRetention);
+            return Objects.hash(indexTemplates, rolloverConfiguration);
         }
 
         @Override
@@ -203,7 +224,5 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.endObject();
             return builder;
         }
-
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java
index 0f088e046dd52..fcc053b8181fa 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java
@@ -16,14 +16,13 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -36,7 +35,6 @@ public class TransportGetComponentTemplateAction extends TransportMasterNodeRead
     GetComponentTemplateAction.Response> {
 
     private final ClusterSettings clusterSettings;
-    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
 
     @Inject
     public TransportGetComponentTemplateAction(
@@ -44,8 +42,7 @@ public TransportGetComponentTemplateAction(
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver,
-        DataStreamGlobalRetentionResolver globalRetentionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver
     ) {
         super(
             GetComponentTemplateAction.NAME,
@@ -59,7 +56,6 @@ public TransportGetComponentTemplateAction(
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         clusterSettings = clusterService.getClusterSettings();
-        this.globalRetentionResolver = globalRetentionResolver;
     }
 
     @Override
@@ -100,12 +96,11 @@ protected void masterOperation(
             listener.onResponse(
                 new GetComponentTemplateAction.Response(
                     results,
-                    clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING),
-                    globalRetentionResolver.resolve(state)
+                    clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING)
                 )
             );
         } else {
-            listener.onResponse(new GetComponentTemplateAction.Response(results, globalRetentionResolver.resolve(state)));
+            listener.onResponse(new GetComponentTemplateAction.Response(results));
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java
index 6d6de026ced5a..e2ce172a1bf0b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java
@@ -16,14 +16,13 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -36,7 +35,6 @@ public class TransportGetComposableIndexTemplateAction extends TransportMasterNo
     GetComposableIndexTemplateAction.Response> {
 
     private final ClusterSettings clusterSettings;
-    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
 
     @Inject
     public TransportGetComposableIndexTemplateAction(
@@ -44,8 +42,7 @@ public TransportGetComposableIndexTemplateAction(
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver,
-        DataStreamGlobalRetentionResolver globalRetentionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver
     ) {
         super(
             GetComposableIndexTemplateAction.NAME,
@@ -59,7 +56,6 @@ public TransportGetComposableIndexTemplateAction(
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         clusterSettings = clusterService.getClusterSettings();
-        this.globalRetentionResolver = globalRetentionResolver;
     }
 
     @Override
@@ -98,12 +94,11 @@ protected void masterOperation(
             listener.onResponse(
                 new GetComposableIndexTemplateAction.Response(
                     results,
-                    clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING),
-                    globalRetentionResolver.resolve(state)
+                    clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING)
                 )
             );
         } else {
-            listener.onResponse(new GetComposableIndexTemplateAction.Response(results, globalRetentionResolver.resolve(state)));
+            listener.onResponse(new GetComposableIndexTemplateAction.Response(results));
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
index ae5bbe38de801..40815e87d26f7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
index 5d0a4a293ea4f..a27defd2c655c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java
@@ -46,27 +46,19 @@ public class SimulateIndexTemplateResponse extends ActionResponse implements ToX
 
     @Nullable
     private final RolloverConfiguration rolloverConfiguration;
-    @Nullable
-    private final DataStreamGlobalRetention globalRetention;
 
-    public SimulateIndexTemplateResponse(
-        @Nullable Template resolvedTemplate,
-        @Nullable Map<String, List<String>> overlappingTemplates,
-        DataStreamGlobalRetention globalRetention
-    ) {
-        this(resolvedTemplate, overlappingTemplates, null, globalRetention);
+    public SimulateIndexTemplateResponse(@Nullable Template resolvedTemplate, @Nullable Map<String, List<String>> overlappingTemplates) {
+        this(resolvedTemplate, overlappingTemplates, null);
     }
 
     public SimulateIndexTemplateResponse(
         @Nullable Template resolvedTemplate,
         @Nullable Map<String, List<String>> overlappingTemplates,
-        @Nullable RolloverConfiguration rolloverConfiguration,
-        @Nullable DataStreamGlobalRetention globalRetention
+        @Nullable RolloverConfiguration rolloverConfiguration
     ) {
         this.resolvedTemplate = resolvedTemplate;
         this.overlappingTemplates = overlappingTemplates;
         this.rolloverConfiguration = rolloverConfiguration;
-        this.globalRetention = globalRetention;
     }
 
     public RolloverConfiguration getRolloverConfiguration() {
@@ -89,9 +81,10 @@ public SimulateIndexTemplateResponse(StreamInput in) throws IOException {
         rolloverConfiguration = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)
             ? in.readOptionalWriteable(RolloverConfiguration::new)
             : null;
-        globalRetention = in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)
-            ? in.readOptionalWriteable(DataStreamGlobalRetention::read)
-            : null;
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
+            && in.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) {
+            in.readOptionalWriteable(DataStreamGlobalRetention::read);
+        }
     }
 
     @Override
@@ -110,8 +103,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
             out.writeOptionalWriteable(rolloverConfiguration);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) {
-            out.writeOptionalWriteable(globalRetention);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
+            && out.getTransportVersion().before(TransportVersions.REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES)) {
+            out.writeOptionalWriteable(null);
         }
     }
 
@@ -147,13 +141,12 @@ public boolean equals(Object o) {
         SimulateIndexTemplateResponse that = (SimulateIndexTemplateResponse) o;
         return Objects.equals(resolvedTemplate, that.resolvedTemplate)
             && Objects.deepEquals(overlappingTemplates, that.overlappingTemplates)
-            && Objects.equals(rolloverConfiguration, that.rolloverConfiguration)
-            && Objects.equals(globalRetention, that.globalRetention);
+            && Objects.equals(rolloverConfiguration, that.rolloverConfiguration);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(resolvedTemplate, overlappingTemplates, rolloverConfiguration, globalRetention);
+        return Objects.hash(resolvedTemplate, overlappingTemplates, rolloverConfiguration);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java
index eb90e95cb08be..6fcaad47e0d72 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java
@@ -16,8 +16,6 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -28,7 +26,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -41,6 +38,7 @@
 import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -74,7 +72,6 @@ public class TransportSimulateIndexTemplateAction extends TransportMasterNodeRea
     private final Set<IndexSettingProvider> indexSettingProviders;
     private final ClusterSettings clusterSettings;
     private final boolean isDslOnlyMode;
-    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
 
     @Inject
     public TransportSimulateIndexTemplateAction(
@@ -87,8 +84,7 @@ public TransportSimulateIndexTemplateAction(
         NamedXContentRegistry xContentRegistry,
         IndicesService indicesService,
         SystemIndices systemIndices,
-        IndexSettingProviders indexSettingProviders,
-        DataStreamGlobalRetentionResolver globalRetentionResolver
+        IndexSettingProviders indexSettingProviders
     ) {
         super(
             SimulateIndexTemplateAction.NAME,
@@ -108,7 +104,6 @@ public TransportSimulateIndexTemplateAction(
         this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders();
         this.clusterSettings = clusterService.getClusterSettings();
         this.isDslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings());
-        this.globalRetentionResolver = globalRetentionResolver;
     }
 
     @Override
@@ -118,7 +113,6 @@ protected void masterOperation(
         ClusterState state,
         ActionListener<SimulateIndexTemplateResponse> listener
     ) throws Exception {
-        final DataStreamGlobalRetention globalRetention = globalRetentionResolver.resolve(state);
         final ClusterState stateWithTemplate;
         if (request.getIndexTemplateRequest() != null) {
             // we'll "locally" add the template defined by the user in the cluster state (as if it existed in the system)
@@ -144,7 +138,7 @@ protected void masterOperation(
 
         String matchingTemplate = findV2Template(stateWithTemplate.metadata(), request.getIndexName(), false);
         if (matchingTemplate == null) {
-            listener.onResponse(new SimulateIndexTemplateResponse(null, null, null));
+            listener.onResponse(new SimulateIndexTemplateResponse(null, null));
             return;
         }
 
@@ -172,12 +166,11 @@ protected void masterOperation(
                 new SimulateIndexTemplateResponse(
                     template,
                     overlapping,
-                    clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING),
-                    globalRetention
+                    clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING)
                 )
             );
         } else {
-            listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping, globalRetention));
+            listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping));
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java
index b1d6d2814f157..ead00dc858a47 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java
@@ -15,21 +15,19 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettingProviders;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -60,7 +58,6 @@ public class TransportSimulateTemplateAction extends TransportMasterNodeReadActi
     private final Set<IndexSettingProvider> indexSettingProviders;
     private final ClusterSettings clusterSettings;
     private final boolean isDslOnlyMode;
-    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
 
     @Inject
     public TransportSimulateTemplateAction(
@@ -73,8 +70,7 @@ public TransportSimulateTemplateAction(
         NamedXContentRegistry xContentRegistry,
         IndicesService indicesService,
         SystemIndices systemIndices,
-        IndexSettingProviders indexSettingProviders,
-        DataStreamGlobalRetentionResolver globalRetentionResolver
+        IndexSettingProviders indexSettingProviders
     ) {
         super(
             SimulateTemplateAction.NAME,
@@ -94,7 +90,6 @@ public TransportSimulateTemplateAction(
         this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders();
         this.clusterSettings = clusterService.getClusterSettings();
         this.isDslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings());
-        this.globalRetentionResolver = globalRetentionResolver;
     }
 
     @Override
@@ -104,7 +99,6 @@ protected void masterOperation(
         ClusterState state,
         ActionListener<SimulateIndexTemplateResponse> listener
     ) throws Exception {
-        final DataStreamGlobalRetention globalRetention = globalRetentionResolver.resolve(state);
         String uuid = UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT);
         final String temporaryIndexName = "simulate_template_index_" + uuid;
         final ClusterState stateWithTemplate;
@@ -182,12 +176,11 @@ protected void masterOperation(
                 new SimulateIndexTemplateResponse(
                     template,
                     overlapping,
-                    clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING),
-                    globalRetention
+                    clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING)
                 )
             );
         } else {
-            listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping, globalRetention));
+            listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping));
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
index 335c0781fb884..38e9918aae853 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
@@ -22,10 +22,10 @@
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java
index 86c6109469477..34a0d2e358acf 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java
@@ -28,12 +28,12 @@
 import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
index 5e053a1e9fc7c..8611d2688f5d2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
@@ -22,10 +22,10 @@
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
index 1c3d168b8889d..5e0d9c9338e0c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
@@ -26,13 +26,13 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.Rewriteable;
 import org.elasticsearch.indices.IndexClosedException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContext;
@@ -121,7 +121,11 @@ protected void doExecute(Task task, ValidateQueryRequest request, ActionListener
         if (request.query() == null) {
             rewriteListener.onResponse(request.query());
         } else {
-            Rewriteable.rewriteAndFetch(request.query(), searchService.getRewriteContext(timeProvider, resolvedIndices), rewriteListener);
+            Rewriteable.rewriteAndFetch(
+                request.query(),
+                searchService.getRewriteContext(timeProvider, resolvedIndices, null),
+                rewriteListener
+            );
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java
index 425461d1f4ba1..7c1304f92eefd 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java
@@ -101,11 +101,11 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(primaryResponse);
     }
 
-    public void writeThin(StreamOutput out) throws IOException {
-        out.writeVInt(id);
-        DocWriteRequest.writeDocumentRequestThin(out, request);
-        out.writeOptionalWriteable(primaryResponse == null ? null : primaryResponse::writeThin);
-    }
+    public static final Writer<BulkItemRequest> THIN_WRITER = (out, item) -> {
+        out.writeVInt(item.id);
+        DocWriteRequest.writeDocumentRequestThin(out, item.request);
+        out.writeOptional(BulkItemResponse.THIN_WRITER, item.primaryResponse);
+    };
 
     @Override
     public long ramBytesUsed() {
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
index 151e8795d0f82..d3e550eaf05b3 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
@@ -264,7 +264,7 @@ public String toString() {
         id = in.readVInt();
         opType = OpType.fromId(in.readByte());
         response = readResponse(shardId, in);
-        failure = in.readBoolean() ? new Failure(in) : null;
+        failure = in.readOptionalWriteable(Failure::new);
         assertConsistent();
     }
 
@@ -272,7 +272,7 @@ public String toString() {
         id = in.readVInt();
         opType = OpType.fromId(in.readByte());
         response = readResponse(in);
-        failure = in.readBoolean() ? new Failure(in) : null;
+        failure = in.readOptionalWriteable(Failure::new);
         assertConsistent();
     }
 
@@ -384,31 +384,21 @@ public void writeTo(StreamOutput out) throws IOException {
             writeResponseType(out);
             response.writeTo(out);
         }
-        if (failure == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            failure.writeTo(out);
-        }
+        out.writeOptionalWriteable(failure);
     }
 
-    public void writeThin(StreamOutput out) throws IOException {
-        out.writeVInt(id);
-        out.writeByte(opType.getId());
+    public static final Writer<BulkItemResponse> THIN_WRITER = (out, item) -> {
+        out.writeVInt(item.id);
+        out.writeByte(item.opType.getId());
 
-        if (response == null) {
+        if (item.response == null) {
             out.writeByte((byte) 2);
         } else {
-            writeResponseType(out);
-            response.writeThin(out);
+            item.writeResponseType(out);
+            item.response.writeThin(out);
         }
-        if (failure == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            failure.writeTo(out);
-        }
-    }
+        out.writeOptionalWriteable(item.failure);
+    };
 
     private void writeResponseType(StreamOutput out) throws IOException {
         if (response instanceof SimulateIndexResponse) {
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java
index 258e5b4c9a58d..813203afe42c5 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java
@@ -10,7 +10,9 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
@@ -91,6 +93,7 @@ final class BulkOperation extends ActionRunnable<BulkResponse> {
     private final OriginSettingClient rolloverClient;
     private final Set<String> failureStoresToBeRolledOver = ConcurrentCollections.newConcurrentSet();
     private final Set<Integer> failedRolloverRequests = ConcurrentCollections.newConcurrentSet();
+    private final FailureStoreMetrics failureStoreMetrics;
 
     BulkOperation(
         Task task,
@@ -104,7 +107,8 @@ final class BulkOperation extends ActionRunnable<BulkResponse> {
         IndexNameExpressionResolver indexNameExpressionResolver,
         LongSupplier relativeTimeProvider,
         long startTimeNanos,
-        ActionListener<BulkResponse> listener
+        ActionListener<BulkResponse> listener,
+        FailureStoreMetrics failureStoreMetrics
     ) {
         this(
             task,
@@ -120,7 +124,8 @@ final class BulkOperation extends ActionRunnable<BulkResponse> {
             startTimeNanos,
             listener,
             new ClusterStateObserver(clusterService, bulkRequest.timeout(), logger, threadPool.getThreadContext()),
-            new FailureStoreDocumentConverter()
+            new FailureStoreDocumentConverter(),
+            failureStoreMetrics
         );
     }
 
@@ -138,7 +143,8 @@ final class BulkOperation extends ActionRunnable<BulkResponse> {
         long startTimeNanos,
         ActionListener<BulkResponse> listener,
         ClusterStateObserver observer,
-        FailureStoreDocumentConverter failureStoreDocumentConverter
+        FailureStoreDocumentConverter failureStoreDocumentConverter,
+        FailureStoreMetrics failureStoreMetrics
     ) {
         super(listener);
         this.task = task;
@@ -156,6 +162,7 @@ final class BulkOperation extends ActionRunnable<BulkResponse> {
         this.observer = observer;
         this.failureStoreDocumentConverter = failureStoreDocumentConverter;
         this.rolloverClient = new OriginSettingClient(client, LAZY_ROLLOVER_ORIGIN);
+        this.failureStoreMetrics = failureStoreMetrics;
     }
 
     @Override
@@ -437,17 +444,11 @@ public void onResponse(BulkShardResponse bulkShardResponse) {
                 for (int idx = 0; idx < bulkShardResponse.getResponses().length; idx++) {
                     // We zip the requests and responses together so that we can identify failed documents and potentially store them
                     BulkItemResponse bulkItemResponse = bulkShardResponse.getResponses()[idx];
+                    BulkItemRequest bulkItemRequest = bulkShardRequest.items()[idx];
 
                     if (bulkItemResponse.isFailed()) {
-                        BulkItemRequest bulkItemRequest = bulkShardRequest.items()[idx];
                         assert bulkItemRequest.id() == bulkItemResponse.getItemId() : "Bulk items were returned out of order";
-
-                        DataStream failureStoreReference = getRedirectTarget(bulkItemRequest.request(), getClusterState().metadata());
-                        if (failureStoreReference != null) {
-                            maybeMarkFailureStoreForRollover(failureStoreReference);
-                            var cause = bulkItemResponse.getFailure().getCause();
-                            addDocumentToRedirectRequests(bulkItemRequest, cause, failureStoreReference.getName());
-                        }
+                        processFailure(bulkItemRequest, bulkItemResponse.getFailure().getCause());
                         addFailure(bulkItemResponse);
                     } else {
                         bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
@@ -464,11 +465,7 @@ public void onFailure(Exception e) {
                     final String indexName = request.index();
                     DocWriteRequest<?> docWriteRequest = request.request();
 
-                    DataStream failureStoreReference = getRedirectTarget(docWriteRequest, getClusterState().metadata());
-                    if (failureStoreReference != null) {
-                        maybeMarkFailureStoreForRollover(failureStoreReference);
-                        addDocumentToRedirectRequests(request, e, failureStoreReference.getName());
-                    }
+                    processFailure(request, e);
                     addFailure(docWriteRequest, request.id(), indexName, e);
                 }
                 completeShardOperation();
@@ -479,45 +476,56 @@ private void completeShardOperation() {
                 clusterState = null;
                 releaseOnFinish.close();
             }
+
+            private void processFailure(BulkItemRequest bulkItemRequest, Exception cause) {
+                var errorType = ElasticsearchException.getExceptionName(ExceptionsHelper.unwrapCause(cause));
+                DocWriteRequest<?> docWriteRequest = bulkItemRequest.request();
+                DataStream failureStoreCandidate = getRedirectTargetCandidate(docWriteRequest, getClusterState().metadata());
+                // If the candidate is not null, the BulkItemRequest targets a data stream, but we'll still have to check if
+                // it has the failure store enabled.
+                if (failureStoreCandidate != null) {
+                    // Do not redirect documents to a failure store that were already headed to one.
+                    var isFailureStoreDoc = docWriteRequest instanceof IndexRequest indexRequest && indexRequest.isWriteToFailureStore();
+                    if (isFailureStoreDoc == false && failureStoreCandidate.isFailureStoreEnabled()) {
+                        // Redirect to failure store.
+                        maybeMarkFailureStoreForRollover(failureStoreCandidate);
+                        addDocumentToRedirectRequests(bulkItemRequest, cause, failureStoreCandidate.getName());
+                        failureStoreMetrics.incrementFailureStore(
+                            bulkItemRequest.index(),
+                            errorType,
+                            FailureStoreMetrics.ErrorLocation.SHARD
+                        );
+                    } else {
+                        // If we can't redirect to a failure store (because either the data stream doesn't have the failure store enabled
+                        // or this request was already targeting a failure store), we increment the rejected counter.
+                        failureStoreMetrics.incrementRejected(
+                            bulkItemRequest.index(),
+                            errorType,
+                            FailureStoreMetrics.ErrorLocation.SHARD,
+                            isFailureStoreDoc
+                        );
+                    }
+                }
+            }
         });
     }
 
     /**
-     * Determines if the write request can be redirected if it fails. Write requests can be redirected IFF they are targeting a data stream
-     * with a failure store and are not already redirected themselves. If the document can be redirected, the data stream name to use for
-     * the redirection is returned.
+     * Tries to find a <i>candidate</i> redirect target for this write request. A candidate redirect target is a data stream that may or
+     * may not have the failure store enabled.
      *
      * @param docWriteRequest the write request to check
      * @param metadata cluster state metadata for resolving index abstractions
-     * @return a data stream if the write request points to a data stream that has the failure store enabled, or {@code null} if it does not
+     * @return a data stream if the write request points to a data stream, or {@code null} if it does not
      */
-    private static DataStream getRedirectTarget(DocWriteRequest<?> docWriteRequest, Metadata metadata) {
+    private static DataStream getRedirectTargetCandidate(DocWriteRequest<?> docWriteRequest, Metadata metadata) {
         // Feature flag guard
         if (DataStream.isFailureStoreFeatureFlagEnabled() == false) {
             return null;
         }
-        // Do not resolve a failure store for documents that were already headed to one
-        if (docWriteRequest instanceof IndexRequest indexRequest && indexRequest.isWriteToFailureStore()) {
-            return null;
-        }
         // If there is no index abstraction, then the request is using a pattern of some sort, which data streams do not support
         IndexAbstraction ia = metadata.getIndicesLookup().get(docWriteRequest.index());
-        if (ia == null) {
-            return null;
-        }
-        if (ia.isDataStreamRelated()) {
-            // The index abstraction could be an alias. Alias abstractions (even for data streams) only keep track of which _index_ they
-            // will write to, not which _data stream_.
-            // We work backward to find the data stream from the concrete write index to cover this case.
-            Index concreteIndex = ia.getWriteIndex();
-            IndexAbstraction writeIndexAbstraction = metadata.getIndicesLookup().get(concreteIndex.getName());
-            DataStream parentDataStream = writeIndexAbstraction.getParentDataStream();
-            if (parentDataStream != null && parentDataStream.isFailureStoreEnabled()) {
-                // Keep the data stream name around to resolve the redirect to failure store if the shard level request fails.
-                return parentDataStream;
-            }
-        }
-        return null;
+        return DataStream.resolveDataStream(ia, metadata);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java
index ac3a6be70d29f..28eef30f9185d 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkPrimaryExecutionContext.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.translog.Translog;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
 
 import java.util.Arrays;
 import java.util.List;
@@ -63,7 +62,6 @@ enum ItemProcessingState {
     private BulkItemResponse executionResult;
     private int updateRetryCounter;
     private long noopMappingUpdateRetryForMappingVersion;
-    private DocumentSizeObserver documentSizeObserver = DocumentSizeObserver.EMPTY_INSTANCE;
 
     BulkPrimaryExecutionContext(BulkShardRequest request, IndexShard primary) {
         this.request = request;
@@ -369,12 +367,4 @@ private boolean assertInvariants(ItemProcessingState... expectedCurrentState) {
         }
         return true;
     }
-
-    public void setDocumentSizeObserver(DocumentSizeObserver documentSizeObserver) {
-        this.documentSizeObserver = documentSizeObserver;
-    }
-
-    public DocumentSizeObserver getDocumentSizeObserver() {
-        return documentSizeObserver;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
index 83b572afb2853..1a8bdb1c885c6 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
@@ -87,6 +87,9 @@ public BulkRequest(StreamInput in) throws IOException {
         requests.addAll(in.readCollectionAsList(i -> DocWriteRequest.readDocumentRequest(null, i)));
         refreshPolicy = RefreshPolicy.readFrom(in);
         timeout = in.readTimeValue();
+        for (DocWriteRequest<?> request : requests) {
+            indices.add(Objects.requireNonNull(request.index(), "request index must not be null"));
+        }
     }
 
     public BulkRequest(@Nullable String globalIndex) {
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java
index 75ab08de942dc..898bfd0e1652c 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java
@@ -139,7 +139,7 @@ public void parse(
         XContent xContent = xContentType.xContent();
         int line = 0;
         int from = 0;
-        byte marker = xContent.streamSeparator();
+        byte marker = xContent.bulkSeparator();
         // Bulk requests can contain a lot of repeated strings for the index, pipeline and routing parameters. This map is used to
         // deduplicate duplicate strings parsed for these parameters. While it does not prevent instantiating the duplicate strings, it
         // reduces their lifetime to the lifetime of this parse call instead of the lifetime of the full bulk request.
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
index 0d2942e688382..f7860c47d8b73 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
@@ -130,14 +130,7 @@ public void writeTo(StreamOutput out) throws IOException {
             throw new IllegalStateException("Inference metadata should have been consumed before writing to the stream");
         }
         super.writeTo(out);
-        out.writeArray((o, item) -> {
-            if (item != null) {
-                o.writeBoolean(true);
-                item.writeThin(o);
-            } else {
-                o.writeBoolean(false);
-            }
-        }, items);
+        out.writeArray((o, item) -> o.writeOptional(BulkItemRequest.THIN_WRITER, item), items);
         if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_VALIDATES_MAPPINGS)) {
             out.writeBoolean(isSimulated);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java
index 3eeb96546c9b0..eb1bb0468c9bb 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java
@@ -56,6 +56,6 @@ public void setForcedRefresh(boolean forcedRefresh) {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         shardId.writeTo(out);
-        out.writeArray((o, item) -> item.writeThin(o), responses);
+        out.writeArray(BulkItemResponse.THIN_WRITER, responses);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java
index cc9f9b8ee1ce7..527a886905aaf 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java
@@ -32,6 +32,8 @@
  */
 public class FailureStoreDocumentConverter {
 
+    private static final int STACKTRACE_PRINT_DEPTH = 2;
+
     private static final Set<String> INGEST_EXCEPTION_HEADERS = Set.of(
         PIPELINE_ORIGIN_EXCEPTION_HEADER,
         PROCESSOR_TAG_EXCEPTION_HEADER,
@@ -68,20 +70,14 @@ public IndexRequest transformFailedRequest(
         Supplier<Long> timeSupplier
     ) throws IOException {
         return new IndexRequest().index(targetIndexName)
-            .source(createSource(source, exception, targetIndexName, timeSupplier))
+            .source(createSource(source, exception, timeSupplier))
             .opType(DocWriteRequest.OpType.CREATE)
             .setWriteToFailureStore(true);
     }
 
-    private static XContentBuilder createSource(
-        IndexRequest source,
-        Exception exception,
-        String targetIndexName,
-        Supplier<Long> timeSupplier
-    ) throws IOException {
+    private static XContentBuilder createSource(IndexRequest source, Exception exception, Supplier<Long> timeSupplier) throws IOException {
         Objects.requireNonNull(source, "source must not be null");
         Objects.requireNonNull(exception, "exception must not be null");
-        Objects.requireNonNull(targetIndexName, "targetIndexName must not be null");
         Objects.requireNonNull(timeSupplier, "timeSupplier must not be null");
         Throwable unwrapped = ExceptionsHelper.unwrapCause(exception);
         XContentBuilder builder = JsonXContent.contentBuilder();
@@ -96,7 +92,9 @@ private static XContentBuilder createSource(
                 if (source.routing() != null) {
                     builder.field("routing", source.routing());
                 }
-                builder.field("index", targetIndexName);
+                if (source.index() != null) {
+                    builder.field("index", source.index());
+                }
                 // Unmapped source field
                 builder.startObject("source");
                 {
@@ -109,7 +107,7 @@ private static XContentBuilder createSource(
             {
                 builder.field("type", ElasticsearchException.getExceptionName(unwrapped));
                 builder.field("message", unwrapped.getMessage());
-                builder.field("stack_trace", ExceptionsHelper.stackTrace(unwrapped));
+                builder.field("stack_trace", ExceptionsHelper.limitedStackTrace(unwrapped, STACKTRACE_PRINT_DEPTH));
                 // Try to find the IngestProcessorException somewhere in the stack trace. Since IngestProcessorException is package-private,
                 // we can't instantiate it in tests, so we'll have to check for the headers directly.
                 var ingestException = ExceptionsHelper.<ElasticsearchException>unwrapCausesAndSuppressed(
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreMetrics.java b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreMetrics.java
new file mode 100644
index 0000000000000..5a36f10785790
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreMetrics.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.telemetry.metric.LongCounter;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+import java.util.Map;
+
+/**
+ * A class containing APM metrics for failure stores. See the JavaDoc on the individual methods for an explanation on what they're tracking.
+ * General notes:
+ * <ul>
+ *     <li>When a document is rerouted in a pipeline, the destination data stream is used for the metric attribute(s).</li>
+ * </ul>
+ */
+public class FailureStoreMetrics {
+
+    public static final FailureStoreMetrics NOOP = new FailureStoreMetrics(MeterRegistry.NOOP);
+
+    public static final String METRIC_TOTAL = "es.data_stream.ingest.documents.total";
+    public static final String METRIC_FAILURE_STORE = "es.data_stream.ingest.documents.failure_store.total";
+    public static final String METRIC_REJECTED = "es.data_stream.ingest.documents.rejected.total";
+
+    private final LongCounter totalCounter;
+    private final LongCounter failureStoreCounter;
+    private final LongCounter rejectedCounter;
+
+    public FailureStoreMetrics(MeterRegistry meterRegistry) {
+        totalCounter = meterRegistry.registerLongCounter(METRIC_TOTAL, "total number of documents that were sent to a data stream", "unit");
+        failureStoreCounter = meterRegistry.registerLongCounter(
+            METRIC_FAILURE_STORE,
+            "number of documents that got redirected to the failure store",
+            "unit"
+        );
+        rejectedCounter = meterRegistry.registerLongCounter(METRIC_REJECTED, "number of documents that were rejected", "unit");
+    }
+
+    /**
+     * This counter tracks the number of documents that we <i>tried</i> to index into a data stream. This includes documents
+     * that were dropped by a pipeline. This counter will only be incremented once for every incoming document (even when it gets
+     * redirected to the failure store and/or gets rejected).
+     * @param dataStream the name of the data stream
+     */
+    public void incrementTotal(String dataStream) {
+        totalCounter.incrementBy(1, Map.of("data_stream", dataStream));
+    }
+
+    /**
+     * This counter tracks the number of documents that we <i>tried</i> to store into a failure store. This includes both pipeline and
+     * shard-level failures.
+     * @param dataStream the name of the data stream
+     * @param errorType the error type (i.e. the name of the exception that was thrown)
+     * @param errorLocation where this failure occurred
+     */
+    public void incrementFailureStore(String dataStream, String errorType, ErrorLocation errorLocation) {
+        failureStoreCounter.incrementBy(
+            1,
+            Map.of("data_stream", dataStream, "error_type", errorType, "error_location", errorLocation.name())
+        );
+    }
+
+    /**
+     * This counter tracks the number of documents that failed to get stored in Elasticsearch. Meaning, any document that did not get
+     * stored in the data stream or in its failure store.
+     * @param dataStream the name of the data stream
+     * @param errorType the error type (i.e. the name of the exception that was thrown)
+     * @param errorLocation where this failure occurred
+     * @param failureStore whether this failure occurred while trying to ingest into a failure store (<code>true</code>) or in the data
+     * stream itself (<code>false</code>)
+     */
+    public void incrementRejected(String dataStream, String errorType, ErrorLocation errorLocation, boolean failureStore) {
+        rejectedCounter.incrementBy(
+            1,
+            Map.of(
+                "data_stream",
+                dataStream,
+                "error_type",
+                errorType,
+                "error_location",
+                errorLocation.name(),
+                "failure_store",
+                failureStore
+            )
+        );
+    }
+
+    public enum ErrorLocation {
+        PIPELINE,
+        SHARD;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java
index ff306cfb08745..74864abe3ec50 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java
@@ -56,7 +56,7 @@ public abstract class TransportAbstractBulkAction extends HandledTransportAction
     protected final SystemIndices systemIndices;
     private final IngestService ingestService;
     private final IngestActionForwarder ingestForwarder;
-    protected final LongSupplier relativeTimeProvider;
+    protected final LongSupplier relativeTimeNanosProvider;
     protected final Executor writeExecutor;
     protected final Executor systemWriteExecutor;
     private final ActionType<BulkResponse> bulkAction;
@@ -71,7 +71,7 @@ public TransportAbstractBulkAction(
         IngestService ingestService,
         IndexingPressure indexingPressure,
         SystemIndices systemIndices,
-        LongSupplier relativeTimeProvider
+        LongSupplier relativeTimeNanosProvider
     ) {
         super(action.name(), transportService, actionFilters, requestReader, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.threadPool = threadPool;
@@ -83,7 +83,7 @@ public TransportAbstractBulkAction(
         this.systemWriteExecutor = threadPool.executor(ThreadPool.Names.SYSTEM_WRITE);
         this.ingestForwarder = new IngestActionForwarder(transportService);
         clusterService.addStateApplier(this.ingestForwarder);
-        this.relativeTimeProvider = relativeTimeProvider;
+        this.relativeTimeNanosProvider = relativeTimeNanosProvider;
         this.bulkAction = action;
     }
 
@@ -216,13 +216,13 @@ private void processBulkIndexIngestRequest(
         Metadata metadata,
         ActionListener<BulkResponse> listener
     ) {
-        final long ingestStartTimeInNanos = System.nanoTime();
+        final long ingestStartTimeInNanos = relativeTimeNanos();
         final BulkRequestModifier bulkRequestModifier = new BulkRequestModifier(original);
         getIngestService(original).executeBulkRequest(
             original.numberOfActions(),
             () -> bulkRequestModifier,
             bulkRequestModifier::markItemAsDropped,
-            (indexName) -> shouldStoreFailure(indexName, metadata, threadPool.absoluteTimeInMillis()),
+            (indexName) -> resolveFailureStore(indexName, metadata, threadPool.absoluteTimeInMillis()),
             bulkRequestModifier::markItemForFailureStore,
             bulkRequestModifier::markItemAsFailed,
             (originalThread, exception) -> {
@@ -230,7 +230,7 @@ private void processBulkIndexIngestRequest(
                     logger.debug("failed to execute pipeline for a bulk request", exception);
                     listener.onFailure(exception);
                 } else {
-                    long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos);
+                    long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(relativeTimeNanos() - ingestStartTimeInNanos);
                     BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest();
                     ActionListener<BulkResponse> actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(
                         ingestTookInMillis,
@@ -274,13 +274,15 @@ public boolean isForceExecution() {
     /**
      * Determines if an index name is associated with either an existing data stream or a template
      * for one that has the failure store enabled.
+     *
      * @param indexName The index name to check.
      * @param metadata Cluster state metadata.
      * @param epochMillis A timestamp to use when resolving date math in the index name.
      * @return true if this is not a simulation, and the given index name corresponds to a data stream with a failure store
-     * or if it matches a template that has a data stream failure store enabled.
+     * or if it matches a template that has a data stream failure store enabled. Returns false if the index name corresponds to a
+     * data stream, but it doesn't have the failure store enabled. Returns null when it doesn't correspond to a data stream.
      */
-    protected abstract boolean shouldStoreFailure(String indexName, Metadata metadata, long epochMillis);
+    protected abstract Boolean resolveFailureStore(String indexName, Metadata metadata, long epochMillis);
 
     /**
      * Retrieves the {@link IndexRequest} from the provided {@link DocWriteRequest} for index or upsert actions.  Upserts are
@@ -307,12 +309,12 @@ protected IngestService getIngestService(BulkRequest request) {
         return ingestService;
     }
 
-    protected long relativeTime() {
-        return relativeTimeProvider.getAsLong();
+    protected long relativeTimeNanos() {
+        return relativeTimeNanosProvider.getAsLong();
     }
 
     protected long buildTookInMillis(long startTimeNanos) {
-        return TimeUnit.NANOSECONDS.toMillis(relativeTime() - startTimeNanos);
+        return TimeUnit.NANOSECONDS.toMillis(relativeTimeNanos() - startTimeNanos);
     }
 
     private void applyPipelinesAndDoInternalExecute(
@@ -321,9 +323,9 @@ private void applyPipelinesAndDoInternalExecute(
         Executor executor,
         ActionListener<BulkResponse> listener
     ) {
-        final long relativeStartTime = threadPool.relativeTimeInMillis();
+        final long relativeStartTimeNanos = relativeTimeNanos();
         if (applyPipelines(task, bulkRequest, executor, listener) == false) {
-            doInternalExecute(task, bulkRequest, executor, listener, relativeStartTime);
+            doInternalExecute(task, bulkRequest, executor, listener, relativeStartTimeNanos);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index d9d5bc92a24d1..bdda4ff487f6b 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -39,16 +39,15 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexingPressure;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -57,7 +56,6 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.concurrent.Executor;
@@ -82,6 +80,7 @@ public class TransportBulkAction extends TransportAbstractBulkAction {
     private final NodeClient client;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final OriginSettingClient rolloverClient;
+    private final FailureStoreMetrics failureStoreMetrics;
 
     @Inject
     public TransportBulkAction(
@@ -94,7 +93,8 @@ public TransportBulkAction(
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
         IndexingPressure indexingPressure,
-        SystemIndices systemIndices
+        SystemIndices systemIndices,
+        FailureStoreMetrics failureStoreMetrics
     ) {
         this(
             threadPool,
@@ -107,7 +107,8 @@ public TransportBulkAction(
             indexNameExpressionResolver,
             indexingPressure,
             systemIndices,
-            System::nanoTime
+            threadPool::relativeTimeInNanos,
+            failureStoreMetrics
         );
     }
 
@@ -122,7 +123,8 @@ public TransportBulkAction(
         IndexNameExpressionResolver indexNameExpressionResolver,
         IndexingPressure indexingPressure,
         SystemIndices systemIndices,
-        LongSupplier relativeTimeProvider
+        LongSupplier relativeTimeProvider,
+        FailureStoreMetrics failureStoreMetrics
     ) {
         this(
             TYPE,
@@ -137,7 +139,8 @@ public TransportBulkAction(
             indexNameExpressionResolver,
             indexingPressure,
             systemIndices,
-            relativeTimeProvider
+            relativeTimeProvider,
+            failureStoreMetrics
         );
     }
 
@@ -154,7 +157,8 @@ public TransportBulkAction(
         IndexNameExpressionResolver indexNameExpressionResolver,
         IndexingPressure indexingPressure,
         SystemIndices systemIndices,
-        LongSupplier relativeTimeProvider
+        LongSupplier relativeTimeProvider,
+        FailureStoreMetrics failureStoreMetrics
     ) {
         super(
             bulkAction,
@@ -173,6 +177,7 @@ public TransportBulkAction(
         this.client = client;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.rolloverClient = new OriginSettingClient(client, LAZY_ROLLOVER_ORIGIN);
+        this.failureStoreMetrics = failureStoreMetrics;
     }
 
     public static <Response extends ReplicationResponse & WriteResponse> ActionListener<BulkResponse> unwrappingSingleItemBulkResponse(
@@ -197,8 +202,10 @@ protected void doInternalExecute(
         BulkRequest bulkRequest,
         Executor executor,
         ActionListener<BulkResponse> listener,
-        long relativeStartTime
+        long relativeStartTimeNanos
     ) {
+        trackIndexRequests(bulkRequest);
+
         Map<String, CreateIndexRequest> indicesToAutoCreate = new HashMap<>();
         Set<String> dataStreamsToBeRolledOver = new HashSet<>();
         Set<String> failureStoresToBeRolledOver = new HashSet<>();
@@ -212,10 +219,31 @@ protected void doInternalExecute(
             indicesToAutoCreate,
             dataStreamsToBeRolledOver,
             failureStoresToBeRolledOver,
-            relativeStartTime
+            relativeStartTimeNanos
         );
     }
 
+    /**
+     * Track the number of index requests in our APM metrics. We'll track almost all docs here (pipeline or no pipeline,
+     * failure store or original), but some docs don't reach this place (dropped and rejected docs), so we increment for those docs in
+     * different places.
+     */
+    private void trackIndexRequests(BulkRequest bulkRequest) {
+        final Metadata metadata = clusterService.state().metadata();
+        for (DocWriteRequest<?> request : bulkRequest.requests) {
+            if (request instanceof IndexRequest == false) {
+                continue;
+            }
+            String resolvedIndexName = IndexNameExpressionResolver.resolveDateMathExpression(request.index());
+            IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(resolvedIndexName);
+            DataStream dataStream = DataStream.resolveDataStream(indexAbstraction, metadata);
+            // We only track index requests into data streams.
+            if (dataStream != null) {
+                failureStoreMetrics.incrementTotal(dataStream.getName());
+            }
+        }
+    }
+
     /**
      * Determine all the targets (i.e. indices, data streams, failure stores) that require an action before we can proceed with the bulk
      * request. Indices might need to be created, and data streams and failure stores might need to be rolled over when they're marked
@@ -309,19 +337,19 @@ protected void createMissingIndicesAndIndexData(
         Map<String, CreateIndexRequest> indicesToAutoCreate,
         Set<String> dataStreamsToBeRolledOver,
         Set<String> failureStoresToBeRolledOver,
-        long startTime
+        long startTimeNanos
     ) {
         final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
         // Optimizing when there are no prerequisite actions
         if (indicesToAutoCreate.isEmpty() && dataStreamsToBeRolledOver.isEmpty() && failureStoresToBeRolledOver.isEmpty()) {
-            executeBulk(task, bulkRequest, startTime, listener, executor, responses, Map.of());
+            executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, Map.of());
             return;
         }
         final Map<String, IndexNotFoundException> indicesThatCannotBeCreated = new HashMap<>();
         Runnable executeBulkRunnable = () -> executor.execute(new ActionRunnable<>(listener) {
             @Override
             protected void doRun() {
-                executeBulk(task, bulkRequest, startTime, listener, executor, responses, indicesThatCannotBeCreated);
+                executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, indicesThatCannotBeCreated);
             }
         });
         try (RefCountingRunnable refs = new RefCountingRunnable(executeBulkRunnable)) {
@@ -533,31 +561,31 @@ void executeBulk(
             responses,
             indicesThatCannotBeCreated,
             indexNameExpressionResolver,
-            relativeTimeProvider,
+            relativeTimeNanosProvider,
             startTimeNanos,
-            listener
+            listener,
+            failureStoreMetrics
         ).run();
     }
 
     /**
-     * Determines if an index name is associated with either an existing data stream or a template
-     * for one that has the failure store enabled.
-     * @param indexName The index name to check.
-     * @param metadata Cluster state metadata.
-     * @param epochMillis A timestamp to use when resolving date math in the index name.
-     * @return true if the given index name corresponds to a data stream with a failure store,
-     * or if it matches a template that has a data stream failure store enabled.
+     * See {@link #resolveFailureStore(String, Metadata, long)}
      */
-    static boolean shouldStoreFailureInternal(String indexName, Metadata metadata, long epochMillis) {
-        return DataStream.isFailureStoreFeatureFlagEnabled()
-            && resolveFailureStoreFromMetadata(indexName, metadata, epochMillis).or(
-                () -> resolveFailureStoreFromTemplate(indexName, metadata)
-            ).orElse(false);
+    // Visibility for testing
+    static Boolean resolveFailureInternal(String indexName, Metadata metadata, long epochMillis) {
+        if (DataStream.isFailureStoreFeatureFlagEnabled() == false) {
+            return null;
+        }
+        var resolution = resolveFailureStoreFromMetadata(indexName, metadata, epochMillis);
+        if (resolution != null) {
+            return resolution;
+        }
+        return resolveFailureStoreFromTemplate(indexName, metadata);
     }
 
     @Override
-    protected boolean shouldStoreFailure(String indexName, Metadata metadata, long time) {
-        return shouldStoreFailureInternal(indexName, metadata, time);
+    protected Boolean resolveFailureStore(String indexName, Metadata metadata, long time) {
+        return resolveFailureInternal(indexName, metadata, time);
     }
 
     /**
@@ -567,30 +595,24 @@ protected boolean shouldStoreFailure(String indexName, Metadata metadata, long t
      * @param epochMillis A timestamp to use when resolving date math in the index name.
      * @return true if the given index name corresponds to an existing data stream with a failure store enabled.
      */
-    private static Optional<Boolean> resolveFailureStoreFromMetadata(String indexName, Metadata metadata, long epochMillis) {
+    private static Boolean resolveFailureStoreFromMetadata(String indexName, Metadata metadata, long epochMillis) {
         if (indexName == null) {
-            return Optional.empty();
+            return null;
         }
 
         // Get index abstraction, resolving date math if it exists
         IndexAbstraction indexAbstraction = metadata.getIndicesLookup()
             .get(IndexNameExpressionResolver.resolveDateMathExpression(indexName, epochMillis));
-
-        // We only store failures if the failure is being written to a data stream,
-        // not when directly writing to backing indices/failure stores
         if (indexAbstraction == null || indexAbstraction.isDataStreamRelated() == false) {
-            return Optional.empty();
+            return null;
         }
 
-        // Locate the write index for the abstraction, and check if it has a data stream associated with it.
-        // This handles alias resolution as well as data stream resolution.
-        Index writeIndex = indexAbstraction.getWriteIndex();
-        assert writeIndex != null : "Could not resolve write index for resource [" + indexName + "]";
-        IndexAbstraction writeAbstraction = metadata.getIndicesLookup().get(writeIndex.getName());
-        DataStream targetDataStream = writeAbstraction.getParentDataStream();
+        // We only store failures if the failure is being written to a data stream,
+        // not when directly writing to backing indices/failure stores
+        DataStream targetDataStream = DataStream.resolveDataStream(indexAbstraction, metadata);
 
         // We will store the failure if the write target belongs to a data stream with a failure store.
-        return Optional.of(targetDataStream != null && targetDataStream.isFailureStoreEnabled());
+        return targetDataStream != null && targetDataStream.isFailureStoreEnabled();
     }
 
     /**
@@ -599,9 +621,9 @@ private static Optional<Boolean> resolveFailureStoreFromMetadata(String indexNam
      * @param metadata Cluster state metadata.
      * @return true if the given index name corresponds to an index template with a data stream failure store enabled.
      */
-    private static Optional<Boolean> resolveFailureStoreFromTemplate(String indexName, Metadata metadata) {
+    private static Boolean resolveFailureStoreFromTemplate(String indexName, Metadata metadata) {
         if (indexName == null) {
-            return Optional.empty();
+            return null;
         }
 
         // Check to see if the index name matches any templates such that an index would have been attributed
@@ -612,11 +634,11 @@ private static Optional<Boolean> resolveFailureStoreFromTemplate(String indexNam
             ComposableIndexTemplate composableIndexTemplate = metadata.templatesV2().get(template);
             if (composableIndexTemplate.getDataStreamTemplate() != null) {
                 // Check if the data stream has the failure store enabled
-                return Optional.of(composableIndexTemplate.getDataStreamTemplate().hasFailureStore());
+                return composableIndexTemplate.getDataStreamTemplate().hasFailureStore();
             }
         }
 
         // Could not locate a failure store via template
-        return Optional.empty();
+        return null;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index fc9df7bbf73b9..ac9ceb44e5b76 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -57,9 +56,10 @@
 import org.elasticsearch.indices.ExecutorSelector;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.plugins.internal.DocumentParsingProvider;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
+import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -117,7 +117,8 @@ public TransportShardBulkAction(
             ExecutorSelector.getWriteExecutorForShard(threadPool),
             PrimaryActionExecution.RejectOnOverload,
             indexingPressure,
-            systemIndices
+            systemIndices,
+            ReplicaActionExecution.SubjectToCircuitBreaker
         );
         this.updateHelper = updateHelper;
         this.mappingUpdatedAction = mappingUpdatedAction;
@@ -363,16 +364,14 @@ static boolean executeBulkItemRequest(
         } else {
             final IndexRequest request = context.getRequestToExecute();
 
-            DocumentSizeObserver documentSizeObserver = documentParsingProvider.newDocumentSizeObserver(request);
-
-            context.setDocumentSizeObserver(documentSizeObserver);
+            XContentMeteringParserDecorator meteringParserDecorator = documentParsingProvider.newMeteringParserDecorator(request);
             final SourceToParse sourceToParse = new SourceToParse(
                 request.id(),
                 request.source(),
                 request.getContentType(),
                 request.routing(),
                 request.getDynamicTemplates(),
-                documentSizeObserver
+                meteringParserDecorator
             );
             result = primary.applyIndexOperationOnPrimary(
                 version,
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
index c08ed6413a7a1..2312a75b91084 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexingPressure;
@@ -31,7 +30,8 @@
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.ingest.SimulateIngestService;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -68,7 +68,7 @@ public TransportSimulateBulkAction(
             ingestService,
             indexingPressure,
             systemIndices,
-            System::nanoTime
+            threadPool::relativeTimeInNanos
         );
         this.indicesService = indicesService;
     }
@@ -79,7 +79,7 @@ protected void doInternalExecute(
         BulkRequest bulkRequest,
         Executor executor,
         ActionListener<BulkResponse> listener,
-        long relativeStartTime
+        long relativeStartTimeNanos
     ) {
         final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
         for (int i = 0; i < bulkRequest.requests.size(); i++) {
@@ -105,7 +105,7 @@ protected void doInternalExecute(
             );
         }
         listener.onResponse(
-            new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(relativeStartTime))
+            new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(relativeStartTimeNanos))
         );
     }
 
@@ -122,7 +122,7 @@ private Exception validateMappings(IndexRequest request) {
             request.getContentType(),
             request.routing(),
             request.getDynamicTemplates(),
-            DocumentSizeObserver.EMPTY_INSTANCE
+            XContentMeteringParserDecorator.NOOP
         );
 
         ClusterState state = clusterService.state();
@@ -166,8 +166,8 @@ protected IngestService getIngestService(BulkRequest request) {
     }
 
     @Override
-    protected boolean shouldStoreFailure(String indexName, Metadata metadata, long time) {
+    protected Boolean resolveFailureStore(String indexName, Metadata metadata, long epochMillis) {
         // A simulate bulk request should not change any persistent state in the system, so we never write to the failure store
-        return false;
+        return null;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java
index 40060d5e5d927..7c788b10405fc 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java
@@ -14,9 +14,11 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -35,18 +37,23 @@ public static class Request extends AcknowledgedRequest<Request> implements Indi
         private final String name;
         private final long startTime;
 
-        public Request(String name) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String name) {
+            super(masterNodeTimeout, ackTimeout);
             this.name = name;
             this.startTime = System.currentTimeMillis();
         }
 
-        public Request(String name, long startTime) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String name, long startTime) {
+            super(masterNodeTimeout, ackTimeout);
             this.name = name;
             this.startTime = startTime;
         }
 
+        @Deprecated(forRemoval = true) // temporary compatibility shim
+        public Request(String name) {
+            this(MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, name);
+        }
+
         public String getName() {
             return name;
         }
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java
index 5b79eae0cebfd..1a62e347012fe 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -46,12 +47,17 @@ public static class Request extends MasterNodeRequest<Request> implements Indice
         private final boolean wildcardExpressionsOriginallySpecified;
         private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
 
-        public Request(String... names) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, String... names) {
+            super(masterNodeTimeout);
             this.names = Objects.requireNonNull(names);
             this.wildcardExpressionsOriginallySpecified = Arrays.stream(names).anyMatch(Regex::isSimpleMatchPattern);
         }
 
+        @Deprecated(forRemoval = true) // temporary compatibility shim
+        public Request(String... names) {
+            this(MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, names);
+        }
+
         public String[] getNames() {
             return names;
         }
@@ -107,7 +113,7 @@ public IndicesOptions indicesOptions() {
             return indicesOptions;
         }
 
-        public IndicesRequest indicesOptions(IndicesOptions options) {
+        public Request indicesOptions(IndicesOptions options) {
             this.indicesOptions = options;
             return this;
         }
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
index 841a2df5eada6..d09a90f9de4f0 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.DataStream;
@@ -25,6 +26,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.DateFieldMapper;
@@ -57,17 +59,22 @@ public static class Request extends MasterNodeReadRequest<Request> implements In
         private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
         private boolean includeDefaults = false;
 
-        public Request(String[] names) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, String[] names) {
+            super(masterNodeTimeout);
             this.names = names;
         }
 
-        public Request(String[] names, boolean includeDefaults) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, String[] names, boolean includeDefaults) {
+            super(masterNodeTimeout);
             this.names = names;
             this.includeDefaults = includeDefaults;
         }
 
+        @Deprecated(forRemoval = true) // temporary compatibility shim
+        public Request(String[] names) {
+            this(MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, names);
+        }
+
         public String[] getNames() {
             return names;
         }
@@ -312,8 +319,7 @@ public XContentBuilder toXContent(
                 }
                 if (dataStream.getLifecycle() != null) {
                     builder.field(LIFECYCLE_FIELD.getPreferredName());
-                    dataStream.getLifecycle()
-                        .toXContent(builder, params, rolloverConfiguration, dataStream.isSystem() ? null : globalRetention);
+                    dataStream.getLifecycle().toXContent(builder, params, rolloverConfiguration, globalRetention, dataStream.isInternal());
                 }
                 if (ilmPolicyName != null) {
                     builder.field(ILM_POLICY_FIELD.getPreferredName(), ilmPolicyName);
@@ -511,7 +517,7 @@ public Response(StreamInput in) throws IOException {
             this(
                 in.readCollectionAsList(DataStreamInfo::new),
                 in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) ? in.readOptionalWriteable(RolloverConfiguration::new) : null,
-                in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)
+                in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
                     ? in.readOptionalWriteable(DataStreamGlobalRetention::read)
                     : null
             );
@@ -537,7 +543,7 @@ public void writeTo(StreamOutput out) throws IOException {
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
                 out.writeOptionalWriteable(rolloverConfiguration);
             }
-            if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 out.writeOptionalWriteable(globalRetention);
             }
         }
@@ -549,7 +555,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             for (DataStreamInfo dataStream : dataStreams) {
                 dataStream.toXContent(
                     builder,
-                    DataStreamLifecycle.maybeAddEffectiveRetentionParams(params),
+                    DataStreamLifecycle.addEffectiveRetentionParams(params),
                     rolloverConfiguration,
                     globalRetention
                 );
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/MigrateToDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/MigrateToDataStreamAction.java
index 226b8d44f636c..894708a282ea9 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/MigrateToDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/MigrateToDataStreamAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -34,8 +35,8 @@ public static class Request extends AcknowledgedRequest<MigrateToDataStreamActio
 
         private final String aliasName;
 
-        public Request(String aliasName) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String aliasName) {
+            super(masterNodeTimeout, ackTimeout);
             this.aliasName = aliasName;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/ModifyDataStreamsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/ModifyDataStreamsAction.java
index c2fdf5729eb59..dcc1e1897bc3d 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/ModifyDataStreamsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/ModifyDataStreamsAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.DataStreamAction;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -69,8 +70,8 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeCollection(actions);
         }
 
-        public Request(List<DataStreamAction> actions) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, List<DataStreamAction> actions) {
+            super(masterNodeTimeout, ackTimeout);
             this.actions = Collections.unmodifiableList(actions);
         }
 
@@ -98,13 +99,22 @@ public ActionRequestValidationException validate() {
             return null;
         }
 
+        public interface Factory {
+            Request create(List<DataStreamAction> actions);
+        }
+
         @SuppressWarnings("unchecked")
-        public static final ConstructingObjectParser<Request, Void> PARSER = new ConstructingObjectParser<>(
+        public static final ConstructingObjectParser<Request, Factory> PARSER = new ConstructingObjectParser<>(
             "data_stream_actions",
-            args -> new Request(((List<DataStreamAction>) args[0]))
+            false,
+            (args, factory) -> factory.create((List<DataStreamAction>) args[0])
         );
         static {
-            PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), DataStreamAction.PARSER, new ParseField("actions"));
+            PARSER.declareObjectArray(
+                ConstructingObjectParser.constructorArg(),
+                (p, c) -> DataStreamAction.PARSER.parse(p, null),
+                new ParseField("actions")
+            );
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/PromoteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/PromoteDataStreamAction.java
index 0853d30d22de4..f423969b617f7 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/PromoteDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/PromoteDataStreamAction.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -34,8 +35,8 @@ public static class Request extends MasterNodeRequest<PromoteDataStreamAction.Re
 
         private final String name;
 
-        public Request(String name) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, String name) {
+            super(masterNodeTimeout);
             this.name = Objects.requireNonNull(name);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java
index 579fb83a89f7e..d51f00681bb5e 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
 
@@ -49,12 +50,12 @@ public static class Request extends MasterNodeReadRequest<Request> implements In
         private boolean includeDefaults;
         private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
 
-        public Request(String[] names) {
-            this(names, false);
+        public Request(TimeValue masterNodeTimeout, String[] names) {
+            this(masterNodeTimeout, names, false);
         }
 
-        public Request(String[] names, boolean includeDefaults) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, String[] names, boolean includeDefaults) {
+            super(masterNodeTimeout);
             this.names = names;
             this.includeDefaults = includeDefaults;
         }
@@ -161,7 +162,7 @@ public Response(StreamInput in) throws IOException {
             super(in);
             this.indices = in.readCollectionAsList(ExplainIndexDataStreamLifecycle::new);
             this.rolloverConfiguration = in.readOptionalWriteable(RolloverConfiguration::new);
-            this.globalRetention = in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)
+            this.globalRetention = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
                 ? in.readOptionalWriteable(DataStreamGlobalRetention::read)
                 : null;
         }
@@ -182,7 +183,7 @@ public DataStreamGlobalRetention getGlobalRetention() {
         public void writeTo(StreamOutput out) throws IOException {
             out.writeCollection(indices);
             out.writeOptionalWriteable(rolloverConfiguration);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 out.writeOptionalWriteable(globalRetention);
             }
         }
@@ -216,7 +217,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
                 builder.field(explainIndexDataLifecycle.getIndex());
                 explainIndexDataLifecycle.toXContent(
                     builder,
-                    DataStreamLifecycle.maybeAddEffectiveRetentionParams(outerParams),
+                    DataStreamLifecycle.addEffectiveRetentionParams(outerParams),
                     rolloverConfiguration,
                     globalRetention
                 );
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java
index 32be73a7b0960..962c2975f5998 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java
@@ -45,7 +45,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj
 
     private final String index;
     private final boolean managedByLifecycle;
-    private final boolean isSystemDataStream;
+    private final boolean isInternalDataStream;
     @Nullable
     private final Long indexCreationDate;
     @Nullable
@@ -61,7 +61,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj
     public ExplainIndexDataStreamLifecycle(
         String index,
         boolean managedByLifecycle,
-        boolean isSystemDataStream,
+        boolean isInternalDataStream,
         @Nullable Long indexCreationDate,
         @Nullable Long rolloverDate,
         @Nullable TimeValue generationDate,
@@ -70,7 +70,7 @@ public ExplainIndexDataStreamLifecycle(
     ) {
         this.index = index;
         this.managedByLifecycle = managedByLifecycle;
-        this.isSystemDataStream = isSystemDataStream;
+        this.isInternalDataStream = isInternalDataStream;
         this.indexCreationDate = indexCreationDate;
         this.rolloverDate = rolloverDate;
         this.generationDateMillis = generationDate == null ? null : generationDate.millis();
@@ -82,9 +82,9 @@ public ExplainIndexDataStreamLifecycle(StreamInput in) throws IOException {
         this.index = in.readString();
         this.managedByLifecycle = in.readBoolean();
         if (in.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
-            this.isSystemDataStream = in.readBoolean();
+            this.isInternalDataStream = in.readBoolean();
         } else {
-            this.isSystemDataStream = false;
+            this.isInternalDataStream = false;
         }
         if (managedByLifecycle) {
             this.indexCreationDate = in.readOptionalLong();
@@ -141,7 +141,7 @@ public XContentBuilder toXContent(
             }
             if (this.lifecycle != null) {
                 builder.field(LIFECYCLE_FIELD.getPreferredName());
-                lifecycle.toXContent(builder, params, rolloverConfiguration, isSystemDataStream ? null : globalRetention);
+                lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, isInternalDataStream);
             }
             if (this.error != null) {
                 if (error.firstOccurrenceTimestamp() != -1L && error.recordedTimestamp() != -1L && error.retryCount() != -1) {
@@ -161,7 +161,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(index);
         out.writeBoolean(managedByLifecycle);
         if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
-            out.writeBoolean(isSystemDataStream);
+            out.writeBoolean(isInternalDataStream);
         }
         if (managedByLifecycle) {
             out.writeOptionalLong(indexCreationDate);
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
index 9907310a8acbd..52af1341692eb 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -48,13 +49,13 @@ public static class Request extends MasterNodeReadRequest<Request> implements In
         private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
         private boolean includeDefaults = false;
 
-        public Request(String[] names) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, String[] names) {
+            super(masterNodeTimeout);
             this.names = names;
         }
 
-        public Request(String[] names, boolean includeDefaults) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, String[] names, boolean includeDefaults) {
+            super(masterNodeTimeout);
             this.names = names;
             this.includeDefaults = includeDefaults;
         }
@@ -142,7 +143,7 @@ public static class Response extends ActionResponse implements ChunkedToXContent
         public record DataStreamLifecycle(
             String dataStreamName,
             @Nullable org.elasticsearch.cluster.metadata.DataStreamLifecycle lifecycle,
-            boolean isSystemDataStream
+            boolean isInternalDataStream
         ) implements Writeable, ToXContentObject {
 
             public static final ParseField NAME_FIELD = new ParseField("name");
@@ -161,7 +162,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeString(dataStreamName);
                 out.writeOptionalWriteable(lifecycle);
                 if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) {
-                    out.writeBoolean(isSystemDataStream);
+                    out.writeBoolean(isInternalDataStream);
                 }
             }
 
@@ -186,9 +187,10 @@ public XContentBuilder toXContent(
                     builder.field(LIFECYCLE_FIELD.getPreferredName());
                     lifecycle.toXContent(
                         builder,
-                        org.elasticsearch.cluster.metadata.DataStreamLifecycle.maybeAddEffectiveRetentionParams(params),
+                        org.elasticsearch.cluster.metadata.DataStreamLifecycle.addEffectiveRetentionParams(params),
                         rolloverConfiguration,
-                        isSystemDataStream ? null : globalRetention
+                        globalRetention,
+                        isInternalDataStream
                     );
                 }
                 builder.endObject();
@@ -220,7 +222,7 @@ public Response(StreamInput in) throws IOException {
             this(
                 in.readCollectionAsList(DataStreamLifecycle::new),
                 in.readOptionalWriteable(RolloverConfiguration::new),
-                in.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)
+                in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
                     ? in.readOptionalWriteable(DataStreamGlobalRetention::read)
                     : null
             );
@@ -243,7 +245,7 @@ public DataStreamGlobalRetention getGlobalRetention() {
         public void writeTo(StreamOutput out) throws IOException {
             out.writeCollection(dataStreamLifecycles);
             out.writeOptionalWriteable(rolloverConfiguration);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.USE_DATA_STREAM_GLOBAL_RETENTION)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 out.writeOptionalWriteable(globalRetention);
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java
index 7bb63ae27b526..c32fa849d9fd3 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java
@@ -45,9 +45,14 @@ private PutDataStreamLifecycleAction() {/* no instances */}
 
     public static final class Request extends AcknowledgedRequest<Request> implements IndicesRequest.Replaceable, ToXContentObject {
 
-        public static final ConstructingObjectParser<Request, Void> PARSER = new ConstructingObjectParser<>(
+        public interface Factory {
+            Request create(@Nullable TimeValue dataRetention, @Nullable Boolean enabled, @Nullable Downsampling downsampling);
+        }
+
+        public static final ConstructingObjectParser<Request, Factory> PARSER = new ConstructingObjectParser<>(
             "put_data_stream_lifecycle_request",
-            args -> new Request(null, ((TimeValue) args[0]), (Boolean) args[1], (Downsampling) args[2])
+            false,
+            (args, factory) -> factory.create((TimeValue) args[0], (Boolean) args[1], (Downsampling) args[2])
         );
 
         static {
@@ -62,13 +67,13 @@ public static final class Request extends AcknowledgedRequest<Request> implement
                 if (p.currentToken() == XContentParser.Token.VALUE_NULL) {
                     return Downsampling.NULL;
                 } else {
-                    return new Downsampling(AbstractObjectParser.parseArray(p, c, Downsampling.Round::fromXContent));
+                    return new Downsampling(AbstractObjectParser.parseArray(p, null, Downsampling.Round::fromXContent));
                 }
             }, DOWNSAMPLING_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_NULL);
         }
 
-        public static Request parseRequest(XContentParser parser) {
-            return PARSER.apply(parser, null);
+        public static Request parseRequest(XContentParser parser, Factory factory) {
+            return PARSER.apply(parser, factory);
         }
 
         private String[] names;
@@ -90,22 +95,35 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeWriteable(lifecycle);
         }
 
-        public Request(String[] names, @Nullable TimeValue dataRetention) {
-            this(names, dataRetention, null, null);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, @Nullable TimeValue dataRetention) {
+            this(masterNodeTimeout, ackTimeout, names, dataRetention, null, null);
         }
 
-        public Request(String[] names, DataStreamLifecycle lifecycle) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, DataStreamLifecycle lifecycle) {
+            super(masterNodeTimeout, ackTimeout);
             this.names = names;
             this.lifecycle = lifecycle;
         }
 
-        public Request(String[] names, @Nullable TimeValue dataRetention, @Nullable Boolean enabled) {
-            this(names, dataRetention, enabled, null);
-        }
-
-        public Request(String[] names, @Nullable TimeValue dataRetention, @Nullable Boolean enabled, @Nullable Downsampling downsampling) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(
+            TimeValue masterNodeTimeout,
+            TimeValue ackTimeout,
+            String[] names,
+            @Nullable TimeValue dataRetention,
+            @Nullable Boolean enabled
+        ) {
+            this(masterNodeTimeout, ackTimeout, names, dataRetention, enabled, null);
+        }
+
+        public Request(
+            TimeValue masterNodeTimeout,
+            TimeValue ackTimeout,
+            String[] names,
+            @Nullable TimeValue dataRetention,
+            @Nullable Boolean enabled,
+            @Nullable Downsampling downsampling
+        ) {
+            super(masterNodeTimeout, ackTimeout);
             this.names = names;
             this.lifecycle = DataStreamLifecycle.newBuilder()
                 .dataRetention(dataRetention)
diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java
index f2b1dc7cd556c..40810f004b0de 100644
--- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java
@@ -30,6 +30,7 @@ public class DeleteRequestBuilder extends ReplicationRequestBuilder<DeleteReques
     private Long term;
     private WriteRequest.RefreshPolicy refreshPolicy;
 
+    @SuppressWarnings("this-escape")
     public DeleteRequestBuilder(ElasticsearchClient client, @Nullable String index) {
         super(client, TransportDeleteAction.TYPE);
         setIndex(index);
diff --git a/server/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/server/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
index 6abe033637c56..44f89ba01edb7 100644
--- a/server/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
+++ b/server/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.action.bulk.TransportBulkAction;
 import org.elasticsearch.action.bulk.TransportSingleItemBulkWriteAction;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 
 /**
diff --git a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
index bece9922f3e46..47e9624d0ec12 100644
--- a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
+++ b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.index.IndexService;
@@ -29,6 +28,7 @@
 import org.elasticsearch.index.query.Rewriteable;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContext;
@@ -98,7 +98,7 @@ protected void doExecute(Task task, ExplainRequest request, ActionListener<Expla
 
         assert request.query() != null;
         LongSupplier timeProvider = () -> request.nowInMillis;
-        Rewriteable.rewriteAndFetch(request.query(), searchService.getRewriteContext(timeProvider, resolvedIndices), rewriteListener);
+        Rewriteable.rewriteAndFetch(request.query(), searchService.getRewriteContext(timeProvider, resolvedIndices, null), rewriteListener);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java
index 51cb05f981177..e435655668882 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java
@@ -108,15 +108,15 @@ private FieldCapabilitiesIndexResponse doFetch(
             null,
             runtimeFields
         );
-
+        var indexMode = searchExecutionContext.getIndexSettings().getMode();
         if (searcher != null && canMatchShard(shardId, indexFilter, nowInMillis, searchExecutionContext) == false) {
-            return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), null, Collections.emptyMap(), false);
+            return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), null, Collections.emptyMap(), false, indexMode);
         }
 
         final MappingMetadata mapping = indexService.getMetadata().mapping();
         String indexMappingHash;
         if (includeEmptyFields || enableFieldHasValue == false) {
-            indexMappingHash = mapping != null ? mapping.getSha256() : null;
+            indexMappingHash = mapping != null ? mapping.getSha256() + indexMode : null;
         } else {
             // even if the mapping is the same if we return only fields with values we need
             // to make sure that we consider all the shard-mappings pair, that is why we
@@ -129,7 +129,7 @@ private FieldCapabilitiesIndexResponse doFetch(
             indexMappingHash = fieldPredicate.modifyHash(indexMappingHash);
             final Map<String, IndexFieldCapabilities> existing = indexMappingHashToResponses.get(indexMappingHash);
             if (existing != null) {
-                return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), indexMappingHash, existing, true);
+                return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), indexMappingHash, existing, true, indexMode);
             }
         }
         task.ensureNotCancelled();
@@ -145,7 +145,7 @@ private FieldCapabilitiesIndexResponse doFetch(
         if (indexMappingHash != null) {
             indexMappingHashToResponses.put(indexMappingHash, responseMap);
         }
-        return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), indexMappingHash, responseMap, true);
+        return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), indexMappingHash, responseMap, true, indexMode);
     }
 
     static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java
index cc72dd80dceac..5a50ed4c9f573 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexMode;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -33,18 +34,21 @@ public final class FieldCapabilitiesIndexResponse implements Writeable {
     private final Map<String, IndexFieldCapabilities> responseMap;
     private final boolean canMatch;
     private final transient TransportVersion originVersion;
+    private final IndexMode indexMode;
 
     public FieldCapabilitiesIndexResponse(
         String indexName,
         @Nullable String indexMappingHash,
         Map<String, IndexFieldCapabilities> responseMap,
-        boolean canMatch
+        boolean canMatch,
+        IndexMode indexMode
     ) {
         this.indexName = indexName;
         this.indexMappingHash = indexMappingHash;
         this.responseMap = responseMap;
         this.canMatch = canMatch;
         this.originVersion = TransportVersion.current();
+        this.indexMode = indexMode;
     }
 
     FieldCapabilitiesIndexResponse(StreamInput in) throws IOException {
@@ -57,6 +61,11 @@ public FieldCapabilitiesIndexResponse(
         } else {
             this.indexMappingHash = null;
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) {
+            this.indexMode = IndexMode.readFrom(in);
+        } else {
+            this.indexMode = IndexMode.STANDARD;
+        }
     }
 
     @Override
@@ -67,9 +76,12 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(MAPPING_HASH_VERSION)) {
             out.writeOptionalString(indexMappingHash);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) {
+            IndexMode.writeTo(indexMode, out);
+        }
     }
 
-    private record CompressedGroup(String[] indices, String mappingHash, int[] fields) {}
+    private record CompressedGroup(String[] indices, IndexMode indexMode, String mappingHash, int[] fields) {}
 
     static List<FieldCapabilitiesIndexResponse> readList(StreamInput input) throws IOException {
         if (input.getTransportVersion().before(MAPPING_HASH_VERSION)) {
@@ -92,10 +104,12 @@ static List<FieldCapabilitiesIndexResponse> readList(StreamInput input) throws I
     private static void collectCompressedResponses(StreamInput input, int groups, ArrayList<FieldCapabilitiesIndexResponse> responses)
         throws IOException {
         final CompressedGroup[] compressedGroups = new CompressedGroup[groups];
+        final boolean readIndexMode = input.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE);
         for (int i = 0; i < groups; i++) {
             final String[] indices = input.readStringArray();
+            final IndexMode indexMode = readIndexMode ? IndexMode.readFrom(input) : IndexMode.STANDARD;
             final String mappingHash = input.readString();
-            compressedGroups[i] = new CompressedGroup(indices, mappingHash, input.readIntArray());
+            compressedGroups[i] = new CompressedGroup(indices, indexMode, mappingHash, input.readIntArray());
         }
         final IndexFieldCapabilities[] ifcLookup = input.readArray(IndexFieldCapabilities::readFrom, IndexFieldCapabilities[]::new);
         for (CompressedGroup compressedGroup : compressedGroups) {
@@ -105,7 +119,7 @@ private static void collectCompressedResponses(StreamInput input, int groups, Ar
                 ifc.put(val.name(), val);
             }
             for (String index : compressedGroup.indices) {
-                responses.add(new FieldCapabilitiesIndexResponse(index, compressedGroup.mappingHash, ifc, true));
+                responses.add(new FieldCapabilitiesIndexResponse(index, compressedGroup.mappingHash, ifc, true, compressedGroup.indexMode));
             }
         }
     }
@@ -117,7 +131,7 @@ private static void collectResponsesLegacyFormat(StreamInput input, int groups,
             final String mappingHash = input.readString();
             final Map<String, IndexFieldCapabilities> ifc = input.readMap(IndexFieldCapabilities::readFrom);
             for (String index : indices) {
-                responses.add(new FieldCapabilitiesIndexResponse(index, mappingHash, ifc, true));
+                responses.add(new FieldCapabilitiesIndexResponse(index, mappingHash, ifc, true, IndexMode.STANDARD));
             }
         }
     }
@@ -164,6 +178,9 @@ private static void writeCompressedResponses(StreamOutput output, Map<String, Li
         output.writeCollection(groupedResponsesMap.values(), (o, fieldCapabilitiesIndexResponses) -> {
             o.writeCollection(fieldCapabilitiesIndexResponses, (oo, r) -> oo.writeString(r.indexName));
             var first = fieldCapabilitiesIndexResponses.get(0);
+            if (output.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) {
+                IndexMode.writeTo(first.indexMode, o);
+            }
             o.writeString(first.indexMappingHash);
             o.writeVInt(first.responseMap.size());
             for (IndexFieldCapabilities ifc : first.responseMap.values()) {
@@ -192,6 +209,10 @@ public String getIndexMappingHash() {
         return indexMappingHash;
     }
 
+    public IndexMode getIndexMode() {
+        return indexMode;
+    }
+
     public boolean canMatch() {
         return canMatch;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
index 41bf42b4e4e9c..bb97b0dc48c42 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.action.fieldcaps;
 
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
@@ -26,7 +27,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -34,6 +34,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.search.SearchService;
@@ -173,7 +174,13 @@ private void doExecuteForked(
             if (resp.canMatch() && resp.getIndexMappingHash() != null) {
                 FieldCapabilitiesIndexResponse curr = indexMappingHashToResponses.putIfAbsent(resp.getIndexMappingHash(), resp);
                 if (curr != null) {
-                    resp = new FieldCapabilitiesIndexResponse(resp.getIndexName(), curr.getIndexMappingHash(), curr.get(), true);
+                    resp = new FieldCapabilitiesIndexResponse(
+                        resp.getIndexName(),
+                        curr.getIndexMappingHash(),
+                        curr.get(),
+                        true,
+                        curr.getIndexMode()
+                    );
                 }
             }
             if (request.includeEmptyFields()) {
@@ -185,7 +192,13 @@ private void doExecuteForked(
                     }
                     Map<String, IndexFieldCapabilities> mergedCaps = new HashMap<>(a.get());
                     mergedCaps.putAll(b.get());
-                    return new FieldCapabilitiesIndexResponse(a.getIndexName(), a.getIndexMappingHash(), mergedCaps, true);
+                    return new FieldCapabilitiesIndexResponse(
+                        a.getIndexName(),
+                        a.getIndexMappingHash(),
+                        mergedCaps,
+                        true,
+                        a.getIndexMode()
+                    );
                 });
             }
             if (fieldCapTask.isCancelled()) {
@@ -248,7 +261,13 @@ private void doExecuteForked(
                     for (FieldCapabilitiesIndexResponse resp : response.getIndexResponses()) {
                         String indexName = RemoteClusterAware.buildRemoteIndexName(clusterAlias, resp.getIndexName());
                         handleIndexResponse.accept(
-                            new FieldCapabilitiesIndexResponse(indexName, resp.getIndexMappingHash(), resp.get(), resp.canMatch())
+                            new FieldCapabilitiesIndexResponse(
+                                indexName,
+                                resp.getIndexMappingHash(),
+                                resp.get(),
+                                resp.canMatch(),
+                                resp.getIndexMode()
+                            )
                         );
                     }
                     for (FieldCapabilitiesFailure failure : response.getFailures()) {
@@ -557,7 +576,12 @@ public void messageReceived(FieldCapabilitiesNodeRequest request, TransportChann
                     .stream()
                     .collect(Collectors.groupingBy(ShardId::getIndexName));
                 final FieldCapabilitiesFetcher fetcher = new FieldCapabilitiesFetcher(indicesService, request.includeEmptyFields());
-                final Predicate<String> fieldNameFilter = Regex.simpleMatcher(request.fields());
+                Predicate<String> fieldNameFilter;
+                try {
+                    fieldNameFilter = Regex.simpleMatcher(request.fields());
+                } catch (TooComplexToDeterminizeException e) {
+                    throw new IllegalArgumentException("The field names are too complex to process. " + e.getMessage());
+                }
                 for (List<ShardId> shardIds : groupedShardIds.values()) {
                     final Map<ShardId, Exception> failures = new HashMap<>();
                     final Set<ShardId> unmatched = new HashSet<>();
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
index db26da382d3e1..330a97e27f278 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.cluster.routing.PlainShardIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -42,6 +41,7 @@
 import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.indices.ExecutorSelector;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java
index f84e35e5303d1..0b63bdc0cf980 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -31,6 +30,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
index fcb10f3deef60..671ba29db242c 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
@@ -19,12 +19,12 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
index a5433618441c3..07b586713f152 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.cluster.routing.PlainShardIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -43,6 +42,7 @@
 import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.indices.ExecutorSelector;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index 5463f9fec4d2a..61d610c9eda4e 100644
--- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -38,7 +38,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.ingest.IngestService;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
+import org.elasticsearch.plugins.internal.XContentParserDecorator;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
@@ -147,6 +147,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
     private Object rawTimestamp;
     private long normalisedBytesParsed = -1;
     private boolean originatesFromUpdateByScript;
+    private boolean originatesFromUpdateByDoc;
 
     public IndexRequest(StreamInput in) throws IOException {
         this(null, in);
@@ -204,6 +205,12 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio
         } else {
             originatesFromUpdateByScript = false;
         }
+
+        if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_DOC_ORIGIN)) {
+            originatesFromUpdateByDoc = in.readBoolean();
+        } else {
+            originatesFromUpdateByDoc = false;
+        }
     }
 
     public IndexRequest() {
@@ -407,8 +414,8 @@ public Map<String, Object> sourceAsMap() {
         return XContentHelper.convertToMap(source, false, contentType).v2();
     }
 
-    public Map<String, Object> sourceAsMap(DocumentSizeObserver documentSizeObserver) {
-        return XContentHelper.convertToMap(source, false, contentType, documentSizeObserver).v2();
+    public Map<String, Object> sourceAsMap(XContentParserDecorator parserDecorator) {
+        return XContentHelper.convertToMap(source, false, contentType, parserDecorator).v2();
     }
 
     /**
@@ -768,6 +775,10 @@ private void writeBody(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_SCRIPT_ORIGIN)) {
             out.writeBoolean(originatesFromUpdateByScript);
         }
+
+        if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_REQUEST_UPDATE_BY_DOC_ORIGIN)) {
+            out.writeBoolean(originatesFromUpdateByDoc);
+        }
     }
 
     @Override
@@ -931,15 +942,6 @@ public IndexRequest setNormalisedBytesParsed(long normalisedBytesParsed) {
         return this;
     }
 
-    /**
-     * when observing document size while parsing, this method indicates that this request should not be recorded.
-     * @return an index request
-     */
-    public IndexRequest noParsedBytesToReport() {
-        this.normalisedBytesParsed = 0;
-        return this;
-    }
-
     /**
      * Adds the pipeline to the list of executed pipelines, if listExecutedPipelines is true
      *
@@ -977,6 +979,15 @@ public IndexRequest setOriginatesFromUpdateByScript(boolean originatesFromUpdate
     }
 
     public boolean originatesFromUpdateByScript() {
-        return this.originatesFromUpdateByScript;
+        return originatesFromUpdateByScript;
+    }
+
+    public boolean originatesFromUpdateByDoc() {
+        return originatesFromUpdateByDoc;
+    }
+
+    public IndexRequest setOriginatesFromUpdateByDoc(boolean originatesFromUpdateByDoc) {
+        this.originatesFromUpdateByDoc = originatesFromUpdateByDoc;
+        return this;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
index 0cb04fbdba1a6..3f3d97e2115cb 100644
--- a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
@@ -54,6 +54,7 @@ public IndexRequestBuilder(ElasticsearchClient client) {
         this(client, null);
     }
 
+    @SuppressWarnings("this-escape")
     public IndexRequestBuilder(ElasticsearchClient client, @Nullable String index) {
         super(client, TransportIndexAction.TYPE);
         setIndex(index);
diff --git a/server/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/server/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
index cc61f40d303d3..304f37401f5fd 100644
--- a/server/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.action.bulk.TransportBulkAction;
 import org.elasticsearch.action.bulk.TransportSingleItemBulkWriteAction;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 
 /**
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
index a8eef9f94b884..99a8cf5a38f13 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
index 8ba251e911431..834e6e5991b26 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
index 5a97596aa00ff..46d5ad0017321 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
@@ -21,9 +21,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
index f38dff3f8c83c..2fc59086b54a6 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
@@ -17,12 +17,12 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequestOptions;
diff --git a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java
index 5a891f33480fa..e3f0f5842cfb3 100644
--- a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java
+++ b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexingPressure;
@@ -30,6 +29,7 @@
 import org.elasticsearch.indices.ExecutorSelector;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportResponseHandler;
@@ -73,7 +73,8 @@ public TransportResyncReplicationAction(
             ExecutorSelector.getWriteExecutorForShard(threadPool),
             PrimaryActionExecution.Force, /* we should never reject resync because of thread pool capacity on primary */
             indexingPressure,
-            systemIndices
+            systemIndices,
+            ReplicaActionExecution.SubjectToCircuitBreaker
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 3ff820c0e0354..1e5b5ebbefe48 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -707,9 +707,11 @@ public void sendSearchResponse(SearchResponseSections internalSearchResponse, At
             final String scrollId = request.scroll() != null ? TransportSearchHelper.buildScrollId(queryResults) : null;
             final BytesReference searchContextId;
             if (buildPointInTimeFromSearchResults()) {
-                searchContextId = SearchContextId.encode(queryResults.asList(), aliasFilter, minTransportVersion);
+                searchContextId = SearchContextId.encode(queryResults.asList(), aliasFilter, minTransportVersion, failures);
             } else {
-                if (request.source() != null && request.source().pointInTimeBuilder() != null) {
+                if (request.source() != null
+                    && request.source().pointInTimeBuilder() != null
+                    && request.source().pointInTimeBuilder().singleSession() == false) {
                     searchContextId = request.source().pointInTimeBuilder().getEncodedId();
                 } else {
                     searchContextId = null;
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
index 04573f72068f3..965b19a69b858 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
@@ -166,6 +166,10 @@ public static void closeContexts(
             final var successes = new AtomicInteger();
             try (RefCountingRunnable refs = new RefCountingRunnable(() -> l.onResponse(successes.get()))) {
                 for (SearchContextIdForNode contextId : contextIds) {
+                    if (contextId.getNode() == null) {
+                        // the shard was missing when creating the PIT, ignore.
+                        continue;
+                    }
                     final DiscoveryNode node = nodeLookup.apply(contextId.getClusterAlias(), contextId.getNode());
                     if (node != null) {
                         try {
diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
index 9ddac7f13eb51..7a33eaa59eb03 100644
--- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
@@ -155,7 +155,8 @@ ShardSearchRequest rewriteShardSearchRequest(ShardSearchRequest request) {
             QueryBuilder query = new KnnScoreDocQueryBuilder(
                 scoreDocs.toArray(Lucene.EMPTY_SCORE_DOCS),
                 source.knnSearch().get(i).getField(),
-                source.knnSearch().get(i).getQueryVector()
+                source.knnSearch().get(i).getQueryVector(),
+                source.knnSearch().get(i).getSimilarity()
             ).boost(source.knnSearch().get(i).boost()).queryName(source.knnSearch().get(i).queryName());
             if (nestedPath != null) {
                 query = new NestedQueryBuilder(nestedPath, query, ScoreMode.Max).innerHit(source.knnSearch().get(i).innerHit());
diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
index 4bf8206861a4d..fa22cb36b70ab 100644
--- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
@@ -206,7 +206,7 @@ public static void readMultiLineFormat(
         TriFunction<String, Object, SearchRequest, Boolean> extraParamParser
     ) throws IOException {
         int from = 0;
-        byte marker = xContent.streamSeparator();
+        byte marker = xContent.bulkSeparator();
         while (true) {
             int nextMarker = findNextMarker(marker, from, data);
             if (nextMarker == -1) {
@@ -343,7 +343,7 @@ public static byte[] writeMultiLineFormat(MultiSearchRequest multiSearchRequest,
                 writeSearchRequestParams(request, xContentBuilder);
                 BytesReference.bytes(xContentBuilder).writeTo(output);
             }
-            output.write(xContent.streamSeparator());
+            output.write(xContent.bulkSeparator());
             try (XContentBuilder xContentBuilder = XContentBuilder.builder(xContent)) {
                 if (request.source() != null) {
                     request.source().toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
@@ -353,7 +353,7 @@ public static byte[] writeMultiLineFormat(MultiSearchRequest multiSearchRequest,
                 }
                 BytesReference.bytes(xContentBuilder).writeTo(output);
             }
-            output.write(xContent.streamSeparator());
+            output.write(xContent.bulkSeparator());
         }
         return output.toByteArray();
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java
index a1cd4df25a25c..146418839f063 100644
--- a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java
@@ -41,6 +41,8 @@ public final class OpenPointInTimeRequest extends ActionRequest implements Indic
 
     private QueryBuilder indexFilter;
 
+    private boolean allowPartialSearchResults = false;
+
     public static final IndicesOptions DEFAULT_INDICES_OPTIONS = SearchRequest.DEFAULT_INDICES_OPTIONS;
 
     public OpenPointInTimeRequest(String... indices) {
@@ -60,6 +62,9 @@ public OpenPointInTimeRequest(StreamInput in) throws IOException {
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
             this.indexFilter = in.readOptionalNamedWriteable(QueryBuilder.class);
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) {
+            this.allowPartialSearchResults = in.readBoolean();
+        }
     }
 
     @Override
@@ -76,6 +81,11 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
             out.writeOptionalWriteable(indexFilter);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) {
+            out.writeBoolean(allowPartialSearchResults);
+        } else if (allowPartialSearchResults) {
+            throw new IOException("[allow_partial_search_results] is not supported on nodes with version " + out.getTransportVersion());
+        }
     }
 
     @Override
@@ -180,6 +190,15 @@ public boolean includeDataStreams() {
         return true;
     }
 
+    public boolean allowPartialSearchResults() {
+        return allowPartialSearchResults;
+    }
+
+    public OpenPointInTimeRequest allowPartialSearchResults(boolean allowPartialSearchResults) {
+        this.allowPartialSearchResults = allowPartialSearchResults;
+        return this;
+    }
+
     @Override
     public String getDescription() {
         return "open search context: indices [" + String.join(",", indices) + "] keep_alive [" + keepAlive + "]";
@@ -200,6 +219,8 @@ public String toString() {
             + ", preference='"
             + preference
             + '\''
+            + ", allowPartialSearchResults="
+            + allowPartialSearchResults
             + '}';
     }
 
@@ -218,12 +239,13 @@ public boolean equals(Object o) {
             && indicesOptions.equals(that.indicesOptions)
             && keepAlive.equals(that.keepAlive)
             && Objects.equals(routing, that.routing)
-            && Objects.equals(preference, that.preference);
+            && Objects.equals(preference, that.preference)
+            && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults);
     }
 
     @Override
     public int hashCode() {
-        int result = Objects.hash(indicesOptions, keepAlive, maxConcurrentShardRequests, routing, preference);
+        int result = Objects.hash(indicesOptions, keepAlive, maxConcurrentShardRequests, routing, preference, allowPartialSearchResults);
         result = 31 * result + Arrays.hashCode(indices);
         return result;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java
index dafcee894c9a6..4a4c0252fb109 100644
--- a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -18,22 +19,46 @@
 import java.util.Base64;
 import java.util.Objects;
 
+import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
+
 public final class OpenPointInTimeResponse extends ActionResponse implements ToXContentObject {
     private final BytesReference pointInTimeId;
 
-    public OpenPointInTimeResponse(BytesReference pointInTimeId) {
+    private final int totalShards;
+    private final int successfulShards;
+    private final int failedShards;
+    private final int skippedShards;
+
+    public OpenPointInTimeResponse(
+        BytesReference pointInTimeId,
+        int totalShards,
+        int successfulShards,
+        int failedShards,
+        int skippedShards
+    ) {
         this.pointInTimeId = Objects.requireNonNull(pointInTimeId, "Point in time parameter must be not null");
+        this.totalShards = totalShards;
+        this.successfulShards = successfulShards;
+        this.failedShards = failedShards;
+        this.skippedShards = skippedShards;
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeBytesReference(pointInTimeId);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) {
+            out.writeVInt(totalShards);
+            out.writeVInt(successfulShards);
+            out.writeVInt(failedShards);
+            out.writeVInt(skippedShards);
+        }
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field("id", Base64.getUrlEncoder().encodeToString(BytesReference.toBytes(pointInTimeId)));
+        buildBroadcastShardsHeader(builder, params, totalShards, successfulShards, failedShards, skippedShards, null);
         builder.endObject();
         return builder;
     }
@@ -42,4 +67,19 @@ public BytesReference getPointInTimeId() {
         return pointInTimeId;
     }
 
+    public int getTotalShards() {
+        return totalShards;
+    }
+
+    public int getSuccessfulShards() {
+        return successfulShards;
+    }
+
+    public int getFailedShards() {
+        return failedShards;
+    }
+
+    public int getSkippedShards() {
+        return skippedShards;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
index 5b42afcb86928..0f7cbd65a63c2 100644
--- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
@@ -28,8 +28,8 @@
 
 /**
  * This search phase is responsible for executing any re-ranking needed for the given search request, iff that is applicable.
- * It starts by retrieving {@code num_shards * window_size} results from the query phase and reduces them to a global list of
- * the top {@code window_size} results. It then reaches out to the shards to extract the needed feature data,
+ * It starts by retrieving {@code num_shards * rank_window_size} results from the query phase and reduces them to a global list of
+ * the top {@code rank_window_size} results. It then reaches out to the shards to extract the needed feature data,
  * and finally passes all this information to the appropriate {@code RankFeatureRankCoordinatorContext} which is responsible for reranking
  * the results. If no rank query is specified, it proceeds directly to the next phase (FetchSearchPhase) by first reducing the results.
  */
@@ -88,7 +88,7 @@ public void onFailure(Exception e) {
 
     void innerRun() throws Exception {
         // if the RankBuilder specifies a QueryPhaseCoordinatorContext, it will be called as part of the reduce call
-        // to operate on the first `window_size * num_shards` results and merge them appropriately.
+        // to operate on the first `rank_window_size * num_shards` results and merge them appropriately.
         SearchPhaseController.ReducedQueryPhase reducedQueryPhase = queryPhaseResults.reduce();
         RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext = coordinatorContext(context.getRequest().source());
         if (rankFeaturePhaseRankCoordinatorContext != null) {
diff --git a/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java
index 0e7f3f9111842..5966a1c924745 100644
--- a/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java
@@ -47,6 +47,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         openRequest.routing(request.param("routing"));
         openRequest.preference(request.param("preference"));
         openRequest.keepAlive(TimeValue.parseTimeValue(request.param("keep_alive"), null, "keep_alive"));
+        openRequest.allowPartialSearchResults(request.paramAsBoolean("allow_partial_search_results", false));
         if (request.hasParam("max_concurrent_shard_requests")) {
             final int maxConcurrentShardRequests = request.paramAsInt(
                 "max_concurrent_shard_requests",
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java
index 95d22e8a9034e..2e4dc724413ea 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -58,12 +59,30 @@ public boolean contains(ShardSearchContextId contextId) {
     public static BytesReference encode(
         List<SearchPhaseResult> searchPhaseResults,
         Map<String, AliasFilter> aliasFilter,
-        TransportVersion version
+        TransportVersion version,
+        ShardSearchFailure[] shardFailures
     ) {
+        assert shardFailures.length == 0 || version.onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)
+            : "[allow_partial_search_results] cannot be enabled on a cluster that has not been fully upgraded to version ["
+                + TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT
+                + "] or higher.";
         try (var out = new BytesStreamOutput()) {
             out.setTransportVersion(version);
             TransportVersion.writeVersion(version, out);
-            out.writeCollection(searchPhaseResults, SearchContextId::writeSearchPhaseResult);
+            boolean allowNullContextId = out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT);
+            int shardSize = searchPhaseResults.size() + (allowNullContextId ? shardFailures.length : 0);
+            out.writeVInt(shardSize);
+            for (var searchResult : searchPhaseResults) {
+                final SearchShardTarget target = searchResult.getSearchShardTarget();
+                target.getShardId().writeTo(out);
+                new SearchContextIdForNode(target.getClusterAlias(), target.getNodeId(), searchResult.getContextId()).writeTo(out);
+            }
+            if (allowNullContextId) {
+                for (var failure : shardFailures) {
+                    failure.shard().getShardId().writeTo(out);
+                    new SearchContextIdForNode(failure.shard().getClusterAlias(), null, null).writeTo(out);
+                }
+            }
             out.writeMap(aliasFilter, StreamOutput::writeWriteable);
             return out.bytes();
         } catch (IOException e) {
@@ -72,12 +91,6 @@ public static BytesReference encode(
         }
     }
 
-    private static void writeSearchPhaseResult(StreamOutput out, SearchPhaseResult searchPhaseResult) throws IOException {
-        final SearchShardTarget target = searchPhaseResult.getSearchShardTarget();
-        target.getShardId().writeTo(out);
-        new SearchContextIdForNode(target.getClusterAlias(), target.getNodeId(), searchPhaseResult.getContextId()).writeTo(out);
-    }
-
     public static SearchContextId decode(NamedWriteableRegistry namedWriteableRegistry, BytesReference id) {
         try (var in = new NamedWriteableAwareStreamInput(id.streamInput(), namedWriteableRegistry)) {
             final TransportVersion version = TransportVersion.readVersion(in);
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
index 3071362f552ea..a70ddf6ee14b9 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -21,25 +22,59 @@ public final class SearchContextIdForNode implements Writeable {
     private final ShardSearchContextId searchContextId;
     private final String clusterAlias;
 
-    SearchContextIdForNode(@Nullable String clusterAlias, String node, ShardSearchContextId searchContextId) {
+    /**
+     * Contains the details required to retrieve a {@link ShardSearchContextId} for a shard on a specific node.
+     *
+     * @param clusterAlias The alias of the cluster, or {@code null} if the shard is local.
+     * @param node The target node where the search context ID is defined, or {@code null} if the shard is missing or unavailable.
+     * @param searchContextId The {@link ShardSearchContextId}, or {@code null} if the shard is missing or unavailable.
+     */
+    SearchContextIdForNode(@Nullable String clusterAlias, @Nullable String node, @Nullable ShardSearchContextId searchContextId) {
         this.node = node;
         this.clusterAlias = clusterAlias;
         this.searchContextId = searchContextId;
     }
 
     SearchContextIdForNode(StreamInput in) throws IOException {
-        this.node = in.readString();
+        boolean allowNull = in.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT);
+        this.node = allowNull ? in.readOptionalString() : in.readString();
         this.clusterAlias = in.readOptionalString();
-        this.searchContextId = new ShardSearchContextId(in);
+        this.searchContextId = allowNull ? in.readOptionalWriteable(ShardSearchContextId::new) : new ShardSearchContextId(in);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(node);
+        boolean allowNull = out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT);
+        if (allowNull) {
+            out.writeOptionalString(node);
+        } else {
+            if (node == null) {
+                // We should never set a null node if the cluster is not fully upgraded to a version that can handle it.
+                throw new IOException(
+                    "Cannot write null node value to a node in version "
+                        + out.getTransportVersion()
+                        + ". The target node must be specified to retrieve the ShardSearchContextId."
+                );
+            }
+            out.writeString(node);
+        }
         out.writeOptionalString(clusterAlias);
-        searchContextId.writeTo(out);
+        if (allowNull) {
+            out.writeOptionalWriteable(searchContextId);
+        } else {
+            if (searchContextId == null) {
+                // We should never set a null search context id if the cluster is not fully upgraded to a version that can handle it.
+                throw new IOException(
+                    "Cannot write null search context ID to a node in version "
+                        + out.getTransportVersion()
+                        + ". A valid search context ID is required to identify the shard's search context in this version."
+                );
+            }
+            searchContextId.writeTo(out);
+        }
     }
 
+    @Nullable
     public String getNode() {
         return node;
     }
@@ -49,6 +84,7 @@ public String getClusterAlias() {
         return clusterAlias;
     }
 
+    @Nullable
     public ShardSearchContextId getSearchContextId() {
         return searchContextId;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java
index 7ad81154691c0..da8479873a4b6 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java
@@ -42,7 +42,13 @@ public void start() {
         }
     }
 
-    static void doCheckNoMissingShards(String phaseName, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts) {
+    protected String missingShardsErrorMessage(StringBuilder missingShards) {
+        return "Search rejected due to missing shards ["
+            + missingShards
+            + "]. Consider using `allow_partial_search_results` setting to bypass this error.";
+    }
+
+    protected void doCheckNoMissingShards(String phaseName, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts) {
         assert request.allowPartialSearchResults() != null : "SearchRequest missing setting for allowPartialSearchResults";
         if (request.allowPartialSearchResults() == false) {
             final StringBuilder missingShards = new StringBuilder();
@@ -58,9 +64,7 @@ static void doCheckNoMissingShards(String phaseName, SearchRequest request, Grou
             }
             if (missingShards.isEmpty() == false) {
                 // Status red - shard is missing all copies and would produce partial results for an index search
-                final String msg = "Search rejected due to missing shards ["
-                    + missingShards
-                    + "]. Consider using `allow_partial_search_results` setting to bypass this error.";
+                final String msg = missingShardsErrorMessage(missingShards);
                 throw new SearchPhaseExecutionException(phaseName, msg, null, ShardSearchFailure.EMPTY_ARRAY);
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 514e8d10eeca1..415a2ba7da9f8 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -320,9 +320,10 @@ public void writeTo(StreamOutput out) throws IOException {
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
         boolean scroll = scroll() != null;
+        boolean allowPartialSearchResults = allowPartialSearchResults() != null && allowPartialSearchResults();
 
         if (source != null) {
-            validationException = source.validate(validationException, scroll);
+            validationException = source.validate(validationException, scroll, allowPartialSearchResults);
         }
         if (scroll) {
             if (requestCache != null && requestCache) {
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 45cb118691082..8d70e2dd6bb66 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -47,6 +47,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.BiFunction;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
@@ -701,6 +702,13 @@ public Cluster getCluster(String clusterAlias) {
             return clusterInfo.get(clusterAlias);
         }
 
+        /**
+         * @return collection of cluster aliases in the search response (including "(local)" if was searched).
+         */
+        public Set<String> getClusterAliases() {
+            return clusterInfo.keySet();
+        }
+
         /**
          * Utility to swap a Cluster object. Guidelines for the remapping function:
          * <ul>
@@ -803,6 +811,7 @@ public boolean hasClusterObjects() {
         public boolean hasRemoteClusters() {
             return total > 1 || clusterInfo.keySet().stream().anyMatch(alias -> alias != RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
         }
+
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTask.java b/server/src/main/java/org/elasticsearch/action/search/SearchTask.java
index 3bf72313c4c21..cc5d60ad0b0c0 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTask.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTask.java
@@ -69,4 +69,11 @@ public Supplier<SearchResponseMerger> getSearchResponseMergerSupplier() {
     public void setSearchResponseMergerSupplier(Supplier<SearchResponseMerger> supplier) {
         this.searchResponseMergerSupplier = supplier;
     }
+
+    /**
+     * Is this async search?
+     */
+    public boolean isAsync() {
+        return false;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index fb3c49d83cb93..52d4542faaf77 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -24,9 +24,12 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.ThrottledTaskRunner;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Releasable;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.dfs.DfsSearchResult;
@@ -60,6 +63,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.Executor;
 import java.util.function.BiFunction;
 
 import static org.elasticsearch.action.search.SearchTransportAPMMetrics.ACTION_ATTRIBUTE_NAME;
@@ -455,9 +459,10 @@ public static void registerRequestHandler(
             boolean freed = searchService.freeReaderContext(request.id());
             channel.sendResponse(SearchFreeContextResponse.of(freed));
         };
+        final Executor freeContextExecutor = buildFreeContextExecutor(transportService);
         transportService.registerRequestHandler(
             FREE_CONTEXT_SCROLL_ACTION_NAME,
-            transportService.getThreadPool().generic(),
+            freeContextExecutor,
             ScrollFreeContextRequest::new,
             instrumentedHandler(FREE_CONTEXT_SCROLL_ACTION_METRIC, transportService, searchTransportMetrics, freeContextHandler)
         );
@@ -470,7 +475,7 @@ public static void registerRequestHandler(
 
         transportService.registerRequestHandler(
             FREE_CONTEXT_ACTION_NAME,
-            transportService.getThreadPool().generic(),
+            freeContextExecutor,
             SearchFreeContextRequest::new,
             instrumentedHandler(FREE_CONTEXT_ACTION_METRIC, transportService, searchTransportMetrics, freeContextHandler)
         );
@@ -478,7 +483,7 @@ public static void registerRequestHandler(
 
         transportService.registerRequestHandler(
             CLEAR_SCROLL_CONTEXTS_ACTION_NAME,
-            transportService.getThreadPool().generic(),
+            freeContextExecutor,
             ClearScrollContextsRequest::new,
             instrumentedHandler(CLEAR_SCROLL_CONTEXTS_ACTION_METRIC, transportService, searchTransportMetrics, (request, channel, task) -> {
                 searchService.freeAllScrollContexts();
@@ -626,6 +631,32 @@ public static void registerRequestHandler(
         TransportActionProxy.registerProxyAction(transportService, QUERY_CAN_MATCH_NODE_NAME, true, CanMatchNodeResponse::new);
     }
 
+    private static Executor buildFreeContextExecutor(TransportService transportService) {
+        final ThrottledTaskRunner throttledTaskRunner = new ThrottledTaskRunner(
+            "free_context",
+            1,
+            transportService.getThreadPool().generic()
+        );
+        return r -> throttledTaskRunner.enqueueTask(new ActionListener<>() {
+            @Override
+            public void onResponse(Releasable releasable) {
+                try (releasable) {
+                    r.run();
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                if (r instanceof AbstractRunnable abstractRunnable) {
+                    abstractRunnable.onFailure(e);
+                }
+                // should be impossible, GENERIC pool doesn't reject anything
+                logger.error("unexpected failure running " + r, e);
+                assert false : new AssertionError("unexpected failure running " + r, e);
+            }
+        });
+    }
+
     private static <Request extends TransportRequest> TransportRequestHandler<Request> instrumentedHandler(
         String actionQualifier,
         TransportService transportService,
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
index 65284d5d55585..cc6223453195f 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportClosePointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportClosePointInTimeAction.java
index f507b2e1136a8..7fd81a7b8ea29 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportClosePointInTimeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportClosePointInTimeAction.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
index a2c41042871ba..9d2f97e4f051c 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
@@ -15,14 +15,15 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -39,7 +40,6 @@ public class TransportMultiSearchAction extends HandledTransportAction<MultiSear
     public static final ActionType<MultiSearchResponse> TYPE = new ActionType<>(NAME);
     private static final Logger logger = LogManager.getLogger(TransportMultiSearchAction.class);
     private final int allocatedProcessors;
-    private final ThreadPool threadPool;
     private final ClusterService clusterService;
     private final LongSupplier relativeTimeProvider;
     private final NodeClient client;
@@ -47,14 +47,12 @@ public class TransportMultiSearchAction extends HandledTransportAction<MultiSear
     @Inject
     public TransportMultiSearchAction(
         Settings settings,
-        ThreadPool threadPool,
         TransportService transportService,
         ClusterService clusterService,
         ActionFilters actionFilters,
         NodeClient client
     ) {
         super(TYPE.name(), transportService, actionFilters, MultiSearchRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
-        this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.allocatedProcessors = EsExecutors.allocatedProcessors(settings);
         this.relativeTimeProvider = System::nanoTime;
@@ -62,7 +60,6 @@ public TransportMultiSearchAction(
     }
 
     TransportMultiSearchAction(
-        ThreadPool threadPool,
         ActionFilters actionFilters,
         TransportService transportService,
         ClusterService clusterService,
@@ -71,7 +68,6 @@ public TransportMultiSearchAction(
         NodeClient client
     ) {
         super(TYPE.name(), transportService, actionFilters, MultiSearchRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
-        this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.allocatedProcessors = allocatedProcessors;
         this.relativeTimeProvider = relativeTimeProvider;
@@ -134,31 +130,39 @@ void executeSearch(
         final ActionListener<MultiSearchResponse> listener,
         final long relativeStartTime
     ) {
+        /*
+         * The number of times that we poll an item from the queue here is the minimum of the number of requests and the maximum number
+         * of concurrent requests. At first glance, it appears that we should never poll from the queue and not obtain a request given
+         * that we only poll here no more times than the number of requests. However, this is not the only consumer of this queue as
+         * earlier requests that have already completed will poll from the queue too, and they could complete before later polls are
+         * invoked here. Thus, it can be the case that we poll here and the queue was empty.
+         */
         SearchRequestSlot request = requests.poll();
-        if (request == null) {
-            /*
-             * The number of times that we poll an item from the queue here is the minimum of the number of requests and the maximum number
-             * of concurrent requests. At first glance, it appears that we should never poll from the queue and not obtain a request given
-             * that we only poll here no more times than the number of requests. However, this is not the only consumer of this queue as
-             * earlier requests that have already completed will poll from the queue too and they could complete before later polls are
-             * invoked here. Thus, it can be the case that we poll here and the queue was empty.
-             */
-            return;
+        // If we have another request to execute, we execute it. If the execution forked #doExecuteSearch will return false and will
+        // recursively call this method again eventually. If it did not fork and was able to execute the search right away #doExecuteSearch
+        // will return true, in which case we continue and run the next search request here.
+        while (request != null && doExecuteSearch(requests, responses, responseCounter, relativeStartTime, request, listener)) {
+            request = requests.poll();
         }
+    }
 
-        /*
-         * With a request in hand, we are now prepared to execute the search request. There are two possibilities, either we go asynchronous
-         * or we do not (this can happen if the request does not resolve to any shards). If we do not go asynchronous, we are going to come
-         * back on the same thread that attempted to execute the search request. At this point, or any other point where we come back on the
-         * same thread as when the request was submitted, we should not recurse lest we might descend into a stack overflow. To avoid this,
-         * when we handle the response rather than going recursive, we fork to another thread, otherwise we recurse.
-         */
-        final Thread thread = Thread.currentThread();
-        client.search(request.request, new ActionListener<>() {
+    private boolean doExecuteSearch(
+        Queue<SearchRequestSlot> requests,
+        AtomicArray<MultiSearchResponse.Item> responses,
+        AtomicInteger responseCounter,
+        long relativeStartTime,
+        SearchRequestSlot request,
+        ActionListener<MultiSearchResponse> listener
+    ) {
+        final SubscribableListener<MultiSearchResponse.Item> subscribeListener = new SubscribableListener<>();
+        client.search(request.request, subscribeListener.safeMap(searchResponse -> {
+            searchResponse.mustIncRef(); // acquire reference on behalf of MultiSearchResponse.Item below
+            return new MultiSearchResponse.Item(searchResponse, null);
+        }));
+        final ActionListener<MultiSearchResponse.Item> responseListener = new ActionListener<>() {
             @Override
-            public void onResponse(final SearchResponse searchResponse) {
-                searchResponse.mustIncRef(); // acquire reference on behalf of MultiSearchResponse.Item below
-                handleResponse(request.responseSlot, new MultiSearchResponse.Item(searchResponse, null));
+            public void onResponse(final MultiSearchResponse.Item searchResponse) {
+                handleResponse(request.responseSlot, searchResponse);
             }
 
             @Override
@@ -174,15 +178,6 @@ private void handleResponse(final int responseSlot, final MultiSearchResponse.It
                 if (responseCounter.decrementAndGet() == 0) {
                     assert requests.isEmpty();
                     finish();
-                } else {
-                    if (thread == Thread.currentThread()) {
-                        // we are on the same thread, we need to fork to another thread to avoid recursive stack overflow on a single thread
-                        threadPool.generic()
-                            .execute(() -> executeSearch(requests, responses, responseCounter, listener, relativeStartTime));
-                    } else {
-                        // we are on a different thread (we went asynchronous), it's safe to recurse
-                        executeSearch(requests, responses, responseCounter, listener, relativeStartTime);
-                    }
                 }
             }
 
@@ -199,7 +194,19 @@ private void finish() {
             private long buildTookInMillis() {
                 return TimeUnit.NANOSECONDS.toMillis(relativeTimeProvider.getAsLong() - relativeStartTime);
             }
-        });
+        };
+        if (subscribeListener.isDone()) {
+            subscribeListener.addListener(responseListener);
+            return true;
+        }
+        // we went forked and have to check if there's more searches to execute after we're done with this search
+        subscribeListener.addListener(
+            ActionListener.runAfter(
+                responseListener,
+                () -> executeSearch(requests, responses, responseCounter, listener, relativeStartTime)
+            )
+        );
+        return false;
     }
 
     record SearchRequestSlot(SearchRequest request, int responseSlot) {
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java
index 91784ba331857..717b1805547be 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java
@@ -10,6 +10,8 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.ActionType;
@@ -21,7 +23,7 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -29,6 +31,8 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.SearchShardTarget;
@@ -50,6 +54,8 @@
 import java.util.concurrent.Executor;
 import java.util.function.BiFunction;
 
+import static org.elasticsearch.core.Strings.format;
+
 public class TransportOpenPointInTimeAction extends HandledTransportAction<OpenPointInTimeRequest, OpenPointInTimeResponse> {
 
     private static final Logger logger = LogManager.getLogger(TransportOpenPointInTimeAction.class);
@@ -62,6 +68,7 @@ public class TransportOpenPointInTimeAction extends HandledTransportAction<OpenP
     private final NamedWriteableRegistry namedWriteableRegistry;
     private final TransportService transportService;
     private final SearchService searchService;
+    private final ClusterService clusterService;
 
     @Inject
     public TransportOpenPointInTimeAction(
@@ -70,7 +77,8 @@ public TransportOpenPointInTimeAction(
         ActionFilters actionFilters,
         TransportSearchAction transportSearchAction,
         SearchTransportService searchTransportService,
-        NamedWriteableRegistry namedWriteableRegistry
+        NamedWriteableRegistry namedWriteableRegistry,
+        ClusterService clusterService
     ) {
         super(TYPE.name(), transportService, actionFilters, OpenPointInTimeRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.transportService = transportService;
@@ -78,6 +86,7 @@ public TransportOpenPointInTimeAction(
         this.searchService = searchService;
         this.searchTransportService = searchTransportService;
         this.namedWriteableRegistry = namedWriteableRegistry;
+        this.clusterService = clusterService;
         transportService.registerRequestHandler(
             OPEN_SHARD_READER_CONTEXT_NAME,
             EsExecutors.DIRECT_EXECUTOR_SERVICE,
@@ -94,17 +103,38 @@ public TransportOpenPointInTimeAction(
 
     @Override
     protected void doExecute(Task task, OpenPointInTimeRequest request, ActionListener<OpenPointInTimeResponse> listener) {
+        final ClusterState clusterState = clusterService.state();
+        // Check if all the nodes in this cluster know about the service
+        if (request.allowPartialSearchResults()
+            && clusterState.getMinTransportVersion().before(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) {
+            listener.onFailure(
+                new ElasticsearchStatusException(
+                    format(
+                        "The [allow_partial_search_results] parameter cannot be used while the cluster is still upgrading. "
+                            + "Please wait until the upgrade is fully completed and try again."
+                    ),
+                    RestStatus.BAD_REQUEST
+                )
+            );
+            return;
+        }
         final SearchRequest searchRequest = new SearchRequest().indices(request.indices())
             .indicesOptions(request.indicesOptions())
             .preference(request.preference())
             .routing(request.routing())
-            .allowPartialSearchResults(false)
+            .allowPartialSearchResults(request.allowPartialSearchResults())
             .source(new SearchSourceBuilder().query(request.indexFilter()));
         searchRequest.setMaxConcurrentShardRequests(request.maxConcurrentShardRequests());
         searchRequest.setCcsMinimizeRoundtrips(false);
         transportSearchAction.executeRequest((SearchTask) task, searchRequest, listener.map(r -> {
             assert r.pointInTimeId() != null : r;
-            return new OpenPointInTimeResponse(r.pointInTimeId());
+            return new OpenPointInTimeResponse(
+                r.pointInTimeId(),
+                r.getTotalShards(),
+                r.getSuccessfulShards(),
+                r.getFailedShards(),
+                r.getSkippedShards()
+            );
         }), searchListener -> new OpenPointInTimePhase(request, searchListener));
     }
 
@@ -213,6 +243,13 @@ SearchPhase openPointInTimePhase(
                 searchRequest.getMaxConcurrentShardRequests(),
                 clusters
             ) {
+                @Override
+                protected String missingShardsErrorMessage(StringBuilder missingShards) {
+                    return "[open_point_in_time] action requires all shards to be available. Missing shards: ["
+                        + missingShards
+                        + "].  Consider using `allow_partial_search_results` setting to bypass this error.";
+                }
+
                 @Override
                 protected void executePhaseOnShard(
                     SearchShardIterator shardIt,
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 84d233ec9710a..32ee9c331295c 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -23,9 +23,12 @@
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
 import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
+import org.elasticsearch.action.admin.cluster.stats.CCSUsage;
+import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -42,10 +45,10 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.ArrayUtils;
@@ -64,10 +67,12 @@
 import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.indices.ExecutorSelector;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.action.search.SearchResponseMetrics;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.aggregations.AggregationReduceContext;
+import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContext;
@@ -82,6 +87,7 @@
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.usage.UsageService;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentFactory;
 
@@ -154,6 +160,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
     private final boolean ccsCheckCompatibility;
     private final SearchResponseMetrics searchResponseMetrics;
     private final Client client;
+    private final UsageService usageService;
 
     @Inject
     public TransportSearchAction(
@@ -170,7 +177,8 @@ public TransportSearchAction(
         ExecutorSelector executorSelector,
         SearchTransportAPMMetrics searchTransportMetrics,
         SearchResponseMetrics searchResponseMetrics,
-        Client client
+        Client client,
+        UsageService usageService
     ) {
         super(TYPE.name(), transportService, actionFilters, SearchRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.threadPool = threadPool;
@@ -189,6 +197,7 @@ public TransportSearchAction(
         this.ccsCheckCompatibility = SearchService.CCS_VERSION_CHECK_SETTING.get(clusterService.getSettings());
         this.searchResponseMetrics = searchResponseMetrics;
         this.client = client;
+        this.usageService = usageService;
     }
 
     private Map<String, OriginalIndices> buildPerIndexOriginalIndices(
@@ -303,43 +312,7 @@ public long buildTookInMillis() {
 
     @Override
     protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<SearchResponse> listener) {
-        ActionListener<SearchResponse> loggingAndMetrics = new ActionListener<>() {
-            @Override
-            public void onResponse(SearchResponse searchResponse) {
-                try {
-                    searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis());
-                    SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus =
-                        SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS;
-                    if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) {
-                        // Deduplicate failures by exception message and index
-                        ShardOperationFailedException[] groupedFailures = ExceptionsHelper.groupBy(searchResponse.getShardFailures());
-                        for (ShardOperationFailedException f : groupedFailures) {
-                            boolean causeHas500Status = false;
-                            if (f.getCause() != null) {
-                                causeHas500Status = ExceptionsHelper.status(f.getCause()).getStatus() >= 500;
-                            }
-                            if ((f.status().getStatus() >= 500 || causeHas500Status)
-                                && ExceptionsHelper.isNodeOrShardUnavailableTypeException(f.getCause()) == false) {
-                                logger.warn("TransportSearchAction shard failure (partial results response)", f);
-                                responseCountTotalStatus = SearchResponseMetrics.ResponseCountTotalStatus.PARTIAL_FAILURE;
-                            }
-                        }
-                    }
-                    listener.onResponse(searchResponse);
-                    // increment after the delegated onResponse to ensure we don't
-                    // record both a success and a failure if there is an exception
-                    searchResponseMetrics.incrementResponseCount(responseCountTotalStatus);
-                } catch (Exception e) {
-                    onFailure(e);
-                }
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                searchResponseMetrics.incrementResponseCount(SearchResponseMetrics.ResponseCountTotalStatus.FAILURE);
-                listener.onFailure(e);
-            }
-        };
+        ActionListener<SearchResponse> loggingAndMetrics = new SearchResponseActionListener((SearchTask) task, listener);
         executeRequest((SearchTask) task, searchRequest, loggingAndMetrics, AsyncSearchActionProvider::new);
     }
 
@@ -394,8 +367,32 @@ void executeRequest(
                     searchPhaseProvider.apply(delegate)
                 );
             } else {
+                if ((listener instanceof TelemetryListener tl) && CCS_TELEMETRY_FEATURE_FLAG.isEnabled()) {
+                    tl.setRemotes(resolvedIndices.getRemoteClusterIndices().size());
+                    if (task.isAsync()) {
+                        tl.setFeature(CCSUsageTelemetry.ASYNC_FEATURE);
+                    }
+                    String client = task.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER);
+                    if (client != null) {
+                        tl.setClient(client);
+                    }
+                    // Check if any of the index patterns are wildcard patterns
+                    var localIndices = resolvedIndices.getLocalIndices();
+                    if (localIndices != null && Arrays.stream(localIndices.indices()).anyMatch(Regex::isSimpleMatchPattern)) {
+                        tl.setFeature(CCSUsageTelemetry.WILDCARD_FEATURE);
+                    }
+                    if (resolvedIndices.getRemoteClusterIndices()
+                        .values()
+                        .stream()
+                        .anyMatch(indices -> Arrays.stream(indices.indices()).anyMatch(Regex::isSimpleMatchPattern))) {
+                        tl.setFeature(CCSUsageTelemetry.WILDCARD_FEATURE);
+                    }
+                }
                 final TaskId parentTaskId = task.taskInfo(clusterService.localNode().getId(), false).taskId();
                 if (shouldMinimizeRoundtrips(rewritten)) {
+                    if ((listener instanceof TelemetryListener tl) && CCS_TELEMETRY_FEATURE_FLAG.isEnabled()) {
+                        tl.setFeature(CCSUsageTelemetry.MRT_FEATURE);
+                    }
                     final AggregationReduceContext.Builder aggregationReduceContextBuilder = rewritten.source() != null
                         && rewritten.source().aggregations() != null
                             ? searchService.aggReduceContextBuilder(task::isCancelled, rewritten.source().aggregations())
@@ -495,11 +492,72 @@ void executeRequest(
                 }
             }
         });
+        final SearchSourceBuilder source = original.source();
+        if (shouldOpenPIT(source)) {
+            openPIT(client, original, searchService.getDefaultKeepAliveInMillis(), listener.delegateFailureAndWrap((delegate, resp) -> {
+                // We set the keep alive to -1 to indicate that we don't need the pit id in the response.
+                // This is needed since we delete the pit prior to sending the response so the id doesn't exist anymore.
+                source.pointInTimeBuilder(new PointInTimeBuilder(resp.getPointInTimeId()).setKeepAlive(TimeValue.MINUS_ONE));
+                executeRequest(task, original, new ActionListener<>() {
+                    @Override
+                    public void onResponse(SearchResponse response) {
+                        // we need to close the PIT first so we delay the release of the response to after the closing
+                        response.incRef();
+                        closePIT(
+                            client,
+                            original.source().pointInTimeBuilder(),
+                            () -> ActionListener.respondAndRelease(listener, response)
+                        );
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        closePIT(client, original.source().pointInTimeBuilder(), () -> listener.onFailure(e));
+                    }
+                }, searchPhaseProvider);
+            }));
+        } else {
+            Rewriteable.rewriteAndFetch(
+                original,
+                searchService.getRewriteContext(timeProvider::absoluteStartMillis, resolvedIndices, original.pointInTimeBuilder()),
+                rewriteListener
+            );
+        }
+    }
 
-        Rewriteable.rewriteAndFetch(
-            original,
-            searchService.getRewriteContext(timeProvider::absoluteStartMillis, resolvedIndices),
-            rewriteListener
+    /**
+     * Returns true if the provided source needs to open a shared point in time prior to executing the request.
+     */
+    private boolean shouldOpenPIT(SearchSourceBuilder source) {
+        if (source == null) {
+            return false;
+        }
+        if (source.pointInTimeBuilder() != null) {
+            return false;
+        }
+        var retriever = source.retriever();
+        return retriever != null && retriever.isCompound();
+    }
+
+    static void openPIT(Client client, SearchRequest request, long keepAliveMillis, ActionListener<OpenPointInTimeResponse> listener) {
+        OpenPointInTimeRequest pitReq = new OpenPointInTimeRequest(request.indices()).indicesOptions(request.indicesOptions())
+            .preference(request.preference())
+            .routing(request.routing())
+            .keepAlive(TimeValue.timeValueMillis(keepAliveMillis));
+        client.execute(TransportOpenPointInTimeAction.TYPE, pitReq, listener);
+    }
+
+    static void closePIT(Client client, PointInTimeBuilder pit, Runnable next) {
+        client.execute(
+            TransportClosePointInTimeAction.TYPE,
+            new ClosePointInTimeRequest(pit.getEncodedId()),
+            ActionListener.runAfter(new ActionListener<>() {
+                @Override
+                public void onResponse(ClosePointInTimeResponse closePointInTimeResponse) {}
+
+                @Override
+                public void onFailure(Exception e) {}
+            }, next)
         );
     }
 
@@ -742,27 +800,26 @@ static void collectSearchShards(
         for (Map.Entry<String, OriginalIndices> entry : remoteIndicesByCluster.entrySet()) {
             final String clusterAlias = entry.getKey();
             boolean skipUnavailable = remoteClusterService.isSkipUnavailable(clusterAlias);
-            TransportSearchAction.CCSActionListener<SearchShardsResponse, Map<String, SearchShardsResponse>> singleListener =
-                new TransportSearchAction.CCSActionListener<>(
-                    clusterAlias,
-                    skipUnavailable,
-                    responsesCountDown,
-                    exceptions,
-                    clusters,
-                    listener
-                ) {
-                    @Override
-                    void innerOnResponse(SearchShardsResponse searchShardsResponse) {
-                        assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION);
-                        ccsClusterInfoUpdate(searchShardsResponse, clusters, clusterAlias, timeProvider);
-                        searchShardsResponses.put(clusterAlias, searchShardsResponse);
-                    }
+            CCSActionListener<SearchShardsResponse, Map<String, SearchShardsResponse>> singleListener = new CCSActionListener<>(
+                clusterAlias,
+                skipUnavailable,
+                responsesCountDown,
+                exceptions,
+                clusters,
+                listener
+            ) {
+                @Override
+                void innerOnResponse(SearchShardsResponse searchShardsResponse) {
+                    assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION);
+                    ccsClusterInfoUpdate(searchShardsResponse, clusters, clusterAlias, timeProvider);
+                    searchShardsResponses.put(clusterAlias, searchShardsResponse);
+                }
 
-                    @Override
-                    Map<String, SearchShardsResponse> createFinalResponse() {
-                        return searchShardsResponses;
-                    }
-                };
+                @Override
+                Map<String, SearchShardsResponse> createFinalResponse() {
+                    return searchShardsResponses;
+                }
+            };
             remoteClusterService.maybeEnsureConnectedAndGetConnection(
                 clusterAlias,
                 skipUnavailable == false,
@@ -789,9 +846,10 @@ Map<String, SearchShardsResponse> createFinalResponse() {
                         );
                     } else {
                         // does not do a can-match
-                        ClusterSearchShardsRequest searchShardsRequest = new ClusterSearchShardsRequest(indices).indicesOptions(
-                            indicesOptions
-                        ).local(true).preference(preference).routing(routing);
+                        ClusterSearchShardsRequest searchShardsRequest = new ClusterSearchShardsRequest(
+                            MasterNodeRequest.infiniteMasterNodeTimeout(connection.getTransportVersion()),
+                            indices
+                        ).indicesOptions(indicesOptions).local(true).preference(preference).routing(routing);
                         transportService.sendRequest(
                             connection,
                             TransportClusterSearchShardsAction.TYPE.name(),
@@ -1052,11 +1110,16 @@ static List<SearchShardIterator> getRemoteShardsIteratorFromPointInTime(
                 final String clusterAlias = entry.getKey();
                 assert clusterAlias.equals(perNode.getClusterAlias()) : clusterAlias + " != " + perNode.getClusterAlias();
                 final List<String> targetNodes = new ArrayList<>(group.allocatedNodes().size());
-                targetNodes.add(perNode.getNode());
-                if (perNode.getSearchContextId().getSearcherId() != null) {
-                    for (String node : group.allocatedNodes()) {
-                        if (node.equals(perNode.getNode()) == false) {
-                            targetNodes.add(node);
+                if (perNode.getNode() != null) {
+                    // If the shard was available when the PIT was created, it's included.
+                    // Otherwise, we add the shard iterator without a target node, allowing a partial search failure to
+                    // be thrown when a search phase attempts to access it.
+                    targetNodes.add(perNode.getNode());
+                    if (perNode.getSearchContextId().getSearcherId() != null) {
+                        for (String node : group.allocatedNodes()) {
+                            if (node.equals(perNode.getNode()) == false) {
+                                targetNodes.add(node);
+                            }
                         }
                     }
                 }
@@ -1152,7 +1215,7 @@ private void executeSearch(
             assert searchRequest.pointInTimeBuilder() != null;
             aliasFilter = resolvedIndices.getSearchContextId().aliasFilter();
             concreteLocalIndices = resolvedIndices.getLocalIndices() == null ? new String[0] : resolvedIndices.getLocalIndices().indices();
-            localShardIterators = getLocalLocalShardsIteratorFromPointInTime(
+            localShardIterators = getLocalShardsIteratorFromPointInTime(
                 clusterState,
                 searchRequest.indicesOptions(),
                 searchRequest.getLocalClusterAlias(),
@@ -1659,7 +1722,7 @@ private static RemoteTransportException wrapRemoteClusterFailure(String clusterA
         return new RemoteTransportException("error while communicating with remote cluster [" + clusterAlias + "]", e);
     }
 
-    static List<SearchShardIterator> getLocalLocalShardsIteratorFromPointInTime(
+    static List<SearchShardIterator> getLocalShardsIteratorFromPointInTime(
         ClusterState clusterState,
         IndicesOptions indicesOptions,
         String localClusterAlias,
@@ -1673,25 +1736,30 @@ static List<SearchShardIterator> getLocalLocalShardsIteratorFromPointInTime(
             if (Strings.isEmpty(perNode.getClusterAlias())) {
                 final ShardId shardId = entry.getKey();
                 final List<String> targetNodes = new ArrayList<>(2);
-                try {
-                    final ShardIterator shards = OperationRouting.getShards(clusterState, shardId);
-                    // Prefer executing shard requests on nodes that are part of PIT first.
-                    if (clusterState.nodes().nodeExists(perNode.getNode())) {
-                        targetNodes.add(perNode.getNode());
-                    }
-                    if (perNode.getSearchContextId().getSearcherId() != null) {
-                        for (ShardRouting shard : shards) {
-                            if (shard.currentNodeId().equals(perNode.getNode()) == false) {
-                                targetNodes.add(shard.currentNodeId());
+                if (perNode.getNode() != null) {
+                    // If the shard was available when the PIT was created, it's included.
+                    // Otherwise, we add the shard iterator without a target node, allowing a partial search failure to
+                    // be thrown when a search phase attempts to access it.
+                    try {
+                        final ShardIterator shards = OperationRouting.getShards(clusterState, shardId);
+                        // Prefer executing shard requests on nodes that are part of PIT first.
+                        if (clusterState.nodes().nodeExists(perNode.getNode())) {
+                            targetNodes.add(perNode.getNode());
+                        }
+                        if (perNode.getSearchContextId().getSearcherId() != null) {
+                            for (ShardRouting shard : shards) {
+                                if (shard.currentNodeId().equals(perNode.getNode()) == false) {
+                                    targetNodes.add(shard.currentNodeId());
+                                }
                             }
                         }
-                    }
-                } catch (IndexNotFoundException | ShardNotFoundException e) {
-                    // We can hit these exceptions if the index was deleted after creating PIT or the cluster state on
-                    // this coordinating node is outdated. It's fine to ignore these extra "retry-able" target shards
-                    // when allowPartialSearchResults is false
-                    if (allowPartialSearchResults == false) {
-                        throw e;
+                    } catch (IndexNotFoundException | ShardNotFoundException e) {
+                        // We can hit these exceptions if the index was deleted after creating PIT or the cluster state on
+                        // this coordinating node is outdated. It's fine to ignore these extra "retry-able" target shards
+                        // when allowPartialSearchResults is false
+                        if (allowPartialSearchResults == false) {
+                            throw e;
+                        }
                     }
                 }
                 OriginalIndices finalIndices = new OriginalIndices(new String[] { shardId.getIndexName() }, indicesOptions);
@@ -1750,4 +1818,112 @@ List<SearchShardIterator> getLocalShardsIterator(
         // the returned list must support in-place sorting, so this is the most memory efficient we can do here
         return Arrays.asList(list);
     }
+
+    private interface TelemetryListener {
+        void setRemotes(int count);
+
+        void setFeature(String feature);
+
+        void setClient(String client);
+    }
+
+    private class SearchResponseActionListener implements ActionListener<SearchResponse>, TelemetryListener {
+        private final SearchTask task;
+        private final ActionListener<SearchResponse> listener;
+        private final CCSUsage.Builder usageBuilder;
+
+        SearchResponseActionListener(SearchTask task, ActionListener<SearchResponse> listener) {
+            this.task = task;
+            this.listener = listener;
+            usageBuilder = new CCSUsage.Builder();
+        }
+
+        /**
+         * Should we collect telemetry for this search?
+         */
+        private boolean collectTelemetry() {
+            return CCS_TELEMETRY_FEATURE_FLAG.isEnabled() && usageBuilder.getRemotesCount() > 0;
+        }
+
+        public void setRemotes(int count) {
+            usageBuilder.setRemotesCount(count);
+        }
+
+        @Override
+        public void setFeature(String feature) {
+            usageBuilder.setFeature(feature);
+        }
+
+        @Override
+        public void setClient(String client) {
+            usageBuilder.setClient(client);
+        }
+
+        @Override
+        public void onResponse(SearchResponse searchResponse) {
+            try {
+                searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis());
+                SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus =
+                    SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS;
+                if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) {
+                    // Deduplicate failures by exception message and index
+                    ShardOperationFailedException[] groupedFailures = ExceptionsHelper.groupBy(searchResponse.getShardFailures());
+                    for (ShardOperationFailedException f : groupedFailures) {
+                        boolean causeHas500Status = false;
+                        if (f.getCause() != null) {
+                            causeHas500Status = ExceptionsHelper.status(f.getCause()).getStatus() >= 500;
+                        }
+                        if ((f.status().getStatus() >= 500 || causeHas500Status)
+                            && ExceptionsHelper.isNodeOrShardUnavailableTypeException(f.getCause()) == false) {
+                            logger.warn("TransportSearchAction shard failure (partial results response)", f);
+                            responseCountTotalStatus = SearchResponseMetrics.ResponseCountTotalStatus.PARTIAL_FAILURE;
+                        }
+                    }
+                }
+                searchResponseMetrics.incrementResponseCount(responseCountTotalStatus);
+
+                if (collectTelemetry()) {
+                    extractCCSTelemetry(searchResponse);
+                    recordTelemetry();
+                }
+            } catch (Exception e) {
+                onFailure(e);
+                return;
+            }
+            // This is last because we want to collect telemetry before returning the response.
+            listener.onResponse(searchResponse);
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            searchResponseMetrics.incrementResponseCount(SearchResponseMetrics.ResponseCountTotalStatus.FAILURE);
+            if (collectTelemetry()) {
+                usageBuilder.setFailure(e);
+                recordTelemetry();
+            }
+            listener.onFailure(e);
+        }
+
+        private void recordTelemetry() {
+            usageService.getCcsUsageHolder().updateUsage(usageBuilder.build());
+        }
+
+        /**
+         * Extract telemetry data from the search response.
+         * @param searchResponse The final response from the search.
+         */
+        private void extractCCSTelemetry(SearchResponse searchResponse) {
+            usageBuilder.took(searchResponse.getTookInMillis());
+            for (String clusterAlias : searchResponse.getClusters().getClusterAliases()) {
+                SearchResponse.Cluster cluster = searchResponse.getClusters().getCluster(clusterAlias);
+                if (cluster.getStatus() == SearchResponse.Cluster.Status.SKIPPED) {
+                    usageBuilder.skippedRemote(clusterAlias);
+                } else {
+                    usageBuilder.perClusterUsage(clusterAlias, cluster.getTook());
+                }
+            }
+
+        }
+
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
index d60033786abeb..1e2c445b22f3c 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.action.search.SearchResponseMetrics;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java
index c3eea1fe557e7..81b999a1239e5 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java
@@ -18,10 +18,10 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.Rewriteable;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -87,6 +87,14 @@ public TransportSearchShardsAction(
 
     @Override
     protected void doExecute(Task task, SearchShardsRequest searchShardsRequest, ActionListener<SearchShardsResponse> listener) {
+        searchShards(task, searchShardsRequest, listener);
+    }
+
+    /**
+     * Notes that this method does not perform authorization for the search shards action.
+     * Callers must ensure that the request was properly authorized before calling this method.
+     */
+    public void searchShards(Task task, SearchShardsRequest searchShardsRequest, ActionListener<SearchShardsResponse> listener) {
         final long relativeStartNanos = System.nanoTime();
         SearchRequest original = new SearchRequest(searchShardsRequest.indices()).indicesOptions(searchShardsRequest.indicesOptions())
             .routing(searchShardsRequest.routing())
@@ -115,7 +123,7 @@ protected void doExecute(Task task, SearchShardsRequest searchShardsRequest, Act
 
         Rewriteable.rewriteAndFetch(
             original,
-            searchService.getRewriteContext(timeProvider::absoluteStartMillis, resolvedIndices),
+            searchService.getRewriteContext(timeProvider::absoluteStartMillis, resolvedIndices, null),
             listener.delegateFailureAndWrap((delegate, searchRequest) -> {
                 Index[] concreteIndices = resolvedIndices.getConcreteLocalIndices();
                 final Set<String> indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, searchRequest.indices());
diff --git a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
index 69bdfdea31ae4..b9200a0e32736 100644
--- a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
@@ -40,14 +40,14 @@ protected HandledTransportAction(
         Writeable.Reader<Request> requestReader,
         Executor executor
     ) {
-        super(actionName, actionFilters, transportService.getTaskManager());
+        super(actionName, actionFilters, transportService.getTaskManager(), executor);
         transportService.registerRequestHandler(
             actionName,
             executor,
             false,
             canTripCircuitBreaker,
             requestReader,
-            (request, channel, task) -> execute(task, request, new ChannelActionListener<>(channel))
+            (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel))
         );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index 33b64a9388c00..7c9dcb608ec84 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -958,7 +958,7 @@ public void writeIndicesOptions(StreamOutput out) throws IOException {
         if (ignoreUnavailable()) {
             backwardsCompatibleOptions.add(Option.ALLOW_UNAVAILABLE_CONCRETE_TARGETS);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ADD_FAILURE_STORE_INDICES_OPTIONS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             if (allowFailureIndices()) {
                 backwardsCompatibleOptions.add(Option.ALLOW_FAILURE_INDICES);
             }
@@ -976,7 +976,7 @@ public void writeIndicesOptions(StreamOutput out) throws IOException {
             states.add(WildcardStates.HIDDEN);
         }
         out.writeEnumSet(states);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ADD_FAILURE_STORE_INDICES_OPTIONS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             failureStoreOptions.writeTo(out);
         }
     }
@@ -989,7 +989,7 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti
             options.contains(Option.EXCLUDE_ALIASES)
         );
         boolean allowFailureIndices = true;
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ADD_FAILURE_STORE_INDICES_OPTIONS)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             allowFailureIndices = options.contains(Option.ALLOW_FAILURE_INDICES);
         }
         GatekeeperOptions gatekeeperOptions = GatekeeperOptions.builder()
@@ -998,7 +998,7 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti
             .allowFailureIndices(allowFailureIndices)
             .ignoreThrottled(options.contains(Option.IGNORE_THROTTLED))
             .build();
-        FailureStoreOptions failureStoreOptions = in.getTransportVersion().onOrAfter(TransportVersions.ADD_FAILURE_STORE_INDICES_OPTIONS)
+        FailureStoreOptions failureStoreOptions = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
             ? FailureStoreOptions.read(in)
             : FailureStoreOptions.DEFAULT;
         return new IndicesOptions(
diff --git a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java
index ee4433369f689..4022ef3873771 100644
--- a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java
+++ b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java
@@ -381,12 +381,12 @@ private boolean assertCompleteAllowed() {
     }
 
     // only used in assertions
-    boolean allowedExecutors(Thread thread1, Thread thread2) {
+    boolean allowedExecutors(Thread blockedThread, Thread completingThread) {
         // this should only be used to validate thread interactions, like not waiting for a future completed on the same
         // executor, hence calling it with the same thread indicates a bug in the assertion using this.
-        assert thread1 != thread2 : "only call this for different threads";
-        String thread1Name = EsExecutors.executorName(thread1);
-        String thread2Name = EsExecutors.executorName(thread2);
-        return thread1Name == null || thread2Name == null || thread1Name.equals(thread2Name) == false;
+        assert blockedThread != completingThread : "only call this for different threads";
+        String blockedThreadName = EsExecutors.executorName(blockedThread);
+        String completingThreadName = EsExecutors.executorName(completingThread);
+        return blockedThreadName == null || completingThreadName == null || blockedThreadName.equals(completingThreadName) == false;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java b/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java
index 41949c7ce3c22..6a2673e3276fc 100644
--- a/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java
+++ b/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java
@@ -410,6 +410,32 @@ public void complete(ActionListener<?> listener) {
         }
     }
 
+    /**
+     * Creates and returns a new {@link SubscribableListener} {@code L} and subscribes {@code nextStep} to this listener such that if this
+     * listener is completed successfully then the result is discarded and {@code nextStep} is invoked with argument {@code L}. If this
+     * listener is completed with exception {@code E} then so is {@code L}.
+     * <p>
+     * This can be used to construct a sequence of async actions, each ignoring the result of the previous ones:
+     * <pre>
+     * l.andThen(l1 -> forkAction1(args1, l1)).andThen(l2 -> forkAction2(args2, l2)).addListener(finalListener);
+     * </pre>
+     * After creating this chain, completing {@code l} with a successful response will call {@code forkAction1}, which will on completion
+     * call {@code forkAction2}, which will in turn pass its response to {@code finalListener}. A failure of any step will bypass the
+     * remaining steps and ultimately fail {@code finalListener}.
+     * <p>
+     * The threading of the {@code nextStep} callback is the same as for listeners added with {@link #addListener}: if this listener is
+     * already complete then {@code nextStep} is invoked on the thread calling {@link #andThen} and in its thread context, but if this
+     * listener is incomplete then {@code nextStep} is invoked on the completing thread and in its thread context. In other words, if you
+     * want to ensure that {@code nextStep} is invoked using a particular executor, then you must do both of:
+     * <ul>
+     * <li>Ensure that this {@link SubscribableListener} is always completed using that executor, and</li>
+     * <li>Invoke {@link #andThen} using that executor.</li>
+     * </ul>
+     */
+    public <U> SubscribableListener<U> andThen(CheckedConsumer<ActionListener<U>, ? extends Exception> nextStep) {
+        return newForked(l -> addListener(l.delegateFailureIgnoreResponseAndWrap(nextStep)));
+    }
+
     /**
      * Creates and returns a new {@link SubscribableListener} {@code L} and subscribes {@code nextStep} to this listener such that if this
      * listener is completed successfully with result {@code R} then {@code nextStep} is invoked with arguments {@code L} and {@code R}. If
diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java
index 222941981f05a..65a7e2302b9ae 100644
--- a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java
@@ -14,11 +14,14 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskManager;
 
+import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public abstract class TransportAction<Request extends ActionRequest, Response extends ActionResponse> {
@@ -26,22 +29,46 @@ public abstract class TransportAction<Request extends ActionRequest, Response ex
     public final String actionName;
     private final ActionFilter[] filters;
     protected final TaskManager taskManager;
+    private final Executor executor;
     /**
      * @deprecated declare your own logger.
      */
     @Deprecated
     protected Logger logger = LogManager.getLogger(getClass());
 
-    protected TransportAction(String actionName, ActionFilters actionFilters, TaskManager taskManager) {
+    interface TransportActionHandler<Request extends ActionRequest, Response extends ActionResponse> {
+        void execute(Task task, Request request, ActionListener<Response> listener);
+    }
+
+    protected TransportAction(String actionName, ActionFilters actionFilters, TaskManager taskManager, Executor executor) {
         this.actionName = actionName;
         this.filters = actionFilters.filters();
         this.taskManager = taskManager;
+        this.executor = executor;
     }
 
     /**
      * Use this method when the transport action should continue to run in the context of the current task
      */
+    protected final void executeDirect(Task task, Request request, ActionListener<Response> listener) {
+        handleExecution(task, request, listener, this::doExecute);
+    }
+
     public final void execute(Task task, Request request, ActionListener<Response> listener) {
+        handleExecution(
+            task,
+            request,
+            listener,
+            executor == EsExecutors.DIRECT_EXECUTOR_SERVICE ? this::doExecute : this::doExecuteForking
+        );
+    }
+
+    private void handleExecution(
+        Task task,
+        Request request,
+        ActionListener<Response> listener,
+        TransportActionHandler<Request, Response> handler
+    ) {
         final ActionRequestValidationException validationException;
         try {
             validationException = request.validate();
@@ -64,10 +91,14 @@ public final void execute(Task task, Request request, ActionListener<Response> l
         // Releasables#releaseOnce to avoid a double-release.
         request.mustIncRef();
         final var releaseRef = Releasables.releaseOnce(request::decRef);
-        RequestFilterChain<Request, Response> requestFilterChain = new RequestFilterChain<>(this, logger, releaseRef);
+        RequestFilterChain<Request, Response> requestFilterChain = new RequestFilterChain<>(this, logger, handler, releaseRef);
         requestFilterChain.proceed(task, actionName, request, ActionListener.runBefore(listener, releaseRef::close));
     }
 
+    private void doExecuteForking(Task task, Request request, ActionListener<Response> listener) {
+        executor.execute(ActionRunnable.wrap(listener, l -> doExecute(task, request, listener)));
+    }
+
     protected abstract void doExecute(Task task, Request request, ActionListener<Response> listener);
 
     private static class RequestFilterChain<Request extends ActionRequest, Response extends ActionResponse>
@@ -75,13 +106,20 @@ private static class RequestFilterChain<Request extends ActionRequest, Response
             ActionFilterChain<Request, Response> {
 
         private final TransportAction<Request, Response> action;
+        private final TransportActionHandler<Request, Response> handler;
         private final AtomicInteger index = new AtomicInteger();
         private final Logger logger;
         private final Releasable releaseRef;
 
-        private RequestFilterChain(TransportAction<Request, Response> action, Logger logger, Releasable releaseRef) {
+        private RequestFilterChain(
+            TransportAction<Request, Response> action,
+            Logger logger,
+            TransportActionHandler<Request, Response> handler,
+            Releasable releaseRef
+        ) {
             this.action = action;
             this.logger = logger;
+            this.handler = handler;
             this.releaseRef = releaseRef;
         }
 
@@ -93,7 +131,7 @@ public void proceed(Task task, String actionName, Request request, ActionListene
                     this.action.filters[i].apply(task, actionName, request, listener, this);
                 } else if (i == this.action.filters.length) {
                     try (releaseRef) {
-                        this.action.doExecute(task, request, listener);
+                        handler.execute(task, request, listener);
                     }
                 } else {
                     listener.onFailure(new IllegalStateException("proceed was called too many times"));
@@ -103,7 +141,6 @@ public void proceed(Task task, String actionName, Request request, ActionListene
                 listener.onFailure(e);
             }
         }
-
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java
index 8aa6bc4de109a..b76dfe07e18ed 100644
--- a/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java
+++ b/server/src/main/java/org/elasticsearch/action/support/UnsafePlainActionFuture.java
@@ -10,36 +10,33 @@
 
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 
-import java.util.Objects;
+import java.util.Set;
 
 /**
  * An unsafe future. You should not need to use this for new code, rather you should be able to convert that code to be async
  * or use a clear hierarchy of thread pool executors around the future.
- *
+ * <p>
  * This future is unsafe, since it allows notifying the future on the same thread pool executor that it is being waited on. This
  * is a common deadlock scenario, since all threads may be waiting and thus no thread may be able to complete the future.
+ * <p>
+ * Note that the deadlock protection in {@link PlainActionFuture} is very weak. In general there's a risk of deadlock if there's any cycle
+ * of threads which block/complete on each other's futures, or dispatch work to each other, but this is much harder to detect.
  */
 @Deprecated(forRemoval = true)
 public class UnsafePlainActionFuture<T> extends PlainActionFuture<T> {
-
-    private final String unsafeExecutor;
-    private final String unsafeExecutor2;
-
-    public UnsafePlainActionFuture(String unsafeExecutor) {
-        this(unsafeExecutor, null);
-    }
-
-    public UnsafePlainActionFuture(String unsafeExecutor, String unsafeExecutor2) {
-        Objects.requireNonNull(unsafeExecutor);
-        this.unsafeExecutor = unsafeExecutor;
-        this.unsafeExecutor2 = unsafeExecutor2;
+    private final Set<String> unsafeExecutors;
+
+    /**
+     * Create a future which permits any of the given named executors to be used unsafely (i.e. used for both waiting for the future's
+     * completion and completing the future).
+     */
+    public UnsafePlainActionFuture(String... unsafeExecutors) {
+        assert unsafeExecutors.length > 0 : "use PlainActionFuture if there are no executors to use unsafely";
+        this.unsafeExecutors = Set.of(unsafeExecutors);
     }
 
     @Override
-    boolean allowedExecutors(Thread thread1, Thread thread2) {
-        return super.allowedExecutors(thread1, thread2)
-            || unsafeExecutor.equals(EsExecutors.executorName(thread1))
-            || unsafeExecutor2 == null
-            || unsafeExecutor2.equals(EsExecutors.executorName(thread1));
+    boolean allowedExecutors(Thread blockedThread, Thread completingThread) {
+        return super.allowedExecutors(blockedThread, completingThread) || unsafeExecutors.contains(EsExecutors.executorName(blockedThread));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
index 269ebd80fb36a..d547f26bd2061 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.support.master;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -32,7 +33,8 @@ public abstract class MasterNodeRequest<Request extends MasterNodeRequest<Reques
      * timeout.
      * <p>
      * For internally-generated requests, choose an appropriate timeout. Often this will be {@link TimeValue#MAX_VALUE} (or {@link
-     * TimeValue#MINUS_ONE} which means an infinite timeout in 8.15.0 onwards) since usually we want internal requests to wait for as long
+     * TimeValue#MINUS_ONE} which means an infinite timeout in 8.14.0 onwards (see <a
+     * href="https://github.com/elastic/elasticsearch/pull/107050">#107050</a>) since usually we want internal requests to wait for as long
      * as necessary to complete.
      *
      * @deprecated all requests should specify a timeout, see <a href="https://github.com/elastic/elasticsearch/issues/107984">#107984</a>.
@@ -51,15 +53,20 @@ public abstract class MasterNodeRequest<Request extends MasterNodeRequest<Reques
 
     /**
      * @param masterNodeTimeout Specifies how long to wait when the master has not been discovered yet, or is disconnected, or is busy
-     *                          processing other tasks. The value {@link TimeValue#MINUS_ONE} means to wait forever in 8.15.0 onwards.
-     *                          <p>
-     *                          For requests which originate in the REST layer, use {@link
-     *                          org.elasticsearch.rest.RestUtils#getMasterNodeTimeout} to determine the timeout.
-     *                          <p>
-     *                          For internally-generated requests, choose an appropriate timeout. Often this will be {@link
-     *                          TimeValue#MAX_VALUE} (or {@link TimeValue#MINUS_ONE} which means an infinite timeout in 8.15.0 onwards)
-     *                          since usually we want internal requests to wait for as long as necessary to complete.
+     *                          processing other tasks:
+     *                          <ul>
+     *                          <li>
+     *                              For requests which originate in the REST layer, use
+     *                              {@link org.elasticsearch.rest.RestUtils#getMasterNodeTimeout} to determine the timeout.
+     *                          </li>
+     *                          <li>
+     *                              For internally-generated requests, choose an appropriate timeout. Often this will be an infinite
+     *                              timeout, see {@link #infiniteMasterNodeTimeout}, since it is reasonable to wait for as long as necessary
+     *                              for internal requests to complete.
+     *                          </li>
+     *                          </ul>
      */
+    // TODO forbid TimeValue#MAX_VALUE once support for version prior to 8.14 dropped
     protected MasterNodeRequest(TimeValue masterNodeTimeout) {
         this.masterNodeTimeout = Objects.requireNonNull(masterNodeTimeout);
         this.masterTerm = 0L;
@@ -127,4 +134,14 @@ public final TimeValue masterNodeTimeout() {
     public final long masterTerm() {
         return masterTerm;
     }
+
+    /**
+     * @return a {@link TimeValue} which represents an infinite master-node timeout, suitable for sending using the given transport version.
+     *         Versions prior to 8.14 did not reliably support {@link TimeValue#MINUS_ONE} for this purpose so for these versions we use
+     *         {@link TimeValue#MAX_VALUE} as the best available alternative.
+     * @see <a href="https://github.com/elastic/elasticsearch/pull/107050">#107050</a>
+     */
+    public static TimeValue infiniteMasterNodeTimeout(TransportVersion transportVersion) {
+        return transportVersion.onOrAfter(TransportVersions.V_8_14_0) ? TimeValue.MINUS_ONE : TimeValue.MAX_VALUE;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index fcd513b175bb1..347edd0916fc5 100644
--- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -78,7 +78,9 @@ protected TransportNodesAction(
         Writeable.Reader<NodeRequest> nodeRequest,
         Executor executor
     ) {
-        super(actionName, actionFilters, transportService.getTaskManager());
+        // Only part of this action execution needs to be forked off - coordination can run on SAME because it's only O(#nodes) work.
+        // Hence the separate "finalExecutor", and why we run the whole TransportAction.execute on SAME.
+        super(actionName, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         assert executor.equals(EsExecutors.DIRECT_EXECUTOR_SERVICE) == false
             : "TransportNodesAction must always fork off the transport thread";
         this.clusterService = Objects.requireNonNull(clusterService);
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index c2d7e173fd0bf..7e3e5bdee206d 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -117,6 +117,20 @@ protected enum SyncGlobalCheckpointAfterOperation {
         AttemptAfterSuccess
     }
 
+    /**
+     * Execution of the replica action
+     */
+    protected enum ReplicaActionExecution {
+        /**
+         * Will only execute when permitted by the configured circuit breakers
+         */
+        SubjectToCircuitBreaker,
+        /**
+         * Will bypass the configured circuit breaker checks
+         */
+        BypassCircuitBreaker
+    }
+
     /**
      * The timeout for retrying replication requests.
      */
@@ -170,11 +184,14 @@ protected TransportReplicationAction(
         Writeable.Reader<ReplicaRequest> replicaRequestReader,
         Executor executor,
         SyncGlobalCheckpointAfterOperation syncGlobalCheckpointAfterOperation,
-        PrimaryActionExecution primaryActionExecution
+        PrimaryActionExecution primaryActionExecution,
+        ReplicaActionExecution replicaActionExecution
     ) {
-        super(actionName, actionFilters, transportService.getTaskManager());
+        // TODO: consider passing the executor, investigate doExecute and let InboundHandler/TransportAction handle concurrency.
+        super(actionName, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         assert syncGlobalCheckpointAfterOperation != null : "Must specify global checkpoint sync behaviour";
         assert primaryActionExecution != null : "Must specify primary action execution behaviour";
+        assert replicaActionExecution != null : "Must specify replica action execution behaviour";
         this.threadPool = threadPool;
         this.transportService = transportService;
         this.clusterService = clusterService;
@@ -208,12 +225,15 @@ protected TransportReplicationAction(
             this::handlePrimaryRequest
         );
 
-        // we must never reject on because of thread pool capacity on replicas
+        boolean canTripCircuitBreakerOnReplica = switch (replicaActionExecution) {
+            case BypassCircuitBreaker -> false;
+            case SubjectToCircuitBreaker -> true;
+        };
         transportService.registerRequestHandler(
             transportReplicaAction,
             executor,
-            true,
-            true,
+            true, // we must never reject because of thread pool capacity on replicas
+            canTripCircuitBreakerOnReplica,
             in -> new ConcreteReplicaRequest<>(replicaRequestReader, in),
             this::handleReplicaRequest
         );
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
index f380710cc0794..90b636ed69e24 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
@@ -78,7 +78,8 @@ protected TransportWriteAction(
         BiFunction<ExecutorSelector, IndexShard, Executor> executorFunction,
         PrimaryActionExecution primaryActionExecution,
         IndexingPressure indexingPressure,
-        SystemIndices systemIndices
+        SystemIndices systemIndices,
+        ReplicaActionExecution replicaActionExecution
     ) {
         // We pass ThreadPool.Names.SAME to the super class as we control the dispatching to the
         // ThreadPool.Names.WRITE/ThreadPool.Names.SYSTEM_WRITE thread pools in this class.
@@ -95,7 +96,8 @@ protected TransportWriteAction(
             replicaRequest,
             EsExecutors.DIRECT_EXECUTOR_SERVICE,
             SyncGlobalCheckpointAfterOperation.AttemptAfterSuccess,
-            primaryActionExecution
+            primaryActionExecution,
+            replicaActionExecution
         );
         this.executorFunction = executorFunction;
         this.indexingPressure = indexingPressure;
diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
index a6a6b2c332a0a..180aa3b336149 100644
--- a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
@@ -72,7 +72,8 @@ protected TransportSingleShardAction(
         Writeable.Reader<Request> request,
         Executor executor
     ) {
-        super(actionName, actionFilters, transportService.getTaskManager());
+        // TODO: consider passing the executor, remove it from doExecute and let InboundHandler/TransportAction handle concurrency.
+        super(actionName, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.transportService = transportService;
@@ -250,7 +251,7 @@ private class TransportHandler implements TransportRequestHandler<Request> {
         @Override
         public void messageReceived(Request request, final TransportChannel channel, Task task) throws Exception {
             // if we have a local operation, execute it on a thread since we don't spawn
-            execute(task, request, new ChannelActionListener<>(channel));
+            executeDirect(task, request, new ChannelActionListener<>(channel));
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymRuleAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymRuleAction.java
index 68a6caa4b0550..f0c4e17abb382 100644
--- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymRuleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymRuleAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.synonyms.SynonymsManagementAPIService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymsAction.java
index d05ae9c0637b6..c22bb308f42de 100644
--- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportDeleteSynonymsAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.synonyms.SynonymsManagementAPIService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymRuleAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymRuleAction.java
index c5deaf21e6acf..5a574def93e0d 100644
--- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymRuleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymRuleAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.synonyms.SynonymsManagementAPIService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsAction.java
index 24b31fd38a8be..09d087e85e9e2 100644
--- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.synonyms.SynonymsManagementAPIService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsSetsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsSetsAction.java
index a0d74270e0694..13177f475354d 100644
--- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsSetsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportGetSynonymsSetsAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.synonyms.SynonymsManagementAPIService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymRuleAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymRuleAction.java
index 46a9a35beae4b..eb0b87ae0211a 100644
--- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymRuleAction.java
+++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymRuleAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.synonyms.SynonymsManagementAPIService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymsAction.java b/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymsAction.java
index d4f3d04e47e5d..4f5589bd85ed7 100644
--- a/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/synonyms/TransportPutSynonymsAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.synonyms.SynonymsManagementAPIService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
index 89129a1aec6a7..b4095d04baff2 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
@@ -17,10 +17,10 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
index d0277e8ce8c80..84f6d433af882 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
@@ -16,13 +16,13 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
index 1375ae7fb6c3a..75bbbf96136a9 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
@@ -16,13 +16,13 @@
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
index cd899d732e916..00d33ec9d2eca 100644
--- a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
@@ -34,7 +34,6 @@
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -49,6 +48,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
index 6b54654d7fbe9..056eb70c53269 100644
--- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
+++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.plugins.internal.DocumentParsingProvider;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
+import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.UpdateCtxMap;
@@ -181,14 +181,14 @@ static String calculateRouting(GetResult getResult, @Nullable IndexRequest updat
     Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResult getResult, boolean detectNoop) {
         final IndexRequest currentRequest = request.doc();
         final String routing = calculateRouting(getResult, currentRequest);
-        final DocumentSizeObserver documentSizeObserver = documentParsingProvider.newDocumentSizeObserver(request);
+        final XContentMeteringParserDecorator meteringParserDecorator = documentParsingProvider.newMeteringParserDecorator(request);
         final Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true);
         final XContentType updateSourceContentType = sourceAndContent.v1();
         final Map<String, Object> updatedSourceAsMap = sourceAndContent.v2();
 
         final boolean noop = XContentHelper.update(
             updatedSourceAsMap,
-            currentRequest.sourceAsMap(documentSizeObserver),
+            currentRequest.sourceAsMap(meteringParserDecorator),
             detectNoop
         ) == false;
 
@@ -226,8 +226,8 @@ Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResu
                 .waitForActiveShards(request.waitForActiveShards())
                 .timeout(request.timeout())
                 .setRefreshPolicy(request.getRefreshPolicy())
-                .setNormalisedBytesParsed(documentSizeObserver.normalisedBytesParsed());
-
+                .setOriginatesFromUpdateByDoc(true);
+            finalIndexRequest.setNormalisedBytesParsed(meteringParserDecorator.meteredDocumentSize().ingestedBytes());
             return new Result(finalIndexRequest, DocWriteResponse.Result.UPDATED, updatedSourceAsMap, updateSourceContentType);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
index c1ee0f7b8af37..587ed2ef75eba 100644
--- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
@@ -68,6 +68,7 @@ public UpdateRequestBuilder(ElasticsearchClient client) {
         this(client, null, null);
     }
 
+    @SuppressWarnings("this-escape")
     public UpdateRequestBuilder(ElasticsearchClient client, String index, String id) {
         super(client, TransportUpdateAction.TYPE);
         setIndex(index);
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/StartupException.java b/server/src/main/java/org/elasticsearch/bootstrap/StartupException.java
index 5d7d764fdae1f..14027a417e2f6 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/StartupException.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/StartupException.java
@@ -8,8 +8,8 @@
 
 package org.elasticsearch.bootstrap;
 
-import org.elasticsearch.common.inject.CreationException;
-import org.elasticsearch.common.inject.spi.Message;
+import org.elasticsearch.injection.guice.CreationException;
+import org.elasticsearch.injection.guice.spi.Message;
 
 import java.io.PrintStream;
 import java.util.Objects;
@@ -25,7 +25,7 @@ public final class StartupException extends Exception {
     /** maximum length of a stacktrace, before we truncate it */
     static final int STACKTRACE_LIMIT = 30;
     /** all lines from this package are RLE-compressed */
-    static final String GUICE_PACKAGE = "org.elasticsearch.common.inject";
+    static final String GUICE_PACKAGE = "org.elasticsearch.injection.guice";
 
     public StartupException(Throwable cause) {
         super(Objects.requireNonNull(cause));
diff --git a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java
index f14a2f6fb5247..4d5a670925b5b 100644
--- a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java
+++ b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java
@@ -64,10 +64,6 @@
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequestBuilder;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
-import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
-import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequestBuilder;
-import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
-import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
 import org.elasticsearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequestBuilder;
 import org.elasticsearch.action.admin.cluster.snapshots.clone.TransportCloneSnapshotAction;
@@ -98,16 +94,6 @@
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
 import org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequestBuilder;
-import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequestBuilder;
-import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequestBuilder;
-import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.action.ingest.DeletePipelineRequestBuilder;
 import org.elasticsearch.action.ingest.DeletePipelineTransportAction;
@@ -122,9 +108,7 @@
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineRequestBuilder;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
-import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.TaskId;
@@ -286,14 +270,6 @@ public CancelTasksRequestBuilder prepareCancelTasks(String... nodesIds) {
         return new CancelTasksRequestBuilder(this).setNodesIds(nodesIds);
     }
 
-    public void searchShards(final ClusterSearchShardsRequest request, final ActionListener<ClusterSearchShardsResponse> listener) {
-        execute(TransportClusterSearchShardsAction.TYPE, request, listener);
-    }
-
-    public ClusterSearchShardsRequestBuilder prepareSearchShards(String... indices) {
-        return new ClusterSearchShardsRequestBuilder(this).setIndices(indices);
-    }
-
     public void putRepository(PutRepositoryRequest request, ActionListener<AcknowledgedResponse> listener) {
         execute(TransportPutRepositoryAction.TYPE, request, listener);
     }
@@ -302,15 +278,6 @@ public PutRepositoryRequestBuilder preparePutRepository(TimeValue masterNodeTime
         return new PutRepositoryRequestBuilder(this, masterNodeTimeout, ackTimeout, name);
     }
 
-    @Deprecated(forRemoval = true) // temporary compatibility shim
-    public PutRepositoryRequestBuilder preparePutRepository(String name) {
-        return preparePutRepository(
-            MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT,
-            AcknowledgedRequest.DEFAULT_ACK_TIMEOUT,
-            name
-        );
-    }
-
     public void deleteRepository(DeleteRepositoryRequest request, ActionListener<AcknowledgedResponse> listener) {
         execute(TransportDeleteRepositoryAction.TYPE, request, listener);
     }
@@ -319,15 +286,6 @@ public DeleteRepositoryRequestBuilder prepareDeleteRepository(TimeValue masterNo
         return new DeleteRepositoryRequestBuilder(this, masterNodeTimeout, ackTimeout, name);
     }
 
-    @Deprecated(forRemoval = true) // temporary compatibility shim
-    public DeleteRepositoryRequestBuilder prepareDeleteRepository(String name) {
-        return prepareDeleteRepository(
-            MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT,
-            AcknowledgedRequest.DEFAULT_ACK_TIMEOUT,
-            name
-        );
-    }
-
     public void getRepositories(GetRepositoriesRequest request, ActionListener<GetRepositoriesResponse> listener) {
         execute(GetRepositoriesAction.INSTANCE, request, listener);
     }
@@ -360,11 +318,6 @@ public void createSnapshot(CreateSnapshotRequest request, ActionListener<CreateS
         execute(TransportCreateSnapshotAction.TYPE, request, listener);
     }
 
-    @Deprecated(forRemoval = true) // temporary compatibility shim
-    public CreateSnapshotRequestBuilder prepareCreateSnapshot(String repository, String name) {
-        return prepareCreateSnapshot(MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, repository, name);
-    }
-
     public CreateSnapshotRequestBuilder prepareCreateSnapshot(TimeValue masterNodeTimeout, String repository, String name) {
         return new CreateSnapshotRequestBuilder(this, masterNodeTimeout, repository, name);
     }
@@ -385,20 +338,10 @@ public GetSnapshotsRequestBuilder prepareGetSnapshots(TimeValue masterNodeTimeou
         return new GetSnapshotsRequestBuilder(this, masterNodeTimeout, repositories);
     }
 
-    @Deprecated(forRemoval = true) // temporary compatibility shim
-    public GetSnapshotsRequestBuilder prepareGetSnapshots(String... repositories) {
-        return prepareGetSnapshots(MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, repositories);
-    }
-
     public void deleteSnapshot(DeleteSnapshotRequest request, ActionListener<AcknowledgedResponse> listener) {
         execute(TransportDeleteSnapshotAction.TYPE, request, listener);
     }
 
-    @Deprecated(forRemoval = true) // temporary compatibility shim
-    public DeleteSnapshotRequestBuilder prepareDeleteSnapshot(String repository, String... names) {
-        return prepareDeleteSnapshot(MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, repository, names);
-    }
-
     public DeleteSnapshotRequestBuilder prepareDeleteSnapshot(TimeValue masterNodeTimeout, String repository, String... names) {
         return new DeleteSnapshotRequestBuilder(this, masterNodeTimeout, repository, names);
     }
@@ -411,11 +354,6 @@ public void restoreSnapshot(RestoreSnapshotRequest request, ActionListener<Resto
         execute(TransportRestoreSnapshotAction.TYPE, request, listener);
     }
 
-    @Deprecated(forRemoval = true) // temporary compatibility shim
-    public RestoreSnapshotRequestBuilder prepareRestoreSnapshot(String repository, String snapshot) {
-        return prepareRestoreSnapshot(MasterNodeRequest.TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, repository, snapshot);
-    }
-
     public RestoreSnapshotRequestBuilder prepareRestoreSnapshot(TimeValue masterNodeTimeout, String repository, String snapshot) {
         return new RestoreSnapshotRequestBuilder(this, masterNodeTimeout, repository, snapshot);
     }
@@ -475,29 +413,4 @@ public ActionFuture<SimulatePipelineResponse> simulatePipeline(SimulatePipelineR
     public SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source, XContentType xContentType) {
         return new SimulatePipelineRequestBuilder(this, source, xContentType);
     }
-
-    public PutStoredScriptRequestBuilder preparePutStoredScript() {
-        return new PutStoredScriptRequestBuilder(this);
-    }
-
-    public void deleteStoredScript(DeleteStoredScriptRequest request, ActionListener<AcknowledgedResponse> listener) {
-        execute(TransportDeleteStoredScriptAction.TYPE, request, listener);
-    }
-
-    public DeleteStoredScriptRequestBuilder prepareDeleteStoredScript(String id) {
-        return new DeleteStoredScriptRequestBuilder(client).setId(id);
-    }
-
-    public void putStoredScript(final PutStoredScriptRequest request, ActionListener<AcknowledgedResponse> listener) {
-        execute(TransportPutStoredScriptAction.TYPE, request, listener);
-
-    }
-
-    public GetStoredScriptRequestBuilder prepareGetStoredScript(String id) {
-        return new GetStoredScriptRequestBuilder(this).setId(id);
-    }
-
-    public void getStoredScript(final GetStoredScriptRequest request, final ActionListener<GetStoredScriptResponse> listener) {
-        execute(GetStoredScriptAction.INSTANCE, request, listener);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java
index f4e86c8a4eca6..c2268bfd9dc62 100644
--- a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java
+++ b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java
@@ -93,6 +93,7 @@ public abstract class AbstractClient implements Client {
     private final ThreadPool threadPool;
     private final AdminClient admin;
 
+    @SuppressWarnings("this-escape")
     public AbstractClient(Settings settings, ThreadPool threadPool) {
         this.settings = settings;
         this.threadPool = threadPool;
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 29933ad20ef10..3fba3a7bdbe13 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -58,7 +58,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
@@ -72,10 +71,12 @@
 import org.elasticsearch.health.node.selection.HealthNodeTaskExecutor;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.ingest.IngestMetadata;
+import org.elasticsearch.injection.guice.AbstractModule;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksNodeService;
 import org.elasticsearch.plugins.ClusterPlugin;
 import org.elasticsearch.script.ScriptMetadata;
+import org.elasticsearch.snapshots.RegisteredPolicySnapshots;
 import org.elasticsearch.snapshots.SnapshotsInfoService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskResultsService;
@@ -234,6 +235,12 @@ public static List<Entry> getNamedWriteables() {
         registerMetadataCustom(entries, NodesShutdownMetadata.TYPE, NodesShutdownMetadata::new, NodesShutdownMetadata::readDiffFrom);
         registerMetadataCustom(entries, FeatureMigrationResults.TYPE, FeatureMigrationResults::new, FeatureMigrationResults::readDiffFrom);
         registerMetadataCustom(entries, DesiredNodesMetadata.TYPE, DesiredNodesMetadata::new, DesiredNodesMetadata::readDiffFrom);
+        registerMetadataCustom(
+            entries,
+            RegisteredPolicySnapshots.TYPE,
+            RegisteredPolicySnapshots::new,
+            RegisteredPolicySnapshots.RegisteredSnapshotsDiff::new
+        );
 
         // Task Status (not Diffable)
         entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterName.java b/server/src/main/java/org/elasticsearch/cluster/ClusterName.java
index 711c2a7fee8e0..dd4194b60e6ac 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterName.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterName.java
@@ -39,7 +39,8 @@ public ClusterName(StreamInput input) throws IOException {
     }
 
     public ClusterName(String value) {
-        this.value = value.intern();
+        // cluster name string is most likely part of a setting so we can speed things up over outright interning here
+        this.value = Settings.internKeyOrValue(value);
     }
 
     public String value() {
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
index c54269da68507..30e9a9a3779d7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -1081,7 +1081,7 @@ public void writeTo(StreamOutput out) throws IOException {
         routingTable.writeTo(out);
         nodes.writeTo(out);
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) {
-            out.writeMap(compatibilityVersions, (streamOutput, versions) -> versions.writeTo(streamOutput));
+            out.writeMap(compatibilityVersions, StreamOutput::writeWriteable);
         }
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
             clusterFeatures.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index cc94137afa322..d765755b5d250 100644
--- a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -17,13 +17,13 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterApplier;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java b/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
index ee1d29ab778f1..aa8ba17a0f05f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.IndicesAdminClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -24,6 +23,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.Mapping;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.xcontent.XContentType;
 
 /**
diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index a01383b3eaa93..ca5fab1087cdc 100644
--- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -46,6 +45,7 @@
 import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardLongFieldRange;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
index 2f604f1b95974..e922d130d7f83 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.cluster.version.CompatibilityVersions;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -223,7 +224,7 @@ public Coordinator(
         this.onJoinValidators = NodeJoinExecutor.addBuiltInJoinValidators(onJoinValidators);
         this.singleNodeDiscovery = DiscoveryModule.isSingleNodeDiscovery(settings);
         this.electionStrategy = electionStrategy;
-        this.joinReasonService = new JoinReasonService(transportService.getThreadPool()::relativeTimeInMillis);
+        this.joinReasonService = new JoinReasonService(transportService.getThreadPool().relativeTimeInMillisSupplier());
         this.joinHelper = new JoinHelper(
             allocationService,
             masterService,
@@ -831,10 +832,12 @@ public void run() {
                                     discover other nodes and form a multi-node cluster via the [{}={}] setting. Fully-formed clusters do \
                                     not attempt to discover other nodes, and nodes with different cluster UUIDs cannot belong to the same \
                                     cluster. The cluster UUID persists across restarts and can only be changed by deleting the contents of \
-                                    the node's data path(s). Remove the discovery configuration to suppress this message.""",
+                                    the node's data path(s). Remove the discovery configuration to suppress this message. See [{}] for \
+                                    more information.""",
                                 applierState.metadata().clusterUUID(),
                                 DISCOVERY_SEED_HOSTS_SETTING.getKey(),
-                                DISCOVERY_SEED_HOSTS_SETTING.get(settings)
+                                DISCOVERY_SEED_HOSTS_SETTING.get(settings),
+                                ReferenceDocs.FORMING_SINGLE_NODE_CLUSTERS
                             );
                         }
                     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistory.java b/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistory.java
index ba7d22fa6a673..720510e50c48d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistory.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistory.java
@@ -59,7 +59,7 @@ public class MasterHistory implements ClusterStateListener {
     @SuppressWarnings("this-escape")
     public MasterHistory(ThreadPool threadPool, ClusterService clusterService) {
         this.masterHistory = new ArrayList<>();
-        this.currentTimeMillisSupplier = threadPool::relativeTimeInMillis;
+        this.currentTimeMillisSupplier = threadPool.relativeTimeInMillisSupplier();
         this.maxHistoryAge = MAX_HISTORY_AGE_SETTING.get(clusterService.getSettings());
         this.clusterService = clusterService;
         clusterService.addListener(this);
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistoryService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistoryService.java
index 8482eb735b0d9..1b2bf81bc85c3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistoryService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/MasterHistoryService.java
@@ -61,7 +61,7 @@ public class MasterHistoryService {
     public MasterHistoryService(TransportService transportService, ThreadPool threadPool, ClusterService clusterService) {
         this.transportService = transportService;
         this.localMasterHistory = new MasterHistory(threadPool, clusterService);
-        this.currentTimeMillisSupplier = threadPool::relativeTimeInMillis;
+        this.currentTimeMillisSupplier = threadPool.relativeTimeInMillisSupplier();
         this.acceptableRemoteHistoryAge = REMOTE_HISTORY_TIME_TO_LIVE_SETTING.get(clusterService.getSettings());
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
index c0102437ad9d0..227e78ed37114 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
@@ -230,9 +230,17 @@ private ClusterState deserializeAndApplyDiff(BytesTransportRequest request, Stre
     private void acceptState(ClusterState incomingState, ActionListener<PublishWithJoinResponse> actionListener) {
         assert incomingState.nodes().isLocalNodeElectedMaster() == false
             : "should handle local publications locally, but got " + incomingState;
-        clusterCoordinationExecutor.execute(
-            ActionRunnable.supply(actionListener, () -> handlePublishRequest.apply(new PublishRequest(incomingState)))
-        );
+        clusterCoordinationExecutor.execute(ActionRunnable.supply(actionListener, new CheckedSupplier<>() {
+            @Override
+            public PublishWithJoinResponse get() {
+                return handlePublishRequest.apply(new PublishRequest(incomingState));
+            }
+
+            @Override
+            public String toString() {
+                return "acceptState[term=" + incomingState.term() + ",version=" + incomingState.version() + "]";
+            }
+        }));
     }
 
     public PublicationContext newPublicationContext(ClusterStatePublicationEvent clusterStatePublicationEvent) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java
index d025ddab26af6..c53395b5d76c1 100644
--- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java
+++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java
@@ -19,7 +19,7 @@ public enum ClusterHealthStatus implements Writeable {
     YELLOW((byte) 1),
     RED((byte) 2);
 
-    private byte value;
+    private final byte value;
 
     ClusterHealthStatus(byte value) {
         this.value = value;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java
index a0f4a929dafdb..ff412d629b3b1 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java
@@ -396,6 +396,8 @@ public static AliasMetadata fromXContent(XContentParser parser) throws IOExcepti
                     } else if ("is_hidden".equals(currentFieldName)) {
                         builder.isHidden(parser.booleanValue());
                     }
+                } else if (token == null) {
+                    throw new IllegalArgumentException("unexpected null token while parsing alias");
                 }
             }
             return builder.build();
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
index 03b23c462ecec..8acee4f6be821 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
@@ -69,7 +69,7 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
 
     public static final FeatureFlag FAILURE_STORE_FEATURE_FLAG = new FeatureFlag("failure_store");
     public static final TransportVersion ADDED_FAILURE_STORE_TRANSPORT_VERSION = TransportVersions.V_8_12_0;
-    public static final TransportVersion ADDED_AUTO_SHARDING_EVENT_VERSION = TransportVersions.DATA_STREAM_AUTO_SHARDING_EVENT;
+    public static final TransportVersion ADDED_AUTO_SHARDING_EVENT_VERSION = TransportVersions.V_8_14_0;
 
     public static boolean isFailureStoreFeatureFlagEnabled() {
         return FAILURE_STORE_FEATURE_FLAG.isEnabled();
@@ -277,6 +277,18 @@ public boolean rolloverOnWrite() {
         return backingIndices.rolloverOnWrite;
     }
 
+    /**
+     * We define that a data stream is considered internal either if it is a system index or if
+     * its name starts with a dot.
+     *
+     * Note: Dot-prefixed internal data streams is a naming convention for internal data streams,
+     * but it's not yet enforced.
+     * @return true if it's a system index or has a dot-prefixed name.
+     */
+    public boolean isInternal() {
+        return isSystem() || name.charAt(0) == '.';
+    }
+
     /**
      * @param timestamp The timestamp used to select a backing index based on its start and end time.
      * @param metadata  The metadata that is used to fetch the start and end times for backing indices of this data stream.
@@ -796,12 +808,12 @@ public List<Index> getIndicesPastRetention(
     ) {
         if (lifecycle == null
             || lifecycle.isEnabled() == false
-            || lifecycle.getEffectiveDataRetention(isSystem() ? null : globalRetention) == null) {
+            || lifecycle.getEffectiveDataRetention(globalRetention, isInternal()) == null) {
             return List.of();
         }
 
         List<Index> indicesPastRetention = getNonWriteIndicesOlderThan(
-            lifecycle.getEffectiveDataRetention(isSystem() ? null : globalRetention),
+            lifecycle.getEffectiveDataRetention(globalRetention, isInternal()),
             indexMetadataSupplier,
             this::isIndexManagedByDataStreamLifecycle,
             nowSupplier
@@ -1202,7 +1214,7 @@ public XContentBuilder toXContent(
         }
         if (lifecycle != null) {
             builder.field(LIFECYCLE.getPreferredName());
-            lifecycle.toXContent(builder, params, rolloverConfiguration, isSystem() ? null : globalRetention);
+            lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, isInternal());
         }
         builder.field(ROLLOVER_ON_WRITE_FIELD.getPreferredName(), backingIndices.rolloverOnWrite);
         if (backingIndices.autoShardingEvent != null) {
@@ -1376,6 +1388,25 @@ private static Instant getTimestampFromParser(BytesReference source, XContentTyp
         }
     }
 
+    /**
+     * Resolve the index abstraction to a data stream. This handles alias resolution as well as data stream resolution. This does <b>NOT</b>
+     * resolve a data stream by providing a concrete backing index.
+     */
+    public static DataStream resolveDataStream(IndexAbstraction indexAbstraction, Metadata metadata) {
+        // We do not consider concrete indices - only data streams and data stream aliases.
+        if (indexAbstraction == null || indexAbstraction.isDataStreamRelated() == false) {
+            return null;
+        }
+
+        // Locate the write index for the abstraction, and check if it has a data stream associated with it.
+        Index writeIndex = indexAbstraction.getWriteIndex();
+        if (writeIndex == null) {
+            return null;
+        }
+        IndexAbstraction writeAbstraction = metadata.getIndicesLookup().get(writeIndex.getName());
+        return writeAbstraction.getParentDataStream();
+    }
+
     /**
      * Modifies the passed Instant object to be used as a bound for a timestamp field in TimeSeries. It needs to be called in both backing
      * index construction (rollover) and index selection for doc insertion. Failure to do so may lead to errors due to document timestamps
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java
index f260b48cd7b7a..32bf46ce45919 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAction.java
@@ -88,7 +88,7 @@ public DataStreamAction(StreamInput in) throws IOException {
         this.type = Type.fromValue(in.readByte());
         this.dataStream = in.readString();
         this.index = in.readString();
-        this.failureStore = in.getTransportVersion().onOrAfter(TransportVersions.MODIFY_DATA_STREAM_FAILURE_STORES) && in.readBoolean();
+        this.failureStore = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) && in.readBoolean();
     }
 
     private DataStreamAction(Type type, String dataStream, String index, boolean failureStore) {
@@ -155,7 +155,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeByte(type.value());
         out.writeString(dataStream);
         out.writeString(index);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.MODIFY_DATA_STREAM_FAILURE_STORES)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeBoolean(failureStore);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFactoryRetention.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFactoryRetention.java
index 5b96f92193e98..be42916b07956 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFactoryRetention.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFactoryRetention.java
@@ -17,7 +17,9 @@
  * Holds the factory retention configuration. Factory retention is the global retention configuration meant to be
  * used if a user hasn't provided other retention configuration via {@link DataStreamGlobalRetention} metadata in the
  * cluster state.
+ * @deprecated This interface is deprecated, please use {@link DataStreamGlobalRetentionSettings}.
  */
+@Deprecated
 public interface DataStreamFactoryRetention {
 
     @Nullable
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java
index f691151eee95e..185f625f6f91f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java
@@ -8,50 +8,26 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.TransportVersion;
-import org.elasticsearch.TransportVersions;
-import org.elasticsearch.cluster.AbstractNamedDiffable;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.NamedDiff;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.features.NodeFeature;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
-import java.util.Iterator;
-import java.util.Objects;
 
 /**
- * A cluster state entry that contains global retention settings that are configurable by the user. These settings include:
- * - default retention, applied on any data stream managed by DSL that does not have an explicit retention defined
- * - max retention, applied on every data stream managed by DSL
+ * Wrapper class for the {@link DataStreamGlobalRetentionSettings}.
  */
-public final class DataStreamGlobalRetention extends AbstractNamedDiffable<ClusterState.Custom> implements ClusterState.Custom {
-
-    public static final String TYPE = "data-stream-global-retention";
+public record DataStreamGlobalRetention(@Nullable TimeValue defaultRetention, @Nullable TimeValue maxRetention) implements Writeable {
 
     public static final NodeFeature GLOBAL_RETENTION = new NodeFeature("data_stream.lifecycle.global_retention");
-
-    public static final ParseField DEFAULT_RETENTION_FIELD = new ParseField("default_retention");
-    public static final ParseField MAX_RETENTION_FIELD = new ParseField("max_retention");
-
-    public static final DataStreamGlobalRetention EMPTY = new DataStreamGlobalRetention(null, null);
     public static final TimeValue MIN_RETENTION_VALUE = TimeValue.timeValueSeconds(10);
 
-    @Nullable
-    private final TimeValue defaultRetention;
-    @Nullable
-    private final TimeValue maxRetention;
-
     /**
      * @param defaultRetention the default retention or null if it's undefined
-     * @param maxRetention the max retention or null if it's undefined
+     * @param maxRetention     the max retention or null if it's undefined
      * @throws IllegalArgumentException when the default retention is greater than the max retention.
      */
     public DataStreamGlobalRetention(TimeValue defaultRetention, TimeValue maxRetention) {
@@ -79,78 +55,12 @@ public static DataStreamGlobalRetention read(StreamInput in) throws IOException
         return new DataStreamGlobalRetention(in.readOptionalTimeValue(), in.readOptionalTimeValue());
     }
 
-    @Override
-    public String getWriteableName() {
-        return TYPE;
-    }
-
-    @Override
-    public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ADD_DATA_STREAM_GLOBAL_RETENTION;
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalTimeValue(defaultRetention);
         out.writeOptionalTimeValue(maxRetention);
     }
 
-    public static NamedDiff<ClusterState.Custom> readDiffFrom(StreamInput in) throws IOException {
-        return readDiffFrom(ClusterState.Custom.class, TYPE, in);
-    }
-
-    @Override
-    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
-        return Iterators.single(this::toXContentFragment);
-    }
-
-    /**
-     * Adds to the XContentBuilder the two fields when they are not null.
-     */
-    public XContentBuilder toXContentFragment(XContentBuilder builder, ToXContent.Params params) throws IOException {
-        if (defaultRetention != null) {
-            builder.field(DEFAULT_RETENTION_FIELD.getPreferredName(), defaultRetention.getStringRep());
-        }
-        if (maxRetention != null) {
-            builder.field(MAX_RETENTION_FIELD.getPreferredName(), maxRetention.getStringRep());
-        }
-        return builder;
-    }
-
-    /**
-     * Returns the metadata found in the cluster state or null. When trying to retrieve the effective global retention,
-     * prefer to use the {@link DataStreamGlobalRetentionResolver#resolve(ClusterState)} because it takes into account
-     * the factory retention settings as well. Only use this, if you only want to know the global retention settings
-     * stored in the cluster metadata.
-     */
-    @Nullable
-    public static DataStreamGlobalRetention getFromClusterState(ClusterState clusterState) {
-        return clusterState.custom(DataStreamGlobalRetention.TYPE);
-    }
-
-    @Nullable
-    public TimeValue getDefaultRetention() {
-        return defaultRetention;
-    }
-
-    @Nullable
-    public TimeValue getMaxRetention() {
-        return maxRetention;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        DataStreamGlobalRetention that = (DataStreamGlobalRetention) o;
-        return Objects.equals(defaultRetention, that.defaultRetention) && Objects.equals(maxRetention, that.maxRetention);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(defaultRetention, maxRetention);
-    }
-
     @Override
     public String toString() {
         return "DataStreamGlobalRetention{"
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionResolver.java
deleted file mode 100644
index ab44595e37c13..0000000000000
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionResolver.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.cluster.metadata;
-
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.core.Nullable;
-
-/**
- * Resolves the global retention configuration for data stream lifecycle taking into consideration the
- * metadata in the cluster state and the factory settings.
- * Currently, we give precedence to the configuration in the metadata and fallback to the factory settings when it's not present.
- */
-public class DataStreamGlobalRetentionResolver {
-
-    private final DataStreamFactoryRetention factoryRetention;
-
-    public DataStreamGlobalRetentionResolver(DataStreamFactoryRetention factoryRetention) {
-        this.factoryRetention = factoryRetention;
-    }
-
-    /**
-     * Return the global retention configuration as found in the metadata. If the metadata is null, then it falls back
-     * to the factory retention. Returns null if both the cluster metadata for global retention and the factory retention
-     * are null.
-     */
-    @Nullable
-    public DataStreamGlobalRetention resolve(ClusterState clusterState) {
-        DataStreamGlobalRetention globalRetentionFromClusterState = DataStreamGlobalRetention.getFromClusterState(clusterState);
-        if (globalRetentionFromClusterState != null || factoryRetention.isDefined() == false) {
-            return globalRetentionFromClusterState;
-        }
-        return new DataStreamGlobalRetention(factoryRetention.getDefaultRetention(), factoryRetention.getMaxRetention());
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSettings.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSettings.java
new file mode 100644
index 0000000000000..a1fcf56a92726
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSettings.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This class holds the data stream global retention settings. It defines, validates and monitors the settings.
+ * <p>
+ * The global retention settings apply to non-system data streams that are managed by the data stream lifecycle. They consist of:
+ * - The default retention which applies to data streams that do not have a retention defined.
+ * - The max retention which applies to all data streams that do not have retention or their retention has exceeded this value.
+ * <p>
+ * Temporarily, we fall back to {@link DataStreamFactoryRetention} to facilitate a smooth transition to these settings.
+ */
+public class DataStreamGlobalRetentionSettings {
+
+    private static final Logger logger = LogManager.getLogger(DataStreamGlobalRetentionSettings.class);
+    public static final TimeValue MIN_RETENTION_VALUE = TimeValue.timeValueSeconds(10);
+
+    public static final Setting<TimeValue> DATA_STREAMS_DEFAULT_RETENTION_SETTING = Setting.timeSetting(
+        "data_streams.lifecycle.retention.default",
+        TimeValue.MINUS_ONE,
+        new Setting.Validator<>() {
+            @Override
+            public void validate(TimeValue value) {}
+
+            @Override
+            public void validate(final TimeValue settingValue, final Map<Setting<?>, Object> settings) {
+                TimeValue defaultRetention = getSettingValueOrNull(settingValue);
+                TimeValue maxRetention = getSettingValueOrNull((TimeValue) settings.get(DATA_STREAMS_MAX_RETENTION_SETTING));
+                validateIsolatedRetentionValue(defaultRetention, DATA_STREAMS_DEFAULT_RETENTION_SETTING.getKey());
+                validateGlobalRetentionConfiguration(defaultRetention, maxRetention);
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final List<Setting<?>> settings = List.of(DATA_STREAMS_MAX_RETENTION_SETTING);
+                return settings.iterator();
+            }
+        },
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    public static final Setting<TimeValue> DATA_STREAMS_MAX_RETENTION_SETTING = Setting.timeSetting(
+        "data_streams.lifecycle.retention.max",
+        TimeValue.MINUS_ONE,
+        new Setting.Validator<>() {
+            @Override
+            public void validate(TimeValue value) {}
+
+            @Override
+            public void validate(final TimeValue settingValue, final Map<Setting<?>, Object> settings) {
+                TimeValue defaultRetention = getSettingValueOrNull((TimeValue) settings.get(DATA_STREAMS_DEFAULT_RETENTION_SETTING));
+                TimeValue maxRetention = getSettingValueOrNull(settingValue);
+                validateIsolatedRetentionValue(maxRetention, DATA_STREAMS_MAX_RETENTION_SETTING.getKey());
+                validateGlobalRetentionConfiguration(defaultRetention, maxRetention);
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final List<Setting<?>> settings = List.of(DATA_STREAMS_DEFAULT_RETENTION_SETTING);
+                return settings.iterator();
+            }
+        },
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    private final DataStreamFactoryRetention factoryRetention;
+
+    @Nullable
+    private volatile TimeValue defaultRetention;
+    @Nullable
+    private volatile TimeValue maxRetention;
+
+    private DataStreamGlobalRetentionSettings(DataStreamFactoryRetention factoryRetention) {
+        this.factoryRetention = factoryRetention;
+    }
+
+    @Nullable
+    public TimeValue getMaxRetention() {
+        return shouldFallbackToFactorySettings() ? factoryRetention.getMaxRetention() : maxRetention;
+    }
+
+    @Nullable
+    public TimeValue getDefaultRetention() {
+        return shouldFallbackToFactorySettings() ? factoryRetention.getDefaultRetention() : defaultRetention;
+    }
+
+    public boolean areDefined() {
+        return getDefaultRetention() != null || getMaxRetention() != null;
+    }
+
+    private boolean shouldFallbackToFactorySettings() {
+        return defaultRetention == null && maxRetention == null;
+    }
+
+    /**
+     * Creates an instance and initialises the cluster settings listeners
+     * @param clusterSettings it will register the cluster settings listeners to monitor for changes
+     * @param factoryRetention for migration purposes, it will be removed shortly
+     */
+    public static DataStreamGlobalRetentionSettings create(ClusterSettings clusterSettings, DataStreamFactoryRetention factoryRetention) {
+        DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings = new DataStreamGlobalRetentionSettings(factoryRetention);
+        clusterSettings.initializeAndWatch(DATA_STREAMS_DEFAULT_RETENTION_SETTING, dataStreamGlobalRetentionSettings::setDefaultRetention);
+        clusterSettings.initializeAndWatch(DATA_STREAMS_MAX_RETENTION_SETTING, dataStreamGlobalRetentionSettings::setMaxRetention);
+        return dataStreamGlobalRetentionSettings;
+    }
+
+    private void setMaxRetention(TimeValue maxRetention) {
+        this.maxRetention = getSettingValueOrNull(maxRetention);
+        logger.info("Updated max factory retention to [{}]", this.maxRetention == null ? null : maxRetention.getStringRep());
+    }
+
+    private void setDefaultRetention(TimeValue defaultRetention) {
+        this.defaultRetention = getSettingValueOrNull(defaultRetention);
+        logger.info("Updated default factory retention to [{}]", this.defaultRetention == null ? null : defaultRetention.getStringRep());
+    }
+
+    private static void validateIsolatedRetentionValue(@Nullable TimeValue retention, String settingName) {
+        if (retention != null && retention.getMillis() < MIN_RETENTION_VALUE.getMillis()) {
+            throw new IllegalArgumentException(
+                "Setting '" + settingName + "' should be greater than " + MIN_RETENTION_VALUE.getStringRep()
+            );
+        }
+    }
+
+    private static void validateGlobalRetentionConfiguration(@Nullable TimeValue defaultRetention, @Nullable TimeValue maxRetention) {
+        if (defaultRetention != null && maxRetention != null && defaultRetention.getMillis() > maxRetention.getMillis()) {
+            throw new IllegalArgumentException(
+                "Setting ["
+                    + DATA_STREAMS_DEFAULT_RETENTION_SETTING.getKey()
+                    + "="
+                    + defaultRetention.getStringRep()
+                    + "] cannot be greater than ["
+                    + DATA_STREAMS_MAX_RETENTION_SETTING.getKey()
+                    + "="
+                    + maxRetention.getStringRep()
+                    + "]."
+            );
+        }
+    }
+
+    @Nullable
+    public DataStreamGlobalRetention get() {
+        if (areDefined() == false) {
+            return null;
+        }
+        return new DataStreamGlobalRetention(getDefaultRetention(), getMaxRetention());
+    }
+
+    /**
+     * Time value settings do not accept null as a value. To represent an undefined retention as a setting we use the value
+     * of <code>-1</code> and this method converts this to null.
+     *
+     * @param value the retention as parsed from the setting
+     * @return the value when it is not -1 and null otherwise
+     */
+    @Nullable
+    private static TimeValue getSettingValueOrNull(TimeValue value) {
+        return value == null || value.equals(TimeValue.MINUS_ONE) ? null : value;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java
index 3fb5e92cb3359..bd9a65735be05 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.xcontent.AbstractObjectParser;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -55,6 +54,7 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
 
     // Versions over the wire
     public static final TransportVersion ADDED_ENABLED_FLAG_VERSION = TransportVersions.V_8_10_X;
+    public static final String EFFECTIVE_RETENTION_REST_API_CAPABILITY = "data_stream_lifecycle_effective_retention";
 
     public static final String DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME = "data_streams.lifecycle_only.mode";
     // The following XContent params are used to enrich the DataStreamLifecycle json with effective retention information
@@ -65,6 +65,7 @@ public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>,
         DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME,
         "true"
     );
+    public static final Tuple<TimeValue, RetentionSource> INFINITE_RETENTION = Tuple.tuple(null, RetentionSource.DATA_STREAM_CONFIGURATION);
 
     /**
      * Check if {@link #DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME} is present and set to {@code true}, indicating that
@@ -145,41 +146,46 @@ public boolean isEnabled() {
     }
 
     /**
-     * The least amount of time data should be kept by elasticsearch. If a caller does not want the global retention considered (for
-     * example, when evaluating the effective retention for a system data stream or a template) then null should be given for
-     * globalRetention.
-     * @param globalRetention The global retention, or null if global retention does not exist or should not be applied
+     * The least amount of time data should be kept by elasticsearch. The effective retention is a function with three parameters,
+     * the {@link DataStreamLifecycle#dataRetention}, the global retention and whether this lifecycle is associated with an internal
+     * data stream.
+     * @param globalRetention The global retention, or null if global retention does not exist.
+     * @param isInternalDataStream A flag denoting if this lifecycle is associated with an internal data stream or not
      * @return the time period or null, null represents that data should never be deleted.
      */
     @Nullable
-    public TimeValue getEffectiveDataRetention(@Nullable DataStreamGlobalRetention globalRetention) {
-        return getEffectiveDataRetentionWithSource(globalRetention).v1();
+    public TimeValue getEffectiveDataRetention(@Nullable DataStreamGlobalRetention globalRetention, boolean isInternalDataStream) {
+        return getEffectiveDataRetentionWithSource(globalRetention, isInternalDataStream).v1();
     }
 
     /**
-     * The least amount of time data should be kept by elasticsearch. If a caller does not want the global retention considered (for
-     * example, when evaluating the effective retention for a system data stream or a template) then null should be given for
-     * globalRetention.
-     * @param globalRetention The global retention, or null if global retention does not exist or should not be applied
+     * The least amount of time data should be kept by elasticsearch.. The effective retention is a function with three parameters,
+     * the {@link DataStreamLifecycle#dataRetention}, the global retention and whether this lifecycle is associated with an internal
+     * data stream.
+     * @param globalRetention The global retention, or null if global retention does not exist.
+     * @param isInternalDataStream A flag denoting if this lifecycle is associated with an internal data stream or not
      * @return A tuple containing the time period or null as v1 (where null represents that data should never be deleted), and the non-null
      * retention source as v2.
      */
-    public Tuple<TimeValue, RetentionSource> getEffectiveDataRetentionWithSource(@Nullable DataStreamGlobalRetention globalRetention) {
+    public Tuple<TimeValue, RetentionSource> getEffectiveDataRetentionWithSource(
+        @Nullable DataStreamGlobalRetention globalRetention,
+        boolean isInternalDataStream
+    ) {
         // If lifecycle is disabled there is no effective retention
         if (enabled == false) {
-            return Tuple.tuple(null, RetentionSource.DATA_STREAM_CONFIGURATION);
+            return INFINITE_RETENTION;
         }
         var dataStreamRetention = getDataStreamRetention();
-        if (globalRetention == null) {
+        if (globalRetention == null || isInternalDataStream) {
             return Tuple.tuple(dataStreamRetention, RetentionSource.DATA_STREAM_CONFIGURATION);
         }
         if (dataStreamRetention == null) {
-            return globalRetention.getDefaultRetention() != null
-                ? Tuple.tuple(globalRetention.getDefaultRetention(), RetentionSource.DEFAULT_GLOBAL_RETENTION)
-                : Tuple.tuple(globalRetention.getMaxRetention(), RetentionSource.MAX_GLOBAL_RETENTION);
+            return globalRetention.defaultRetention() != null
+                ? Tuple.tuple(globalRetention.defaultRetention(), RetentionSource.DEFAULT_GLOBAL_RETENTION)
+                : Tuple.tuple(globalRetention.maxRetention(), RetentionSource.MAX_GLOBAL_RETENTION);
         }
-        if (globalRetention.getMaxRetention() != null && globalRetention.getMaxRetention().getMillis() < dataStreamRetention.getMillis()) {
-            return Tuple.tuple(globalRetention.getMaxRetention(), RetentionSource.MAX_GLOBAL_RETENTION);
+        if (globalRetention.maxRetention() != null && globalRetention.maxRetention().getMillis() < dataStreamRetention.getMillis()) {
+            return Tuple.tuple(globalRetention.maxRetention(), RetentionSource.MAX_GLOBAL_RETENTION);
         } else {
             return Tuple.tuple(dataStreamRetention, RetentionSource.DATA_STREAM_CONFIGURATION);
         }
@@ -187,7 +193,7 @@ public Tuple<TimeValue, RetentionSource> getEffectiveDataRetentionWithSource(@Nu
 
     /**
      * The least amount of time data the data stream is requesting es to keep the data.
-     * NOTE: this can be overridden by the {@link DataStreamLifecycle#getEffectiveDataRetention(DataStreamGlobalRetention)}.
+     * NOTE: this can be overridden by the {@link DataStreamLifecycle#getEffectiveDataRetention(DataStreamGlobalRetention,boolean)}.
      * @return the time period or null, null represents that data should never be deleted.
      */
     @Nullable
@@ -199,12 +205,16 @@ public TimeValue getDataStreamRetention() {
      * This method checks if the effective retention is matching what the user has configured; if the effective retention
      * does not match then it adds a warning informing the user about the effective retention and the source.
      */
-    public void addWarningHeaderIfDataRetentionNotEffective(@Nullable DataStreamGlobalRetention globalRetention) {
-        if (globalRetention == null) {
+    public void addWarningHeaderIfDataRetentionNotEffective(
+        @Nullable DataStreamGlobalRetention globalRetention,
+        boolean isInternalDataStream
+    ) {
+        if (globalRetention == null || isInternalDataStream) {
             return;
         }
         Tuple<TimeValue, DataStreamLifecycle.RetentionSource> effectiveDataRetentionWithSource = getEffectiveDataRetentionWithSource(
-            globalRetention
+            globalRetention,
+            isInternalDataStream
         );
         if (effectiveDataRetentionWithSource.v1() == null) {
             return;
@@ -318,7 +328,7 @@ public String toString() {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return toXContent(builder, params, null, null);
+        return toXContent(builder, params, null, null, false);
     }
 
     /**
@@ -326,12 +336,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
      * and injects the RolloverConditions if they exist.
      * In order to request the effective retention you need to set {@link #INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME} to true
      * in the XContent params.
+     * NOTE: this is used for serialising user output and the result is never deserialised in elasticsearch.
      */
     public XContentBuilder toXContent(
         XContentBuilder builder,
         Params params,
         @Nullable RolloverConfiguration rolloverConfiguration,
-        @Nullable DataStreamGlobalRetention globalRetention
+        @Nullable DataStreamGlobalRetention globalRetention,
+        boolean isInternalDataStream
     ) throws IOException {
         builder.startObject();
         builder.field(ENABLED_FIELD.getPreferredName(), enabled);
@@ -342,11 +354,14 @@ public XContentBuilder toXContent(
                 builder.field(DATA_RETENTION_FIELD.getPreferredName(), dataRetention.value().getStringRep());
             }
         }
+        Tuple<TimeValue, RetentionSource> effectiveDataRetentionWithSource = getEffectiveDataRetentionWithSource(
+            globalRetention,
+            isInternalDataStream
+        );
         if (params.paramAsBoolean(INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false)) {
-            Tuple<TimeValue, RetentionSource> effectiveRetention = getEffectiveDataRetentionWithSource(globalRetention);
-            if (effectiveRetention.v1() != null) {
-                builder.field(EFFECTIVE_RETENTION_FIELD.getPreferredName(), effectiveRetention.v1().getStringRep());
-                builder.field(RETENTION_SOURCE_FIELD.getPreferredName(), effectiveRetention.v2().displayName());
+            if (effectiveDataRetentionWithSource.v1() != null) {
+                builder.field(EFFECTIVE_RETENTION_FIELD.getPreferredName(), effectiveDataRetentionWithSource.v1().getStringRep());
+                builder.field(RETENTION_SOURCE_FIELD.getPreferredName(), effectiveDataRetentionWithSource.v2().displayName());
             }
         }
 
@@ -356,25 +371,29 @@ public XContentBuilder toXContent(
         }
         if (rolloverConfiguration != null) {
             builder.field(ROLLOVER_FIELD.getPreferredName());
-            rolloverConfiguration.evaluateAndConvertToXContent(builder, params, getEffectiveDataRetention(globalRetention));
+            rolloverConfiguration.evaluateAndConvertToXContent(builder, params, effectiveDataRetentionWithSource.v1());
         }
         builder.endObject();
         return builder;
     }
 
+    /**
+     * This method deserialises XContent format as it was generated ONLY by {@link DataStreamLifecycle#toXContent(XContentBuilder, Params)}.
+     * It does not support the output of
+     * {@link DataStreamLifecycle#toXContent(XContentBuilder, Params, RolloverConfiguration, DataStreamGlobalRetention, boolean)} because
+     * this output is enriched with derived fields we do not handle in this deserialisation.
+     */
     public static DataStreamLifecycle fromXContent(XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }
 
     /**
-     * Adds a retention param to signal that this serialisation should include the effective retention metadata
+     * Adds a retention param to signal that this serialisation should include the effective retention metadata.
+     * @param params the XContent params to be extended with the new flag
+     * @return XContent params with `include_effective_retention` set to true. If the flag exists it will override it.
      */
-    public static ToXContent.Params maybeAddEffectiveRetentionParams(ToXContent.Params params) {
-        boolean shouldAddEffectiveRetention = Objects.equals(params.param(RestRequest.PATH_RESTRICTED), "serverless");
-        return new DelegatingMapParams(
-            Map.of(INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, Boolean.toString(shouldAddEffectiveRetention)),
-            params
-        );
+    public static ToXContent.Params addEffectiveRetentionParams(ToXContent.Params params) {
+        return new DelegatingMapParams(INCLUDE_EFFECTIVE_RETENTION_PARAMS, params);
     }
 
     public static Builder newBuilder(DataStreamLifecycle lifecycle) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
index 2b65a68e8d43c..742439c9a2484 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.cluster.routing.allocation.IndexMetadataUpdater;
 import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -129,7 +130,8 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
     );
     public static final ClusterBlock INDEX_READ_ONLY_ALLOW_DELETE_BLOCK = new ClusterBlock(
         12,
-        "disk usage exceeded flood-stage watermark, index has read-only-allow-delete block",
+        "disk usage exceeded flood-stage watermark, index has read-only-allow-delete block; for more information, see "
+            + ReferenceDocs.FLOOD_STAGE_WATERMARK,
         false,
         false,
         true,
@@ -1623,7 +1625,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
             }
             primaryTerms = in.readVLongArray();
             mappings = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), MAPPING_DIFF_VALUE_READER);
-            if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_FIELDS_METADATA)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 inferenceFields = DiffableUtils.readImmutableOpenMapDiff(
                     in,
                     DiffableUtils.getStringKeySerializer(),
@@ -1691,7 +1693,7 @@ public void writeTo(StreamOutput out) throws IOException {
             }
             out.writeVLongArray(primaryTerms);
             mappings.writeTo(out);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_FIELDS_METADATA)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 inferenceFields.writeTo(out);
             }
             aliases.writeTo(out);
@@ -1784,7 +1786,7 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String,
                 builder.putMapping(new MappingMetadata(in));
             }
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_FIELDS_METADATA)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             var fields = in.readCollectionAsImmutableList(InferenceFieldMetadata::new);
             fields.stream().forEach(f -> builder.putInferenceField(f));
         }
@@ -1856,7 +1858,7 @@ public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException
                 mapping.writeTo(out);
             }
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_FIELDS_METADATA)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeCollection(inferenceFields.values());
         }
         out.writeCollection(aliases.values());
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
index b5ee0ebd7e387..07dcb7baf0777 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -514,7 +513,6 @@ private ClusterState applyCreateIndexWithTemporaryService(
 
             ClusterState updated = clusterStateCreateIndex(
                 currentState,
-                request.blocks(),
                 indexMetadata,
                 metadataTransformer,
                 allocationService.getShardRoutingRoleStrategy()
@@ -1231,7 +1229,6 @@ public static List<AliasMetadata> resolveAndValidateAliases(
      */
     static ClusterState clusterStateCreateIndex(
         ClusterState currentState,
-        Set<ClusterBlock> clusterBlocks,
         IndexMetadata indexMetadata,
         BiConsumer<Metadata.Builder, IndexMetadata> metadataTransformer,
         ShardRoutingRoleStrategy shardRoutingRoleStrategy
@@ -1245,15 +1242,13 @@ static ClusterState clusterStateCreateIndex(
             newMetadata = currentState.metadata().withAddedIndex(indexMetadata);
         }
 
-        String indexName = indexMetadata.getIndex().getName();
-        ClusterBlocks.Builder blocks = createClusterBlocksBuilder(currentState, indexName, clusterBlocks);
-        blocks.updateBlocks(indexMetadata);
+        var blocksBuilder = ClusterBlocks.builder().blocks(currentState.blocks());
+        blocksBuilder.updateBlocks(indexMetadata);
 
-        ClusterState updatedState = ClusterState.builder(currentState).blocks(blocks).metadata(newMetadata).build();
+        var routingTableBuilder = RoutingTable.builder(shardRoutingRoleStrategy, currentState.routingTable())
+            .addAsNew(newMetadata.index(indexMetadata.getIndex().getName()));
 
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(shardRoutingRoleStrategy, updatedState.routingTable())
-            .addAsNew(updatedState.metadata().index(indexName));
-        return ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build();
+        return ClusterState.builder(currentState).blocks(blocksBuilder).metadata(newMetadata).routingTable(routingTableBuilder).build();
     }
 
     static IndexMetadata buildIndexMetadata(
@@ -1326,16 +1321,6 @@ private static IndexMetadata.Builder createIndexMetadataBuilder(
         return builder;
     }
 
-    private static ClusterBlocks.Builder createClusterBlocksBuilder(ClusterState currentState, String index, Set<ClusterBlock> blocks) {
-        ClusterBlocks.Builder blocksBuilder = ClusterBlocks.builder().blocks(currentState.blocks());
-        if (blocks.isEmpty() == false) {
-            for (ClusterBlock block : blocks) {
-                blocksBuilder.addIndexBlock(index, block);
-            }
-        }
-        return blocksBuilder;
-    }
-
     private static void updateIndexMappingsAndBuildSortOrder(
         IndexService indexService,
         CreateIndexClusterStateUpdateRequest request,
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
index 7363e71d65c72..9e8a99351d84a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
@@ -41,18 +41,18 @@ public class MetadataDataStreamsService {
 
     private final ClusterService clusterService;
     private final IndicesService indicesService;
-    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
+    private final DataStreamGlobalRetentionSettings globalRetentionSettings;
     private final MasterServiceTaskQueue<UpdateLifecycleTask> updateLifecycleTaskQueue;
     private final MasterServiceTaskQueue<SetRolloverOnWriteTask> setRolloverOnWriteTaskQueue;
 
     public MetadataDataStreamsService(
         ClusterService clusterService,
         IndicesService indicesService,
-        DataStreamGlobalRetentionResolver globalRetentionResolver
+        DataStreamGlobalRetentionSettings globalRetentionSettings
     ) {
         this.clusterService = clusterService;
         this.indicesService = indicesService;
-        this.globalRetentionResolver = globalRetentionResolver;
+        this.globalRetentionSettings = globalRetentionSettings;
         ClusterStateTaskExecutor<UpdateLifecycleTask> updateLifecycleExecutor = new SimpleBatchedAckListenerTaskExecutor<>() {
 
             @Override
@@ -214,17 +214,15 @@ static ClusterState modifyDataStream(
     ClusterState updateDataLifecycle(ClusterState currentState, List<String> dataStreamNames, @Nullable DataStreamLifecycle lifecycle) {
         Metadata metadata = currentState.metadata();
         Metadata.Builder builder = Metadata.builder(metadata);
-        boolean atLeastOneDataStreamIsNotSystem = false;
+        boolean onlyInternalDataStreams = true;
         for (var dataStreamName : dataStreamNames) {
             var dataStream = validateDataStream(metadata, dataStreamName);
             builder.put(dataStream.copy().setLifecycle(lifecycle).build());
-            atLeastOneDataStreamIsNotSystem = atLeastOneDataStreamIsNotSystem || dataStream.isSystem() == false;
+            onlyInternalDataStreams = onlyInternalDataStreams && dataStream.isInternal();
         }
         if (lifecycle != null) {
-            if (atLeastOneDataStreamIsNotSystem) {
-                // We don't issue any warnings if all data streams are system data streams
-                lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetentionResolver.resolve(currentState));
-            }
+            // We don't issue any warnings if all data streams are internal data streams
+            lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get(), onlyInternalDataStreams);
         }
         return ClusterState.builder(currentState).metadata(builder.build()).build();
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java
index 9aebc9a2b810d..a87912f3ffc8d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java
@@ -23,11 +23,11 @@
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.snapshots.SnapshotInProgressException;
 import org.elasticsearch.snapshots.SnapshotsService;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java
index 26a968d1b201f..41ef8ed6aa470 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.CloseUtils;
@@ -33,6 +32,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.io.IOException;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java
index be6d6f3ef1e53..be12198cbaaaa 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java
@@ -50,7 +50,6 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
@@ -67,6 +66,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.ShardLimitValidator;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.snapshots.SnapshotInProgressException;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
index e9658e71f895e..ff23f50ef7afe 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.HeaderWarning;
@@ -55,6 +54,7 @@
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.PipelineConfiguration;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.io.IOException;
@@ -137,7 +137,7 @@ public class MetadataIndexTemplateService {
     private final NamedXContentRegistry xContentRegistry;
     private final SystemIndices systemIndices;
     private final Set<IndexSettingProvider> indexSettingProviders;
-    private final DataStreamGlobalRetentionResolver globalRetentionResolver;
+    private final DataStreamGlobalRetentionSettings globalRetentionSettings;
 
     /**
      * This is the cluster state task executor for all template-based actions.
@@ -183,7 +183,7 @@ public MetadataIndexTemplateService(
         NamedXContentRegistry xContentRegistry,
         SystemIndices systemIndices,
         IndexSettingProviders indexSettingProviders,
-        DataStreamGlobalRetentionResolver globalRetentionResolver
+        DataStreamGlobalRetentionSettings globalRetentionSettings
     ) {
         this.clusterService = clusterService;
         this.taskQueue = clusterService.createTaskQueue("index-templates", Priority.URGENT, TEMPLATE_TASK_EXECUTOR);
@@ -193,7 +193,7 @@ public MetadataIndexTemplateService(
         this.xContentRegistry = xContentRegistry;
         this.systemIndices = systemIndices;
         this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders();
-        this.globalRetentionResolver = globalRetentionResolver;
+        this.globalRetentionSettings = globalRetentionSettings;
     }
 
     public void removeTemplates(
@@ -345,7 +345,7 @@ public ClusterState addComponentTemplate(
                         tempStateWithComponentTemplateAdded.metadata(),
                         composableTemplateName,
                         composableTemplate,
-                        globalRetentionResolver.resolve(currentState)
+                        globalRetentionSettings.get()
                     );
                     validateIndexTemplateV2(composableTemplateName, composableTemplate, tempStateWithComponentTemplateAdded);
                 } catch (Exception e) {
@@ -369,9 +369,8 @@ public ClusterState addComponentTemplate(
         }
 
         if (finalComponentTemplate.template().lifecycle() != null) {
-            finalComponentTemplate.template()
-                .lifecycle()
-                .addWarningHeaderIfDataRetentionNotEffective(globalRetentionResolver.resolve(currentState));
+            // We do not know if this lifecycle will belong to an internal data stream, so we fall back to a non internal.
+            finalComponentTemplate.template().lifecycle().addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get(), false);
         }
 
         logger.info("{} component template [{}]", existing == null ? "adding" : "updating", name);
@@ -732,7 +731,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT
 
         validate(name, templateToValidate);
         validateDataStreamsStillReferenced(currentState, name, templateToValidate);
-        validateLifecycle(currentState.metadata(), name, templateToValidate, globalRetentionResolver.resolve(currentState));
+        validateLifecycle(currentState.metadata(), name, templateToValidate, globalRetentionSettings.get());
 
         if (templateToValidate.isDeprecated() == false) {
             validateUseOfDeprecatedComponentTemplates(name, templateToValidate, currentState.metadata().componentTemplates());
@@ -817,7 +816,12 @@ static void validateLifecycle(
                         + "] specifies lifecycle configuration that can only be used in combination with a data stream"
                 );
             }
-            lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
+            if (globalRetention != null) {
+                // We cannot know for sure if the template will apply to internal data streams, so we use a simpler heuristic:
+                // If all the index patterns start with a dot, we consider that all the connected data streams are internal.
+                boolean isInternalDataStream = template.indexPatterns().stream().allMatch(indexPattern -> indexPattern.charAt(0) == '.');
+                lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention, isInternalDataStream);
+            }
         }
     }
 
@@ -1309,7 +1313,12 @@ static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(Met
         for (Map.Entry<String, ComposableIndexTemplate> entry : metadata.templatesV2().entrySet()) {
             final String name = entry.getKey();
             final ComposableIndexTemplate template = entry.getValue();
-            if (isHidden == false) {
+            /*
+             * We do not ordinarily return match-all templates for hidden indices. But all backing indices for data streams are hidden,
+             * and we do want to return even match-all templates for those. Not doing so can result in a situation where a data stream is
+             * built with a template that none of its indices match.
+             */
+            if (isHidden == false || template.getDataStreamTemplate() != null) {
                 final boolean matched = template.indexPatterns().stream().anyMatch(patternMatchPredicate);
                 if (matched) {
                     candidates.add(Tuple.tuple(name, template));
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java
index 4ed18489c44b0..1c956c7f78f32 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
@@ -33,6 +32,7 @@
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 
 import java.util.ArrayList;
 import java.util.HashMap;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java
index 08b0d56da782a..38eefa4085527 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java
@@ -125,6 +125,14 @@ public boolean contains(String nodeId, SingleNodeShutdownMetadata.Type type) {
         return get(nodeId, type) != null;
     }
 
+    /**
+     * Checks if the provided node is scheduled for being permanently removed from the cluster.
+     */
+    public boolean isNodeMarkedForRemoval(String nodeId) {
+        var singleNodeShutdownMetadata = get(nodeId);
+        return singleNodeShutdownMetadata != null && singleNodeShutdownMetadata.getType().isRemovalType();
+    }
+
     /**
      * Add or update the shutdown metadata for a single node.
      * @param nodeShutdownMetadata The single node shutdown metadata to add or update.
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java
index 70440adc4ebbe..0a045261e07b8 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java
@@ -70,7 +70,11 @@ public class Template implements SimpleDiffable<Template>, ToXContentObject {
         }, MAPPINGS, ObjectParser.ValueType.VALUE_OBJECT_ARRAY);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
             Map<String, AliasMetadata> aliasMap = new HashMap<>();
-            while ((p.nextToken()) != XContentParser.Token.END_OBJECT) {
+            XContentParser.Token token;
+            while ((token = p.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == null) {
+                    break;
+                }
                 AliasMetadata alias = AliasMetadata.Builder.fromXContent(p);
                 aliasMap.put(alias.alias(), alias);
             }
@@ -250,7 +254,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla
         }
         if (this.lifecycle != null) {
             builder.field(LIFECYCLE.getPreferredName());
-            lifecycle.toXContent(builder, params, rolloverConfiguration, null);
+            lifecycle.toXContent(builder, params, rolloverConfiguration, null, false);
         }
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
index f11bcc0aff3b4..35a0cb683346d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
@@ -19,11 +19,11 @@
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java
index 6f15eb5f6e49d..e1c70fd5e6a8c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.common.util.ByteUtils;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper;
 import org.elasticsearch.transport.Transports;
@@ -48,6 +49,9 @@
  * Generates the shard id for {@code (id, routing)} pairs.
  */
 public abstract class IndexRouting {
+
+    static final NodeFeature BOOLEAN_ROUTING_PATH = new NodeFeature("routing.boolean_routing_path");
+
     /**
      * Build the routing from {@link IndexMetadata}.
      */
@@ -361,6 +365,7 @@ private void extractItem(String path, XContentParser source) throws IOException
                         break;
                     case VALUE_STRING:
                     case VALUE_NUMBER:
+                    case VALUE_BOOLEAN:
                         hashes.add(new NameAndHash(new BytesRef(path), hash(new BytesRef(source.text()))));
                         source.nextToken();
                         break;
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java
new file mode 100644
index 0000000000000..34194ac35ce6b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.routing;
+
+import org.elasticsearch.features.FeatureSpecification;
+import org.elasticsearch.features.NodeFeature;
+
+import java.util.Set;
+
+public class RoutingFeatures implements FeatureSpecification {
+
+    @Override
+    public Set<NodeFeature> getFeatures() {
+        return Set.of(IndexRouting.BOOLEAN_ROUTING_PATH);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 411143b1aef9d..991aa66d07ff6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -44,6 +43,7 @@
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.gateway.PriorityComparator;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 
 import java.util.ArrayList;
 import java.util.Collections;
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/FrequencyCappedAction.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/FrequencyCappedAction.java
index 3c827923ebefa..85ec145783023 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/FrequencyCappedAction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/FrequencyCappedAction.java
@@ -24,7 +24,7 @@ public class FrequencyCappedAction {
     private long next = -1;
 
     public FrequencyCappedAction(ThreadPool threadPool) {
-        this(threadPool::relativeTimeInMillis);
+        this(threadPool.relativeTimeInMillisSupplier());
     }
 
     public FrequencyCappedAction(LongSupplier currentTimeMillisSupplier) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java
index 8fb91d89417e0..b6c19f331c712 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java
@@ -40,9 +40,11 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.health.Diagnosis;
 import org.elasticsearch.health.HealthIndicatorDetails;
 import org.elasticsearch.health.HealthIndicatorImpact;
@@ -56,6 +58,7 @@
 import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
 import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
 
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -108,11 +111,29 @@ public class ShardsAvailabilityHealthIndicatorService implements HealthIndicator
 
     private static final String DATA_TIER_ALLOCATION_DECIDER_NAME = "data_tier";
 
+    /**
+     * Changes the behavior of isNewlyCreatedAndInitializingReplica so that the
+     * shard_availability health indicator returns YELLOW if a primary
+     * is STARTED, but a replica is still INITIALIZING and the replica has been
+     * unassigned for less than the value of this setting. This function is
+     * only used in serverless, so this setting has no effect in stateless.
+     */
+    public static final Setting<TimeValue> REPLICA_UNASSIGNED_BUFFER_TIME = Setting.timeSetting(
+        "health.shards_availability.replica_unassigned_buffer_time",
+        TimeValue.timeValueSeconds(3),
+        TimeValue.timeValueSeconds(0),
+        TimeValue.timeValueSeconds(20),
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
     private final ClusterService clusterService;
     private final AllocationService allocationService;
 
     private final SystemIndices systemIndices;
 
+    private volatile TimeValue replicaUnassignedBufferTime = TimeValue.timeValueSeconds(0);
+
     public ShardsAvailabilityHealthIndicatorService(
         ClusterService clusterService,
         AllocationService allocationService,
@@ -121,6 +142,11 @@ public ShardsAvailabilityHealthIndicatorService(
         this.clusterService = clusterService;
         this.allocationService = allocationService;
         this.systemIndices = systemIndices;
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(REPLICA_UNASSIGNED_BUFFER_TIME, this::setReplicaUnassignedBufferTime);
+    }
+
+    private void setReplicaUnassignedBufferTime(TimeValue replicaUnassignedBufferTime) {
+        this.replicaUnassignedBufferTime = replicaUnassignedBufferTime;
     }
 
     @Override
@@ -144,7 +170,7 @@ public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResources
         var state = clusterService.state();
         var shutdown = state.getMetadata().custom(NodesShutdownMetadata.TYPE, NodesShutdownMetadata.EMPTY);
         var status = createNewStatus(state.getMetadata());
-        updateShardAllocationStatus(status, state, shutdown, verbose);
+        updateShardAllocationStatus(status, state, shutdown, verbose, replicaUnassignedBufferTime);
         return createIndicator(
             status.getStatus(),
             status.getSymptom(),
@@ -158,14 +184,15 @@ static void updateShardAllocationStatus(
         ShardAllocationStatus status,
         ClusterState state,
         NodesShutdownMetadata shutdown,
-        boolean verbose
+        boolean verbose,
+        TimeValue replicaUnassignedBufferTime
     ) {
         for (IndexRoutingTable indexShardRouting : state.routingTable()) {
             for (int i = 0; i < indexShardRouting.size(); i++) {
                 IndexShardRoutingTable shardRouting = indexShardRouting.shard(i);
                 status.addPrimary(shardRouting.primaryShard(), state, shutdown, verbose);
                 for (ShardRouting replicaShard : shardRouting.replicaShards()) {
-                    status.addReplica(replicaShard, state, shutdown, verbose);
+                    status.addReplica(replicaShard, state, shutdown, verbose, replicaUnassignedBufferTime);
                 }
             }
         }
@@ -438,11 +465,18 @@ public class ShardAllocationCounts {
         public SearchableSnapshotsState searchableSnapshotsState = new SearchableSnapshotsState();
         final Map<Diagnosis.Definition, Set<String>> diagnosisDefinitions = new HashMap<>();
 
-        public void increment(ShardRouting routing, ClusterState state, NodesShutdownMetadata shutdowns, boolean verbose) {
+        public void increment(
+            ShardRouting routing,
+            ClusterState state,
+            NodesShutdownMetadata shutdowns,
+            boolean verbose,
+            TimeValue replicaUnassignedBufferTime
+        ) {
             boolean isNew = isUnassignedDueToNewInitialization(routing, state);
             boolean isRestarting = isUnassignedDueToTimelyRestart(routing, shutdowns);
+            long replicaUnassignedCutoffTime = Instant.now().toEpochMilli() - replicaUnassignedBufferTime.millis();
             boolean allUnavailable = areAllShardsOfThisTypeUnavailable(routing, state)
-                && isNewlyCreatedAndInitializingReplica(routing, state) == false;
+                && isNewlyCreatedAndInitializingReplica(routing, state, replicaUnassignedCutoffTime) == false;
             if (allUnavailable) {
                 indicesWithAllShardsUnavailable.add(routing.getIndexName());
             }
@@ -520,7 +554,7 @@ boolean areAllShardsOfThisTypeUnavailable(ShardRouting routing, ClusterState sta
      * (a newly created index having unassigned replicas for example), we don't want the cluster
      * to turn "unhealthy" for the tiny amount of time before the shards are allocated.
      */
-    static boolean isNewlyCreatedAndInitializingReplica(ShardRouting routing, ClusterState state) {
+    static boolean isNewlyCreatedAndInitializingReplica(ShardRouting routing, ClusterState state, long replicaUnassignedCutoffTime) {
         if (routing.active()) {
             return false;
         }
@@ -528,10 +562,15 @@ static boolean isNewlyCreatedAndInitializingReplica(ShardRouting routing, Cluste
             return false;
         }
         ShardRouting primary = state.routingTable().shardRoutingTable(routing.shardId()).primaryShard();
-        if (primary.active()) {
-            return false;
+        if (primary.active() == false) {
+            return ClusterShardHealth.getInactivePrimaryHealth(primary) == ClusterHealthStatus.YELLOW;
         }
-        return ClusterShardHealth.getInactivePrimaryHealth(primary) == ClusterHealthStatus.YELLOW;
+
+        Optional<UnassignedInfo> ui = Optional.ofNullable(routing.unassignedInfo());
+        return ui.filter(info -> info.failedAllocations() == 0)
+            .filter(info -> info.lastAllocationStatus() != UnassignedInfo.AllocationStatus.DECIDERS_NO)
+            .filter(info -> info.unassignedTimeMillis() > replicaUnassignedCutoffTime)
+            .isPresent();
     }
 
     private static boolean isUnassignedDueToTimelyRestart(ShardRouting routing, NodesShutdownMetadata shutdowns) {
@@ -910,11 +949,17 @@ public ShardAllocationStatus(Metadata clusterMetadata) {
         }
 
         void addPrimary(ShardRouting routing, ClusterState state, NodesShutdownMetadata shutdowns, boolean verbose) {
-            primaries.increment(routing, state, shutdowns, verbose);
+            primaries.increment(routing, state, shutdowns, verbose, TimeValue.MINUS_ONE);
         }
 
-        void addReplica(ShardRouting routing, ClusterState state, NodesShutdownMetadata shutdowns, boolean verbose) {
-            replicas.increment(routing, state, shutdowns, verbose);
+        void addReplica(
+            ShardRouting routing,
+            ClusterState state,
+            NodesShutdownMetadata shutdowns,
+            boolean verbose,
+            TimeValue replicaUnassignedBufferTime
+        ) {
+            replicas.increment(routing, state, shutdowns, verbose, replicaUnassignedBufferTime);
         }
 
         void updateSearchableSnapshotsOfAvailableIndices() {
diff --git a/server/src/main/java/org/elasticsearch/cluster/version/CompatibilityVersions.java b/server/src/main/java/org/elasticsearch/cluster/version/CompatibilityVersions.java
index a92cf1ce2e42c..8ebb24e86105a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/version/CompatibilityVersions.java
+++ b/server/src/main/java/org/elasticsearch/cluster/version/CompatibilityVersions.java
@@ -19,12 +19,9 @@
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * Wraps component version numbers for cluster state
@@ -42,6 +39,8 @@ public record CompatibilityVersions(
     Map<String, org.elasticsearch.indices.SystemIndexDescriptor.MappingsVersion> systemIndexMappingsVersion
 ) implements Writeable, ToXContentFragment {
 
+    public static final CompatibilityVersions EMPTY = new CompatibilityVersions(TransportVersions.MINIMUM_COMPATIBLE, Map.of());
+
     /**
      * Constructs a VersionWrapper collecting all the minimum versions from the values of the map.
      *
@@ -49,18 +48,26 @@ public record CompatibilityVersions(
      * @return Minimum versions for the cluster
      */
     public static CompatibilityVersions minimumVersions(Collection<CompatibilityVersions> compatibilityVersions) {
-        TransportVersion minimumTransport = compatibilityVersions.stream()
-            .map(CompatibilityVersions::transportVersion)
-            .min(Comparator.naturalOrder())
-            // In practice transportVersions is always nonempty (except in tests) but use a conservative default anyway:
-            .orElse(TransportVersions.MINIMUM_COMPATIBLE);
-
-        Map<String, SystemIndexDescriptor.MappingsVersion> minimumMappingsVersions = compatibilityVersions.stream()
-            .flatMap(mv -> mv.systemIndexMappingsVersion().entrySet().stream())
-            .collect(
-                Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (v1, v2) -> Stream.of(v1, v2).min(Comparator.naturalOrder()).get())
-            );
-
+        if (compatibilityVersions.isEmpty()) {
+            return EMPTY;
+        }
+        TransportVersion minimumTransport = null;
+        Map<String, SystemIndexDescriptor.MappingsVersion> minimumMappingsVersions = null;
+        for (CompatibilityVersions cv : compatibilityVersions) {
+            TransportVersion version = cv.transportVersion();
+            if (minimumTransport == null) {
+                minimumTransport = version;
+                minimumMappingsVersions = new HashMap<>(cv.systemIndexMappingsVersion());
+                continue;
+            }
+            if (version.compareTo(minimumTransport) < 0) {
+                minimumTransport = version;
+            }
+            for (Map.Entry<String, SystemIndexDescriptor.MappingsVersion> entry : cv.systemIndexMappingsVersion().entrySet()) {
+                minimumMappingsVersions.merge(entry.getKey(), entry.getValue(), (v1, v2) -> v1.compareTo(v2) < 0 ? v1 : v2);
+            }
+        }
+        // transportVersions is always non-null since we break out on empty above
         return new CompatibilityVersions(minimumTransport, minimumMappingsVersions);
     }
 
@@ -113,7 +120,7 @@ public void writeTo(StreamOutput out) throws IOException {
         TransportVersion.writeVersion(this.transportVersion(), out);
 
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_11_X)) {
-            out.writeMap(this.systemIndexMappingsVersion(), (o, v) -> v.writeTo(o));
+            out.writeMap(this.systemIndexMappingsVersion(), StreamOutput::writeWriteable);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java b/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
index e731cf3bc58be..58c23ab9aa398 100644
--- a/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
+++ b/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
@@ -56,8 +56,10 @@ public static String getBase64UUID(Random random) {
         return Base64.getUrlEncoder().withoutPadding().encodeToString(getUUIDBytes(random));
     }
 
+    static final int SIZE_IN_BYTES = 16;
+
     private static byte[] getUUIDBytes(Random random) {
-        final byte[] randomBytes = new byte[16];
+        final byte[] randomBytes = new byte[SIZE_IN_BYTES];
         random.nextBytes(randomBytes);
         /* Set the version to version 4 (see http://www.ietf.org/rfc/rfc4122.txt)
          * The randomly or pseudo-randomly generated version.
diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
index 65225530b7d38..59c55fb7b624a 100644
--- a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
+++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
@@ -78,6 +78,10 @@ public enum ReferenceDocs {
     SNAPSHOT_REPOSITORY_ANALYSIS,
     S3_COMPATIBLE_REPOSITORIES,
     LUCENE_MAX_DOCS_LIMIT,
+    MAX_SHARDS_PER_NODE,
+    FLOOD_STAGE_WATERMARK,
+    X_OPAQUE_ID,
+    FORMING_SINGLE_NODE_CLUSTERS,
     // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner
     ;
 
diff --git a/server/src/main/java/org/elasticsearch/common/TimeBasedUUIDGenerator.java b/server/src/main/java/org/elasticsearch/common/TimeBasedUUIDGenerator.java
index f7f7f520fec90..d66b0f579ce3e 100644
--- a/server/src/main/java/org/elasticsearch/common/TimeBasedUUIDGenerator.java
+++ b/server/src/main/java/org/elasticsearch/common/TimeBasedUUIDGenerator.java
@@ -47,6 +47,8 @@ protected byte[] macAddress() {
         return SECURE_MUNGED_ADDRESS;
     }
 
+    static final int SIZE_IN_BYTES = 15;
+
     @Override
     public String getBase64UUID() {
         final int sequenceId = sequenceNumber.incrementAndGet() & 0xffffff;
@@ -61,7 +63,7 @@ public String getBase64UUID() {
             sequenceId == 0 ? (lastTimestamp, currentTimeMillis) -> Math.max(lastTimestamp, currentTimeMillis) + 1 : Math::max
         );
 
-        final byte[] uuidBytes = new byte[15];
+        final byte[] uuidBytes = new byte[SIZE_IN_BYTES];
         int i = 0;
 
         // We have auto-generated ids, which are usually used for append-only workloads.
diff --git a/server/src/main/java/org/elasticsearch/common/UUIDs.java b/server/src/main/java/org/elasticsearch/common/UUIDs.java
index 43a232e82510e..ebc0978f38d49 100644
--- a/server/src/main/java/org/elasticsearch/common/UUIDs.java
+++ b/server/src/main/java/org/elasticsearch/common/UUIDs.java
@@ -17,26 +17,50 @@ public class UUIDs {
     private static final RandomBasedUUIDGenerator RANDOM_UUID_GENERATOR = new RandomBasedUUIDGenerator();
     private static final UUIDGenerator TIME_UUID_GENERATOR = new TimeBasedUUIDGenerator();
 
-    /** Generates a time-based UUID (similar to Flake IDs), which is preferred when generating an ID to be indexed into a Lucene index as
-     *  primary key.  The id is opaque and the implementation is free to change at any time! */
+    /**
+     * The length of a UUID string generated by {@link #base64UUID}.
+     */
+    // A 15-byte time-based UUID is base64-encoded as 5 3-byte chunks (each becoming 4 chars after encoding).
+    public static final int TIME_BASED_UUID_STRING_LENGTH = 20;
+
+    /**
+     * Generates a time-based UUID (similar to Flake IDs), which is preferred when generating an ID to be indexed into a Lucene index as
+     * primary key. The id is opaque and the implementation is free to change at any time!
+     * The resulting string has length {@link #TIME_BASED_UUID_STRING_LENGTH}.
+     */
     public static String base64UUID() {
         return TIME_UUID_GENERATOR.getBase64UUID();
     }
 
-    /** Returns a Base64 encoded version of a Version 4.0 compatible UUID as defined here: http://www.ietf.org/rfc/rfc4122.txt, using the
-     *  provided {@code Random} instance */
+    /**
+     * The length of a UUID string generated by {@link #randomBase64UUID} and {@link #randomBase64UUIDSecureString}.
+     */
+    // A 16-byte v4 UUID is base64-encoded as 5 3-byte chunks (each becoming 4 chars after encoding) plus another byte (becomes 2 chars).
+    public static final int RANDOM_BASED_UUID_STRING_LENGTH = 22;
+
+    /**
+     * Returns a Base64 encoded string representing a <a href="http://www.ietf.org/rfc/rfc4122.txt">RFC4122 version 4 UUID</a>, using the
+     * provided {@code Random} instance.
+     * The resulting string has length {@link #RANDOM_BASED_UUID_STRING_LENGTH}.
+     */
     public static String randomBase64UUID(Random random) {
         return RandomBasedUUIDGenerator.getBase64UUID(random);
     }
 
-    /** Returns a Base64 encoded version of a Version 4.0 compatible UUID as defined here: http://www.ietf.org/rfc/rfc4122.txt, using a
-     *  private {@code SecureRandom} instance */
+    /**
+     * Returns a Base64 encoded string representing a <a href="http://www.ietf.org/rfc/rfc4122.txt">RFC4122 version 4 UUID</a>, using a
+     * private {@code SecureRandom} instance.
+     * The resulting string has length {@link #RANDOM_BASED_UUID_STRING_LENGTH}.
+     */
     public static String randomBase64UUID() {
         return RANDOM_UUID_GENERATOR.getBase64UUID();
     }
 
-    /** Returns a Base64 encoded {@link SecureString} of a Version 4.0 compatible UUID as defined here: http://www.ietf.org/rfc/rfc4122.txt,
-     *  using a private {@code SecureRandom} instance */
+    /**
+     * Returns a Base64 encoded {@link SecureString} representing a <a href="http://www.ietf.org/rfc/rfc4122.txt">RFC4122 version 4
+     * UUID</a>, using a private {@code SecureRandom} instance.
+     * The resulting string has length {@link #RANDOM_BASED_UUID_STRING_LENGTH}.
+     */
     public static SecureString randomBase64UUIDSecureString() {
         return RandomBasedUUIDGenerator.getBase64UUIDSecureString();
     }
diff --git a/server/src/main/java/org/elasticsearch/common/inject/ConstantFactory.java b/server/src/main/java/org/elasticsearch/common/inject/ConstantFactory.java
deleted file mode 100644
index cb3bc95136b29..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/ConstantFactory.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.elasticsearch.common.inject;
-
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalContext;
-import org.elasticsearch.common.inject.internal.InternalFactory;
-import org.elasticsearch.common.inject.internal.ToStringBuilder;
-import org.elasticsearch.common.inject.spi.Dependency;
-
-/**
- * @author crazybob@google.com (Bob Lee)
- */
-class ConstantFactory<T> implements InternalFactory<T> {
-
-    private final Initializable<T> initializable;
-
-    ConstantFactory(Initializable<T> initializable) {
-        this.initializable = initializable;
-    }
-
-    @Override
-    public T get(Errors errors, InternalContext context, Dependency<?> dependency) throws ErrorsException {
-        return initializable.get(errors);
-    }
-
-    @Override
-    public String toString() {
-        return new ToStringBuilder(ConstantFactory.class).add("value", initializable).toString();
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Inject.java b/server/src/main/java/org/elasticsearch/common/inject/Inject.java
deleted file mode 100644
index e56c4c21ad39e..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/Inject.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.elasticsearch.common.inject;
-
-import java.lang.annotation.Documented;
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-import static java.lang.annotation.ElementType.CONSTRUCTOR;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-/**
- * Annotates members of your implementation class (constructors, methods
- * and fields) into which the {@link Injector} should inject values.
- * The Injector fulfills injection requests for:
- * <ul>
- * <li>Every instance it constructs. The class being constructed must have
- * exactly one of its constructors marked with {@code @Inject} or must have a
- * constructor taking no parameters. The Injector then proceeds to perform
- * method and field injections.
- * <li>Pre-constructed instances passed to
- * {@link org.elasticsearch.common.inject.binder.LinkedBindingBuilder#toInstance(Object)} and
- * {@link org.elasticsearch.common.inject.binder.LinkedBindingBuilder#toProvider(Provider)}.
- * In this case all constructors are, of course, ignored.
- * </ul>
- * <p>
- * In all cases, a member can be injected regardless of its Java access
- * specifier (private, default, protected, public).
- *
- * @author crazybob@google.com (Bob Lee)
- */
-@Target({ METHOD, CONSTRUCTOR })
-@Retention(RUNTIME)
-@Documented
-public @interface Inject {
-
-    /**
-     * If true, and the appropriate binding is not found,
-     * the Injector will skip injection of this method or field rather than
-     * produce an error. When applied to a field, any default value already
-     * assigned to the field will remain (guice will not actively null out the
-     * field). When applied to a method, the method will only be invoked if
-     * bindings for <i>all</i> parameters are found. When applied to a
-     * constructor, an error will result upon Injector creation.
-     */
-    boolean optional() default false;
-}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Injector.java b/server/src/main/java/org/elasticsearch/common/inject/Injector.java
deleted file mode 100644
index a3060af8f56d4..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/Injector.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.elasticsearch.common.inject;
-
-import java.util.List;
-
-/**
- * Builds the graphs of objects that make up your application. The injector tracks the dependencies
- * for each type and uses bindings to inject them. This is the core of Guice, although you rarely
- * interact with it directly. This "behind-the-scenes" operation is what distinguishes dependency
- * injection from its cousin, the service locator pattern.
- * <p>
- * Contains several default bindings:
- * <ul>
- * <li>This {@link Injector} instance itself
- * <li>A {@code Provider<T>} for each binding of type {@code T}
- * <li>The {@link java.util.logging.Logger} for the class being injected
- * </ul>
- * <p>
- * Injectors are created using the facade class {@link Guice}.
- *
- * @author crazybob@google.com (Bob Lee)
- * @author jessewilson@google.com (Jesse Wilson)
- */
-public interface Injector {
-
-    /**
-     * Returns all explicit bindings for {@code type}.
-     * <p>
-     * This method is part of the Guice SPI and is intended for use by tools and extensions.
-     */
-    <T> List<Binding<T>> findBindingsByType(TypeLiteral<T> type);
-
-    /**
-     * Returns the provider used to obtain instances for the given injection key. When feasible, avoid
-     * using this method, in favor of having Guice inject your dependencies ahead of time.
-     *
-     * @throws ConfigurationException if this injector cannot find or create the provider.
-     * @see Binder#getProvider(Key) for an alternative that offers up front error detection
-     */
-    <T> Provider<T> getProvider(Key<T> key);
-
-    /**
-     * Returns the appropriate instance for the given injection key; equivalent to {@code
-     * getProvider(key).get()}. When feasible, avoid using this method, in favor of having Guice
-     * inject your dependencies ahead of time.
-     *
-     * @throws ConfigurationException if this injector cannot find or create the provider.
-     * @throws ProvisionException     if there was a runtime failure while providing an instance.
-     */
-    <T> T getInstance(Key<T> key);
-
-    /**
-     * Returns the appropriate instance for the given injection type; equivalent to {@code
-     * getProvider(type).get()}. When feasible, avoid using this method, in favor of having Guice
-     * inject your dependencies ahead of time.
-     *
-     * @throws ConfigurationException if this injector cannot find or create the provider.
-     * @throws ProvisionException     if there was a runtime failure while providing an instance.
-     */
-    <T> T getInstance(Class<T> type);
-}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/binder/LinkedBindingBuilder.java b/server/src/main/java/org/elasticsearch/common/inject/binder/LinkedBindingBuilder.java
deleted file mode 100644
index 4c4ba81cf504b..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/binder/LinkedBindingBuilder.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.elasticsearch.common.inject.binder;
-
-import org.elasticsearch.common.inject.Provider;
-
-/**
- * See the EDSL examples at {@link org.elasticsearch.common.inject.Binder}.
- *
- * @author crazybob@google.com (Bob Lee)
- */
-public interface LinkedBindingBuilder<T> extends ScopedBindingBuilder {
-
-    /**
-     * See the EDSL examples at {@link org.elasticsearch.common.inject.Binder}.
-     */
-    ScopedBindingBuilder to(Class<? extends T> implementation);
-
-    /**
-     * See the EDSL examples at {@link org.elasticsearch.common.inject.Binder}.
-     */
-    void toInstance(T instance);
-
-    /**
-     * See the EDSL examples at {@link org.elasticsearch.common.inject.Binder}.
-     */
-    ScopedBindingBuilder toProvider(Provider<? extends T> provider);
-
-}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/binder/package-info.java b/server/src/main/java/org/elasticsearch/common/inject/binder/package-info.java
deleted file mode 100644
index e8acf47290ad7..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/binder/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Interfaces which make up {@link org.elasticsearch.common.inject.Binder}'s
- * expression language.
- */
-package org.elasticsearch.common.inject.binder;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/Annotations.java b/server/src/main/java/org/elasticsearch/common/inject/internal/Annotations.java
deleted file mode 100644
index 24f2539e2574e..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/Annotations.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.elasticsearch.common.inject.internal;
-
-import org.elasticsearch.common.inject.BindingAnnotation;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.TypeLiteral;
-
-import java.lang.annotation.Annotation;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.reflect.Member;
-
-/**
- * Annotation utilities.
- *
- * @author crazybob@google.com (Bob Lee)
- */
-public class Annotations {
-
-    /**
-     * Returns true if the given annotation is retained at runtime.
-     */
-    public static boolean isRetainedAtRuntime(Class<? extends Annotation> annotationType) {
-        Retention retention = annotationType.getAnnotation(Retention.class);
-        return retention != null && retention.value() == RetentionPolicy.RUNTIME;
-    }
-
-    /**
-     * Gets a key for the given type, member and annotations.
-     */
-    public static Key<?> getKey(TypeLiteral<?> type, Member member, Annotation[] annotations, Errors errors) throws ErrorsException {
-        int numErrorsBefore = errors.size();
-        Annotation found = findBindingAnnotation(errors, member, annotations);
-        errors.throwIfNewErrors(numErrorsBefore);
-        return found == null ? Key.get(type) : Key.get(type, found);
-    }
-
-    /**
-     * Returns the binding annotation on {@code member}, or null if there isn't one.
-     */
-    public static Annotation findBindingAnnotation(Errors errors, Member member, Annotation[] annotations) {
-        Annotation found = null;
-
-        for (Annotation annotation : annotations) {
-            if (annotation.annotationType().getAnnotation(BindingAnnotation.class) != null) {
-                if (found != null) {
-                    errors.duplicateBindingAnnotations(member, found.annotationType(), annotation.annotationType());
-                } else {
-                    found = annotation;
-                }
-            }
-        }
-
-        return found;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/package-info.java b/server/src/main/java/org/elasticsearch/common/inject/internal/package-info.java
deleted file mode 100644
index 45beaf4650c28..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <i>Guice</i> (sounds like like "juice")
- */
-
-package org.elasticsearch.common.inject.internal;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/matcher/package-info.java b/server/src/main/java/org/elasticsearch/common/inject/matcher/package-info.java
deleted file mode 100644
index a1e1ef80c442f..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/matcher/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Used for matching things. Primarily used to pick out methods to which to
- * apply interceptors.
- */
-package org.elasticsearch.common.inject.matcher;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/multibindings/Element.java b/server/src/main/java/org/elasticsearch/common/inject/multibindings/Element.java
deleted file mode 100644
index 6fca4135b596f..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/multibindings/Element.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright (C) 2008 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.elasticsearch.common.inject.multibindings;
-
-import org.elasticsearch.common.inject.BindingAnnotation;
-
-import java.lang.annotation.Retention;
-
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-/**
- * An internal binding annotation applied to each element in a multibinding.
- * All elements are assigned a globally-unique id to allow different modules
- * to contribute multibindings independently.
- *
- * @author jessewilson@google.com (Jesse Wilson)
- */
-@Retention(RUNTIME)
-@BindingAnnotation
-@interface Element {
-    int uniqueId();
-}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/multibindings/package-info.java b/server/src/main/java/org/elasticsearch/common/inject/multibindings/package-info.java
deleted file mode 100644
index f42cfc7e5b64f..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/multibindings/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright (C) 2008 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Extension for binding multiple instances in a collection; this extension requires {@code
- * guice-multibindings-2.0.jar}.
- */
-package org.elasticsearch.common.inject.multibindings;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/name/package-info.java b/server/src/main/java/org/elasticsearch/common/inject/name/package-info.java
deleted file mode 100644
index bacd62bc113c3..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/name/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Support for binding to string-based names.
- */
-package org.elasticsearch.common.inject.name;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/package-info.java b/server/src/main/java/org/elasticsearch/common/inject/package-info.java
deleted file mode 100644
index ba292887e7024..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/package-info.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/* @notice
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * Modifications copyright (C) 2020 Elasticsearch B.V.
- */
-
-/**
- * <i>Google Guice</i> (pronounced "juice") is an ultra-lightweight dependency
- * injection framework.  Please refer to the Guice
- * <a href="http://docs.google.com/Doc?id=dd2fhx4z_5df5hw8">User's Guide</a>
- * for a gentle introduction.
- *
- * <p>The principal public APIs in this package are:
- *
- * <dl>
- * <dt>{@link org.elasticsearch.common.inject.Inject}
- * <dd>The annotation you will use in your implementation classes to tell Guice
- *     where and how it should send in ("inject") the objects you depend on
- *     (your "dependencies").
- *
- * <dt>{@link org.elasticsearch.common.inject.Module}
- * <dd>The interface you will implement in order to specify "bindings" --
- *     instructions for how Guice should handle injection -- for a particular
- *     set of interfaces.
- *
- * <dt>{@link org.elasticsearch.common.inject.Binder}
- * <dd>The object that Guice passes into your {@link org.elasticsearch.common.inject.Module}
- *     to collect these bindings.
- *
- * <dt>{@link org.elasticsearch.common.inject.Provider}
- * <dd>The interface you will implement when you need to customize exactly how
- *     Guice creates instances for a particular binding.
- *
- * </dl>
- *
- */
-package org.elasticsearch.common.inject;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/Element.java b/server/src/main/java/org/elasticsearch/common/inject/spi/Element.java
deleted file mode 100644
index 06bdf5e697137..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/Element.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright (C) 2008 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.elasticsearch.common.inject.spi;
-
-/**
- * A core component of a module or injector.
- * <p>
- * The elements of a module can be inspected, validated and rewritten. Use {@link
- * Elements#getElements(org.elasticsearch.common.inject.Module[]) Elements.getElements()} to read the elements
- * from a module to rewrite them.
- * This can be used for static analysis and generation of Guice modules.
- *
- * @author jessewilson@google.com (Jesse Wilson)
- * @author crazybob@google.com (Bob Lee)
- * @since 2.0
- */
-public interface Element {
-
-    /**
-     * Returns an arbitrary object containing information about the "place" where this element was
-     * configured. Used by Guice in the production of descriptive error messages.
-     * <p>
-     * Tools might specially handle types they know about; {@code StackTraceElement} is a good
-     * example. Tools should simply call {@code toString()} on the source object if the type is
-     * unfamiliar.
-     */
-    Object getSource();
-
-    /**
-     * Accepts an element visitor. Invokes the visitor method specific to this element's type.
-     *
-     * @param visitor to call back on
-     */
-    <T> T acceptVisitor(ElementVisitor<T> visitor);
-
-}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/package-info.java b/server/src/main/java/org/elasticsearch/common/inject/spi/package-info.java
deleted file mode 100644
index fc52f724df349..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Guice service provider interface
- */
-
-package org.elasticsearch.common.inject.spi;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/util/Types.java b/server/src/main/java/org/elasticsearch/common/inject/util/Types.java
deleted file mode 100644
index f25e3b728ee0b..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/util/Types.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright (C) 2008 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.elasticsearch.common.inject.util;
-
-import org.elasticsearch.common.inject.Provider;
-import org.elasticsearch.common.inject.internal.MoreTypes;
-import org.elasticsearch.common.inject.internal.MoreTypes.GenericArrayTypeImpl;
-import org.elasticsearch.common.inject.internal.MoreTypes.ParameterizedTypeImpl;
-import org.elasticsearch.common.inject.internal.MoreTypes.WildcardTypeImpl;
-
-import java.lang.reflect.GenericArrayType;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
-import java.lang.reflect.WildcardType;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Static methods for working with types.
- *
- * @author crazybob@google.com (Bob Lee)
- * @since 2.0
- */
-public final class Types {
-    private Types() {}
-
-    /**
-     * Returns a new parameterized type, applying {@code typeArguments} to
-     * {@code rawType}. The returned type does not have an owner type.
-     *
-     * @return a parameterized type.
-     */
-    public static ParameterizedType newParameterizedType(Type rawType, Type... typeArguments) {
-        return newParameterizedTypeWithOwner(null, rawType, typeArguments);
-    }
-
-    /**
-     * Returns a new parameterized type, applying {@code typeArguments} to
-     * {@code rawType} and enclosed by {@code ownerType}.
-     *
-     * @return a parameterized type.
-     */
-    public static ParameterizedType newParameterizedTypeWithOwner(Type ownerType, Type rawType, Type... typeArguments) {
-        return new ParameterizedTypeImpl(ownerType, rawType, typeArguments);
-    }
-
-    /**
-     * Returns an array type whose elements are all instances of
-     * {@code componentType}.
-     *
-     * @return a generic array type.
-     */
-    public static GenericArrayType arrayOf(Type componentType) {
-        return new GenericArrayTypeImpl(componentType);
-    }
-
-    /**
-     * Returns a type that represents an unknown type that extends {@code bound}.
-     * For example, if {@code bound} is {@code CharSequence.class}, this returns
-     * {@code ? extends CharSequence}. If {@code bound} is {@code Object.class},
-     * this returns {@code ?}, which is shorthand for {@code ? extends Object}.
-     */
-    public static WildcardType subtypeOf(Type bound) {
-        return new WildcardTypeImpl(new Type[] { bound }, MoreTypes.EMPTY_TYPE_ARRAY);
-    }
-
-    /**
-     * Returns a type that represents an unknown supertype of {@code bound}. For
-     * example, if {@code bound} is {@code String.class}, this returns {@code ?
-     * super String}.
-     */
-    public static WildcardType supertypeOf(Type bound) {
-        return new WildcardTypeImpl(new Type[] { Object.class }, new Type[] { bound });
-    }
-
-    /**
-     * Returns a type modelling a {@link Set} whose elements are of type
-     * {@code elementType}.
-     *
-     * @return a parameterized type.
-     */
-    public static ParameterizedType setOf(Type elementType) {
-        return newParameterizedType(Set.class, elementType);
-    }
-
-    /**
-     * Returns a type modelling a {@link Map} whose keys are of type
-     * {@code keyType} and whose values are of type {@code valueType}.
-     *
-     * @return a parameterized type.
-     */
-    public static ParameterizedType mapOf(Type keyType, Type valueType) {
-        return newParameterizedType(Map.class, keyType, valueType);
-    }
-
-    // for other custom collections types, use newParameterizedType()
-
-    /**
-     * Returns a type modelling a {@link Provider} that provides elements of type
-     * {@code elementType}.
-     *
-     * @return a parameterized type.
-     */
-    public static ParameterizedType providerOf(Type providedType) {
-        return newParameterizedType(Provider.class, providedType);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/util/package-info.java b/server/src/main/java/org/elasticsearch/common/inject/util/package-info.java
deleted file mode 100644
index c4f740d6dc2c1..0000000000000
--- a/server/src/main/java/org/elasticsearch/common/inject/util/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright (C) 2009 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Helper methods for working with Guice.
- */
-package org.elasticsearch.common.inject.util;
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/ByteArrayStreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/ByteArrayStreamInput.java
index 52eee5af3f6f5..a27eec4c12061 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/ByteArrayStreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/ByteArrayStreamInput.java
@@ -117,4 +117,15 @@ public void readBytes(byte[] b, int offset, int len) {
         System.arraycopy(bytes, pos, b, offset, len);
         pos += len;
     }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        final int available = limit - pos;
+        if (available <= 0) {
+            return -1;
+        }
+        int toRead = Math.min(len, available);
+        readBytes(b, off, toRead);
+        return toRead;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
index c0ef0e0abf39b..b84c67bd8c8a2 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
@@ -97,6 +97,11 @@ public int read() throws IOException {
         return delegate.read();
     }
 
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        return delegate.read(b, off, len);
+    }
+
     @Override
     public void close() throws IOException {
         delegate.close();
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index 60322ea89cbe8..497028ef37c69 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -104,6 +104,10 @@ public void setTransportVersion(TransportVersion version) {
      */
     public abstract void readBytes(byte[] b, int offset, int len) throws IOException;
 
+    // force implementing bulk reads to avoid accidentally slow implementations
+    @Override
+    public abstract int read(byte[] b, int off, int len) throws IOException;
+
     /**
      * Reads a bytes reference from this stream, copying any bytes read to a new {@code byte[]}. Use {@link #readReleasableBytesReference()}
      * when reading large bytes references where possible top avoid needless allocations and copying.
@@ -902,7 +906,16 @@ public final Instant readOptionalInstant() throws IOException {
 
     private ZonedDateTime readZonedDateTime() throws IOException {
         final String timeZoneId = readString();
-        return ZonedDateTime.ofInstant(Instant.ofEpochMilli(readLong()), ZoneId.of(timeZoneId));
+        final Instant instant;
+        if (getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT_BROKEN)) {
+            // epoch seconds can be negative, but it was incorrectly first written as vlong
+            boolean zlong = getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT);
+            long seconds = zlong ? readZLong() : readVLong();
+            instant = Instant.ofEpochSecond(seconds, readInt());
+        } else {
+            instant = Instant.ofEpochMilli(readLong());
+        }
+        return ZonedDateTime.ofInstant(instant, ZoneId.of(timeZoneId));
     }
 
     private OffsetTime readOffsetTime() throws IOException {
@@ -1082,8 +1095,23 @@ public <T> T[] readOptionalArray(Writeable.Reader<T> reader, IntFunction<T[]> ar
         return readBoolean() ? readArray(reader, arraySupplier) : null;
     }
 
+    /**
+     * Reads a possibly-null value using the given {@link org.elasticsearch.common.io.stream.Writeable.Reader}.
+     *
+     * @see StreamOutput#writeOptionalWriteable
+     */
+    // just an alias for readOptional() since we don't actually care whether T extends Writeable
     @Nullable
     public <T extends Writeable> T readOptionalWriteable(Writeable.Reader<T> reader) throws IOException {
+        return readOptional(reader);
+    }
+
+    /**
+     * Reads a possibly-null value using the given {@link org.elasticsearch.common.io.stream.Writeable.Reader}.
+     *
+     * @see StreamOutput#writeOptional
+     */
+    public <T> T readOptional(Writeable.Reader<T> reader) throws IOException {
         if (readBoolean()) {
             T t = reader.read(this);
             if (t == null) {
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index f17e3ee8018a2..5780885473b00 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -766,7 +766,18 @@ public final void writeOptionalInstant(@Nullable Instant instant) throws IOExcep
             o.writeByte((byte) 23);
             final ZonedDateTime zonedDateTime = (ZonedDateTime) v;
             o.writeString(zonedDateTime.getZone().getId());
-            o.writeLong(zonedDateTime.toInstant().toEpochMilli());
+            Instant instant = zonedDateTime.toInstant();
+            if (o.getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT_BROKEN)) {
+                // epoch seconds can be negative, but it was incorrectly first written as vlong
+                if (o.getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT)) {
+                    o.writeZLong(instant.getEpochSecond());
+                } else {
+                    o.writeVLong(instant.getEpochSecond());
+                }
+                o.writeInt(instant.getNano());
+            } else {
+                o.writeLong(instant.toEpochMilli());
+            }
         }),
         entry(Set.class, (o, v) -> {
             if (v instanceof LinkedHashSet) {
@@ -1004,6 +1015,12 @@ public <T extends Writeable> void writeOptionalArray(@Nullable T[] array) throws
         writeOptionalArray(StreamOutput::writeWriteable, array);
     }
 
+    /**
+     * Writes a boolean value indicating whether the given object is {@code null}, followed by the object's serialization if it is not
+     * {@code null}.
+     *
+     * @see StreamInput#readOptionalWriteable
+     */
     public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOException {
         if (writeable != null) {
             writeBoolean(true);
@@ -1013,6 +1030,21 @@ public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOExcep
         }
     }
 
+    /**
+     * Writes a boolean value indicating whether the given object is {@code null}, followed by the object's serialization if it is not
+     * {@code null}.
+     *
+     * @see StreamInput#readOptional
+     */
+    public <T> void writeOptional(Writer<T> writer, @Nullable T maybeItem) throws IOException {
+        if (maybeItem != null) {
+            writeBoolean(true);
+            writer.write(this, maybeItem);
+        } else {
+            writeBoolean(false);
+        }
+    }
+
     /**
      * This method allow to use a method reference when writing collection elements such as
      * {@code out.writeMap(map, StreamOutput::writeString, StreamOutput::writeWriteable)}
diff --git a/server/src/main/java/org/elasticsearch/common/logging/ConsoleThrowablePatternConverter.java b/server/src/main/java/org/elasticsearch/common/logging/ConsoleThrowablePatternConverter.java
index af52636be10ad..d8cbfe2f5ae4c 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/ConsoleThrowablePatternConverter.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/ConsoleThrowablePatternConverter.java
@@ -16,7 +16,7 @@
 import org.apache.logging.log4j.core.pattern.ThrowablePatternConverter;
 import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.bootstrap.StartupException;
-import org.elasticsearch.common.inject.CreationException;
+import org.elasticsearch.injection.guice.CreationException;
 
 /**
  * Outputs a very short version of exceptions for an interactive console, pointing to full log for details.
diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java
index 80aeaffaabec5..7f94e8d3d1824 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java
@@ -62,10 +62,12 @@ private static ESLogMessage getEsLogMessage(
             .field(KEY_FIELD_NAME, key)
             .field("elasticsearch.event.category", category.name().toLowerCase(Locale.ROOT));
 
-        if (Strings.isNullOrEmpty(xOpaqueId)) {
-            return esLogMessage;
+        if (Strings.isNullOrEmpty(xOpaqueId) == false) {
+            esLogMessage.field(X_OPAQUE_ID_FIELD_NAME, xOpaqueId);
         }
-
-        return esLogMessage.field(X_OPAQUE_ID_FIELD_NAME, xOpaqueId).field(ELASTIC_ORIGIN_FIELD_NAME, productOrigin);
+        if (Strings.isNullOrEmpty(productOrigin) == false) {
+            esLogMessage.field(ELASTIC_ORIGIN_FIELD_NAME, productOrigin);
+        }
+        return esLogMessage;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 36b3076c29a31..acdc3e32ea31a 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -73,6 +73,7 @@
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.lucene.grouping.TopFieldGroups;
+import org.elasticsearch.search.retriever.rankdoc.RankDocsSortField;
 import org.elasticsearch.search.sort.ShardDocSortField;
 
 import java.io.IOException;
@@ -551,6 +552,8 @@ private static SortField rewriteMergeSortField(SortField sortField) {
             return newSortField;
         } else if (sortField.getClass() == ShardDocSortField.class) {
             return new SortField(sortField.getField(), SortField.Type.LONG, sortField.getReverse());
+        } else if (sortField.getClass() == RankDocsSortField.class) {
+            return new SortField(sortField.getField(), SortField.Type.INT, sortField.getReverse());
         } else {
             return sortField;
         }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
index 58cbfa2be3f05..6b8a75337b8ee 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
@@ -62,18 +62,24 @@ public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) throws IOEx
         leafScript._setIndexName(indexName);
         leafScript._setShard(shardId);
         return new LeafScoreFunction() {
-            @Override
-            public double score(int docId, float subQueryScore) throws IOException {
+
+            private double score(int docId, float subQueryScore, ScoreScript.ExplanationHolder holder) throws IOException {
                 leafScript.setDocument(docId);
                 scorer.docid = docId;
                 scorer.score = subQueryScore;
-                double result = leafScript.execute(null);
+                double result = leafScript.execute(holder);
+
                 if (result < 0f) {
                     throw new IllegalArgumentException("script score function must not produce negative scores, but got: [" + result + "]");
                 }
                 return result;
             }
 
+            @Override
+            public double score(int docId, float subQueryScore) throws IOException {
+                return score(docId, subQueryScore, null);
+            }
+
             @Override
             public Explanation explainScore(int docId, Explanation subQueryScore) throws IOException {
                 Explanation exp;
@@ -83,11 +89,17 @@ public Explanation explainScore(int docId, Explanation subQueryScore) throws IOE
                     scorer.score = subQueryScore.getValue().floatValue();
                     exp = ((ExplainableScoreScript) leafScript).explain(subQueryScore);
                 } else {
-                    double score = score(docId, subQueryScore.getValue().floatValue());
+                    ScoreScript.ExplanationHolder holder = new ScoreScript.ExplanationHolder();
+                    double score = score(docId, subQueryScore.getValue().floatValue(), holder);
                     // info about params already included in sScript
-                    String explanation = "script score function, computed with script:\"" + sScript + "\"";
                     Explanation scoreExp = Explanation.match(subQueryScore.getValue(), "_score: ", subQueryScore);
-                    return Explanation.match((float) score, explanation, scoreExp);
+                    Explanation customExplanation = holder.get(score, null);
+                    if (customExplanation != null) {
+                        return Explanation.match((float) score, customExplanation.getDescription(), scoreExp);
+                    } else {
+                        String explanation = "script score function, computed with script:\"" + sScript + "\"";
+                        return Explanation.match((float) score, explanation, scoreExp);
+                    }
                 }
                 return exp;
             }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java
index 7ddaa4bc681fa..93837269f2090 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreQuery.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.common.lucene.search.function;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -29,10 +30,13 @@
 import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.ScoreScript.ExplanationHolder;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptTermStats;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.Objects;
+import java.util.Set;
 
 /**
  * A query that uses a script to compute documents' scores.
@@ -86,9 +90,18 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
             return subQuery.createWeight(searcher, scoreMode, boost);
         }
         boolean needsScore = scriptBuilder.needs_score();
-        ScoreMode subQueryScoreMode = needsScore ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES;
+        boolean needsTermStatistics = scriptBuilder.needs_termStats();
+
+        ScoreMode subQueryScoreMode = needsScore || needsTermStatistics ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES;
         Weight subQueryWeight = subQuery.createWeight(searcher, subQueryScoreMode, 1.0f);
 
+        // We collect the different terms used in the child query.
+        final Set<Term> terms = new HashSet<>();
+
+        if (needsTermStatistics) {
+            this.visit(QueryVisitor.termCollector(terms));
+        }
+
         return new Weight(this) {
             @Override
             public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
@@ -167,6 +180,9 @@ private ScoreScript makeScoreScript(LeafReaderContext context) throws IOExceptio
                 final ScoreScript scoreScript = scriptBuilder.newInstance(new DocValuesDocReader(lookup, context));
                 scoreScript._setIndexName(indexName);
                 scoreScript._setShard(shardId);
+                if (needsTermStatistics) {
+                    scoreScript._setTermStats(new ScriptTermStats(searcher, context, scoreScript::_getDocId, terms));
+                }
                 return scoreScript;
             }
 
diff --git a/server/src/main/java/org/elasticsearch/common/regex/Regex.java b/server/src/main/java/org/elasticsearch/common/regex/Regex.java
index 983144c7cee89..f292ac5032f7e 100644
--- a/server/src/main/java/org/elasticsearch/common/regex/Regex.java
+++ b/server/src/main/java/org/elasticsearch/common/regex/Regex.java
@@ -44,10 +44,20 @@ public static boolean isMatchAllPattern(String str) {
         return str.equals("*");
     }
 
+    /**
+     * Returns true if the str ends with "*".
+     */
     public static boolean isSuffixMatchPattern(String str) {
         return str.length() > 1 && str.indexOf('*') == str.length() - 1;
     }
 
+    /**
+     * Returns true if the str ends with ".*".
+     */
+    public static boolean isSuffixWildcard(String str) {
+        return isSuffixMatchPattern(str) && str.endsWith(".*");
+    }
+
     /** Return an {@link Automaton} that matches the given pattern. */
     public static Automaton simpleMatchToAutomaton(String pattern) {
         List<Automaton> automata = new ArrayList<>();
@@ -71,9 +81,12 @@ public static Automaton simpleMatchToAutomaton(String... patterns) {
 
         List<BytesRef> simpleStrings = new ArrayList<>();
         List<Automaton> automata = new ArrayList<>();
+        List<BytesRef> prefixes = new ArrayList<>();
         for (String pattern : patterns) {
             // Strings longer than 1000 characters aren't supported by makeStringUnion
-            if (isSimpleMatchPattern(pattern) || pattern.length() >= 1000) {
+            if (isSuffixWildcard(pattern) && pattern.length() < 1000) {
+                prefixes.add(new BytesRef(pattern.substring(0, pattern.length() - 1)));
+            } else if (isSimpleMatchPattern(pattern) || pattern.length() >= 1000) {
                 automata.add(simpleMatchToAutomaton(pattern));
             } else {
                 simpleStrings.add(new BytesRef(pattern));
@@ -87,11 +100,18 @@ public static Automaton simpleMatchToAutomaton(String... patterns) {
             } else {
                 simpleStringsAutomaton = Automata.makeString(simpleStrings.get(0).utf8ToString());
             }
-            if (automata.isEmpty()) {
+            if (automata.isEmpty() && prefixes.isEmpty()) {
                 return simpleStringsAutomaton;
             }
             automata.add(simpleStringsAutomaton);
         }
+        if (false == prefixes.isEmpty()) {
+            List<Automaton> prefixAutomaton = new ArrayList<>();
+            Collections.sort(prefixes);
+            prefixAutomaton.add(Automata.makeStringUnion(prefixes));
+            prefixAutomaton.add(Automata.makeAnyString());
+            automata.add(Operations.concatenate(prefixAutomaton));
+        }
         return Operations.union(automata);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/common/scheduler/SchedulerEngine.java b/server/src/main/java/org/elasticsearch/common/scheduler/SchedulerEngine.java
index be4d7c741bc92..66b4f3c82e3cf 100644
--- a/server/src/main/java/org/elasticsearch/common/scheduler/SchedulerEngine.java
+++ b/server/src/main/java/org/elasticsearch/common/scheduler/SchedulerEngine.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.core.Nullable;
 
 import java.time.Clock;
 import java.util.Collection;
@@ -39,15 +40,30 @@
  */
 public class SchedulerEngine {
 
-    public static class Job {
-        private final String id;
-        private final Schedule schedule;
-
+    /**
+     * In most cases a Job only requires a `schedule` and an `id`, but an optional `fixedStartTime`
+     * can also be used. This is used as a fixed `startTime` argument for all calls to
+     * `schedule.nextScheduledTimeAfter(startTime, now)`. Interval-based schedules use `startTime`
+     * as a basis time from which all run times are calculated. If a Job does not contain a
+     * `fixedStartTime`, this basis time will be the time at which the Job is added to the SchedulerEngine.
+     * This could change if a master change or restart causes a new SchedulerEngine to be constructed.
+     * But using a `fixedStartTime` populated  from a time stored in cluster state allows the basis time
+     * to remain unchanged across master changes and restarts.
+     *
+     * @param id the id of the job
+     * @param schedule the schedule which is used to calculate when the job runs
+     * @param fixedStartTime a fixed time in the past which the schedule uses to calculate run times,
+     */
+    public record Job(String id, Schedule schedule, @Nullable Long fixedStartTime) {
         public Job(String id, Schedule schedule) {
-            this.id = id;
-            this.schedule = schedule;
+            this(id, schedule, null);
         }
 
+        /**
+         * The following getters are redundant with the getters built in by the record.
+         * Unfortunately, getFieldName form getters are expected by serverless.
+         * These getters are being added back until serverless can be updated for the new getters.
+         */
         public String getId() {
             return id;
         }
@@ -55,19 +71,23 @@ public String getId() {
         public Schedule getSchedule() {
             return schedule;
         }
-    }
 
-    public static class Event {
-        private final String jobName;
-        private final long triggeredTime;
-        private final long scheduledTime;
+        public Long getFixedStartTime() {
+            return fixedStartTime;
+        }
+    }
 
-        public Event(String jobName, long triggeredTime, long scheduledTime) {
-            this.jobName = jobName;
-            this.triggeredTime = triggeredTime;
-            this.scheduledTime = scheduledTime;
+    public record Event(String jobName, long triggeredTime, long scheduledTime) {
+        @Override
+        public String toString() {
+            return "Event[jobName=" + jobName + "," + "triggeredTime=" + triggeredTime + "," + "scheduledTime=" + scheduledTime + "]";
         }
 
+        /**
+         * The following getters are redundant with the getters built in by the record.
+         * Unfortunately, getFieldName form getters are expected by serverless.
+         * These getters are being added back until serverless can be updated for the new getters.
+         */
         public String getJobName() {
             return jobName;
         }
@@ -79,11 +99,6 @@ public long getTriggeredTime() {
         public long getScheduledTime() {
             return scheduledTime;
         }
-
-        @Override
-        public String toString() {
-            return "Event[jobName=" + jobName + "," + "triggeredTime=" + triggeredTime + "," + "scheduledTime=" + scheduledTime + "]";
-        }
     }
 
     public interface Listener {
@@ -159,12 +174,13 @@ public Set<String> scheduledJobIds() {
     }
 
     public void add(Job job) {
-        ActiveSchedule schedule = new ActiveSchedule(job.getId(), job.getSchedule(), clock.millis());
+        final long startTime = job.fixedStartTime() == null ? clock.millis() : job.fixedStartTime();
+        ActiveSchedule schedule = new ActiveSchedule(job.id(), job.schedule(), startTime);
         schedules.compute(schedule.name, (name, previousSchedule) -> {
             if (previousSchedule != null) {
                 previousSchedule.cancel();
             }
-            logger.debug(() -> "added job [" + job.getId() + "]");
+            logger.debug(() -> "added job [" + job.id() + "]");
             return schedule;
         });
     }
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index d5f770ebb95fc..3c60d63f78991 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.cluster.coordination.MasterHistory;
 import org.elasticsearch.cluster.coordination.NoMasterBlockService;
 import org.elasticsearch.cluster.coordination.Reconfigurator;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -54,6 +55,7 @@
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.shards.ShardsAvailabilityHealthIndicatorService;
 import org.elasticsearch.cluster.service.ClusterApplierService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterService;
@@ -129,11 +131,8 @@
 import org.elasticsearch.transport.TransportSettings;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
-import java.util.Objects;
 import java.util.Set;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * Encapsulates all valid cluster level settings.
@@ -204,7 +203,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         }
     }
 
-    public static Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS = Stream.of(
+    public static final Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS = Set.of(
         AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING,
         AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING,
         BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING,
@@ -598,6 +597,9 @@ public void apply(Settings value, Settings current, Settings previous) {
         TDigestExecutionHint.SETTING,
         MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT_SETTING,
         MergePolicyConfig.DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT_SETTING,
-        TransportService.ENABLE_STACK_OVERFLOW_AVOIDANCE
-    ).filter(Objects::nonNull).collect(Collectors.toSet());
+        TransportService.ENABLE_STACK_OVERFLOW_AVOIDANCE,
+        DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING,
+        DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING,
+        ShardsAvailabilityHealthIndicatorService.REPLICA_UNASSIGNED_BUFFER_TIME
+    );
 }
diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java
index e96de685381eb..ad9f933ec0459 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -584,7 +584,7 @@ private T get(Settings settings, boolean validate) {
      */
     public void diff(Settings.Builder builder, Settings source, Settings defaultSettings) {
         if (exists(source) == false) {
-            if (exists(defaultSettings)) {
+            if (existsOrFallbackExists(defaultSettings)) {
                 // If the setting is only in the defaults, use the value from the defaults
                 builder.put(getKey(), getRaw(defaultSettings));
             } else {
diff --git a/server/src/main/java/org/elasticsearch/common/settings/Settings.java b/server/src/main/java/org/elasticsearch/common/settings/Settings.java
index be8292f02bb59..1df7b27304fd0 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -1567,7 +1567,7 @@ private static String toString(Object o) {
      * @param s string to intern
      * @return interned string
      */
-    static String internKeyOrValue(String s) {
+    public static String internKeyOrValue(String s) {
         return settingLiteralDeduplicator.deduplicate(s);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
index 2f2e4367e1930..6c9a217331b84 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
@@ -11,8 +11,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.injection.guice.Binder;
+import org.elasticsearch.injection.guice.Module;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index 7caf570806c0e..7bee1e895bceb 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -11,6 +11,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.client.internal.OriginSettingClient;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -578,7 +579,7 @@ public Map<String, Object> getTransientHeaders() {
     }
 
     /**
-     * Add the {@code value} for the specified {@code key} Any duplicate {@code value} is ignored.
+     * Add the {@code value} for the specified {@code key}. Any duplicate {@code value} is ignored.
      *
      * @param key         the header name
      * @param value       the header value
@@ -798,6 +799,30 @@ private ThreadContextStruct putResponseHeaders(Map<String, Set<String>> headers)
             return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext);
         }
 
+        private void logWarningHeaderThresholdExceeded(long threshold, Setting<?> thresholdSetting) {
+            // If available, log some selected headers to help identifying the source of the request.
+            // Note: Only Task.HEADERS_TO_COPY are guaranteed to be preserved at this point.
+            Map<String, String> selectedHeaders = new HashMap<>(requestHeaders);
+            selectedHeaders.keySet().retainAll(List.of(Task.X_OPAQUE_ID_HTTP_HEADER, Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER));
+
+            boolean sizeExceeded = HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.equals(thresholdSetting);
+
+            StringBuilder message = new StringBuilder().append("Dropping a warning header");
+            if (selectedHeaders.isEmpty() == false) {
+                message.append(" for request [").append(selectedHeaders).append("]");
+            }
+            message.append(", as their total").append(sizeExceeded ? " size" : " count");
+            message.append(" reached the maximum allowed of [").append(threshold).append("]").append(sizeExceeded ? " bytes" : "");
+            message.append(" set in [").append(thresholdSetting.getKey()).append("]!");
+
+            if (selectedHeaders.containsKey(Task.X_OPAQUE_ID_HTTP_HEADER) == false) {
+                message.append(" Add X-Opaque-Id headers to identify the source of this warning");
+                message.append(", see ").append(ReferenceDocs.X_OPAQUE_ID).append(" for more information.");
+            }
+
+            logger.warn(message);
+        }
+
         private ThreadContextStruct putResponse(
             final String key,
             final String value,
@@ -810,24 +835,12 @@ private ThreadContextStruct putResponse(
             // check if we can add another warning header - if max size within limits
             if (key.equals("Warning") && (maxWarningHeaderSize != -1)) { // if size is NOT unbounded, check its limits
                 if (warningHeadersSize > maxWarningHeaderSize) { // if max size has already been reached before
-                    logger.warn(
-                        "Dropping a warning header, as their total size reached the maximum allowed of ["
-                            + maxWarningHeaderSize
-                            + "] bytes set in ["
-                            + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey()
-                            + "]!"
-                    );
+                    logWarningHeaderThresholdExceeded(maxWarningHeaderSize, HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE);
                     return this;
                 }
                 newWarningHeaderSize += "Warning".getBytes(StandardCharsets.UTF_8).length + value.getBytes(StandardCharsets.UTF_8).length;
                 if (newWarningHeaderSize > maxWarningHeaderSize) {
-                    logger.warn(
-                        "Dropping a warning header, as their total size reached the maximum allowed of ["
-                            + maxWarningHeaderSize
-                            + "] bytes set in ["
-                            + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey()
-                            + "]!"
-                    );
+                    logWarningHeaderThresholdExceeded(maxWarningHeaderSize, HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE);
                     return new ThreadContextStruct(
                         requestHeaders,
                         responseHeaders,
@@ -857,13 +870,7 @@ private ThreadContextStruct putResponse(
             if ((key.equals("Warning")) && (maxWarningHeaderCount != -1)) { // if count is NOT unbounded, check its limits
                 final int warningHeaderCount = newResponseHeaders.containsKey("Warning") ? newResponseHeaders.get("Warning").size() : 0;
                 if (warningHeaderCount > maxWarningHeaderCount) {
-                    logger.warn(
-                        "Dropping a warning header, as their total count reached the maximum allowed of ["
-                            + maxWarningHeaderCount
-                            + "] set in ["
-                            + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT.getKey()
-                            + "]!"
-                    );
+                    logWarningHeaderThresholdExceeded(maxWarningHeaderCount, HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT);
                     return this;
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
index 9998cb55064e3..ea2d6e2d6e8e4 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
+import org.elasticsearch.plugins.internal.XContentParserDecorator;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent;
@@ -152,14 +152,14 @@ public static Tuple<XContentType, Map<String, Object>> convertToMap(
         BytesReference bytes,
         boolean ordered,
         XContentType xContentType,
-        DocumentSizeObserver documentSizeObserver
+        XContentParserDecorator parserDecorator
     ) {
         return parseToType(
             ordered ? XContentParser::mapOrdered : XContentParser::map,
             bytes,
             xContentType,
             XContentParserConfiguration.EMPTY,
-            documentSizeObserver
+            parserDecorator
         );
     }
 
@@ -207,7 +207,7 @@ public static <T> Tuple<XContentType, T> parseToType(
         @Nullable XContentType xContentType,
         @Nullable XContentParserConfiguration config
     ) throws ElasticsearchParseException {
-        return parseToType(extractor, bytes, xContentType, config, DocumentSizeObserver.EMPTY_INSTANCE);
+        return parseToType(extractor, bytes, xContentType, config, XContentParserDecorator.NOOP);
     }
 
     public static <T> Tuple<XContentType, T> parseToType(
@@ -215,11 +215,11 @@ public static <T> Tuple<XContentType, T> parseToType(
         BytesReference bytes,
         @Nullable XContentType xContentType,
         @Nullable XContentParserConfiguration config,
-        DocumentSizeObserver documentSizeObserver
+        XContentParserDecorator parserDecorator
     ) throws ElasticsearchParseException {
         config = config != null ? config : XContentParserConfiguration.EMPTY;
         try (
-            XContentParser parser = documentSizeObserver.wrapParser(
+            XContentParser parser = parserDecorator.decorate(
                 xContentType != null ? createParser(config, bytes, xContentType) : createParser(config, bytes)
             )
         ) {
diff --git a/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java b/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
index 0552335ab092d..9a28c7b7f91bb 100644
--- a/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
+++ b/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.cluster.version.CompatibilityVersions;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -39,6 +38,7 @@
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.gateway.GatewayMetaState;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.injection.guice.AbstractModule;
 import org.elasticsearch.monitor.NodeHealthService;
 import org.elasticsearch.plugins.ClusterCoordinationPlugin;
 import org.elasticsearch.plugins.DiscoveryPlugin;
diff --git a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java
index 11f3bbdc13bbf..44aa31524fe97 100644
--- a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java
+++ b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java
@@ -496,6 +496,11 @@ public void onFailure(Exception e) {
                             } // else this Peer has been superseded by a different instance which should be left in place
                         }
                     }
+
+                    @Override
+                    public String toString() {
+                        return "Peer#establishConnection[" + transportAddress + "]";
+                    }
                 })
             );
         }
@@ -584,6 +589,7 @@ public String toString() {
                 + Optional.ofNullable(probeConnectionResult.get())
                     .map(result -> result.getDiscoveryNode().descriptionWithoutAttributes())
                     .orElse("unknown")
+                + "]"
                 + (peersRequestInFlight ? " [request in flight]" : "");
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java b/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
index fb6333c372d70..d3a2af7bfce10 100644
--- a/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
+++ b/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
@@ -14,9 +14,9 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 
 import java.io.IOException;
 import java.util.HashMap;
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
index 43e03d30bd120..0714efc952e26 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
@@ -24,13 +24,13 @@
 import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetadata;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetadata.NodeStoreFilesMetadata;
+import org.elasticsearch.injection.guice.Inject;
 
 import java.util.Collections;
 import java.util.List;
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayModule.java b/server/src/main/java/org/elasticsearch/gateway/GatewayModule.java
index 5562ae7188e18..cd4a62f04f31c 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayModule.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayModule.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.gateway;
 
-import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.injection.guice.AbstractModule;
 
 public class GatewayModule extends AbstractModule {
 
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
index 91280c4da40b6..de37da8b9cd24 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -34,6 +33,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
index 0262e37dd74a9..11a1f6b865fd8 100644
--- a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
+++ b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -34,6 +33,7 @@
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportRequest;
diff --git a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index 599517b481eeb..abc407e50a1a7 100644
--- a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -36,6 +35,7 @@
 import org.elasticsearch.index.shard.ShardStateMetadata;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
diff --git a/server/src/main/java/org/elasticsearch/health/GetHealthAction.java b/server/src/main/java/org/elasticsearch/health/GetHealthAction.java
index eeff060c174da..3aacf3e847605 100644
--- a/server/src/main/java/org/elasticsearch/health/GetHealthAction.java
+++ b/server/src/main/java/org/elasticsearch/health/GetHealthAction.java
@@ -20,11 +20,12 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.health.stats.HealthApiStats;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -199,7 +200,7 @@ public LocalAction(
             NodeClient client,
             HealthApiStats healthApiStats
         ) {
-            super(NAME, actionFilters, transportService.getTaskManager());
+            super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
             this.clusterService = clusterService;
             this.healthService = healthService;
             this.client = client;
diff --git a/server/src/main/java/org/elasticsearch/health/HealthPeriodicLogger.java b/server/src/main/java/org/elasticsearch/health/HealthPeriodicLogger.java
index 8208e4bd70c34..97c0679bed34f 100644
--- a/server/src/main/java/org/elasticsearch/health/HealthPeriodicLogger.java
+++ b/server/src/main/java/org/elasticsearch/health/HealthPeriodicLogger.java
@@ -299,7 +299,7 @@ protected void doClose() throws IOException {
 
     @Override
     public void triggered(SchedulerEngine.Event event) {
-        if (event.getJobName().equals(HEALTH_PERIODIC_LOGGER_JOB_NAME) && this.enabled) {
+        if (event.jobName().equals(HEALTH_PERIODIC_LOGGER_JOB_NAME) && this.enabled) {
             this.tryToLogHealth();
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/health/node/DiskHealthInfo.java b/server/src/main/java/org/elasticsearch/health/node/DiskHealthInfo.java
index f1e085482b72a..244ed03bf098c 100644
--- a/server/src/main/java/org/elasticsearch/health/node/DiskHealthInfo.java
+++ b/server/src/main/java/org/elasticsearch/health/node/DiskHealthInfo.java
@@ -20,6 +20,7 @@
  * The health status of the disk space of this node along with the cause.
  */
 public record DiskHealthInfo(HealthStatus healthStatus, @Nullable Cause cause) implements Writeable {
+
     public DiskHealthInfo(HealthStatus healthStatus) {
         this(healthStatus, null);
     }
diff --git a/server/src/main/java/org/elasticsearch/health/node/FetchHealthInfoCacheAction.java b/server/src/main/java/org/elasticsearch/health/node/FetchHealthInfoCacheAction.java
index f342daa413fd6..8b3ed36911f94 100644
--- a/server/src/main/java/org/elasticsearch/health/node/FetchHealthInfoCacheAction.java
+++ b/server/src/main/java/org/elasticsearch/health/node/FetchHealthInfoCacheAction.java
@@ -15,11 +15,11 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.health.node.action.HealthNodeRequest;
 import org.elasticsearch.health.node.action.TransportHealthNodeAction;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/health/node/UpdateHealthInfoCacheAction.java b/server/src/main/java/org/elasticsearch/health/node/UpdateHealthInfoCacheAction.java
index b0dc5958c7ed0..18cdfe276c487 100644
--- a/server/src/main/java/org/elasticsearch/health/node/UpdateHealthInfoCacheAction.java
+++ b/server/src/main/java/org/elasticsearch/health/node/UpdateHealthInfoCacheAction.java
@@ -16,12 +16,12 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.health.node.action.HealthNodeRequest;
 import org.elasticsearch.health.node.action.TransportHealthNodeAction;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java
index f329cb11430c8..ec6a8b2ffe706 100644
--- a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java
index d6f9f5c395cbf..96598ba38a3fe 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexMode.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java
@@ -12,9 +12,12 @@
 import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService;
 import org.elasticsearch.cluster.routing.IndexRouting;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.DocumentDimensions;
@@ -289,7 +292,7 @@ public boolean shouldValidateTimestamp() {
         @Override
         public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {
             if (sourceFieldMapper.isSynthetic() == false) {
-                throw new IllegalArgumentException("Indices with with index mode [logs] only support synthetic source");
+                throw new IllegalArgumentException("Indices with with index mode [" + IndexMode.LOGSDB + "] only support synthetic source");
             }
         }
 
@@ -297,6 +300,11 @@ public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {
         public boolean isSyntheticSourceEnabled() {
             return true;
         }
+
+        @Override
+        public String getDefaultCodec() {
+            return CodecService.BEST_COMPRESSION_CODEC;
+        }
     };
 
     private static void validateTimeSeriesSettings(Map<Setting<?>, Object> settings) {
@@ -466,6 +474,10 @@ public String getName() {
      */
     public abstract boolean isSyntheticSourceEnabled();
 
+    public String getDefaultCodec() {
+        return CodecService.DEFAULT_CODEC;
+    }
+
     /**
      * Parse a string into an {@link IndexMode}.
      */
@@ -484,6 +496,25 @@ public static IndexMode fromString(String value) {
         };
     }
 
+    public static IndexMode readFrom(StreamInput in) throws IOException {
+        int mode = in.readByte();
+        return switch (mode) {
+            case 0 -> STANDARD;
+            case 1 -> TIME_SERIES;
+            case 2 -> LOGSDB;
+            default -> throw new IllegalStateException("unexpected index mode [" + mode + "]");
+        };
+    }
+
+    public static void writeTo(IndexMode indexMode, StreamOutput out) throws IOException {
+        final int code = switch (indexMode) {
+            case STANDARD -> 0;
+            case TIME_SERIES -> 1;
+            case LOGSDB -> 2;
+        };
+        out.writeByte((byte) code);
+    }
+
     @Override
     public String toString() {
         return getName();
diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java
index 3d81cccc92dc7..cbe94526819d9 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexService.java
@@ -332,10 +332,18 @@ public NodeMappingStats getNodeMappingStats() {
         if (mapperService == null) {
             return null;
         }
-        long totalCount = mapperService().mappingLookup().getTotalMapperCount();
-        long totalEstimatedOverhead = totalCount * 1024L; // 1KiB estimated per mapping
-        NodeMappingStats indexNodeMappingStats = new NodeMappingStats(totalCount, totalEstimatedOverhead);
-        return indexNodeMappingStats;
+        long numFields = mapperService().mappingLookup().getTotalMapperCount();
+        long totalEstimatedOverhead = numFields * 1024L; // 1KiB estimated per mapping
+        // Assume all segments have the same mapping; otherwise, we need to acquire searchers to count the actual fields.
+        int numLeaves = 0;
+        for (IndexShard shard : shards.values()) {
+            try {
+                numLeaves += shard.commitStats().getNumLeaves();
+            } catch (AlreadyClosedException ignored) {
+
+            }
+        }
+        return new NodeMappingStats(numFields, totalEstimatedOverhead, numLeaves, numLeaves * numFields);
     }
 
     public Set<Integer> shardIds() {
@@ -791,6 +799,7 @@ public QueryRewriteContext newQueryRewriteContext(
             valuesSourceRegistry,
             allowExpensiveQueries,
             scriptService,
+            null,
             null
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
index 8bebd390d3440..608d88fdef664 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
@@ -112,6 +112,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
     public static final IndexVersion ADD_SECURITY_MIGRATION = def(8_510_00_0, Version.LUCENE_9_11_0);
     public static final IndexVersion UPGRADE_TO_LUCENE_9_11_1 = def(8_511_00_0, Version.LUCENE_9_11_1);
     public static final IndexVersion INDEX_SORTING_ON_NESTED = def(8_512_00_0, Version.LUCENE_9_11_1);
+    public static final IndexVersion LENIENT_UPDATEABLE_SYNONYMS = def(8_513_00_0, Version.LUCENE_9_11_1);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _
@@ -220,7 +221,7 @@ static Collection<IndexVersion> getAllVersions() {
         return VERSION_IDS.values();
     }
 
-    static final IntFunction<String> VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(IndexVersions.class);
+    static final IntFunction<String> VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(IndexVersions.class, LATEST_DEFINED.id());
 
     // no instance
     private IndexVersions() {}
diff --git a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
index 14c2c5440bd24..b4a7571e96802 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
@@ -90,7 +90,11 @@ public final class IndexingSlowLog implements IndexingOperationListener {
         Property.IndexSettingDeprecatedInV7AndRemovedInV8
     );
 
-    private final Logger indexLogger;
+    private static final Logger indexLogger = LogManager.getLogger(INDEX_INDEXING_SLOWLOG_PREFIX + ".index");
+    static {
+        Loggers.setLevel(indexLogger, Level.TRACE);
+    }
+
     private final Index index;
 
     private boolean reformat;
@@ -127,8 +131,6 @@ public final class IndexingSlowLog implements IndexingOperationListener {
 
     IndexingSlowLog(IndexSettings indexSettings, SlowLogFieldProvider slowLogFieldProvider) {
         this.slowLogFieldProvider = slowLogFieldProvider;
-        this.indexLogger = LogManager.getLogger(INDEX_INDEXING_SLOWLOG_PREFIX + ".index");
-        Loggers.setLevel(this.indexLogger, Level.TRACE);
         this.index = indexSettings.getIndex();
 
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING, this::setReformat);
diff --git a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java
index eb227e6e1136d..6aff86d32c5a3 100644
--- a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java
+++ b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java
@@ -41,12 +41,17 @@ public final class SearchSlowLog implements SearchOperationListener {
     private long fetchDebugThreshold;
     private long fetchTraceThreshold;
 
-    private final Logger queryLogger;
-    private final Logger fetchLogger;
+    static final String INDEX_SEARCH_SLOWLOG_PREFIX = "index.search.slowlog";
 
-    private final SlowLogFieldProvider slowLogFieldProvider;
+    private static final Logger queryLogger = LogManager.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".query");
+    private static final Logger fetchLogger = LogManager.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".fetch");
 
-    static final String INDEX_SEARCH_SLOWLOG_PREFIX = "index.search.slowlog";
+    static {
+        Loggers.setLevel(queryLogger, Level.TRACE);
+        Loggers.setLevel(fetchLogger, Level.TRACE);
+    }
+
+    private final SlowLogFieldProvider slowLogFieldProvider;
 
     public static final Setting<Boolean> INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING = Setting.boolSetting(
         INDEX_SEARCH_SLOWLOG_PREFIX + ".include.user",
@@ -130,12 +135,6 @@ public final class SearchSlowLog implements SearchOperationListener {
     public SearchSlowLog(IndexSettings indexSettings, SlowLogFieldProvider slowLogFieldProvider) {
         slowLogFieldProvider.init(indexSettings);
         this.slowLogFieldProvider = slowLogFieldProvider;
-
-        this.queryLogger = LogManager.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".query");
-        this.fetchLogger = LogManager.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".fetch");
-        Loggers.setLevel(this.fetchLogger, Level.TRACE);
-        Loggers.setLevel(this.queryLogger, Level.TRACE);
-
         indexSettings.getScopedSettings()
             .addSettingsUpdateConsumer(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING, this::setQueryWarnThreshold);
         this.queryWarnThreshold = indexSettings.getValue(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING).nanos();
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerProvider.java
index a52298d148a3d..c812b9ccf38ba 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerProvider.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.elasticsearch.common.inject.Provider;
+import org.elasticsearch.injection.guice.Provider;
 
 public interface AnalyzerProvider<T extends Analyzer> extends Provider<T> {
 
diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecProvider.java b/server/src/main/java/org/elasticsearch/index/codec/CodecProvider.java
new file mode 100644
index 0000000000000..5b1a3bce0080b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/codec/CodecProvider.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec;
+
+import org.apache.lucene.codecs.Codec;
+
+/**
+ * Abstracts codec lookup by name, to make CodecService extensible.
+ */
+public interface CodecProvider {
+    Codec codec(String name);
+
+    /**
+     * Returns all registered available codec names.
+     */
+    String[] availableCodecs();
+}
diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
index a8dfa3079ea27..aa65289616ff6 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
@@ -28,7 +28,7 @@
  * data-structures per field. Elasticsearch exposes the full
  * {@link Codec} capabilities through this {@link CodecService}.
  */
-public class CodecService {
+public class CodecService implements CodecProvider {
 
     public static final FeatureFlag ZSTD_STORED_FIELDS_FEATURE_FLAG = new FeatureFlag("zstd_stored_fields");
 
@@ -86,8 +86,9 @@ public Codec codec(String name) {
     }
 
     /**
-     * Returns all registered available codec names
+     * Returns all registered available codec names.
      */
+    @Override
     public String[] availableCodecs() {
         return codecs.keySet().toArray(new String[0]);
     }
@@ -96,6 +97,7 @@ public static class DeduplicateFieldInfosCodec extends FilterCodec {
 
         private final DeduplicatingFieldInfosFormat deduplicatingFieldInfosFormat;
 
+        @SuppressWarnings("this-escape")
         protected DeduplicateFieldInfosCodec(String name, Codec delegate) {
             super(name, delegate);
             this.deduplicatingFieldInfosFormat = new DeduplicatingFieldInfosFormat(super.fieldInfosFormat());
@@ -106,5 +108,9 @@ public final FieldInfosFormat fieldInfosFormat() {
             return deduplicatingFieldInfosFormat;
         }
 
+        public final Codec delegate() {
+            return delegate;
+        }
+
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java
index 1228c908f7c18..685e9774b04a7 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java
@@ -24,8 +24,6 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 
-import java.util.Objects;
-
 /**
  * Class that encapsulates the logic of figuring out the most appropriate file format for a given field, across postings, doc values and
  * vectors.
@@ -33,7 +31,6 @@
 public class PerFieldFormatSupplier {
 
     private final MapperService mapperService;
-    private final BigArrays bigArrays;
     private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
     private final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
     private final ES87BloomFilterPostingsFormat bloomFilterPostingsFormat;
@@ -43,7 +40,6 @@ public class PerFieldFormatSupplier {
 
     public PerFieldFormatSupplier(MapperService mapperService, BigArrays bigArrays) {
         this.mapperService = mapperService;
-        this.bigArrays = Objects.requireNonNull(bigArrays);
         this.bloomFilterPostingsFormat = new ES87BloomFilterPostingsFormat(bigArrays, this::internalGetPostingsFormatForField);
         this.tsdbDocValuesFormat = new ES87TSDBDocValuesFormat();
         this.es812PostingsFormat = new ES812PostingsFormat();
diff --git a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsReader.java b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsReader.java
index 1aada2a153c3c..3aaf2ee5a8c4b 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsReader.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsReader.java
@@ -874,10 +874,6 @@ public int advance(int target) throws IOException {
         private void skipPositions() throws IOException {
             // Skip positions now:
             int toSkip = posPendingCount - freq;
-            // if (DEBUG) {
-            // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);
-            // }
-
             final int leftInBlock = BLOCK_SIZE - posBufferUpto;
             if (toSkip < leftInBlock) {
                 int end = posBufferUpto + toSkip;
@@ -1010,7 +1006,7 @@ final class BlockImpactsDocsEnum extends ImpactsEnum {
 
         final boolean indexHasFreqs;
 
-        private int docFreq; // number of docs in this posting list
+        private final int docFreq; // number of docs in this posting list
         private int blockUpto; // number of documents in or before the current block
         private int doc; // doc we last read
         private long accum; // accumulator for doc deltas
@@ -1211,8 +1207,8 @@ final class BlockImpactsPostingsEnum extends ImpactsEnum {
         final boolean indexHasOffsets;
         final boolean indexHasPayloads;
 
-        private int docFreq; // number of docs in this posting list
-        private long totalTermFreq; // number of positions in this posting list
+        private final int docFreq; // number of docs in this posting list
+        private final long totalTermFreq; // number of positions in this posting list
         private int docUpto; // how many docs we've read
         private int doc; // doc we last read
         private long accum; // accumulator for doc deltas
@@ -1228,19 +1224,19 @@ final class BlockImpactsPostingsEnum extends ImpactsEnum {
         private long posPendingFP;
 
         // Where this term's postings start in the .doc file:
-        private long docTermStartFP;
+        private final long docTermStartFP;
 
         // Where this term's postings start in the .pos file:
-        private long posTermStartFP;
+        private final long posTermStartFP;
 
         // Where this term's payloads/offsets start in the .pay
         // file:
-        private long payTermStartFP;
+        private final long payTermStartFP;
 
         // File pointer where the last (vInt encoded) pos delta
         // block is. We need this to know whether to bulk
         // decode vs vInt decode the block:
-        private long lastPosBlockFP;
+        private final long lastPosBlockFP;
 
         private int nextSkipDoc = -1;
 
@@ -1507,8 +1503,8 @@ final class BlockImpactsEverythingEnum extends ImpactsEnum {
         final boolean indexHasOffsets;
         final boolean indexHasPayloads;
 
-        private int docFreq; // number of docs in this posting list
-        private long totalTermFreq; // number of positions in this posting list
+        private final int docFreq; // number of docs in this posting list
+        private final long totalTermFreq; // number of positions in this posting list
         private int docUpto; // how many docs we've read
         private int posDocUpTo; // for how many docs we've read positions, offsets, and payloads
         private int doc; // doc we last read
@@ -1528,19 +1524,19 @@ final class BlockImpactsEverythingEnum extends ImpactsEnum {
         private long payPendingFP;
 
         // Where this term's postings start in the .doc file:
-        private long docTermStartFP;
+        private final long docTermStartFP;
 
         // Where this term's postings start in the .pos file:
-        private long posTermStartFP;
+        private final long posTermStartFP;
 
         // Where this term's payloads/offsets start in the .pay
         // file:
-        private long payTermStartFP;
+        private final long payTermStartFP;
 
         // File pointer where the last (vInt encoded) pos delta
         // block is. We need this to know whether to bulk
         // decode vs vInt decode the block:
-        private long lastPosBlockFP;
+        private final long lastPosBlockFP;
 
         private int nextSkipDoc = -1;
 
@@ -1835,10 +1831,6 @@ public int advance(int target) throws IOException {
         private void skipPositions() throws IOException {
             // Skip positions now:
             int toSkip = posPendingCount - (int) freqBuffer[docBufferUpto - 1];
-            // if (DEBUG) {
-            // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);
-            // }
-
             final int leftInBlock = BLOCK_SIZE - posBufferUpto;
             if (toSkip < leftInBlock) {
                 int end = posBufferUpto + toSkip;
diff --git a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812SkipReader.java b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812SkipReader.java
index f9b36114361ca..8dd99392625fd 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812SkipReader.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812SkipReader.java
@@ -54,8 +54,8 @@
  * <p>Therefore, we'll trim df before passing it to the interface. see trim(int)
  */
 class ES812SkipReader extends MultiLevelSkipListReader {
-    private long[] docPointer;
-    private long[] posPointer;
+    private final long[] docPointer;
+    private final long[] posPointer;
     private long[] payPointer;
     private int[] posBufferUpto;
     private int[] payloadByteUpto;
diff --git a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812SkipWriter.java b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812SkipWriter.java
index dbfb7c86a1475..98c516fc890e8 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812SkipWriter.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812SkipWriter.java
@@ -51,8 +51,8 @@
  * uptos(position, payload). 4. start offset.
  */
 final class ES812SkipWriter extends MultiLevelSkipListWriter {
-    private int[] lastSkipDoc;
-    private long[] lastSkipDocPointer;
+    private final int[] lastSkipDoc;
+    private final long[] lastSkipDocPointer;
     private long[] lastSkipPosPointer;
     private long[] lastSkipPayPointer;
 
@@ -66,7 +66,7 @@ final class ES812SkipWriter extends MultiLevelSkipListWriter {
     private long curPayPointer;
     private int curPosBufferUpto;
     private int curPayloadByteUpto;
-    private CompetitiveImpactAccumulator[] curCompetitiveFreqNorms;
+    private final CompetitiveImpactAccumulator[] curCompetitiveFreqNorms;
     private boolean fieldHasPositions;
     private boolean fieldHasOffsets;
     private boolean fieldHasPayloads;
@@ -197,7 +197,7 @@ protected void writeSkipData(int level, DataOutput skipBuffer) throws IOExceptio
         }
 
         CompetitiveImpactAccumulator competitiveFreqNorms = curCompetitiveFreqNorms[level];
-        assert competitiveFreqNorms.getCompetitiveFreqNormPairs().size() > 0;
+        assert competitiveFreqNorms.getCompetitiveFreqNormPairs().isEmpty() == false;
         if (level + 1 < numberOfSkipLevels) {
             curCompetitiveFreqNorms[level + 1].addAll(competitiveFreqNorms);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java
index fb90327770674..b6e1bb503045c 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java
@@ -342,14 +342,10 @@ public BytesRef lookupOrd(int ord) throws IOException {
         @Override
         public int lookupTerm(BytesRef key) throws IOException {
             TermsEnum.SeekStatus status = termsEnum.seekCeil(key);
-            switch (status) {
-                case FOUND:
-                    return Math.toIntExact(termsEnum.ord());
-                case NOT_FOUND:
-                case END:
-                default:
-                    return Math.toIntExact(-1L - termsEnum.ord());
-            }
+            return switch (status) {
+                case FOUND -> Math.toIntExact(termsEnum.ord());
+                default -> Math.toIntExact(-1L - termsEnum.ord());
+            };
         }
 
         @Override
@@ -384,14 +380,10 @@ public BytesRef lookupOrd(long ord) throws IOException {
         @Override
         public long lookupTerm(BytesRef key) throws IOException {
             TermsEnum.SeekStatus status = termsEnum.seekCeil(key);
-            switch (status) {
-                case FOUND:
-                    return termsEnum.ord();
-                case NOT_FOUND:
-                case END:
-                default:
-                    return -1L - termsEnum.ord();
-            }
+            return switch (status) {
+                case FOUND -> termsEnum.ord();
+                default -> -1L - termsEnum.ord();
+            };
         }
 
         @Override
@@ -400,7 +392,7 @@ public TermsEnum termsEnum() throws IOException {
         }
     }
 
-    private class TermsDict extends BaseTermsEnum {
+    private static class TermsDict extends BaseTermsEnum {
         static final int LZ4_DECOMPRESSOR_PADDING = 7;
 
         final TermsDictEntry entry;
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java
index 701bf5dc98552..420cdbe016552 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java
@@ -15,7 +15,6 @@
 import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
 import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
 import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
-import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat;
 import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FloatVectorValues;
@@ -45,8 +44,7 @@ public ES813Int8FlatVectorFormat() {
      */
     public ES813Int8FlatVectorFormat(Float confidenceInterval, int bits, boolean compress) {
         super(NAME);
-        // TODO can we just switch this to ES814ScalarQuantizedVectorsFormat ?
-        this.format = new Lucene99ScalarQuantizedVectorsFormat(confidenceInterval, bits, compress);
+        this.format = new ES814ScalarQuantizedVectorsFormat(confidenceInterval, bits, compress);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java
index c4b52d26fc6e7..431015b015779 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java
@@ -98,6 +98,8 @@ public String toString() {
             + bits
             + ", compressed="
             + compress
+            + ", flatVectorScorer="
+            + flatVectorScorer
             + ", rawVectorFormat="
             + rawVectorFormat
             + ")";
@@ -234,6 +236,11 @@ static final class ESFlatVectorsScorer implements FlatVectorsScorer {
             factory = VectorScorerFactory.instance().orElse(null);
         }
 
+        @Override
+        public String toString() {
+            return "ESFlatVectorsScorer(" + "delegate=" + delegate + ", factory=" + factory + ')';
+        }
+
         @Override
         public RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction sim, RandomAccessVectorValues values)
             throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java
index b827bb6436f07..840b37611374a 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java
@@ -78,6 +78,10 @@ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOCo
         return super.fieldsWriter(directory, si, context);
     }
 
+    public Mode getMode() {
+        return mode;
+    }
+
     private static class ZstdCompressionMode extends CompressionMode {
         private final int level;
 
diff --git a/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java b/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java
index 16b58e001dcbe..1a03cf6c7bf98 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/CommitStats.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.lucene.index.SegmentInfos;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -27,6 +28,7 @@ public final class CommitStats implements Writeable, ToXContentFragment {
     private final long generation;
     private final String id; // lucene commit id in base 64;
     private final int numDocs;
+    private final int numLeaves;
 
     public CommitStats(SegmentInfos segmentInfos) {
         // clone the map to protect against concurrent changes
@@ -35,6 +37,7 @@ public CommitStats(SegmentInfos segmentInfos) {
         generation = segmentInfos.getLastGeneration();
         id = Base64.getEncoder().encodeToString(segmentInfos.getId());
         numDocs = Lucene.getNumDocs(segmentInfos);
+        numLeaves = segmentInfos.size();
     }
 
     CommitStats(StreamInput in) throws IOException {
@@ -42,6 +45,7 @@ public CommitStats(SegmentInfos segmentInfos) {
         generation = in.readLong();
         id = in.readOptionalString();
         numDocs = in.readInt();
+        numLeaves = in.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS) ? in.readVInt() : 0;
     }
 
     @Override
@@ -49,12 +53,16 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         CommitStats that = (CommitStats) o;
-        return userData.equals(that.userData) && generation == that.generation && Objects.equals(id, that.id) && numDocs == that.numDocs;
+        return userData.equals(that.userData)
+            && generation == that.generation
+            && Objects.equals(id, that.id)
+            && numDocs == that.numDocs
+            && numLeaves == that.numLeaves;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(userData, generation, id, numDocs);
+        return Objects.hash(userData, generation, id, numDocs, numLeaves);
     }
 
     public static CommitStats readOptionalCommitStatsFrom(StreamInput in) throws IOException {
@@ -81,12 +89,19 @@ public int getNumDocs() {
         return numDocs;
     }
 
+    public int getNumLeaves() {
+        return numLeaves;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeMap(userData, StreamOutput::writeString);
         out.writeLong(generation);
         out.writeOptionalString(id);
         out.writeInt(numDocs);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS)) {
+            out.writeVInt(numLeaves);
+        }
     }
 
     static final class Fields {
diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index 6f4511483126f..b07132eea75e8 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -61,7 +61,6 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.DocumentParser;
-import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.LuceneDocument;
 import org.elasticsearch.index.mapper.Mapper;
@@ -69,6 +68,7 @@
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.seqno.SeqNoStats;
@@ -242,19 +242,32 @@ protected final DocsStats docsStats(IndexReader indexReader) {
     /**
      * Returns the {@link DenseVectorStats} for this engine
      */
-    public DenseVectorStats denseVectorStats() {
+    public DenseVectorStats denseVectorStats(MappingLookup mappingLookup) {
+        if (mappingLookup == null) {
+            return new DenseVectorStats(0);
+        }
+
+        List<String> fields = new ArrayList<>();
+        for (Mapper mapper : mappingLookup.fieldMappers()) {
+            if (mapper instanceof DenseVectorFieldMapper) {
+                fields.add(mapper.fullPath());
+            }
+        }
+        if (fields.isEmpty()) {
+            return new DenseVectorStats(0);
+        }
         try (Searcher searcher = acquireSearcher(DOC_STATS_SOURCE, SearcherScope.INTERNAL)) {
-            return denseVectorStats(searcher.getIndexReader());
+            return denseVectorStats(searcher.getIndexReader(), fields);
         }
     }
 
-    protected final DenseVectorStats denseVectorStats(IndexReader indexReader) {
+    protected final DenseVectorStats denseVectorStats(IndexReader indexReader, List<String> fields) {
         long valueCount = 0;
         // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause
         // the next scheduled refresh to go through and refresh the stats as well
         for (LeafReaderContext readerContext : indexReader.leaves()) {
             try {
-                valueCount += getDenseVectorValueCount(readerContext.reader());
+                valueCount += getDenseVectorValueCount(readerContext.reader(), fields);
             } catch (IOException e) {
                 logger.trace(() -> "failed to get dense vector stats for [" + readerContext + "]", e);
             }
@@ -262,9 +275,10 @@ protected final DenseVectorStats denseVectorStats(IndexReader indexReader) {
         return new DenseVectorStats(valueCount);
     }
 
-    private long getDenseVectorValueCount(final LeafReader atomicReader) throws IOException {
+    private long getDenseVectorValueCount(final LeafReader atomicReader, List<String> fields) throws IOException {
         long count = 0;
-        for (FieldInfo info : atomicReader.getFieldInfos()) {
+        for (var field : fields) {
+            var info = atomicReader.getFieldInfos().fieldInfo(field);
             if (info.getVectorDimension() > 0) {
                 switch (info.getVectorEncoding()) {
                     case FLOAT32 -> {
@@ -285,23 +299,31 @@ private long getDenseVectorValueCount(final LeafReader atomicReader) throws IOEx
      * Returns the {@link SparseVectorStats} for this engine
      */
     public SparseVectorStats sparseVectorStats(MappingLookup mappingLookup) {
+        if (mappingLookup == null) {
+            return new SparseVectorStats(0);
+        }
+        List<BytesRef> fields = new ArrayList<>();
+        for (Mapper mapper : mappingLookup.fieldMappers()) {
+            if (mapper instanceof SparseVectorFieldMapper) {
+                fields.add(new BytesRef(mapper.fullPath()));
+            }
+        }
+        if (fields.isEmpty()) {
+            return new SparseVectorStats(0);
+        }
+        Collections.sort(fields);
         try (Searcher searcher = acquireSearcher(DOC_STATS_SOURCE, SearcherScope.INTERNAL)) {
-            return sparseVectorStats(searcher.getIndexReader(), mappingLookup);
+            return sparseVectorStats(searcher.getIndexReader(), fields);
         }
     }
 
-    protected final SparseVectorStats sparseVectorStats(IndexReader indexReader, MappingLookup mappingLookup) {
+    protected final SparseVectorStats sparseVectorStats(IndexReader indexReader, List<BytesRef> fields) {
         long valueCount = 0;
-
-        if (mappingLookup == null) {
-            return new SparseVectorStats(valueCount);
-        }
-
         // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause
         // the next scheduled refresh to go through and refresh the stats as well
         for (LeafReaderContext readerContext : indexReader.leaves()) {
             try {
-                valueCount += getSparseVectorValueCount(readerContext.reader(), mappingLookup);
+                valueCount += getSparseVectorValueCount(readerContext.reader(), fields);
             } catch (IOException e) {
                 logger.trace(() -> "failed to get sparse vector stats for [" + readerContext + "]", e);
             }
@@ -309,28 +331,16 @@ protected final SparseVectorStats sparseVectorStats(IndexReader indexReader, Map
         return new SparseVectorStats(valueCount);
     }
 
-    private long getSparseVectorValueCount(final LeafReader atomicReader, MappingLookup mappingLookup) throws IOException {
+    private long getSparseVectorValueCount(final LeafReader atomicReader, List<BytesRef> fields) throws IOException {
         long count = 0;
-
-        Map<String, FieldMapper> mappers = new HashMap<>();
-        for (Mapper mapper : mappingLookup.fieldMappers()) {
-            if (mapper instanceof FieldMapper fieldMapper) {
-                if (fieldMapper.fieldType() instanceof SparseVectorFieldMapper.SparseVectorFieldType) {
-                    mappers.put(fieldMapper.fullPath(), fieldMapper);
-                }
-            }
-        }
-
-        for (FieldInfo info : atomicReader.getFieldInfos()) {
-            String name = info.name;
-            if (mappers.containsKey(name)) {
-                Terms terms = atomicReader.terms(FieldNamesFieldMapper.NAME);
-                if (terms != null) {
-                    TermsEnum termsEnum = terms.iterator();
-                    if (termsEnum.seekExact(new BytesRef(name))) {
-                        count += termsEnum.docFreq();
-                    }
-                }
+        Terms terms = atomicReader.terms(FieldNamesFieldMapper.NAME);
+        if (terms == null) {
+            return count;
+        }
+        TermsEnum termsEnum = terms.iterator();
+        for (var fieldName : fields) {
+            if (termsEnum.seekExact(fieldName)) {
+                count += termsEnum.docFreq();
             }
         }
         return count;
diff --git a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
index 51840d2fbfcdd..317adcc67cf59 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
@@ -22,7 +22,9 @@
 import org.elasticsearch.common.unit.MemorySizeValue;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.codec.CodecProvider;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.seqno.RetentionLeases;
@@ -60,7 +62,7 @@ public final class EngineConfig {
     private final MergePolicy mergePolicy;
     private final Analyzer analyzer;
     private final Similarity similarity;
-    private final CodecService codecService;
+    private final CodecProvider codecProvider;
     private final Engine.EventListener eventListener;
     private final QueryCache queryCache;
     private final QueryCachingPolicy queryCachingPolicy;
@@ -95,11 +97,16 @@ public Supplier<RetentionLeases> retentionLeasesSupplier() {
      * This setting is also settable on the node and the index level, it's commonly used in hot/cold node archs where index is likely
      * allocated on both `kind` of nodes.
      */
-    public static final Setting<String> INDEX_CODEC_SETTING = new Setting<>("index.codec", "default", s -> {
+    public static final Setting<String> INDEX_CODEC_SETTING = new Setting<>("index.codec", settings -> {
+        IndexMode indexMode = IndexSettings.MODE.get(settings);
+        return indexMode.getDefaultCodec();
+    }, s -> {
         switch (s) {
-            case "default":
-            case "best_compression":
-            case "lucene_default":
+            case CodecService.DEFAULT_CODEC:
+            case CodecService.LEGACY_DEFAULT_CODEC:
+            case CodecService.BEST_COMPRESSION_CODEC:
+            case CodecService.LEGACY_BEST_COMPRESSION_CODEC:
+            case CodecService.LUCENE_DEFAULT_CODEC:
                 return s;
             default:
                 if (Codec.availableCodecs().contains(s) == false) { // we don't error message the not officially supported ones
@@ -148,7 +155,7 @@ public EngineConfig(
         MergePolicy mergePolicy,
         Analyzer analyzer,
         Similarity similarity,
-        CodecService codecService,
+        CodecProvider codecProvider,
         Engine.EventListener eventListener,
         QueryCache queryCache,
         QueryCachingPolicy queryCachingPolicy,
@@ -176,9 +183,9 @@ public EngineConfig(
         this.mergePolicy = mergePolicy;
         this.analyzer = analyzer;
         this.similarity = similarity;
-        this.codecService = codecService;
+        this.codecProvider = codecProvider;
         this.eventListener = eventListener;
-        codecName = indexSettings.getValue(INDEX_CODEC_SETTING);
+        this.codecName = indexSettings.getValue(INDEX_CODEC_SETTING);
         this.mapperService = mapperService;
         // We need to make the indexing buffer for this shard at least as large
         // as the amount of memory that is available for all engines on the
@@ -252,14 +259,22 @@ public boolean isEnableGcDeletes() {
      * </p>
      */
     public Codec getCodec() {
-        return codecService.codec(codecName);
+        return codecProvider.codec(codecName);
     }
 
     /**
-     * @return the {@link CodecService}
+     * @return the {@link CodecProvider}
      */
-    public CodecService getCodecService() {
-        return codecService;
+    public CodecProvider getCodecProvider() {
+        return codecProvider;
+    }
+
+    /**
+     * @return the {@link CodecProvider}
+     */
+    @Deprecated // to avoid breaking serverless, just temporary
+    public CodecProvider getCodecService() {
+        return codecProvider;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 8b1602e2ea2ca..9e0fbd0bb691d 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -263,6 +263,7 @@ public InternalEngine(EngineConfig engineConfig) {
                 assert translog.getGeneration() != null;
                 this.translog = translog;
                 this.totalDiskSpace = new ByteSizeValue(Environment.getFileStore(translog.location()).getTotalSpace(), ByteSizeUnit.BYTES);
+                this.lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
                 this.softDeletesPolicy = newSoftDeletesPolicy();
                 this.combinedDeletionPolicy = new CombinedDeletionPolicy(
                     logger,
@@ -344,7 +345,7 @@ private LocalCheckpointTracker createLocalCheckpointTracker(
         final long maxSeqNo;
         final long localCheckpoint;
         final SequenceNumbers.CommitInfo seqNoStats = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(
-            store.readLastCommittedSegmentsInfo().userData.entrySet()
+            lastCommittedSegmentInfos.userData.entrySet()
         );
         maxSeqNo = seqNoStats.maxSeqNo();
         localCheckpoint = seqNoStats.localCheckpoint();
@@ -363,7 +364,7 @@ protected LongConsumer translogPersistedSeqNoConsumer() {
     }
 
     private SoftDeletesPolicy newSoftDeletesPolicy() throws IOException {
-        final Map<String, String> commitUserData = store.readLastCommittedSegmentsInfo().userData;
+        final Map<String, String> commitUserData = lastCommittedSegmentInfos.userData;
         final long lastMinRetainedSeqNo;
         if (commitUserData.containsKey(Engine.MIN_RETAINED_SEQNO)) {
             lastMinRetainedSeqNo = Long.parseLong(commitUserData.get(Engine.MIN_RETAINED_SEQNO));
@@ -775,7 +776,6 @@ private ExternalReaderManager createReaderManager(RefreshWarmerListener external
         try {
             try {
                 directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId);
-                lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
                 internalReaderManager = createInternalReaderManager(directoryReader);
                 ExternalReaderManager externalReaderManager = new ExternalReaderManager(internalReaderManager, externalRefreshListener);
                 success = true;
@@ -2676,8 +2676,8 @@ private IndexWriter createWriter() throws IOException {
         }
     }
 
-    // pkg-private for testing
-    IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException {
+    // protected for testing
+    protected IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException {
         if (Assertions.ENABLED) {
             return new AssertingIndexWriter(directory, iwc);
         } else {
@@ -2685,17 +2685,19 @@ IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOEx
         }
     }
 
+    // with tests.verbose, lucene sets this up: plumb to align with filesystem stream
+    private static final boolean TESTS_VERBOSE = Boolean.parseBoolean(System.getProperty("tests.verbose"));
+
+    private static final boolean SHUFFLE_FORCE_MERGE = Booleans.parseBoolean(
+        System.getProperty("es.shuffle_forced_merge", Boolean.TRUE.toString())
+    );
+
     private IndexWriterConfig getIndexWriterConfig() {
         final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer());
         iwc.setCommitOnClose(false); // we by default don't commit on close
         iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
         iwc.setIndexDeletionPolicy(combinedDeletionPolicy);
-        // with tests.verbose, lucene sets this up: plumb to align with filesystem stream
-        boolean verbose = false;
-        try {
-            verbose = Boolean.parseBoolean(System.getProperty("tests.verbose"));
-        } catch (Exception ignore) {}
-        iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger));
+        iwc.setInfoStream(TESTS_VERBOSE ? InfoStream.getDefault() : new LoggerInfoStream(logger));
         iwc.setMergeScheduler(mergeScheduler);
         // Give us the opportunity to upgrade old segments while performing
         // background merges
@@ -2712,8 +2714,7 @@ private IndexWriterConfig getIndexWriterConfig() {
                 new PrunePostingsMergePolicy(mergePolicy, IdFieldMapper.NAME)
             )
         );
-        boolean shuffleForcedMerge = Booleans.parseBoolean(System.getProperty("es.shuffle_forced_merge", Boolean.TRUE.toString()));
-        if (shuffleForcedMerge) {
+        if (SHUFFLE_FORCE_MERGE) {
             // We wrap the merge policy for all indices even though it is mostly useful for time-based indices
             // but there should be no overhead for other type of indices so it's simpler than adding a setting
             // to enable it.
@@ -2850,7 +2851,9 @@ public synchronized void afterMerge(OnGoingMerge merge) {
                     @Override
                     public void onFailure(Exception e) {
                         if (isClosed.get() == false) {
-                            logger.warn("failed to flush after merge has finished");
+                            logger.warn("failed to flush after merge has finished", e);
+                        } else {
+                            logger.info("failed to flush after merge has finished during shard close");
                         }
                     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java b/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java
index e514a6d2adac0..1f8caa3cc236e 100644
--- a/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java
+++ b/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java
@@ -34,8 +34,7 @@ public FlushStats(StreamInput in) throws IOException {
         total = in.readVLong();
         totalTimeInMillis = in.readVLong();
         periodic = in.readVLong();
-        totalTimeExcludingWaitingOnLockInMillis = in.getTransportVersion()
-            .onOrAfter(TransportVersions.TRACK_FLUSH_TIME_EXCLUDING_WAITING_ON_LOCKS) ? in.readVLong() : 0L;
+        totalTimeExcludingWaitingOnLockInMillis = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readVLong() : 0L;
     }
 
     public FlushStats(long total, long periodic, long totalTimeInMillis, long totalTimeExcludingWaitingOnLockInMillis) {
@@ -131,7 +130,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(total);
         out.writeVLong(totalTimeInMillis);
         out.writeVLong(periodic);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.TRACK_FLUSH_TIME_EXCLUDING_WAITING_ON_LOCKS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeVLong(totalTimeExcludingWaitingOnLockInMillis);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
index f0cc51f3effa5..1f0088ec96478 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -60,6 +61,8 @@ public class BooleanFieldMapper extends FieldMapper {
 
     public static final String CONTENT_TYPE = "boolean";
 
+    static final NodeFeature BOOLEAN_DIMENSION = new NodeFeature("mapper.boolean_dimension");
+
     public static class Values {
         public static final BytesRef TRUE = new BytesRef("T");
         public static final BytesRef FALSE = new BytesRef("F");
@@ -69,7 +72,7 @@ private static BooleanFieldMapper toType(FieldMapper in) {
         return (BooleanFieldMapper) in;
     }
 
-    public static final class Builder extends FieldMapper.Builder {
+    public static final class Builder extends FieldMapper.DimensionBuilder {
 
         private final Parameter<Boolean> docValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
         private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, true);
@@ -94,6 +97,8 @@ public static final class Builder extends FieldMapper.Builder {
 
         private final IndexVersion indexCreatedVersion;
 
+        private final Parameter<Boolean> dimension;
+
         public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion) {
             super(name);
             this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
@@ -106,15 +111,36 @@ public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalform
             );
             this.script.precludesParameters(ignoreMalformed, nullValue);
             addScriptValidation(script, indexed, docValues);
+            this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension()).addValidator(v -> {
+                if (v && (indexed.getValue() == false || docValues.getValue() == false)) {
+                    throw new IllegalArgumentException(
+                        "Field ["
+                            + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM
+                            + "] requires that ["
+                            + indexed.name
+                            + "] and ["
+                            + docValues.name
+                            + "] are true"
+                    );
+                }
+            });
+        }
+
+        public Builder dimension(boolean dimension) {
+            this.dimension.setValue(dimension);
+            return this;
         }
 
         @Override
         protected Parameter<?>[] getParameters() {
-            return new Parameter<?>[] { meta, docValues, indexed, nullValue, stored, script, onScriptError, ignoreMalformed };
+            return new Parameter<?>[] { meta, docValues, indexed, nullValue, stored, script, onScriptError, ignoreMalformed, dimension };
         }
 
         @Override
         public BooleanFieldMapper build(MapperBuilderContext context) {
+            if (inheritDimensionParameterFromParentObject(context)) {
+                dimension(true);
+            }
             MappedFieldType ft = new BooleanFieldType(
                 context.buildFullName(leafName()),
                 indexed.getValue() && indexCreatedVersion.isLegacyIndexVersion() == false,
@@ -122,7 +148,8 @@ public BooleanFieldMapper build(MapperBuilderContext context) {
                 docValues.getValue(),
                 nullValue.getValue(),
                 scriptValues(),
-                meta.getValue()
+                meta.getValue(),
+                dimension.getValue()
             );
             return new BooleanFieldMapper(
                 leafName(),
@@ -158,6 +185,7 @@ public static final class BooleanFieldType extends TermBasedFieldType {
 
         private final Boolean nullValue;
         private final FieldValues<Boolean> scriptValues;
+        private final boolean isDimension;
 
         public BooleanFieldType(
             String name,
@@ -166,11 +194,13 @@ public BooleanFieldType(
             boolean hasDocValues,
             Boolean nullValue,
             FieldValues<Boolean> scriptValues,
-            Map<String, String> meta
+            Map<String, String> meta,
+            boolean isDimension
         ) {
             super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
             this.nullValue = nullValue;
             this.scriptValues = scriptValues;
+            this.isDimension = isDimension;
         }
 
         public BooleanFieldType(String name) {
@@ -182,7 +212,7 @@ public BooleanFieldType(String name, boolean isIndexed) {
         }
 
         public BooleanFieldType(String name, boolean isIndexed, boolean hasDocValues) {
-            this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap());
+            this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap(), false);
         }
 
         @Override
@@ -195,6 +225,11 @@ public boolean isSearchable() {
             return isIndexed() || hasDocValues();
         }
 
+        @Override
+        public boolean isDimension() {
+            return isDimension;
+        }
+
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
             if (format != null) {
@@ -455,6 +490,10 @@ private void indexValue(DocumentParserContext context, Boolean value) {
         if (value == null) {
             return;
         }
+
+        if (fieldType().isDimension()) {
+            context.getDimensions().addBoolean(fieldType().name(), value).validate(context.indexSettings());
+        }
         if (indexed) {
             context.doc().add(new StringField(fieldType().name(), value ? Values.TRUE : Values.FALSE, Field.Store.NO));
         }
@@ -480,7 +519,17 @@ protected void indexScriptValues(
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).init(this);
+        return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(fieldType().isDimension())
+            .init(this);
+    }
+
+    @Override
+    public void doValidate(MappingLookup lookup) {
+        if (fieldType().isDimension() && null != lookup.nestedLookup().getNestedParent(fullPath())) {
+            throw new IllegalArgumentException(
+                TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + " can't be configured in nested field [" + fullPath() + "]"
+            );
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java
index efc3c7b507300..7bb1f99e81705 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java
@@ -15,6 +15,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -28,29 +29,44 @@
  * stored in a different field in case of ignore_malformed being enabled.
  */
 public class CompositeSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader {
-    private final String fieldName;
+    private final String leafFieldName;
     private final String fullFieldName;
-    private final SyntheticFieldLoaderLayer[] parts;
-    private boolean hasValue;
+    private final Collection<Layer> parts;
+    private boolean storedFieldLoadersHaveValues;
+    private boolean docValuesLoadersHaveValues;
 
-    public CompositeSyntheticFieldLoader(String fieldName, String fullFieldName, SyntheticFieldLoaderLayer... parts) {
-        this.fieldName = fieldName;
+    public CompositeSyntheticFieldLoader(String leafFieldName, String fullFieldName, Layer... parts) {
+        this(leafFieldName, fullFieldName, Arrays.asList(parts));
+    }
+
+    public CompositeSyntheticFieldLoader(String leafFieldName, String fullFieldName, Collection<Layer> parts) {
+        this.leafFieldName = leafFieldName;
         this.fullFieldName = fullFieldName;
         this.parts = parts;
-        this.hasValue = false;
+        this.storedFieldLoadersHaveValues = false;
+        this.docValuesLoadersHaveValues = false;
     }
 
     @Override
     public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
-        return Arrays.stream(parts).flatMap(SyntheticFieldLoaderLayer::storedFieldLoaders).map(e -> Map.entry(e.getKey(), values -> {
-            hasValue = true;
-            e.getValue().load(values);
+        return parts.stream().flatMap(Layer::storedFieldLoaders).map(e -> Map.entry(e.getKey(), new StoredFieldLoader() {
+            @Override
+            public void advanceToDoc(int docId) {
+                storedFieldLoadersHaveValues = false;
+                e.getValue().advanceToDoc(docId);
+            }
+
+            @Override
+            public void load(List<Object> newValues) {
+                storedFieldLoadersHaveValues = true;
+                e.getValue().load(newValues);
+            }
         }));
     }
 
     @Override
     public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
-        var loaders = new ArrayList<DocValuesLoader>(parts.length);
+        var loaders = new ArrayList<DocValuesLoader>(parts.size());
         for (var part : parts) {
             var partLoader = part.docValuesLoader(leafReader, docIdsInLeaf);
             if (partLoader != null) {
@@ -63,38 +79,40 @@ public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf
         }
 
         return docId -> {
+            this.docValuesLoadersHaveValues = false;
+
             boolean hasDocs = false;
             for (var loader : loaders) {
                 hasDocs |= loader.advanceToDoc(docId);
             }
 
-            this.hasValue |= hasDocs;
+            this.docValuesLoadersHaveValues = hasDocs;
             return hasDocs;
         };
     }
 
     @Override
     public boolean hasValue() {
-        return hasValue;
+        return storedFieldLoadersHaveValues || docValuesLoadersHaveValues;
     }
 
     @Override
     public void write(XContentBuilder b) throws IOException {
-        var totalCount = Arrays.stream(parts).mapToLong(SyntheticFieldLoaderLayer::valueCount).sum();
+        var totalCount = parts.stream().mapToLong(Layer::valueCount).sum();
 
         if (totalCount == 0) {
             return;
         }
 
         if (totalCount == 1) {
-            b.field(fieldName);
+            b.field(leafFieldName);
             for (var part : parts) {
                 part.write(b);
             }
             return;
         }
 
-        b.startArray(fieldName);
+        b.startArray(leafFieldName);
         for (var part : parts) {
             part.write(b);
         }
@@ -113,9 +131,9 @@ public String fieldName() {
      * Note that the contract of {@link SourceLoader.SyntheticFieldLoader#write(XContentBuilder)}
      * is slightly different here since it only needs to write field values without encompassing object or array.
      */
-    public interface SyntheticFieldLoaderLayer extends SourceLoader.SyntheticFieldLoader {
+    public interface Layer extends SourceLoader.SyntheticFieldLoader {
         /**
-         * Number of values that this loader will write.
+         * Number of values that this loader will write for a given document.
          * @return
          */
         long valueCount();
@@ -125,12 +143,30 @@ public interface SyntheticFieldLoaderLayer extends SourceLoader.SyntheticFieldLo
      * Layer that loads malformed values stored in a dedicated field with a conventional name.
      * @see IgnoreMalformedStoredValues
      */
-    public static class MalformedValuesLayer implements SyntheticFieldLoaderLayer {
+    public static class MalformedValuesLayer extends StoredFieldLayer {
+        public MalformedValuesLayer(String fieldName) {
+            super(IgnoreMalformedStoredValues.name(fieldName));
+        }
+
+        @Override
+        protected void writeValue(Object value, XContentBuilder b) throws IOException {
+            if (value instanceof BytesRef r) {
+                XContentDataHelper.decodeAndWrite(b, r);
+            } else {
+                b.value(value);
+            }
+        }
+    }
+
+    /**
+     * Layer that loads field values from a provided stored field.
+     */
+    public abstract static class StoredFieldLayer implements Layer {
         private final String fieldName;
         private List<Object> values;
 
-        public MalformedValuesLayer(String fieldName) {
-            this.fieldName = IgnoreMalformedStoredValues.name(fieldName);
+        public StoredFieldLayer(String fieldName) {
+            this.fieldName = fieldName;
             this.values = emptyList();
         }
 
@@ -141,7 +177,17 @@ public long valueCount() {
 
         @Override
         public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
-            return Stream.of(Map.entry(fieldName, values -> this.values = values));
+            return Stream.of(Map.entry(fieldName, new SourceLoader.SyntheticFieldLoader.StoredFieldLoader() {
+                @Override
+                public void advanceToDoc(int docId) {
+                    values = emptyList();
+                }
+
+                @Override
+                public void load(List<Object> newValues) {
+                    values = newValues;
+                }
+            }));
         }
 
         @Override
@@ -157,15 +203,15 @@ public boolean hasValue() {
         @Override
         public void write(XContentBuilder b) throws IOException {
             for (Object v : values) {
-                if (v instanceof BytesRef r) {
-                    XContentDataHelper.decodeAndWrite(b, r);
-                } else {
-                    b.value(v);
-                }
+                writeValue(v, b);
             }
-            values = emptyList();
         }
 
+        /**
+         * Write a value read from stored field using appropriate format.
+         */
+        protected abstract void writeValue(Object value, XContentBuilder b) throws IOException;
+
         @Override
         public String fieldName() {
             return fieldName;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java
index 3cf90f2385525..aa69e4db50e76 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java
@@ -44,6 +44,8 @@ default DocumentDimensions addString(String fieldName, String value) {
 
     DocumentDimensions addUnsignedLong(String fieldName, long value);
 
+    DocumentDimensions addBoolean(String fieldName, boolean value);
+
     DocumentDimensions validate(IndexSettings settings);
 
     /**
@@ -83,6 +85,12 @@ public DocumentDimensions addUnsignedLong(String fieldName, long value) {
             return this;
         }
 
+        @Override
+        public DocumentDimensions addBoolean(String fieldName, boolean value) {
+            add(fieldName);
+            return this;
+        }
+
         @Override
         public DocumentDimensions validate(final IndexSettings settings) {
             // DO NOTHING
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 8bf7f3f4e72a3..35f0130c58706 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
+import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -41,6 +41,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Consumer;
 
 import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT;
@@ -76,9 +77,9 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL
         final RootDocumentParserContext context;
         final XContentType xContentType = source.getXContentType();
 
-        DocumentSizeObserver documentSizeObserver = source.getDocumentSizeObserver();
+        XContentMeteringParserDecorator meteringParserDecorator = source.getDocumentSizeObserver();
         try (
-            XContentParser parser = documentSizeObserver.wrapParser(
+            XContentParser parser = meteringParserDecorator.decorate(
                 XContentHelper.createParser(parserConfiguration, source.source(), xContentType)
             )
         ) {
@@ -106,7 +107,7 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL
             context.sourceToParse().source(),
             context.sourceToParse().getXContentType(),
             dynamicUpdate,
-            documentSizeObserver
+            meteringParserDecorator.meteredDocumentSize()
         ) {
             @Override
             public String documentDescription() {
@@ -274,7 +275,7 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio
                 context.addIgnoredField(
                     new IgnoredSourceFieldMapper.NameValue(
                         context.parent().fullPath(),
-                        context.parent().fullPath().indexOf(currentFieldName),
+                        context.parent().fullPath().lastIndexOf(currentFieldName),
                         XContentDataHelper.encodeToken(parser),
                         context.doc()
                     )
@@ -301,7 +302,7 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio
                 context.addIgnoredField(
                     new IgnoredSourceFieldMapper.NameValue(
                         context.parent().fullPath(),
-                        context.parent().fullPath().indexOf(context.parent().leafName()),
+                        context.parent().fullPath().lastIndexOf(context.parent().leafName()),
                         XContentDataHelper.encodeXContentBuilder(tuple.v2()),
                         context.doc()
                     )
@@ -476,7 +477,7 @@ static void parseObjectOrField(DocumentParserContext context, Mapper mapper) thr
 
     private static boolean shouldFlattenObject(DocumentParserContext context, FieldMapper fieldMapper) {
         return context.parser().currentToken() == XContentParser.Token.START_OBJECT
-            && context.parent().subobjects() == false
+            && context.parent().subobjects() != ObjectMapper.Subobjects.ENABLED
             && fieldMapper.supportsParsingObject() == false;
     }
 
@@ -517,7 +518,7 @@ private static void parseObject(final DocumentParserContext context, String curr
     private static void doParseObject(DocumentParserContext context, String currentFieldName, Mapper objectMapper) throws IOException {
         context.path().add(currentFieldName);
         boolean withinLeafObject = context.path().isWithinLeafObject();
-        if (objectMapper instanceof ObjectMapper objMapper && objMapper.subobjects() == false) {
+        if (objectMapper instanceof ObjectMapper objMapper && objMapper.subobjects() != ObjectMapper.Subobjects.ENABLED) {
             context.path().setWithinLeafObject(true);
         }
         parseObjectOrField(context, objectMapper);
@@ -563,7 +564,7 @@ private static void parseObjectDynamic(DocumentParserContext context, String cur
             } else {
                 dynamicObjectMapper = DynamicFieldsBuilder.createDynamicObjectMapper(context, currentFieldName);
             }
-            if (context.parent().subobjects() == false) {
+            if (context.parent().subobjects() == ObjectMapper.Subobjects.DISABLED) {
                 if (dynamicObjectMapper instanceof NestedObjectMapper) {
                     throw new DocumentParsingException(
                         context.parser().getTokenLocation(),
@@ -1012,7 +1013,7 @@ private static class NoOpObjectMapper extends ObjectMapper {
                 name,
                 fullPath,
                 Explicit.IMPLICIT_TRUE,
-                Explicit.IMPLICIT_TRUE,
+                Optional.empty(),
                 Explicit.IMPLICIT_FALSE,
                 Dynamic.RUNTIME,
                 Collections.emptyMap()
@@ -1051,7 +1052,7 @@ private static class RootDocumentParserContext extends DocumentParserContext {
                 mappingLookup.getMapping().getRoot(),
                 ObjectMapper.Dynamic.getRootDynamic(mappingLookup)
             );
-            if (mappingLookup.getMapping().getRoot().subobjects()) {
+            if (mappingLookup.getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.ENABLED) {
                 this.parser = DotExpandingXContentParser.expandDots(parser, this.path);
             } else {
                 this.parser = parser;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java
index d479cb97e3fd3..6eb1920df02c8 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java
@@ -161,7 +161,7 @@ static Mapper createDynamicObjectMapper(DocumentParserContext context, String na
         Mapper mapper = createObjectMapperFromTemplate(context, name);
         return mapper != null
             ? mapper
-            : new ObjectMapper.Builder(name, ObjectMapper.Defaults.SUBOBJECTS).enabled(ObjectMapper.Defaults.ENABLED)
+            : new ObjectMapper.Builder(name, context.parent().subobjects).enabled(ObjectMapper.Defaults.ENABLED)
                 .build(context.createDynamicMapperBuilderContext());
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java
index 3d2c51fb5b8af..9265fb31a48d2 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java
@@ -107,7 +107,17 @@ private static class Stored extends IgnoreMalformedStoredValues {
 
         @Override
         public Stream<Map.Entry<String, SourceLoader.SyntheticFieldLoader.StoredFieldLoader>> storedFieldLoaders() {
-            return Stream.of(Map.entry(name(fieldName), values -> this.values = values));
+            return Stream.of(Map.entry(name(fieldName), new SourceLoader.SyntheticFieldLoader.StoredFieldLoader() {
+                @Override
+                public void advanceToDoc(int docId) {
+                    values = emptyList();
+                }
+
+                @Override
+                public void load(List<Object> newValues) {
+                    values = newValues;
+                }
+            }));
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
index efbc75490550d..f94a05b2a8658 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java
@@ -83,7 +83,7 @@ void write(XContentBuilder builder) throws IOException {
             XContentDataHelper.decodeAndWrite(builder, value());
         }
 
-        private String getFieldName() {
+        String getFieldName() {
             return parentOffset() == 0 ? name() : name().substring(parentOffset());
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
index b69c2b4606ed6..d5819f5eb4338 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
@@ -47,6 +47,7 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.time.ZoneId;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -625,7 +626,9 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
                 "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        return new SortedSetDocValuesSyntheticFieldLoader(fullPath(), leafName(), null, ignoreMalformed) {
+
+        var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+        layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) {
             @Override
             protected BytesRef convert(BytesRef value) {
                 byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length);
@@ -637,6 +640,12 @@ protected BytesRef preserve(BytesRef value) {
                 // No need to copy because convert has made a deep copy
                 return value;
             }
-        };
+        });
+
+        if (ignoreMalformed) {
+            layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
+        }
+
+        return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
index 16aa827e6a251..d130f37c3e8eb 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -67,6 +67,7 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -88,6 +89,7 @@ public final class KeywordFieldMapper extends FieldMapper {
     public static final String CONTENT_TYPE = "keyword";
 
     static final NodeFeature KEYWORD_DIMENSION_IGNORE_ABOVE = new NodeFeature("mapper.keyword_dimension_ignore_above");
+    static final NodeFeature KEYWORD_NORMALIZER_SYNTHETIC_SOURCE = new NodeFeature("mapper.keyword_normalizer_synthetic_source");
 
     public static class Defaults {
         public static final FieldType FIELD_TYPE;
@@ -855,7 +857,7 @@ public boolean hasNormalizer() {
     private final Script script;
     private final ScriptCompiler scriptCompiler;
     private final IndexVersion indexCreatedVersion;
-    private final boolean storeIgnored;
+    private final boolean isSyntheticSource;
 
     private final IndexAnalyzers indexAnalyzers;
 
@@ -865,7 +867,7 @@ private KeywordFieldMapper(
         KeywordFieldType mappedFieldType,
         MultiFields multiFields,
         CopyTo copyTo,
-        boolean storeIgnored,
+        boolean isSyntheticSource,
         Builder builder
     ) {
         super(simpleName, mappedFieldType, multiFields, copyTo, builder.script.get() != null, builder.onScriptError.getValue());
@@ -880,7 +882,7 @@ private KeywordFieldMapper(
         this.indexAnalyzers = builder.indexAnalyzers;
         this.scriptCompiler = builder.scriptCompiler;
         this.indexCreatedVersion = builder.indexCreatedVersion;
-        this.storeIgnored = storeIgnored;
+        this.isSyntheticSource = isSyntheticSource;
     }
 
     @Override
@@ -915,7 +917,7 @@ private void indexValue(DocumentParserContext context, String value) {
 
         if (value.length() > fieldType().ignoreAbove()) {
             context.addIgnoredField(fullPath());
-            if (storeIgnored) {
+            if (isSyntheticSource) {
                 // Save a copy of the field so synthetic source can load it
                 context.doc().add(new StoredField(originalName(), new BytesRef(value)));
             }
@@ -1025,6 +1027,11 @@ private String originalName() {
 
     @Override
     protected SyntheticSourceMode syntheticSourceMode() {
+        if (hasNormalizer()) {
+            // NOTE: no matter if we have doc values or not we use a stored field to reconstruct the original value
+            // whose doc values would be altered by the normalizer
+            return SyntheticSourceMode.FALLBACK;
+        }
         if (fieldType.stored() || hasDocValues) {
             return SyntheticSourceMode.NATIVE;
         }
@@ -1046,33 +1053,22 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String simpleName)
                 "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        if (hasNormalizer()) {
-            throw new IllegalArgumentException(
-                "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares a normalizer"
-            );
+
+        if (syntheticSourceMode() != SyntheticSourceMode.NATIVE) {
+            return super.syntheticFieldLoader();
         }
 
+        var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
         if (fieldType.stored()) {
-            return new StringStoredFieldFieldLoader(
-                fullPath(),
-                simpleName,
-                fieldType().ignoreAbove == Defaults.IGNORE_ABOVE ? null : originalName()
-            ) {
+            layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(fullPath()) {
                 @Override
-                protected void write(XContentBuilder b, Object value) throws IOException {
+                protected void writeValue(Object value, XContentBuilder b) throws IOException {
                     BytesRef ref = (BytesRef) value;
                     b.utf8Value(ref.bytes, ref.offset, ref.length);
                 }
-            };
-        }
-
-        if (hasDocValues) {
-            return new SortedSetDocValuesSyntheticFieldLoader(
-                fullPath(),
-                simpleName,
-                fieldType().ignoreAbove == Defaults.IGNORE_ABOVE ? null : originalName(),
-                false
-            ) {
+            });
+        } else if (hasDocValues) {
+            layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) {
 
                 @Override
                 protected BytesRef convert(BytesRef value) {
@@ -1084,10 +1080,19 @@ protected BytesRef preserve(BytesRef value) {
                     // Preserve must make a deep copy because convert gets a shallow copy from the iterator
                     return BytesRef.deepCopyOf(value);
                 }
-            };
+            });
         }
 
-        return super.syntheticFieldLoader();
-    }
+        if (fieldType().ignoreAbove != Defaults.IGNORE_ABOVE) {
+            layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) {
+                @Override
+                protected void writeValue(Object value, XContentBuilder b) throws IOException {
+                    BytesRef ref = (BytesRef) value;
+                    b.utf8Value(ref.bytes, ref.offset, ref.length);
+                }
+            });
+        }
 
+        return new CompositeSyntheticFieldLoader(simpleName, fullPath(), layers);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java
index 9c60ee96c6fd4..2c1e01c3cd196 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java
@@ -26,6 +26,7 @@ public abstract static class Builder {
 
         private String leafName;
 
+        @SuppressWarnings("this-escape")
         protected Builder(String leafName) {
             setLeafName(leafName);
         }
@@ -55,6 +56,7 @@ default boolean supportsVersion(IndexVersion indexCreatedVersion) {
 
     private final String leafName;
 
+    @SuppressWarnings("this-escape")
     public Mapper(String leafName) {
         Objects.requireNonNull(leafName);
         this.leafName = internFieldName(leafName);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
index a5f54ab89602b..63bbef061c61f 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
@@ -29,7 +29,12 @@ public Set<NodeFeature> getFeatures() {
             DenseVectorFieldMapper.BIT_VECTORS,
             DocumentMapper.INDEX_SORTING_ON_NESTED,
             KeywordFieldMapper.KEYWORD_DIMENSION_IGNORE_ABOVE,
-            IndexModeFieldMapper.QUERYING_INDEX_MODE
+            IndexModeFieldMapper.QUERYING_INDEX_MODE,
+            NodeMappingStats.SEGMENT_LEVEL_FIELDS_STATS,
+            BooleanFieldMapper.BOOLEAN_DIMENSION,
+            ObjectMapper.SUBOBJECTS_AUTO,
+            KeywordFieldMapper.KEYWORD_NORMALIZER_SYNTHETIC_SOURCE,
+            SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX
         );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
index 76212f9899f5c..f61f91250516a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
@@ -28,6 +28,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
@@ -49,7 +50,7 @@ public static class Builder extends ObjectMapper.Builder {
         private final Function<Query, BitSetProducer> bitSetProducer;
 
         public Builder(String name, IndexVersion indexCreatedVersion, Function<Query, BitSetProducer> bitSetProducer) {
-            super(name, Explicit.IMPLICIT_TRUE);
+            super(name, Optional.empty());
             this.indexCreatedVersion = indexCreatedVersion;
             this.bitSetProducer = bitSetProducer;
         }
@@ -121,7 +122,7 @@ public static class TypeParser extends ObjectMapper.TypeParser {
         @Override
         public Mapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext)
             throws MapperParsingException {
-            if (parseSubobjects(node).explicit()) {
+            if (parseSubobjects(node).isPresent()) {
                 throw new MapperParsingException("Nested type [" + name + "] does not support [subobjects] parameter");
             }
             NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(
@@ -209,7 +210,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) {
         Query nestedTypeFilter,
         Function<Query, BitSetProducer> bitsetProducer
     ) {
-        super(name, fullPath, enabled, Explicit.IMPLICIT_TRUE, storeArraySource, dynamic, mappers);
+        super(name, fullPath, enabled, Optional.empty(), storeArraySource, dynamic, mappers);
         this.parentTypeFilter = parentTypeFilter;
         this.nestedTypePath = nestedTypePath;
         this.nestedTypeFilter = nestedTypeFilter;
@@ -440,8 +441,8 @@ public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf
         }
 
         private List<Integer> collectChildren(int parentDoc, BitSet parentDocs, DocIdSetIterator childIt) throws IOException {
-            assert parentDocs.get(parentDoc) : "wrong context, doc " + parentDoc + " is not a parent of " + nestedTypePath;
-            final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
+            assert parentDoc < 0 || parentDocs.get(parentDoc) : "wrong context, doc " + parentDoc + " is not a parent of " + nestedTypePath;
+            final int prevParentDoc = parentDoc > 0 ? parentDocs.prevSetBit(parentDoc - 1) : -1;
             int childDocId = childIt.docID();
             if (childDocId <= prevParentDoc) {
                 childDocId = childIt.advance(prevParentDoc + 1);
@@ -463,17 +464,14 @@ public boolean hasValue() {
         public void write(XContentBuilder b) throws IOException {
             assert (children != null && children.size() > 0);
             if (children.size() == 1) {
-                b.startObject(leafName());
+                b.field(leafName());
                 leafStoredFieldLoader.advanceTo(children.get(0));
                 leafSourceLoader.write(leafStoredFieldLoader, children.get(0), b);
-                b.endObject();
             } else {
                 b.startArray(leafName());
                 for (int childId : children) {
-                    b.startObject();
                     leafStoredFieldLoader.advanceTo(childId);
                     leafSourceLoader.write(leafStoredFieldLoader, childId, b);
-                    b.endObject();
                 }
                 b.endArray();
             }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java b/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java
index 87a4f1367e108..77f86816d6fb7 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java
@@ -8,11 +8,13 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -25,15 +27,22 @@
  */
 public class NodeMappingStats implements Writeable, ToXContentFragment {
 
+    public static final NodeFeature SEGMENT_LEVEL_FIELDS_STATS = new NodeFeature("mapper.segment_level_fields_stats");
+
     private static final class Fields {
         static final String MAPPINGS = "mappings";
         static final String TOTAL_COUNT = "total_count";
         static final String TOTAL_ESTIMATED_OVERHEAD = "total_estimated_overhead";
         static final String TOTAL_ESTIMATED_OVERHEAD_IN_BYTES = "total_estimated_overhead_in_bytes";
+        static final String TOTAL_SEGMENTS = "total_segments";
+        static final String TOTAL_SEGMENT_FIELDS = "total_segment_fields";
+        static final String AVERAGE_FIELDS_PER_SEGMENT = "average_fields_per_segment";
     }
 
     private long totalCount;
     private long totalEstimatedOverhead;
+    private long totalSegments;
+    private long totalSegmentFields;
 
     public NodeMappingStats() {
 
@@ -42,17 +51,25 @@ public NodeMappingStats() {
     public NodeMappingStats(StreamInput in) throws IOException {
         totalCount = in.readVLong();
         totalEstimatedOverhead = in.readVLong();
+        if (in.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS)) {
+            totalSegments = in.readVLong();
+            totalSegmentFields = in.readVLong();
+        }
     }
 
-    public NodeMappingStats(long totalCount, long totalEstimatedOverhead) {
+    public NodeMappingStats(long totalCount, long totalEstimatedOverhead, long totalSegments, long totalSegmentFields) {
         this.totalCount = totalCount;
         this.totalEstimatedOverhead = totalEstimatedOverhead;
+        this.totalSegments = totalSegments;
+        this.totalSegmentFields = totalSegmentFields;
     }
 
     public void add(@Nullable NodeMappingStats other) {
         if (other == null) return;
         this.totalCount += other.totalCount;
         this.totalEstimatedOverhead += other.totalEstimatedOverhead;
+        this.totalSegments += other.totalSegments;
+        this.totalSegmentFields += other.totalSegmentFields;
     }
 
     public long getTotalCount() {
@@ -63,10 +80,22 @@ public ByteSizeValue getTotalEstimatedOverhead() {
         return ByteSizeValue.ofBytes(totalEstimatedOverhead);
     }
 
+    public long getTotalSegments() {
+        return totalSegments;
+    }
+
+    public long getTotalSegmentFields() {
+        return totalSegmentFields;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(totalCount);
         out.writeVLong(totalEstimatedOverhead);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_LEVEL_FIELDS_STATS)) {
+            out.writeVLong(totalSegments);
+            out.writeVLong(totalSegmentFields);
+        }
     }
 
     @Override
@@ -74,6 +103,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject(Fields.MAPPINGS);
         builder.field(Fields.TOTAL_COUNT, getTotalCount());
         builder.humanReadableField(Fields.TOTAL_ESTIMATED_OVERHEAD_IN_BYTES, Fields.TOTAL_ESTIMATED_OVERHEAD, getTotalEstimatedOverhead());
+        builder.field(Fields.TOTAL_SEGMENTS, totalSegments);
+        builder.field(Fields.TOTAL_SEGMENT_FIELDS, totalSegmentFields);
+        builder.field(Fields.AVERAGE_FIELDS_PER_SEGMENT, totalSegments == 0 ? 0 : totalSegmentFields / totalSegments);
         builder.endObject();
         return builder;
     }
@@ -83,11 +115,14 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         NodeMappingStats that = (NodeMappingStats) o;
-        return totalCount == that.totalCount && totalEstimatedOverhead == that.totalEstimatedOverhead;
+        return totalCount == that.totalCount
+            && totalEstimatedOverhead == that.totalEstimatedOverhead
+            && totalSegments == that.totalSegments
+            && totalSegmentFields == that.totalSegmentFields;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(totalCount, totalEstimatedOverhead);
+        return Objects.hash(totalCount, totalEstimatedOverhead, totalSegments, totalSegmentFields);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
index a3d5999a3dcd2..163f76ef47034 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
@@ -9,13 +9,14 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
@@ -32,6 +33,8 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.TreeMap;
 import java.util.stream.Stream;
 
@@ -40,10 +43,50 @@ public class ObjectMapper extends Mapper {
 
     public static final String CONTENT_TYPE = "object";
     static final String STORE_ARRAY_SOURCE_PARAM = "store_array_source";
+    static final NodeFeature SUBOBJECTS_AUTO = new NodeFeature("mapper.subobjects_auto");
+
+    /**
+     * Enhances the previously boolean option for subobjects support with an intermediate mode `auto` that uses
+     * any objects that are present in the mappings and flattens any fields defined outside the predefined objects.
+     */
+    public enum Subobjects {
+        ENABLED(Boolean.TRUE),
+        DISABLED(Boolean.FALSE),
+        AUTO("auto");
+
+        private final Object printedValue;
+
+        Subobjects(Object printedValue) {
+            this.printedValue = printedValue;
+        }
+
+        static Subobjects from(Object node) {
+            if (node instanceof Boolean value) {
+                return value ? Subobjects.ENABLED : Subobjects.DISABLED;
+            }
+            if (node instanceof String value) {
+                if (value.equalsIgnoreCase("true")) {
+                    return ENABLED;
+                }
+                if (value.equalsIgnoreCase("false")) {
+                    return DISABLED;
+                }
+                if (value.equalsIgnoreCase("auto")) {
+                    return AUTO;
+                }
+            }
+            throw new ElasticsearchParseException("unknown subobjects value: " + node);
+        }
+
+        @Override
+        public String toString() {
+            return printedValue.toString();
+        }
+    }
 
     public static class Defaults {
         public static final boolean ENABLED = true;
-        public static final Explicit<Boolean> SUBOBJECTS = Explicit.IMPLICIT_TRUE;
+        public static final Optional<Subobjects> SUBOBJECTS = Optional.empty();
         public static final Explicit<Boolean> STORE_ARRAY_SOURCE = Explicit.IMPLICIT_FALSE;
         public static final Dynamic DYNAMIC = Dynamic.TRUE;
     }
@@ -74,19 +117,19 @@ DynamicFieldsBuilder getDynamicFieldsBuilder() {
          * If no dynamic settings are explicitly configured, we default to {@link #TRUE}
          */
         static Dynamic getRootDynamic(MappingLookup mappingLookup) {
-            ObjectMapper.Dynamic rootDynamic = mappingLookup.getMapping().getRoot().dynamic;
+            Dynamic rootDynamic = mappingLookup.getMapping().getRoot().dynamic;
             return rootDynamic == null ? Defaults.DYNAMIC : rootDynamic;
         }
     }
 
     public static class Builder extends Mapper.Builder {
-        protected final Explicit<Boolean> subobjects;
+        protected Optional<Subobjects> subobjects;
         protected Explicit<Boolean> enabled = Explicit.IMPLICIT_TRUE;
         protected Explicit<Boolean> storeArraySource = Defaults.STORE_ARRAY_SOURCE;
         protected Dynamic dynamic;
         protected final List<Mapper.Builder> mappersBuilders = new ArrayList<>();
 
-        public Builder(String name, Explicit<Boolean> subobjects) {
+        public Builder(String name, Optional<Subobjects> subobjects) {
             super(name);
             this.subobjects = subobjects;
         }
@@ -131,24 +174,31 @@ public Mapper build(MapperBuilderContext context) {
         public final void addDynamic(String name, String prefix, Mapper mapper, DocumentParserContext context) {
             // If the mapper to add has no dots, or the current object mapper has subobjects set to false,
             // we just add it as it is for sure a leaf mapper
-            if (name.contains(".") == false || subobjects.value() == false) {
+            if (name.contains(".") == false || (subobjects.isPresent() && (subobjects.get() == Subobjects.DISABLED))) {
                 add(name, mapper);
-            }
-            // otherwise we strip off the first object path of the mapper name, load or create
-            // the relevant object mapper, and then recurse down into it, passing the remainder
-            // of the mapper name. So for a mapper 'foo.bar.baz', we locate 'foo' and then
-            // call addDynamic on it with the name 'bar.baz', and next call addDynamic on 'bar' with the name 'baz'.
-            else {
+            } else {
+                // We strip off the first object path of the mapper name, load or create
+                // the relevant object mapper, and then recurse down into it, passing the remainder
+                // of the mapper name. So for a mapper 'foo.bar.baz', we locate 'foo' and then
+                // call addDynamic on it with the name 'bar.baz', and next call addDynamic on 'bar' with the name 'baz'.
                 int firstDotIndex = name.indexOf('.');
                 String immediateChild = name.substring(0, firstDotIndex);
                 String immediateChildFullName = prefix == null ? immediateChild : prefix + "." + immediateChild;
-                ObjectMapper.Builder parentBuilder = findObjectBuilder(immediateChildFullName, context);
-                parentBuilder.addDynamic(name.substring(firstDotIndex + 1), immediateChildFullName, mapper, context);
-                add(parentBuilder);
+                Builder parentBuilder = findObjectBuilder(immediateChildFullName, context);
+                if (parentBuilder != null) {
+                    parentBuilder.addDynamic(name.substring(firstDotIndex + 1), immediateChildFullName, mapper, context);
+                    add(parentBuilder);
+                } else if (subobjects.isPresent() && subobjects.get() == Subobjects.AUTO) {
+                    // No matching parent object was found, the mapper is added as a leaf - similar to subobjects false.
+                    add(name, mapper);
+                } else {
+                    // Expected to find a matching parent object but got null.
+                    throw new IllegalStateException("Missing intermediate object " + immediateChildFullName);
+                }
             }
         }
 
-        private static ObjectMapper.Builder findObjectBuilder(String fullName, DocumentParserContext context) {
+        private static Builder findObjectBuilder(String fullName, DocumentParserContext context) {
             // does the object mapper already exist? if so, use that
             ObjectMapper objectMapper = context.mappingLookup().objectMappers().get(fullName);
             if (objectMapper != null) {
@@ -159,7 +209,8 @@ private static ObjectMapper.Builder findObjectBuilder(String fullName, DocumentP
             if (objectMapper != null) {
                 return objectMapper.newBuilder(context.indexSettings().getIndexVersionCreated());
             }
-            throw new IllegalStateException("Missing intermediate object " + fullName);
+            // no object mapper found
+            return null;
         }
 
         protected final Map<String, Mapper> buildMappers(MapperBuilderContext mapperBuilderContext) {
@@ -175,7 +226,7 @@ protected final Map<String, Mapper> buildMappers(MapperBuilderContext mapperBuil
                     // mix of object notation and dot notation.
                     mapper = existing.merge(mapper, MapperMergeContext.from(mapperBuilderContext, Long.MAX_VALUE));
                 }
-                if (subobjects.value() == false && mapper instanceof ObjectMapper objectMapper) {
+                if (subobjects.isPresent() && subobjects.get() == Subobjects.DISABLED && mapper instanceof ObjectMapper objectMapper) {
                     // We're parsing a mapping that has set `subobjects: false` but has defined sub-objects
                     objectMapper.asFlattenedFieldMappers(mapperBuilderContext).forEach(m -> mappers.put(m.leafName(), m));
                 } else {
@@ -214,8 +265,8 @@ public boolean supportsVersion(IndexVersion indexCreatedVersion) {
         public Mapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext)
             throws MapperParsingException {
             parserContext.incrementMappingObjectDepth(); // throws MapperParsingException if depth limit is exceeded
-            Explicit<Boolean> subobjects = parseSubobjects(node);
-            ObjectMapper.Builder builder = new Builder(name, subobjects);
+            Optional<Subobjects> subobjects = parseSubobjects(node);
+            Builder builder = new Builder(name, subobjects);
             parseObjectFields(node, parserContext, builder);
             parserContext.decrementMappingObjectDepth();
             return builder;
@@ -237,7 +288,7 @@ protected static boolean parseObjectOrDocumentTypeProperties(
             String fieldName,
             Object fieldNode,
             MappingParserContext parserContext,
-            ObjectMapper.Builder builder
+            Builder builder
         ) {
             if (fieldName.equals("dynamic")) {
                 String value = fieldNode.toString();
@@ -276,19 +327,15 @@ protected static boolean parseObjectOrDocumentTypeProperties(
             return false;
         }
 
-        protected static Explicit<Boolean> parseSubobjects(Map<String, Object> node) {
+        protected static Optional<Subobjects> parseSubobjects(Map<String, Object> node) {
             Object subobjectsNode = node.remove("subobjects");
             if (subobjectsNode != null) {
-                return Explicit.explicitBoolean(XContentMapValues.nodeBooleanValue(subobjectsNode, "subobjects.subobjects"));
+                return Optional.of(Subobjects.from(subobjectsNode));
             }
             return Defaults.SUBOBJECTS;
         }
 
-        protected static void parseProperties(
-            ObjectMapper.Builder objBuilder,
-            Map<String, Object> propsNode,
-            MappingParserContext parserContext
-        ) {
+        protected static void parseProperties(Builder objBuilder, Map<String, Object> propsNode, MappingParserContext parserContext) {
             Iterator<Map.Entry<String, Object>> iterator = propsNode.entrySet().iterator();
             while (iterator.hasNext()) {
                 Map.Entry<String, Object> entry = iterator.next();
@@ -320,7 +367,9 @@ protected static void parseProperties(
                         }
                     }
 
-                    if (objBuilder.subobjects.value() == false && type.equals(NestedObjectMapper.CONTENT_TYPE)) {
+                    if (objBuilder.subobjects.isPresent()
+                        && objBuilder.subobjects.get() == Subobjects.DISABLED
+                        && type.equals(NestedObjectMapper.CONTENT_TYPE)) {
                         throw new MapperParsingException(
                             "Tried to add nested object ["
                                 + fieldName
@@ -334,7 +383,7 @@ protected static void parseProperties(
                         throw new MapperParsingException("No handler for type [" + type + "] declared on field [" + fieldName + "]");
                     }
                     Mapper.Builder fieldBuilder;
-                    if (objBuilder.subobjects.value() == false) {
+                    if (objBuilder.subobjects.isPresent() && objBuilder.subobjects.get() != Subobjects.ENABLED) {
                         fieldBuilder = typeParser.parse(fieldName, propNode, parserContext);
                     } else {
                         String[] fieldNameParts = fieldName.split("\\.");
@@ -347,7 +396,7 @@ protected static void parseProperties(
                         for (int i = fieldNameParts.length - 2; i >= 0; --i) {
                             String intermediateObjectName = fieldNameParts[i];
                             validateFieldName(intermediateObjectName, parserContext.indexVersionCreated());
-                            ObjectMapper.Builder intermediate = new ObjectMapper.Builder(intermediateObjectName, Defaults.SUBOBJECTS);
+                            Builder intermediate = new Builder(intermediateObjectName, Defaults.SUBOBJECTS);
                             intermediate.add(fieldBuilder);
                             fieldBuilder = intermediate;
                         }
@@ -382,7 +431,7 @@ private static void validateFieldName(String fieldName, IndexVersion indexCreate
     private final String fullPath;
 
     protected final Explicit<Boolean> enabled;
-    protected final Explicit<Boolean> subobjects;
+    protected final Optional<Subobjects> subobjects;
     protected final Explicit<Boolean> storeArraySource;
     protected final Dynamic dynamic;
 
@@ -392,7 +441,7 @@ private static void validateFieldName(String fieldName, IndexVersion indexCreate
         String name,
         String fullPath,
         Explicit<Boolean> enabled,
-        Explicit<Boolean> subobjects,
+        Optional<Subobjects> subobjects,
         Explicit<Boolean> storeArraySource,
         Dynamic dynamic,
         Map<String, Mapper> mappers
@@ -410,15 +459,17 @@ private static void validateFieldName(String fieldName, IndexVersion indexCreate
         } else {
             this.mappers = Map.copyOf(mappers);
         }
-        assert subobjects.value() || this.mappers.values().stream().noneMatch(m -> m instanceof ObjectMapper)
+        assert subobjects.isEmpty()
+            || subobjects.get() != Subobjects.DISABLED
+            || this.mappers.values().stream().noneMatch(m -> m instanceof ObjectMapper)
             : "When subobjects is false, mappers must not contain an ObjectMapper";
     }
 
     /**
      * @return a Builder that will produce an empty ObjectMapper with the same configuration as this one
      */
-    public ObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) {
-        ObjectMapper.Builder builder = new ObjectMapper.Builder(leafName(), subobjects);
+    public Builder newBuilder(IndexVersion indexVersionCreated) {
+        Builder builder = new Builder(leafName(), subobjects);
         builder.enabled = this.enabled;
         builder.dynamic = this.dynamic;
         return builder;
@@ -463,8 +514,8 @@ public final Dynamic dynamic() {
         return dynamic;
     }
 
-    public final boolean subobjects() {
-        return subobjects.value();
+    public final Subobjects subobjects() {
+        return subobjects.orElse(Subobjects.ENABLED);
     }
 
     public final boolean storeArraySource() {
@@ -473,9 +524,6 @@ public final boolean storeArraySource() {
 
     @Override
     public void validate(MappingLookup mappers) {
-        if (storeArraySource() && mappers.isSourceSynthetic() == false) {
-            throw new MapperParsingException("Parameter [" + STORE_ARRAY_SOURCE_PARAM + "] can only be set in synthetic source mode.");
-        }
         for (Mapper mapper : this.mappers.values()) {
             mapper.validate(mappers);
         }
@@ -508,9 +556,9 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex
 
     protected record MergeResult(
         Explicit<Boolean> enabled,
-        Explicit<Boolean> subObjects,
+        Optional<Subobjects> subObjects,
         Explicit<Boolean> trackArraySource,
-        ObjectMapper.Dynamic dynamic,
+        Dynamic dynamic,
         Map<String, Mapper> mappers
     ) {
         static MergeResult build(ObjectMapper existing, ObjectMapper mergeWithObject, MapperMergeContext parentMergeContext) {
@@ -529,11 +577,11 @@ static MergeResult build(ObjectMapper existing, ObjectMapper mergeWithObject, Ma
             } else {
                 enabled = existing.enabled;
             }
-            final Explicit<Boolean> subObjects;
-            if (mergeWithObject.subobjects.explicit()) {
+            final Optional<Subobjects> subObjects;
+            if (mergeWithObject.subobjects.isPresent()) {
                 if (reason == MergeReason.INDEX_TEMPLATE) {
                     subObjects = mergeWithObject.subobjects;
-                } else if (existing.subobjects != mergeWithObject.subobjects) {
+                } else if (existing.subobjects() != mergeWithObject.subobjects()) {
                     throw new MapperException(
                         "the [subobjects] parameter can't be updated for the object mapping [" + existing.fullPath() + "]"
                     );
@@ -558,7 +606,7 @@ static MergeResult build(ObjectMapper existing, ObjectMapper mergeWithObject, Ma
                 trackArraySource = existing.storeArraySource;
             }
             MapperMergeContext objectMergeContext = existing.createChildContext(parentMergeContext, existing.leafName());
-            Map<String, Mapper> mergedMappers = buildMergedMappers(existing, mergeWithObject, objectMergeContext, subObjects.value());
+            Map<String, Mapper> mergedMappers = buildMergedMappers(existing, mergeWithObject, objectMergeContext, subObjects);
             return new MergeResult(
                 enabled,
                 subObjects,
@@ -572,11 +620,13 @@ private static Map<String, Mapper> buildMergedMappers(
             ObjectMapper existing,
             ObjectMapper mergeWithObject,
             MapperMergeContext objectMergeContext,
-            boolean subobjects
+            Optional<Subobjects> subobjects
         ) {
             Map<String, Mapper> mergedMappers = new HashMap<>();
             for (Mapper childOfExistingMapper : existing.mappers.values()) {
-                if (subobjects == false && childOfExistingMapper instanceof ObjectMapper objectMapper) {
+                if (subobjects.isPresent()
+                    && subobjects.get() == Subobjects.DISABLED
+                    && childOfExistingMapper instanceof ObjectMapper objectMapper) {
                     // An existing mapping with sub-objects is merged with a mapping that has set `subobjects: false`
                     objectMapper.asFlattenedFieldMappers(objectMergeContext.getMapperBuilderContext())
                         .forEach(m -> mergedMappers.put(m.leafName(), m));
@@ -587,7 +637,9 @@ private static Map<String, Mapper> buildMergedMappers(
             for (Mapper mergeWithMapper : mergeWithObject) {
                 Mapper mergeIntoMapper = mergedMappers.get(mergeWithMapper.leafName());
                 if (mergeIntoMapper == null) {
-                    if (subobjects == false && mergeWithMapper instanceof ObjectMapper objectMapper) {
+                    if (subobjects.isPresent()
+                        && subobjects.get() == Subobjects.DISABLED
+                        && mergeWithMapper instanceof ObjectMapper objectMapper) {
                         // An existing mapping that has set `subobjects: false` is merged with a mapping with sub-objects
                         objectMapper.asFlattenedFieldMappers(objectMergeContext.getMapperBuilderContext())
                             .stream()
@@ -599,7 +651,8 @@ private static Map<String, Mapper> buildMergedMappers(
                         putMergedMapper(mergedMappers, truncateObjectMapper(objectMergeContext, om));
                     }
                 } else if (mergeIntoMapper instanceof ObjectMapper objectMapper) {
-                    assert subobjects : "existing object mappers are supposed to be flattened if subobjects is false";
+                    assert subobjects.isEmpty() || subobjects.get() != Subobjects.DISABLED
+                        : "existing object mappers are supposed to be flattened if subobjects is false";
                     putMergedMapper(mergedMappers, objectMapper.merge(mergeWithMapper, objectMergeContext));
                 } else {
                     assert mergeIntoMapper instanceof FieldMapper || mergeIntoMapper instanceof FieldAliasMapper;
@@ -681,7 +734,7 @@ private void ensureFlattenable(MapperBuilderContext context, ContentPath path) {
         if (isEnabled() == false) {
             throwAutoFlatteningException(path, "the value of [enabled] is [false]");
         }
-        if (subobjects.explicit() && subobjects()) {
+        if (subobjects.isPresent() && subobjects.get() == Subobjects.ENABLED) {
             throwAutoFlatteningException(path, "the value of [subobjects] is [true]");
         }
     }
@@ -716,8 +769,8 @@ void toXContent(XContentBuilder builder, Params params, ToXContent custom) throw
         if (isEnabled() != Defaults.ENABLED) {
             builder.field("enabled", enabled.value());
         }
-        if (subobjects != Defaults.SUBOBJECTS) {
-            builder.field("subobjects", subobjects.value());
+        if (subobjects.isPresent()) {
+            builder.field("subobjects", subobjects.get().printedValue);
         }
         if (storeArraySource != Defaults.STORE_ARRAY_SOURCE) {
             builder.field(STORE_ARRAY_SOURCE_PARAM, storeArraySource.value());
@@ -774,7 +827,9 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
     private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader {
         private final List<SourceLoader.SyntheticFieldLoader> fields;
         private final boolean isFragment;
-        private boolean hasValue;
+        private boolean storedFieldLoadersHaveValues;
+        private boolean docValuesLoadersHaveValues;
+        private boolean ignoredValuesPresent;
         private List<IgnoredSourceFieldMapper.NameValue> ignoredValues;
 
         private SyntheticSourceFieldLoader(List<SourceLoader.SyntheticFieldLoader> fields, boolean isFragment) {
@@ -784,17 +839,28 @@ private SyntheticSourceFieldLoader(List<SourceLoader.SyntheticFieldLoader> field
 
         @Override
         public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
-            return fields.stream().flatMap(SourceLoader.SyntheticFieldLoader::storedFieldLoaders).map(e -> Map.entry(e.getKey(), values -> {
-                hasValue = true;
-                e.getValue().load(values);
-            }));
+            return fields.stream()
+                .flatMap(SourceLoader.SyntheticFieldLoader::storedFieldLoaders)
+                .map(e -> Map.entry(e.getKey(), new StoredFieldLoader() {
+                    @Override
+                    public void advanceToDoc(int docId) {
+                        storedFieldLoadersHaveValues = false;
+                        e.getValue().advanceToDoc(docId);
+                    }
+
+                    @Override
+                    public void load(List<Object> newValues) {
+                        storedFieldLoadersHaveValues = true;
+                        e.getValue().load(newValues);
+                    }
+                }));
         }
 
         @Override
         public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
-            List<SourceLoader.SyntheticFieldLoader.DocValuesLoader> loaders = new ArrayList<>();
+            List<DocValuesLoader> loaders = new ArrayList<>();
             for (SourceLoader.SyntheticFieldLoader field : fields) {
-                SourceLoader.SyntheticFieldLoader.DocValuesLoader loader = field.docValuesLoader(leafReader, docIdsInLeaf);
+                DocValuesLoader loader = field.docValuesLoader(leafReader, docIdsInLeaf);
                 if (loader != null) {
                     loaders.add(loader);
                 }
@@ -806,7 +872,7 @@ public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf
         }
 
         private class ObjectDocValuesLoader implements DocValuesLoader {
-            private final List<SourceLoader.SyntheticFieldLoader.DocValuesLoader> loaders;
+            private final List<DocValuesLoader> loaders;
 
             private ObjectDocValuesLoader(List<DocValuesLoader> loaders) {
                 this.loaders = loaders;
@@ -814,24 +880,26 @@ private ObjectDocValuesLoader(List<DocValuesLoader> loaders) {
 
             @Override
             public boolean advanceToDoc(int docId) throws IOException {
+                docValuesLoadersHaveValues = false;
+
                 boolean anyLeafHasDocValues = false;
-                for (SourceLoader.SyntheticFieldLoader.DocValuesLoader docValueLoader : loaders) {
+                for (DocValuesLoader docValueLoader : loaders) {
                     boolean leafHasValue = docValueLoader.advanceToDoc(docId);
                     anyLeafHasDocValues |= leafHasValue;
                 }
-                hasValue |= anyLeafHasDocValues;
+                docValuesLoadersHaveValues = anyLeafHasDocValues;
                 return anyLeafHasDocValues;
             }
         }
 
         @Override
         public boolean hasValue() {
-            return hasValue;
+            return storedFieldLoadersHaveValues || docValuesLoadersHaveValues || ignoredValuesPresent;
         }
 
         @Override
         public void write(XContentBuilder b) throws IOException {
-            if (hasValue == false) {
+            if (hasValue() == false) {
                 return;
             }
             if (isRoot() && isEnabled() == false) {
@@ -843,28 +911,32 @@ public void write(XContentBuilder b) throws IOException {
                 return;
             }
 
-            if (isFragment == false) {
-                if (isRoot()) {
-                    b.startObject();
-                } else {
-                    b.startObject(leafName());
-                }
+            if (isRoot() || isFragment) {
+                b.startObject();
+            } else {
+                b.startObject(leafName());
             }
 
             if (ignoredValues != null && ignoredValues.isEmpty() == false) {
                 // Use an ordered map between field names and writer functions, to order writing by field name.
-                Map<String, CheckedConsumer<XContentBuilder, IOException>> orderedFields = new TreeMap<>();
+                Map<String, FieldWriter> orderedFields = new TreeMap<>();
                 for (IgnoredSourceFieldMapper.NameValue value : ignoredValues) {
-                    orderedFields.put(value.name(), value::write);
+                    var existing = orderedFields.get(value.name());
+                    if (existing == null) {
+                        orderedFields.put(value.name(), new FieldWriter.IgnoredSource(value));
+                    } else if (existing instanceof FieldWriter.IgnoredSource isw) {
+                        isw.mergeWith(value);
+                    }
                 }
                 for (SourceLoader.SyntheticFieldLoader field : fields) {
                     if (field.hasValue()) {
                         // Skip if the field source is stored separately, to avoid double-printing.
-                        orderedFields.putIfAbsent(field.fieldName(), field::write);
+                        orderedFields.computeIfAbsent(field.fieldName(), k -> new FieldWriter.FieldLoader(field));
                     }
                 }
+
                 for (var writer : orderedFields.values()) {
-                    writer.accept(b);
+                    writer.writeTo(b);
                 }
                 ignoredValues = null;
             } else {
@@ -874,21 +946,20 @@ public void write(XContentBuilder b) throws IOException {
                     }
                 }
             }
-            hasValue = false;
-            if (isFragment == false) {
-                b.endObject();
-            }
+            b.endObject();
         }
 
         @Override
         public boolean setIgnoredValues(Map<String, List<IgnoredSourceFieldMapper.NameValue>> objectsWithIgnoredFields) {
+            ignoredValuesPresent = false;
+
             if (objectsWithIgnoredFields == null || objectsWithIgnoredFields.isEmpty()) {
                 return false;
             }
             ignoredValues = objectsWithIgnoredFields.remove(ObjectMapper.this.fullPath());
-            hasValue |= ignoredValues != null;
+            ignoredValuesPresent |= ignoredValues != null;
             for (SourceLoader.SyntheticFieldLoader loader : fields) {
-                hasValue |= loader.setIgnoredValues(objectsWithIgnoredFields);
+                ignoredValuesPresent |= loader.setIgnoredValues(objectsWithIgnoredFields);
             }
             return this.ignoredValues != null;
         }
@@ -897,6 +968,42 @@ public boolean setIgnoredValues(Map<String, List<IgnoredSourceFieldMapper.NameVa
         public String fieldName() {
             return ObjectMapper.this.fullPath();
         }
+
+        interface FieldWriter {
+            void writeTo(XContentBuilder builder) throws IOException;
+
+            record FieldLoader(SourceLoader.SyntheticFieldLoader loader) implements FieldWriter {
+                @Override
+                public void writeTo(XContentBuilder builder) throws IOException {
+                    loader.write(builder);
+                }
+            }
+
+            class IgnoredSource implements FieldWriter {
+                private final String fieldName;
+                private final String leafName;
+                private final List<BytesRef> values;
+
+                IgnoredSource(IgnoredSourceFieldMapper.NameValue initialValue) {
+                    this.fieldName = initialValue.name();
+                    this.leafName = initialValue.getFieldName();
+                    this.values = new ArrayList<>();
+                    this.values.add(initialValue.value());
+                }
+
+                @Override
+                public void writeTo(XContentBuilder builder) throws IOException {
+                    XContentDataHelper.writeMerged(builder, leafName, values);
+                }
+
+                public FieldWriter mergeWith(IgnoredSourceFieldMapper.NameValue nameValue) {
+                    assert Objects.equals(nameValue.name(), fieldName) : "IgnoredSource is merged with wrong field data";
+
+                    values.add(nameValue.value());
+                    return this;
+                }
+            }
+        }
     }
 
     protected boolean isRoot() {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
index d500d42e45fae..8076c836cd0cb 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.util.Collections;
@@ -24,7 +23,6 @@
  * The result of parsing a document.
  */
 public class ParsedDocument {
-
     private final Field version;
 
     private final String id;
@@ -34,7 +32,7 @@ public class ParsedDocument {
 
     private final List<LuceneDocument> documents;
 
-    private final DocumentSizeObserver documentSizeObserver;
+    private final DocumentSize normalizedSize;
 
     private BytesReference source;
     private XContentType xContentType;
@@ -62,7 +60,7 @@ public static ParsedDocument noopTombstone(String reason) {
             new BytesArray("{}"),
             XContentType.JSON,
             null,
-            DocumentSizeObserver.EMPTY_INSTANCE
+            DocumentSize.UNKNOWN
         );
     }
 
@@ -87,7 +85,7 @@ public static ParsedDocument deleteTombstone(String id) {
             new BytesArray("{}"),
             XContentType.JSON,
             null,
-            DocumentSizeObserver.EMPTY_INSTANCE
+            DocumentSize.UNKNOWN
         );
     }
 
@@ -100,7 +98,7 @@ public ParsedDocument(
         BytesReference source,
         XContentType xContentType,
         Mapping dynamicMappingsUpdate,
-        DocumentSizeObserver documentSizeObserver
+        DocumentSize normalizedSize
     ) {
         this.version = version;
         this.seqID = seqID;
@@ -110,7 +108,7 @@ public ParsedDocument(
         this.source = source;
         this.dynamicMappingsUpdate = dynamicMappingsUpdate;
         this.xContentType = xContentType;
-        this.documentSizeObserver = documentSizeObserver;
+        this.normalizedSize = normalizedSize;
     }
 
     public String id() {
@@ -179,8 +177,16 @@ public String documentDescription() {
         return "id";
     }
 
-    public DocumentSizeObserver getDocumentSizeObserver() {
-        return documentSizeObserver;
+    public DocumentSize getNormalizedSize() {
+        return normalizedSize;
     }
 
+    /**
+     * Normalized ingested and stored size of a document.
+     * @param ingestedBytes ingest size of the document
+     * @param storedBytes stored retained size of the document
+     */
+    public record DocumentSize(long ingestedBytes, long storedBytes) {
+        public static final DocumentSize UNKNOWN = new DocumentSize(-1, -1);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java
index 0b7f4de157bdc..7370fe3c61772 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java
@@ -18,6 +18,7 @@
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue;
@@ -52,7 +53,7 @@ public static class Builder extends ObjectMapper.Builder {
 
         public Builder(String name) {
             // Subobjects are not currently supported.
-            super(name, Explicit.IMPLICIT_FALSE);
+            super(name, Optional.of(Subobjects.DISABLED));
         }
 
         @Override
@@ -103,7 +104,7 @@ public PassThroughObjectMapper build(MapperBuilderContext context) {
         int priority
     ) {
         // Subobjects are not currently supported.
-        super(name, fullPath, enabled, Explicit.IMPLICIT_FALSE, Explicit.IMPLICIT_FALSE, dynamic, mappers);
+        super(name, fullPath, enabled, Optional.of(Subobjects.DISABLED), Explicit.IMPLICIT_FALSE, dynamic, mappers);
         this.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields;
         this.priority = priority;
         if (priority < 0) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
index 11aabd8726f4f..6c178330e5c9e 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
@@ -34,6 +34,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.function.BiConsumer;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
@@ -75,7 +76,7 @@ public static class Builder extends ObjectMapper.Builder {
         protected Explicit<Boolean> dateDetection = Defaults.DATE_DETECTION;
         protected Explicit<Boolean> numericDetection = Defaults.NUMERIC_DETECTION;
 
-        public Builder(String name, Explicit<Boolean> subobjects) {
+        public Builder(String name, Optional<Subobjects> subobjects) {
             super(name, subobjects);
         }
 
@@ -132,7 +133,7 @@ public RootObjectMapper build(MapperBuilderContext context) {
     RootObjectMapper(
         String name,
         Explicit<Boolean> enabled,
-        Explicit<Boolean> subobjects,
+        Optional<Subobjects> subobjects,
         Explicit<Boolean> trackArraySource,
         Dynamic dynamic,
         Map<String, Mapper> mappers,
@@ -442,7 +443,7 @@ protected boolean isRoot() {
 
     public static RootObjectMapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext)
         throws MapperParsingException {
-        Explicit<Boolean> subobjects = parseSubobjects(node);
+        Optional<Subobjects> subobjects = parseSubobjects(node);
         RootObjectMapper.Builder builder = new Builder(name, subobjects);
         Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
         while (iterator.hasNext()) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericDocValuesSyntheticFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericDocValuesSyntheticFieldLoader.java
index 96ba151472a03..a2568ec91ebdf 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericDocValuesSyntheticFieldLoader.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericDocValuesSyntheticFieldLoader.java
@@ -104,7 +104,6 @@ public void write(XContentBuilder b) throws IOException {
                 values.write(b);
                 ignoreMalformedValues.write(b);
                 b.endArray();
-                return;
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SortedSetDocValuesSyntheticFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SortedSetDocValuesSyntheticFieldLoader.java
deleted file mode 100644
index 335e551365931..0000000000000
--- a/server/src/main/java/org/elasticsearch/index/mapper/SortedSetDocValuesSyntheticFieldLoader.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.index.mapper;
-
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.logging.LogManager;
-import org.elasticsearch.logging.Logger;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-
-import static java.util.Collections.emptyList;
-
-/**
- * Load {@code _source} fields from {@link SortedSetDocValues}.
- */
-public abstract class SortedSetDocValuesSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader {
-    private static final Logger logger = LogManager.getLogger(SortedSetDocValuesSyntheticFieldLoader.class);
-
-    private final String name;
-    private final String simpleName;
-    private DocValuesFieldValues docValues = NO_VALUES;
-
-    /**
-     * Optionally loads stored fields values.
-     */
-    @Nullable
-    private final String storedValuesName;
-    private List<Object> storedValues = emptyList();
-
-    /**
-     * Optionally loads malformed values from stored fields.
-     */
-    private final IgnoreMalformedStoredValues ignoreMalformedValues;
-
-    /**
-     * Build a loader from doc values and, optionally, a stored field.
-     * @param name the name of the field to load from doc values
-     * @param simpleName the name to give the field in the rendered {@code _source}
-     * @param storedValuesName the name of a stored field to load or null if there aren't any stored field for this field
-     * @param loadIgnoreMalformedValues should we load values skipped by {@code ignore_malformed}
-     */
-    public SortedSetDocValuesSyntheticFieldLoader(
-        String name,
-        String simpleName,
-        @Nullable String storedValuesName,
-        boolean loadIgnoreMalformedValues
-    ) {
-        this.name = name;
-        this.simpleName = simpleName;
-        this.storedValuesName = storedValuesName;
-        this.ignoreMalformedValues = loadIgnoreMalformedValues
-            ? IgnoreMalformedStoredValues.stored(name)
-            : IgnoreMalformedStoredValues.empty();
-    }
-
-    @Override
-    public String fieldName() {
-        return name;
-    }
-
-    @Override
-    public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
-        if (storedValuesName == null) {
-            return ignoreMalformedValues.storedFieldLoaders();
-        }
-        return Stream.concat(
-            Stream.of(Map.entry(storedValuesName, values -> this.storedValues = values)),
-            ignoreMalformedValues.storedFieldLoaders()
-        );
-    }
-
-    @Override
-    public DocValuesLoader docValuesLoader(LeafReader reader, int[] docIdsInLeaf) throws IOException {
-        SortedSetDocValues dv = DocValues.getSortedSet(reader, name);
-        if (dv.getValueCount() == 0) {
-            docValues = NO_VALUES;
-            return null;
-        }
-        if (docIdsInLeaf != null && docIdsInLeaf.length > 1) {
-            /*
-             * The singleton optimization is mostly about looking up ordinals
-             * in sorted order and doesn't buy anything if there is only a single
-             * document.
-             */
-            SortedDocValues singleton = DocValues.unwrapSingleton(dv);
-            if (singleton != null) {
-                SingletonDocValuesLoader loader = buildSingletonDocValuesLoader(singleton, docIdsInLeaf);
-                docValues = loader == null ? NO_VALUES : loader;
-                return loader;
-            }
-        }
-        ImmediateDocValuesLoader loader = new ImmediateDocValuesLoader(dv);
-        docValues = loader;
-        return loader;
-    }
-
-    @Override
-    public boolean hasValue() {
-        return docValues.count() > 0 || storedValues.isEmpty() == false || ignoreMalformedValues.count() > 0;
-    }
-
-    @Override
-    public void write(XContentBuilder b) throws IOException {
-        int total = docValues.count() + storedValues.size() + ignoreMalformedValues.count();
-        switch (total) {
-            case 0:
-                return;
-            case 1:
-                b.field(simpleName);
-                if (docValues.count() > 0) {
-                    assert docValues.count() == 1;
-                    assert storedValues.isEmpty();
-                    assert ignoreMalformedValues.count() == 0;
-                    docValues.write(b);
-                } else if (storedValues.isEmpty() == false) {
-                    assert docValues.count() == 0;
-                    assert storedValues.size() == 1;
-                    assert ignoreMalformedValues.count() == 0;
-                    BytesRef converted = convert((BytesRef) storedValues.get(0));
-                    b.utf8Value(converted.bytes, converted.offset, converted.length);
-                    storedValues = emptyList();
-                } else {
-                    assert docValues.count() == 0;
-                    assert storedValues.isEmpty();
-                    assert ignoreMalformedValues.count() == 1;
-                    ignoreMalformedValues.write(b);
-                }
-                return;
-            default:
-                b.startArray(simpleName);
-                docValues.write(b);
-                for (Object v : storedValues) {
-                    BytesRef converted = convert((BytesRef) v);
-                    b.utf8Value(converted.bytes, converted.offset, converted.length);
-                }
-                storedValues = emptyList();
-                ignoreMalformedValues.write(b);
-                b.endArray();
-                return;
-        }
-    }
-
-    private interface DocValuesFieldValues {
-        int count();
-
-        void write(XContentBuilder b) throws IOException;
-    }
-
-    private static final DocValuesFieldValues NO_VALUES = new DocValuesFieldValues() {
-        @Override
-        public int count() {
-            return 0;
-        }
-
-        @Override
-        public void write(XContentBuilder b) {}
-    };
-
-    /**
-     * Load ordinals in line with populating the doc and immediately
-     * convert from ordinals into {@link BytesRef}s.
-     */
-    private class ImmediateDocValuesLoader implements DocValuesLoader, DocValuesFieldValues {
-        private final SortedSetDocValues dv;
-        private boolean hasValue;
-
-        ImmediateDocValuesLoader(SortedSetDocValues dv) {
-            this.dv = dv;
-        }
-
-        @Override
-        public boolean advanceToDoc(int docId) throws IOException {
-            return hasValue = dv.advanceExact(docId);
-        }
-
-        @Override
-        public int count() {
-            return hasValue ? dv.docValueCount() : 0;
-        }
-
-        @Override
-        public void write(XContentBuilder b) throws IOException {
-            if (hasValue == false) {
-                return;
-            }
-            for (int i = 0; i < dv.docValueCount(); i++) {
-                BytesRef c = convert(dv.lookupOrd(dv.nextOrd()));
-                b.utf8Value(c.bytes, c.offset, c.length);
-            }
-        }
-    }
-
-    /**
-     * Load all ordinals for all docs up front and resolve to their string
-     * values in order. This should be much more disk-friendly than
-     * {@link ImmediateDocValuesLoader} because it resolves the ordinals in order and
-     * marginally more cpu friendly because it resolves the ordinals one time.
-     */
-    private SingletonDocValuesLoader buildSingletonDocValuesLoader(SortedDocValues singleton, int[] docIdsInLeaf) throws IOException {
-        int[] ords = new int[docIdsInLeaf.length];
-        int found = 0;
-        for (int d = 0; d < docIdsInLeaf.length; d++) {
-            if (false == singleton.advanceExact(docIdsInLeaf[d])) {
-                ords[d] = -1;
-                continue;
-            }
-            ords[d] = singleton.ordValue();
-            found++;
-        }
-        if (found == 0) {
-            return null;
-        }
-        int[] sortedOrds = ords.clone();
-        Arrays.sort(sortedOrds);
-        int unique = 0;
-        int prev = -1;
-        for (int ord : sortedOrds) {
-            if (ord != prev) {
-                prev = ord;
-                unique++;
-            }
-        }
-        int[] uniqueOrds = new int[unique];
-        BytesRef[] converted = new BytesRef[unique];
-        unique = 0;
-        prev = -1;
-        for (int ord : sortedOrds) {
-            if (ord != prev) {
-                prev = ord;
-                uniqueOrds[unique] = ord;
-                converted[unique] = preserve(convert(singleton.lookupOrd(ord)));
-                unique++;
-            }
-        }
-        logger.debug("loading [{}] on [{}] docs covering [{}] ords", name, docIdsInLeaf.length, uniqueOrds.length);
-        return new SingletonDocValuesLoader(docIdsInLeaf, ords, uniqueOrds, converted);
-    }
-
-    private static class SingletonDocValuesLoader implements DocValuesLoader, DocValuesFieldValues {
-        private final int[] docIdsInLeaf;
-        private final int[] ords;
-        private final int[] uniqueOrds;
-        private final BytesRef[] converted;
-
-        private int idx = -1;
-
-        private SingletonDocValuesLoader(int[] docIdsInLeaf, int[] ords, int[] uniqueOrds, BytesRef[] converted) {
-            this.docIdsInLeaf = docIdsInLeaf;
-            this.ords = ords;
-            this.uniqueOrds = uniqueOrds;
-            this.converted = converted;
-        }
-
-        @Override
-        public boolean advanceToDoc(int docId) throws IOException {
-            idx++;
-            if (docIdsInLeaf[idx] != docId) {
-                throw new IllegalArgumentException(
-                    "expected to be called with [" + docIdsInLeaf[idx] + "] but was called with " + docId + " instead"
-                );
-            }
-            return ords[idx] >= 0;
-        }
-
-        @Override
-        public int count() {
-            return ords[idx] < 0 ? 0 : 1;
-        }
-
-        @Override
-        public void write(XContentBuilder b) throws IOException {
-            if (ords[idx] < 0) {
-                return;
-            }
-            int convertedIdx = Arrays.binarySearch(uniqueOrds, ords[idx]);
-            if (convertedIdx < 0) {
-                throw new IllegalStateException("received unexpected ord [" + ords[idx] + "]. Expected " + Arrays.toString(uniqueOrds));
-            }
-            BytesRef c = converted[convertedIdx];
-            b.utf8Value(c.bytes, c.offset, c.length);
-        }
-    }
-
-    /**
-     * Convert a {@link BytesRef} read from the source into bytes to write
-     * to the xcontent. This shouldn't make a deep copy if the conversion
-     * process itself doesn't require one.
-     */
-    protected abstract BytesRef convert(BytesRef value);
-
-    /**
-     * Preserves {@link BytesRef bytes} returned by {@link #convert}
-     * to by written later. This should make a
-     * {@link BytesRef#deepCopyOf deep copy} if {@link #convert} didn't.
-     */
-    protected abstract BytesRef preserve(BytesRef value);
-}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SortedSetDocValuesSyntheticFieldLoaderLayer.java b/server/src/main/java/org/elasticsearch/index/mapper/SortedSetDocValuesSyntheticFieldLoaderLayer.java
new file mode 100644
index 0000000000000..626e7ecb16f46
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SortedSetDocValuesSyntheticFieldLoaderLayer.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.stream.Stream;
+
+/**
+ * Load {@code _source} fields from {@link SortedSetDocValues}.
+ */
+public abstract class SortedSetDocValuesSyntheticFieldLoaderLayer implements CompositeSyntheticFieldLoader.Layer {
+    private static final Logger logger = LogManager.getLogger(SortedSetDocValuesSyntheticFieldLoaderLayer.class);
+
+    private final String name;
+    private DocValuesFieldValues docValues = NO_VALUES;
+
+    /**
+     * Build a loader from doc values and, optionally, a stored field.
+     * @param name the name of the field to load from doc values
+     */
+    public SortedSetDocValuesSyntheticFieldLoaderLayer(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public String fieldName() {
+        return name;
+    }
+
+    @Override
+    public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
+        return Stream.of();
+    }
+
+    @Override
+    public DocValuesLoader docValuesLoader(LeafReader reader, int[] docIdsInLeaf) throws IOException {
+        SortedSetDocValues dv = DocValues.getSortedSet(reader, name);
+        if (dv.getValueCount() == 0) {
+            docValues = NO_VALUES;
+            return null;
+        }
+        if (docIdsInLeaf != null && docIdsInLeaf.length > 1) {
+            /*
+             * The singleton optimization is mostly about looking up ordinals
+             * in sorted order and doesn't buy anything if there is only a single
+             * document.
+             */
+            SortedDocValues singleton = DocValues.unwrapSingleton(dv);
+            if (singleton != null) {
+                SingletonDocValuesLoader loader = buildSingletonDocValuesLoader(singleton, docIdsInLeaf);
+                docValues = loader == null ? NO_VALUES : loader;
+                return loader;
+            }
+        }
+        ImmediateDocValuesLoader loader = new ImmediateDocValuesLoader(dv);
+        docValues = loader;
+        return loader;
+    }
+
+    @Override
+    public boolean hasValue() {
+        return docValues.count() > 0;
+    }
+
+    @Override
+    public long valueCount() {
+        return docValues.count();
+    }
+
+    @Override
+    public void write(XContentBuilder b) throws IOException {
+        docValues.write(b);
+    }
+
+    private interface DocValuesFieldValues {
+        int count();
+
+        void write(XContentBuilder b) throws IOException;
+    }
+
+    private static final DocValuesFieldValues NO_VALUES = new DocValuesFieldValues() {
+        @Override
+        public int count() {
+            return 0;
+        }
+
+        @Override
+        public void write(XContentBuilder b) {}
+    };
+
+    /**
+     * Load ordinals in line with populating the doc and immediately
+     * convert from ordinals into {@link BytesRef}s.
+     */
+    private class ImmediateDocValuesLoader implements DocValuesLoader, DocValuesFieldValues {
+        private final SortedSetDocValues dv;
+        private boolean hasValue;
+
+        ImmediateDocValuesLoader(SortedSetDocValues dv) {
+            this.dv = dv;
+        }
+
+        @Override
+        public boolean advanceToDoc(int docId) throws IOException {
+            return hasValue = dv.advanceExact(docId);
+        }
+
+        @Override
+        public int count() {
+            return hasValue ? dv.docValueCount() : 0;
+        }
+
+        @Override
+        public void write(XContentBuilder b) throws IOException {
+            if (hasValue == false) {
+                return;
+            }
+            for (int i = 0; i < dv.docValueCount(); i++) {
+                BytesRef c = convert(dv.lookupOrd(dv.nextOrd()));
+                b.utf8Value(c.bytes, c.offset, c.length);
+            }
+        }
+    }
+
+    /**
+     * Load all ordinals for all docs up front and resolve to their string
+     * values in order. This should be much more disk-friendly than
+     * {@link ImmediateDocValuesLoader} because it resolves the ordinals in order and
+     * marginally more cpu friendly because it resolves the ordinals one time.
+     */
+    private SingletonDocValuesLoader buildSingletonDocValuesLoader(SortedDocValues singleton, int[] docIdsInLeaf) throws IOException {
+        int[] ords = new int[docIdsInLeaf.length];
+        int found = 0;
+        for (int d = 0; d < docIdsInLeaf.length; d++) {
+            if (false == singleton.advanceExact(docIdsInLeaf[d])) {
+                ords[d] = -1;
+                continue;
+            }
+            ords[d] = singleton.ordValue();
+            found++;
+        }
+        if (found == 0) {
+            return null;
+        }
+        int[] sortedOrds = ords.clone();
+        Arrays.sort(sortedOrds);
+        int unique = 0;
+        int prev = -1;
+        for (int ord : sortedOrds) {
+            if (ord != prev) {
+                prev = ord;
+                unique++;
+            }
+        }
+        int[] uniqueOrds = new int[unique];
+        BytesRef[] converted = new BytesRef[unique];
+        unique = 0;
+        prev = -1;
+        for (int ord : sortedOrds) {
+            if (ord != prev) {
+                prev = ord;
+                uniqueOrds[unique] = ord;
+                converted[unique] = preserve(convert(singleton.lookupOrd(ord)));
+                unique++;
+            }
+        }
+        logger.debug("loading [{}] on [{}] docs covering [{}] ords", name, docIdsInLeaf.length, uniqueOrds.length);
+        return new SingletonDocValuesLoader(docIdsInLeaf, ords, uniqueOrds, converted);
+    }
+
+    private static class SingletonDocValuesLoader implements DocValuesLoader, DocValuesFieldValues {
+        private final int[] docIdsInLeaf;
+        private final int[] ords;
+        private final int[] uniqueOrds;
+        private final BytesRef[] converted;
+
+        private int idx = -1;
+
+        private SingletonDocValuesLoader(int[] docIdsInLeaf, int[] ords, int[] uniqueOrds, BytesRef[] converted) {
+            this.docIdsInLeaf = docIdsInLeaf;
+            this.ords = ords;
+            this.uniqueOrds = uniqueOrds;
+            this.converted = converted;
+        }
+
+        @Override
+        public boolean advanceToDoc(int docId) throws IOException {
+            idx++;
+            if (docIdsInLeaf[idx] != docId) {
+                throw new IllegalArgumentException(
+                    "expected to be called with [" + docIdsInLeaf[idx] + "] but was called with " + docId + " instead"
+                );
+            }
+            return ords[idx] >= 0;
+        }
+
+        @Override
+        public int count() {
+            return ords[idx] < 0 ? 0 : 1;
+        }
+
+        @Override
+        public void write(XContentBuilder b) throws IOException {
+            if (ords[idx] < 0) {
+                return;
+            }
+            int convertedIdx = Arrays.binarySearch(uniqueOrds, ords[idx]);
+            if (convertedIdx < 0) {
+                throw new IllegalStateException("received unexpected ord [" + ords[idx] + "]. Expected " + Arrays.toString(uniqueOrds));
+            }
+            BytesRef c = converted[convertedIdx];
+            b.utf8Value(c.bytes, c.offset, c.length);
+        }
+    }
+
+    /**
+     * Convert a {@link BytesRef} read from the source into bytes to write
+     * to the xcontent. This shouldn't make a deep copy if the conversion
+     * process itself doesn't require one.
+     */
+    protected abstract BytesRef convert(BytesRef value);
+
+    /**
+     * Preserves {@link BytesRef bytes} returned by {@link #convert}
+     * to by written later. This should make a
+     * {@link BytesRef#deepCopyOf deep copy} if {@link #convert} didn't.
+     */
+    protected abstract BytesRef preserve(BytesRef value);
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
index 908108bce31da..8d34d3188a388 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
@@ -38,6 +38,9 @@
 
 public class SourceFieldMapper extends MetadataFieldMapper {
     public static final NodeFeature SYNTHETIC_SOURCE_FALLBACK = new NodeFeature("mapper.source.synthetic_source_fallback");
+    public static final NodeFeature SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX = new NodeFeature(
+        "mapper.source.synthetic_source_stored_fields_advance_fix"
+    );
 
     public static final String NAME = "_source";
     public static final String RECOVERY_SOURCE_NAME = "_recovery_source";
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java
index c9bea33852a20..934e5aced644b 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java
@@ -185,6 +185,10 @@ public Source source(LeafStoredFieldLoader storedFieldLoader, int docId) throws
 
             @Override
             public void write(LeafStoredFieldLoader storedFieldLoader, int docId, XContentBuilder b) throws IOException {
+                for (var fieldLevelStoredFieldLoader : storedFieldLoaders.values()) {
+                    fieldLevelStoredFieldLoader.advanceToDoc(docId);
+                }
+
                 // Maps the names of existing objects to lists of ignored fields they contain.
                 Map<String, List<IgnoredSourceFieldMapper.NameValue>> objectsWithIgnoredFields = null;
 
@@ -322,6 +326,15 @@ default boolean setIgnoredValues(Map<String, List<IgnoredSourceFieldMapper.NameV
          * Sync for stored field values.
          */
         interface StoredFieldLoader {
+            /**
+             * Signals the loader that values for this document will be loaded next.
+             * Allows loader to discard cached data for previous document.
+             */
+            void advanceToDoc(int docId);
+
+            /**
+             * Loads values read from a corresponding stored field into this loader.
+             */
             void load(List<Object> values);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
index 6a020127019f5..ddf92de647ae5 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
+import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.util.Map;
@@ -28,7 +28,7 @@ public class SourceToParse {
     private final XContentType xContentType;
 
     private final Map<String, String> dynamicTemplates;
-    private final DocumentSizeObserver documentSizeObserver;
+    private final XContentMeteringParserDecorator meteringParserDecorator;
 
     public SourceToParse(
         @Nullable String id,
@@ -36,7 +36,7 @@ public SourceToParse(
         XContentType xContentType,
         @Nullable String routing,
         Map<String, String> dynamicTemplates,
-        DocumentSizeObserver documentSizeObserver
+        XContentMeteringParserDecorator meteringParserDecorator
     ) {
         this.id = id;
         // we always convert back to byte array, since we store it and Field only supports bytes..
@@ -45,15 +45,15 @@ public SourceToParse(
         this.xContentType = Objects.requireNonNull(xContentType);
         this.routing = routing;
         this.dynamicTemplates = Objects.requireNonNull(dynamicTemplates);
-        this.documentSizeObserver = documentSizeObserver;
+        this.meteringParserDecorator = meteringParserDecorator;
     }
 
     public SourceToParse(String id, BytesReference source, XContentType xContentType) {
-        this(id, source, xContentType, null, Map.of(), DocumentSizeObserver.EMPTY_INSTANCE);
+        this(id, source, xContentType, null, Map.of(), XContentMeteringParserDecorator.NOOP);
     }
 
     public SourceToParse(String id, BytesReference source, XContentType xContentType, String routing) {
-        this(id, source, xContentType, routing, Map.of(), DocumentSizeObserver.EMPTY_INSTANCE);
+        this(id, source, xContentType, routing, Map.of(), XContentMeteringParserDecorator.NOOP);
     }
 
     public BytesReference source() {
@@ -90,7 +90,7 @@ public XContentType getXContentType() {
         return this.xContentType;
     }
 
-    public DocumentSizeObserver getDocumentSizeObserver() {
-        return documentSizeObserver;
+    public XContentMeteringParserDecorator getDocumentSizeObserver() {
+        return meteringParserDecorator;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java
index b26aed11233f9..0f2d61374d202 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.LeafReader;
-import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -22,64 +21,49 @@
 public abstract class StringStoredFieldFieldLoader implements SourceLoader.SyntheticFieldLoader {
     private final String name;
     private final String simpleName;
-    private List<Object> values = emptyList();
 
-    @Nullable
-    private final String extraStoredName;
-    private List<Object> extraValues = emptyList();
+    private List<Object> values = emptyList();
 
-    public StringStoredFieldFieldLoader(String name, String simpleName, @Nullable String extraStoredName) {
+    public StringStoredFieldFieldLoader(String name, String simpleName) {
         this.name = name;
         this.simpleName = simpleName;
-        this.extraStoredName = extraStoredName;
     }
 
     @Override
     public final Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
-        Stream<Map.Entry<String, StoredFieldLoader>> standard = Stream.of(Map.entry(name, values -> this.values = values));
-        if (extraStoredName == null) {
-            return standard;
-        }
-        return Stream.concat(standard, Stream.of(Map.entry(extraStoredName, values -> this.extraValues = values)));
+        return Stream.of(Map.entry(name, new SourceLoader.SyntheticFieldLoader.StoredFieldLoader() {
+            @Override
+            public void advanceToDoc(int docId) {
+                values = emptyList();
+            }
+
+            @Override
+            public void load(List<Object> newValues) {
+                values = newValues;
+            }
+        }));
     }
 
     @Override
     public final boolean hasValue() {
-        return values.isEmpty() == false || extraValues.isEmpty() == false;
+        return values.isEmpty() == false;
     }
 
     @Override
     public final void write(XContentBuilder b) throws IOException {
-        int size = values.size() + extraValues.size();
-        switch (size) {
+        switch (values.size()) {
             case 0:
                 return;
             case 1:
                 b.field(simpleName);
-                if (values.size() > 0) {
-                    assert values.size() == 1;
-                    assert extraValues.isEmpty();
-                    write(b, values.get(0));
-                } else {
-                    assert values.isEmpty();
-                    assert extraValues.size() == 1;
-                    write(b, extraValues.get(0));
-                }
-                values = emptyList();
-                extraValues = emptyList();
+                write(b, values.get(0));
                 return;
             default:
                 b.startArray(simpleName);
                 for (Object value : values) {
                     write(b, value);
                 }
-                for (Object value : extraValues) {
-                    write(b, value);
-                }
                 b.endArray();
-                values = emptyList();
-                extraValues = emptyList();
-                return;
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
index 1f0c920c39c8f..9c98647f37bd9 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -1459,7 +1459,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
             );
         }
         if (store) {
-            return new StringStoredFieldFieldLoader(fullPath(), leafName(), null) {
+            return new StringStoredFieldFieldLoader(fullPath(), leafName()) {
                 @Override
                 protected void write(XContentBuilder b, Object value) throws IOException {
                     b.value((String) value);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java
index 112b3ec96b39e..f1c6a072c2d9e 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java
@@ -344,6 +344,18 @@ public DocumentDimensions addUnsignedLong(String fieldName, long value) {
             }
         }
 
+        @Override
+        public DocumentDimensions addBoolean(String fieldName, boolean value) {
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.write((byte) 'b');
+                out.write(value ? 't' : 'f');
+                add(fieldName, out.bytes());
+            } catch (IOException e) {
+                throw new IllegalArgumentException("Dimension field cannot be serialized.", e);
+            }
+            return this;
+        }
+
         @Override
         public DocumentDimensions validate(final IndexSettings settings) {
             if (settings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ID_HASHING)
@@ -415,6 +427,8 @@ public static Map<String, Object> decodeTsidAsMap(StreamInput in) {
                     }
                     case (byte) 'd' -> // parse a double
                         result.put(name, in.readDouble());
+                    case (byte) 'b' -> // parse a boolean
+                        result.put(name, in.read() == 't');
                     default -> throw new IllegalArgumentException("Cannot parse [" + name + "]: Unknown type [" + type + "]");
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java
index d97e03d3874ee..fefafbf13017b 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java
@@ -26,6 +26,8 @@
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
 
 /**
  * Helper class for processing field data of any type, as provided by the {@link XContentParser}.
@@ -92,6 +94,70 @@ static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
         }
     }
 
+    /**
+     * Writes encoded values to provided builder. If there are multiple values they are merged into
+     * a single resulting array.
+     * @param b destination
+     * @param fieldName name of the field that is written
+     * @param encodedParts subset of field data encoded using methods of this class. Can contain arrays which will be flattened.
+     * @throws IOException
+     */
+    static void writeMerged(XContentBuilder b, String fieldName, List<BytesRef> encodedParts) throws IOException {
+        if (encodedParts.isEmpty()) {
+            return;
+        }
+
+        if (encodedParts.size() == 1) {
+            b.field(fieldName);
+            XContentDataHelper.decodeAndWrite(b, encodedParts.get(0));
+            return;
+        }
+
+        b.startArray(fieldName);
+
+        for (var encodedValue : encodedParts) {
+            Optional<XContentType> encodedXContentType = switch ((char) encodedValue.bytes[encodedValue.offset]) {
+                case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> Optional.of(
+                    getXContentType(encodedValue)
+                );
+                default -> Optional.empty();
+            };
+            if (encodedXContentType.isEmpty()) {
+                // This is a plain value, we can just write it
+                XContentDataHelper.decodeAndWrite(b, encodedValue);
+            } else {
+                // Encoded value could be an array which needs to be flattened
+                // since we are already inside an array.
+                try (
+                    XContentParser parser = encodedXContentType.get()
+                        .xContent()
+                        .createParser(
+                            XContentParserConfiguration.EMPTY,
+                            encodedValue.bytes,
+                            encodedValue.offset + 1,
+                            encodedValue.length - 1
+                        )
+                ) {
+                    if (parser.currentToken() == null) {
+                        parser.nextToken();
+                    }
+
+                    // It's an array, we will flatten it.
+                    if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
+                        while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                            b.copyCurrentStructure(parser);
+                        }
+                    } else {
+                        // It is a single complex structure (an object), write it as is.
+                        b.copyCurrentStructure(parser);
+                    }
+                }
+            }
+        }
+
+        b.endArray();
+    }
+
     /**
      * Returns the {@link XContentType} to use for creating an XContentBuilder to decode the passed value.
      */
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java
index 959460758ab30..fd8bbe3b62422 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedSortedSetDocValuesSyntheticFieldLoader.java
@@ -12,12 +12,14 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.mapper.SortedSetDocValuesSyntheticFieldLoader;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Map;
+import java.util.stream.Stream;
 
-public class FlattenedSortedSetDocValuesSyntheticFieldLoader extends SortedSetDocValuesSyntheticFieldLoader {
+public class FlattenedSortedSetDocValuesSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader {
     private DocValuesFieldValues docValues = NO_VALUES;
     private final String fieldFullPath;
     private final String keyedFieldFullPath;
@@ -31,7 +33,6 @@ public class FlattenedSortedSetDocValuesSyntheticFieldLoader extends SortedSetDo
      * @param leafName                the name of the leaf field to use in the rendered {@code _source}
      */
     public FlattenedSortedSetDocValuesSyntheticFieldLoader(String fieldFullPath, String keyedFieldFullPath, String leafName) {
-        super(fieldFullPath, leafName, null, false);
         this.fieldFullPath = fieldFullPath;
         this.keyedFieldFullPath = keyedFieldFullPath;
         this.leafName = leafName;
@@ -42,6 +43,11 @@ public String fieldName() {
         return fieldFullPath;
     }
 
+    @Override
+    public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
+        return Stream.empty();
+    }
+
     @Override
     public DocValuesLoader docValuesLoader(LeafReader reader, int[] docIdsInLeaf) throws IOException {
         final SortedSetDocValues dv = DocValues.getSortedSet(reader, keyedFieldFullPath);
@@ -69,16 +75,6 @@ public void write(XContentBuilder b) throws IOException {
         b.endObject();
     }
 
-    @Override
-    protected BytesRef convert(BytesRef value) {
-        return value;
-    }
-
-    @Override
-    protected BytesRef preserve(BytesRef value) {
-        return BytesRef.deepCopyOf(value);
-    }
-
     private interface DocValuesFieldValues {
         int count();
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
index 8ffe4b4cc4a66..6a231d15f7be8 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
@@ -937,8 +937,8 @@ int parseDimensionCount(DocumentParserContext context) throws IOException {
             }
 
             @Override
-            public void checkDimensions(int dvDims, int qvDims) {
-                if (dvDims != qvDims * Byte.SIZE) {
+            public void checkDimensions(Integer dvDims, int qvDims) {
+                if (dvDims != null && dvDims != qvDims * Byte.SIZE) {
                     throw new IllegalArgumentException(
                         "The query vector has a different number of dimensions ["
                             + qvDims * Byte.SIZE
@@ -972,8 +972,8 @@ abstract void checkVectorMagnitude(
             float squaredMagnitude
         );
 
-        public void checkDimensions(int dvDims, int qvDims) {
-            if (dvDims != qvDims) {
+        public void checkDimensions(Integer dvDims, int qvDims) {
+            if (dvDims != null && dvDims != qvDims) {
                 throw new IllegalArgumentException(
                     "The query vector has a different number of dimensions [" + qvDims + "] than the document vectors [" + dvDims + "]."
                 );
@@ -1789,17 +1789,21 @@ public Query termQuery(Object value, SearchExecutionContext context) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support term queries");
         }
 
-        public Query createExactKnnQuery(VectorData queryVector) {
+        public Query createExactKnnQuery(VectorData queryVector, Float vectorSimilarity) {
             if (isIndexed() == false) {
                 throw new IllegalArgumentException(
                     "to perform knn search on field [" + name() + "], its mapping must have [index] set to [true]"
                 );
             }
-            return switch (elementType) {
+            Query knnQuery = switch (elementType) {
                 case BYTE -> createExactKnnByteQuery(queryVector.asByteVector());
                 case FLOAT -> createExactKnnFloatQuery(queryVector.asFloatVector());
                 case BIT -> createExactKnnBitQuery(queryVector.asByteVector());
             };
+            if (vectorSimilarity != null) {
+                knnQuery = new VectorSimilarityQuery(knnQuery, vectorSimilarity, similarity.score(vectorSimilarity, elementType, dims));
+            }
+            return knnQuery;
         }
 
         private Query createExactKnnBitQuery(byte[] queryVector) {
diff --git a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
index 4b4727bca4198..9856bab64ec6a 100644
--- a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
@@ -410,4 +410,28 @@ private static boolean rewriteClauses(
     public TransportVersion getMinimalSupportedVersion() {
         return TransportVersions.ZERO;
     }
+
+    /**
+     * Create a new builder with the same clauses but modification of
+     * the builder won't modify the original. Modifications of any
+     * of the copy's clauses will modify the original. Don't so that.
+     */
+    public BoolQueryBuilder shallowCopy() {
+        BoolQueryBuilder copy = new BoolQueryBuilder();
+        copy.adjustPureNegative = adjustPureNegative;
+        copy.minimumShouldMatch = minimumShouldMatch;
+        for (QueryBuilder q : mustClauses) {
+            copy.must(q);
+        }
+        for (QueryBuilder q : mustNotClauses) {
+            copy.mustNot(q);
+        }
+        for (QueryBuilder q : filterClauses) {
+            copy.filter(q);
+        }
+        for (QueryBuilder q : shouldClauses) {
+            copy.should(q);
+        }
+        return copy;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java
index 2a1062f8876d2..7cb1b04972bfa 100644
--- a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java
+++ b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java
@@ -9,11 +9,13 @@
 package org.elasticsearch.index.query;
 
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.shard.IndexLongFieldRange;
+import org.elasticsearch.indices.DateFieldRangeInfo;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 
 import java.util.Collections;
@@ -23,19 +25,24 @@
  * Context object used to rewrite {@link QueryBuilder} instances into simplified version in the coordinator.
  * Instances of this object rely on information stored in the {@code IndexMetadata} for certain indices.
  * Right now this context object is able to rewrite range queries that include a known timestamp field
- * (i.e. the timestamp field for DataStreams) into a MatchNoneQueryBuilder and skip the shards that
- * don't hold queried data. See IndexMetadata#getTimestampRange() for more details
+ * (i.e. the timestamp field for DataStreams or the 'event.ingested' field in ECS) into a MatchNoneQueryBuilder
+ * and skip the shards that don't hold queried data. See IndexMetadata for more details.
  */
 public class CoordinatorRewriteContext extends QueryRewriteContext {
-    private final IndexLongFieldRange indexLongFieldRange;
-    private final DateFieldMapper.DateFieldType timestampFieldType;
+    private final DateFieldRangeInfo dateFieldRangeInfo;
 
+    /**
+     * Context for coordinator search rewrites based on time ranges for the @timestamp field and/or 'event.ingested' field
+     * @param parserConfig
+     * @param client
+     * @param nowInMillis
+     * @param dateFieldRangeInfo range and field type info for @timestamp and 'event.ingested'
+     */
     public CoordinatorRewriteContext(
         XContentParserConfiguration parserConfig,
         Client client,
         LongSupplier nowInMillis,
-        IndexLongFieldRange indexLongFieldRange,
-        DateFieldMapper.DateFieldType timestampFieldType
+        DateFieldRangeInfo dateFieldRangeInfo
     ) {
         super(
             parserConfig,
@@ -51,31 +58,40 @@ public CoordinatorRewriteContext(
             null,
             null,
             null,
+            null,
             null
         );
-        this.indexLongFieldRange = indexLongFieldRange;
-        this.timestampFieldType = timestampFieldType;
-    }
-
-    long getMinTimestamp() {
-        return indexLongFieldRange.getMin();
-    }
-
-    long getMaxTimestamp() {
-        return indexLongFieldRange.getMax();
-    }
-
-    boolean hasTimestampData() {
-        return indexLongFieldRange.isComplete() && indexLongFieldRange != IndexLongFieldRange.EMPTY;
+        this.dateFieldRangeInfo = dateFieldRangeInfo;
     }
 
+    /**
+     * @param fieldName Must be one of DataStream.TIMESTAMP_FIELD_FIELD or IndexMetadata.EVENT_INGESTED_FIELD_NAME
+     * @return MappedField with type for the field. Returns null if fieldName is not one of the allowed field names.
+     */
     @Nullable
     public MappedFieldType getFieldType(String fieldName) {
-        if (fieldName.equals(timestampFieldType.name()) == false) {
+        if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName)) {
+            return dateFieldRangeInfo.timestampFieldType();
+        } else if (IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) {
+            return dateFieldRangeInfo.eventIngestedFieldType();
+        } else {
             return null;
         }
+    }
 
-        return timestampFieldType;
+    /**
+     * @param fieldName Must be one of DataStream.TIMESTAMP_FIELD_FIELD or IndexMetadata.EVENT_INGESTED_FIELD_NAME
+     * @return IndexLongFieldRange with min/max ranges for the field. Returns null if fieldName is not one of the allowed field names.
+     */
+    @Nullable
+    public IndexLongFieldRange getFieldRange(String fieldName) {
+        if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName)) {
+            return dateFieldRangeInfo.timestampRange();
+        } else if (IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) {
+            return dateFieldRangeInfo.eventIngestedRange();
+        } else {
+            return null;
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java
index e44861b4afe8a..ec53dfe5c0d05 100644
--- a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.shard.IndexLongFieldRange;
+import org.elasticsearch.indices.DateFieldRangeInfo;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 
 import java.util.function.Function;
@@ -25,14 +26,14 @@ public class CoordinatorRewriteContextProvider {
     private final Client client;
     private final LongSupplier nowInMillis;
     private final Supplier<ClusterState> clusterStateSupplier;
-    private final Function<Index, DateFieldMapper.DateFieldType> mappingSupplier;
+    private final Function<Index, DateFieldRangeInfo> mappingSupplier;
 
     public CoordinatorRewriteContextProvider(
         XContentParserConfiguration parserConfig,
         Client client,
         LongSupplier nowInMillis,
         Supplier<ClusterState> clusterStateSupplier,
-        Function<Index, DateFieldMapper.DateFieldType> mappingSupplier
+        Function<Index, DateFieldRangeInfo> mappingSupplier
     ) {
         this.parserConfig = parserConfig;
         this.client = client;
@@ -49,18 +50,30 @@ public CoordinatorRewriteContext getCoordinatorRewriteContext(Index index) {
         if (indexMetadata == null) {
             return null;
         }
-        DateFieldMapper.DateFieldType dateFieldType = mappingSupplier.apply(index);
-        if (dateFieldType == null) {
+        DateFieldRangeInfo dateFieldRangeInfo = mappingSupplier.apply(index);
+        if (dateFieldRangeInfo == null) {
             return null;
         }
+        DateFieldMapper.DateFieldType timestampFieldType = dateFieldRangeInfo.timestampFieldType();
         IndexLongFieldRange timestampRange = indexMetadata.getTimestampRange();
+        IndexLongFieldRange eventIngestedRange = indexMetadata.getEventIngestedRange();
+
         if (timestampRange.containsAllShardRanges() == false) {
-            timestampRange = indexMetadata.getTimeSeriesTimestampRange(dateFieldType);
-            if (timestampRange == null) {
+            // if @timestamp range is not present or not ready in cluster state, fallback to using time series range (if present)
+            timestampRange = indexMetadata.getTimeSeriesTimestampRange(timestampFieldType);
+            // if timestampRange in the time series is null AND the eventIngestedRange is not ready for use, return null (no coord rewrite)
+            if (timestampRange == null && eventIngestedRange.containsAllShardRanges() == false) {
                 return null;
             }
         }
 
-        return new CoordinatorRewriteContext(parserConfig, client, nowInMillis, timestampRange, dateFieldType);
+        // the DateFieldRangeInfo from the mappingSupplier only has field types, but not ranges
+        // so create a new object with ranges pulled from cluster state
+        return new CoordinatorRewriteContext(
+            parserConfig,
+            client,
+            nowInMillis,
+            new DateFieldRangeInfo(timestampFieldType, timestampRange, dateFieldRangeInfo.eventIngestedFieldType(), eventIngestedRange)
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
index 81adfee36f923..9537aeec6a219 100644
--- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
+++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
@@ -24,6 +25,7 @@
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
+import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 
@@ -62,6 +64,7 @@ public class QueryRewriteContext {
     protected boolean mapUnmappedFieldAsString;
     protected Predicate<String> allowedFields;
     private final ResolvedIndices resolvedIndices;
+    private final PointInTimeBuilder pit;
 
     public QueryRewriteContext(
         final XContentParserConfiguration parserConfiguration,
@@ -77,7 +80,8 @@ public QueryRewriteContext(
         final ValuesSourceRegistry valuesSourceRegistry,
         final BooleanSupplier allowExpensiveQueries,
         final ScriptCompiler scriptService,
-        final ResolvedIndices resolvedIndices
+        final ResolvedIndices resolvedIndices,
+        final PointInTimeBuilder pit
     ) {
 
         this.parserConfiguration = parserConfiguration;
@@ -95,6 +99,7 @@ public QueryRewriteContext(
         this.allowExpensiveQueries = allowExpensiveQueries;
         this.scriptService = scriptService;
         this.resolvedIndices = resolvedIndices;
+        this.pit = pit;
     }
 
     public QueryRewriteContext(final XContentParserConfiguration parserConfiguration, final Client client, final LongSupplier nowInMillis) {
@@ -112,6 +117,7 @@ public QueryRewriteContext(final XContentParserConfiguration parserConfiguration
             null,
             null,
             null,
+            null,
             null
         );
     }
@@ -120,7 +126,8 @@ public QueryRewriteContext(
         final XContentParserConfiguration parserConfiguration,
         final Client client,
         final LongSupplier nowInMillis,
-        final ResolvedIndices resolvedIndices
+        final ResolvedIndices resolvedIndices,
+        final PointInTimeBuilder pit
     ) {
         this(
             parserConfiguration,
@@ -136,7 +143,8 @@ public QueryRewriteContext(
             null,
             null,
             null,
-            resolvedIndices
+            resolvedIndices,
+            pit
         );
     }
 
@@ -390,4 +398,12 @@ public Iterable<Map.Entry<String, MappedFieldType>> getAllFields() {
     public ResolvedIndices getResolvedIndices() {
         return resolvedIndices;
     }
+
+    /**
+     * Returns the {@link PointInTimeBuilder} used by the search request, or null if not specified.
+     */
+    @Nullable
+    public PointInTimeBuilder getPointInTimeBuilder() {
+        return pit;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
index 4d2a6d3eaecdb..8b154b3845964 100644
--- a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -436,15 +437,22 @@ public String getWriteableName() {
     protected MappedFieldType.Relation getRelation(final CoordinatorRewriteContext coordinatorRewriteContext) {
         final MappedFieldType fieldType = coordinatorRewriteContext.getFieldType(fieldName);
         if (fieldType instanceof final DateFieldMapper.DateFieldType dateFieldType) {
-            if (coordinatorRewriteContext.hasTimestampData() == false) {
+            IndexLongFieldRange fieldRange = coordinatorRewriteContext.getFieldRange(fieldName);
+            if (fieldRange.isComplete() == false || fieldRange == IndexLongFieldRange.EMPTY) {
+                // if not all shards for this (frozen) index have reported ranges to cluster state, OR if they
+                // have reported in and the range is empty (no data for that field), then return DISJOINT in order
+                // to rewrite the query to MatchNone
                 return MappedFieldType.Relation.DISJOINT;
             }
-            long minTimestamp = coordinatorRewriteContext.getMinTimestamp();
-            long maxTimestamp = coordinatorRewriteContext.getMaxTimestamp();
+            if (fieldRange == IndexLongFieldRange.UNKNOWN) {
+                // do a full search if UNKNOWN for whatever reason (e.g., event.ingested is UNKNOWN in a
+                // mixed-cluster where nodes with a version before event.ingested was added to cluster state)
+                return MappedFieldType.Relation.INTERSECTS;
+            }
             DateMathParser dateMathParser = getForceDateParser();
             return dateFieldType.isFieldWithinQuery(
-                minTimestamp,
-                maxTimestamp,
+                fieldRange.getMin(),
+                fieldRange.getMax(),
                 from,
                 to,
                 includeLower,
diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java
index 4b16a5833cf41..f25a0c73ac25d 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java
@@ -269,6 +269,7 @@ private SearchExecutionContext(
             valuesSourceRegistry,
             allowExpensiveQueries,
             scriptService,
+            null,
             null
         );
         this.shardId = shardId;
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index 4e708d58a8a63..dd9226b7b2f8a 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -24,6 +24,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.core.TimeValue.timeValueMillis;
@@ -248,7 +249,7 @@ public TimeValue getTimeout() {
      * Timeout to wait for the shards on to be available for each bulk request?
      */
     public Self setTimeout(TimeValue timeout) {
-        this.timeout = timeout;
+        this.timeout = Objects.requireNonNull(timeout);
         return self();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
index 3452c6659a392..411732dcc91e3 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
@@ -20,6 +20,7 @@ public DeleteByQueryRequestBuilder(ElasticsearchClient client) {
         this(client, new SearchRequestBuilder(client));
     }
 
+    @SuppressWarnings("this-escape")
     private DeleteByQueryRequestBuilder(ElasticsearchClient client, SearchRequestBuilder search) {
         super(client, DeleteByQueryAction.INSTANCE, search);
         source().setFetchSource(false);
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
index a051d9c2df430..1c02461409f5d 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
@@ -16,13 +16,13 @@
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -64,7 +64,8 @@ public GlobalCheckpointSyncAction(
             Request::new,
             threadPool.executor(ThreadPool.Names.WRITE),
             SyncGlobalCheckpointAfterOperation.DoNotSync,
-            PrimaryActionExecution.Force
+            PrimaryActionExecution.Force,
+            ReplicaActionExecution.SubjectToCircuitBreaker
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java
index 49548aa1a6353..ff141d771c3c0 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -30,6 +29,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
index 0aa0f0b8d1556..f90d8945857b7 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -27,6 +26,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -83,7 +83,8 @@ public RetentionLeaseBackgroundSyncAction(
             Request::new,
             threadPool.executor(ThreadPool.Names.MANAGEMENT),
             SyncGlobalCheckpointAfterOperation.DoNotSync,
-            PrimaryActionExecution.RejectOnOverload
+            PrimaryActionExecution.RejectOnOverload,
+            ReplicaActionExecution.BypassCircuitBreaker
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
index 0efcf8ac9298b..67ed7c6e4c191 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -39,6 +38,7 @@
 import org.elasticsearch.indices.ExecutorSelector;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -91,9 +91,10 @@ public RetentionLeaseSyncAction(
             RetentionLeaseSyncAction.Request::new,
             RetentionLeaseSyncAction.Request::new,
             new ManagementOnlyExecutorFunction(threadPool),
-            PrimaryActionExecution.RejectOnOverload,
+            PrimaryActionExecution.Force,
             indexingPressure,
-            systemIndices
+            systemIndices,
+            ReplicaActionExecution.BypassCircuitBreaker
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncer.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncer.java
index 6d29c5ea3e565..e5eca2a42af70 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncer.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncer.java
@@ -10,8 +10,8 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 
 import java.util.Objects;
 
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 73cbca36a69c8..b7d1beb4d1e06 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -17,6 +17,7 @@
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.ReferenceManager;
@@ -224,6 +225,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     private final IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier;
     private final Engine.IndexCommitListener indexCommitListener;
     private FieldInfos fieldInfos;
+    private volatile ShardFieldStats shardFieldStats;
+
     // sys prop to disable the field has value feature, defaults to true (enabled) if set to false (disabled) the
     // field caps always returns empty fields ignoring the value of the query param `field_caps_empty_fields_filter`.
     private final boolean enableFieldHasValue = Booleans.parseBoolean(
@@ -1425,7 +1428,8 @@ public CompletionStats completionStats(String... fields) {
 
     public DenseVectorStats denseVectorStats() {
         readAllowed();
-        return getEngine().denseVectorStats();
+        MappingLookup mappingLookup = mapperService != null ? mapperService.mappingLookup() : null;
+        return getEngine().denseVectorStats(mappingLookup);
     }
 
     public SparseVectorStats sparseVectorStats() {
@@ -3490,7 +3494,7 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) {
             translogConfig,
             IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()),
             List.of(refreshListeners, refreshPendingLocationListener, refreshFieldHasValueListener),
-            Collections.singletonList(new RefreshMetricUpdater(refreshMetric)),
+            List.of(new RefreshMetricUpdater(refreshMetric), new RefreshShardFieldStatsListener()),
             indexSort,
             circuitBreakerService,
             globalCheckpointSupplier,
@@ -4060,6 +4064,38 @@ public void afterRefresh(boolean didRefresh) {
         }
     }
 
+    /**
+     * Returns the shard-level field stats, which includes the number of segments in the latest NRT reader of this shard
+     * and the total number of fields across those segments.
+     */
+    public ShardFieldStats getShardFieldStats() {
+        return shardFieldStats;
+    }
+
+    private class RefreshShardFieldStatsListener implements ReferenceManager.RefreshListener {
+        @Override
+        public void beforeRefresh() {
+
+        }
+
+        @Override
+        public void afterRefresh(boolean didRefresh) {
+            if (shardFieldStats == null || didRefresh) {
+                try (var searcher = getEngine().acquireSearcher("shard_field_stats", Engine.SearcherScope.INTERNAL)) {
+                    int numSegments = 0;
+                    int totalFields = 0;
+                    for (LeafReaderContext leaf : searcher.getLeafContexts()) {
+                        numSegments++;
+                        totalFields += leaf.reader().getFieldInfos().size();
+                    }
+                    shardFieldStats = new ShardFieldStats(numSegments, totalFields);
+                } catch (AlreadyClosedException ignored) {
+
+                }
+            }
+        }
+    }
+
     /**
      * Ensures this shard is search active before invoking the provided listener.
      * <p>
diff --git a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
index 956858f094c95..5552c573faec8 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.action.resync.ResyncReplicationResponse;
 import org.elasticsearch.action.resync.TransportResyncReplicationAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -25,6 +24,7 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskManager;
diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java b/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java
new file mode 100644
index 0000000000000..9c53abb1e95e5
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.shard;
+
+/**
+ * A per shard stats including the number of segments and total fields across those segments.
+ * These stats should be recomputed whenever the shard is refreshed.
+ *
+ * @param numSegments the number of segments
+ * @param totalFields the total number of fields across the segments
+ */
+public record ShardFieldStats(int numSegments, int totalFields) {
+
+}
diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index 22a7249ec7237..d9866220558be 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -419,7 +419,7 @@ private void internalRecoverFromStore(IndexShard indexShard, ActionListener<Void
 
             .newForked(indexShard::preRecovery)
 
-            .<Void>andThen((l, ignored) -> {
+            .<Void>andThen(l -> {
                 final RecoveryState recoveryState = indexShard.recoveryState();
                 final boolean indexShouldExists = recoveryState.getRecoverySource().getType() != RecoverySource.Type.EMPTY_STORE;
                 indexShard.prepareForIndexRecovery();
@@ -490,7 +490,7 @@ private void internalRecoverFromStore(IndexShard indexShard, ActionListener<Void
                 indexShard.openEngineAndRecoverFromTranslog(l);
             })
 
-            .<Void>andThen((l, ignored) -> {
+            .<Void>andThen(l -> {
                 indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
                 indexShard.finalizeRecovery();
                 indexShard.postRecovery("post recovery from shard_store", l);
@@ -536,7 +536,7 @@ record ShardAndIndexIds(IndexId indexId, ShardId shardId) {}
 
             .newForked(indexShard::preRecovery)
 
-            .<ShardAndIndexIds>andThen((shardAndIndexIdsListener, ignored) -> {
+            .<ShardAndIndexIds>andThen(shardAndIndexIdsListener -> {
                 final RecoveryState.Translog translogState = indexShard.recoveryState().getTranslog();
                 if (restoreSource == null) {
                     throw new IndexShardRestoreFailedException(shardId, "empty restore source");
@@ -585,7 +585,7 @@ record ShardAndIndexIds(IndexId indexId, ShardId shardId) {}
                 );
             })
 
-            .<Void>andThen((l, ignored) -> {
+            .<Void>andThen(l -> {
                 indexShard.getIndexEventListener().afterFilesRestoredFromRepository(indexShard);
                 final Store store = indexShard.store();
                 bootstrap(indexShard, store);
@@ -594,7 +594,7 @@ record ShardAndIndexIds(IndexId indexId, ShardId shardId) {}
                 indexShard.openEngineAndRecoverFromTranslog(l);
             })
 
-            .<Void>andThen((l, ignored) -> {
+            .<Void>andThen(l -> {
                 indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
                 indexShard.finalizeRecovery();
                 indexShard.postRecovery("restore done", l);
diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
index 2a8fe96151c11..817ecb4601d59 100644
--- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
+++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.gateway.CorruptStateException;
 import org.elasticsearch.index.store.StoreFileMetadata;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentFragment;
@@ -318,7 +319,11 @@ public static FileInfo fromXContent(XContentParser parser) throws IOException {
                     }
                     case WRITER_UUID -> {
                         writerUuid = new BytesRef(parser.binaryValue());
-                        assert writerUuid.length > 0;
+                        assert BlobStoreIndexShardSnapshots.INTEGRITY_ASSERTIONS_ENABLED == false || writerUuid.length > 0;
+                        if (writerUuid.length == 0) {
+                            // we never write UNAVAILABLE_WRITER_UUID, so this must be due to corruption
+                            throw new ElasticsearchParseException("invalid (empty) writer uuid");
+                        }
                     }
                     default -> XContentParserUtils.throwUnknownField(currentFieldName, parser);
                 }
@@ -336,6 +341,12 @@ public static FileInfo fromXContent(XContentParser parser) throws IOException {
             } else if (checksum == null) {
                 throw new ElasticsearchParseException("missing checksum for name [" + name + "]");
             }
+            try {
+                // check for corruption before asserting writtenBy is parseable in the StoreFileMetadata constructor
+                org.apache.lucene.util.Version.parse(writtenBy);
+            } catch (Exception e) {
+                throw new ElasticsearchParseException("invalid written_by [" + writtenBy + "]");
+            }
             return new FileInfo(name, new StoreFileMetadata(physicalName, length, checksum, writtenBy, metaHash, writerUuid), partSize);
         }
 
@@ -566,6 +577,11 @@ public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) th
             }
         }
 
+        // check for corruption before asserting snapshot != null in the BlobStoreIndexShardSnapshot ctor
+        if (snapshot == null) {
+            throw new CorruptStateException("snapshot missing");
+        }
+
         return new BlobStoreIndexShardSnapshot(
             snapshot,
             indexFiles == null ? List.of() : indexFiles,
diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
index b17545a4cbeb6..30fbbba5ed095 100644
--- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
+++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
@@ -264,6 +264,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
+    static volatile boolean INTEGRITY_ASSERTIONS_ENABLED = true;
+
     public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
         if (token == null) { // New parser
@@ -317,7 +319,12 @@ public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) t
             List<FileInfo> fileInfosBuilder = new ArrayList<>();
             for (String file : entry.v2()) {
                 FileInfo fileInfo = files.get(file);
-                assert fileInfo != null;
+                if (fileInfo == null) {
+                    // could happen in production if the repo contents are corrupted
+                    final var exception = new IllegalStateException("shard index inconsistent at file [" + file + "]");
+                    assert INTEGRITY_ASSERTIONS_ENABLED == false : exception;
+                    throw exception;
+                }
                 fileInfosBuilder.add(fileInfo);
             }
             snapshots.add(new SnapshotFiles(entry.v1(), Collections.unmodifiableList(fileInfosBuilder), historyUUIDs.get(entry.v1())));
diff --git a/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java b/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java
index 1316776ec39b2..21ccdd56900f8 100644
--- a/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java
+++ b/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java
@@ -73,7 +73,7 @@ public IndexingPressureStats(StreamInput in) throws IOException {
         this.currentPrimaryOps = 0;
         this.currentReplicaOps = 0;
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.INDEXING_PRESSURE_DOCUMENT_REJECTIONS_COUNT)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             primaryDocumentRejections = in.readVLong();
         } else {
             primaryDocumentRejections = -1L;
@@ -152,7 +152,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(memoryLimit);
         }
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.INDEXING_PRESSURE_DOCUMENT_REJECTIONS_COUNT)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeVLong(primaryDocumentRejections);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java b/server/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java
index 6d1456040c8fa..9420d923107e1 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/BufferedChecksumStreamInput.java
@@ -66,6 +66,15 @@ public void readBytes(byte[] b, int offset, int len) throws IOException {
         digest.update(b, offset, len);
     }
 
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        int read = delegate.read(b, off, len);
+        if (read > 0) {
+            digest.update(b, off, read);
+        }
+        return read;
+    }
+
     private static final ThreadLocal<byte[]> buffer = ThreadLocal.withInitial(() -> new byte[8]);
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
index 3f21c3a26ea04..7e1de5a9bc77c 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
@@ -15,17 +15,15 @@
 import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.store.OutputStreamIndexOutput;
 import org.elasticsearch.common.io.Channels;
+import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
+import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
@@ -166,8 +164,9 @@ public String toString() {
     }
 
     public static Checkpoint read(Path path) throws IOException {
-        try (Directory dir = new NIOFSDirectory(path.getParent())) {
-            try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) {
+        try {
+            final byte[] bytes = Files.readAllBytes(path);
+            try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(path.toString(), bytes, 0, bytes.length)) {
                 // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
                 CodecUtil.checksumEntireFile(indexInput);
                 final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, VERSION_LUCENE_8, CURRENT_VERSION);
@@ -177,9 +176,9 @@ public static Checkpoint read(Path path) throws IOException {
                     return Checkpoint.readCheckpointV4(indexInput);
                 }
                 return readCheckpointV3(indexInput);
-            } catch (CorruptIndexException | NoSuchFileException | IndexFormatTooOldException | IndexFormatTooNewException e) {
-                throw new TranslogCorruptedException(path.toString(), e);
             }
+        } catch (CorruptIndexException | NoSuchFileException | IndexFormatTooOldException | IndexFormatTooNewException e) {
+            throw new TranslogCorruptedException(path.toString(), e);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/indices/DateFieldRangeInfo.java b/server/src/main/java/org/elasticsearch/indices/DateFieldRangeInfo.java
new file mode 100644
index 0000000000000..b631806e3ce95
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/indices/DateFieldRangeInfo.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices;
+
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.shard.IndexLongFieldRange;
+
+/**
+ * Data holder of timestamp fields held in cluster state IndexMetadata.
+ * @param timestampFieldType field type for the @timestamp field
+ * @param timestampRange min/max range for the @timestamp field (in a specific index)
+ * @param eventIngestedFieldType field type for the 'event.ingested' field
+ * @param eventIngestedRange min/max range for the 'event.ingested' field (in a specific index)
+ */
+public record DateFieldRangeInfo(
+    DateFieldMapper.DateFieldType timestampFieldType,
+    IndexLongFieldRange timestampRange,
+    DateFieldMapper.DateFieldType eventIngestedFieldType,
+    IndexLongFieldRange eventIngestedRange
+) {
+
+}
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
index 03df21531d4cc..d6b90c38f5064 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.action.admin.indices.rollover.MinSizeCondition;
 import org.elasticsearch.action.admin.indices.rollover.OptimalShardCountCondition;
 import org.elasticsearch.action.resync.TransportResyncReplicationAction;
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.index.mapper.BinaryFieldMapper;
 import org.elasticsearch.index.mapper.BooleanFieldMapper;
@@ -74,6 +73,7 @@
 import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.store.IndicesStore;
+import org.elasticsearch.injection.guice.AbstractModule;
 import org.elasticsearch.plugins.FieldPredicate;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
index 0d81d24e64646..decc082d314e6 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.ResolvedIndices;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.TransportAutoPutMappingAction;
@@ -98,7 +99,6 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
-import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
@@ -137,6 +137,7 @@
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
+import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
@@ -398,7 +399,7 @@ protected void doStop() {
         final CountDownLatch latch = new CountDownLatch(indices.size());
         for (final Index index : indices) {
             indicesStopExecutor.execute(
-                () -> ActionListener.run(
+                ActionRunnable.wrap(
                     ActionListener.assertOnce(ActionListener.<Void>releasing(latch::countDown)),
                     l -> removeIndex(
                         index,
@@ -1755,8 +1756,8 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, Set<String
     /**
      * Returns a new {@link QueryRewriteContext} with the given {@code now} provider
      */
-    public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis, ResolvedIndices resolvedIndices) {
-        return new QueryRewriteContext(parserConfig, client, nowInMillis, resolvedIndices);
+    public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis, ResolvedIndices resolvedIndices, PointInTimeBuilder pit) {
+        return new QueryRewriteContext(parserConfig, client, nowInMillis, resolvedIndices, pit);
     }
 
     public DataRewriteContext getDataRewriteContext(LongSupplier nowInMillis) {
@@ -1764,7 +1765,13 @@ public DataRewriteContext getDataRewriteContext(LongSupplier nowInMillis) {
     }
 
     public CoordinatorRewriteContextProvider getCoordinatorRewriteContextProvider(LongSupplier nowInMillis) {
-        return new CoordinatorRewriteContextProvider(parserConfig, client, nowInMillis, clusterService::state, this::getTimestampFieldType);
+        return new CoordinatorRewriteContextProvider(
+            parserConfig,
+            client,
+            nowInMillis,
+            clusterService::state,
+            this::getTimestampFieldTypeInfo
+        );
     }
 
     /**
@@ -1854,14 +1861,16 @@ public boolean allPendingDanglingIndicesWritten() {
     }
 
     /**
-     * @return the field type of the {@code @timestamp} field of the given index, or {@code null} if:
+     * @return DateFieldRangeInfo holding the field types of the {@code @timestamp} and {@code event.ingested} fields of the index.
+     * or {@code null} if:
      * - the index is not found,
      * - the field is not found, or
-     * - the field is not a timestamp field.
+     * - the mapping is not known yet, or
+     * - the index does not have a useful timestamp field.
      */
     @Nullable
-    public DateFieldMapper.DateFieldType getTimestampFieldType(Index index) {
-        return timestampFieldMapperService.getTimestampFieldType(index);
+    public DateFieldRangeInfo getTimestampFieldTypeInfo(Index index) {
+        return timestampFieldMapperService.getTimestampFieldTypeInfo(index);
     }
 
     public IndexScopedSettings getIndexScopedSettings() {
diff --git a/server/src/main/java/org/elasticsearch/indices/ShardLimitValidator.java b/server/src/main/java/org/elasticsearch/indices/ShardLimitValidator.java
index 06bb1439be4e6..f58ee757cc511 100644
--- a/server/src/main/java/org/elasticsearch/indices/ShardLimitValidator.java
+++ b/server/src/main/java/org/elasticsearch/indices/ShardLimitValidator.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -296,7 +297,8 @@ static String errorMessageFrom(Result result) {
             + result.maxShardsInCluster
             + "] maximum "
             + result.group
-            + " shards open";
+            + " shards open; for more information, see "
+            + ReferenceDocs.MAX_SHARDS_PER_NODE;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/indices/TimestampFieldMapperService.java b/server/src/main/java/org/elasticsearch/indices/TimestampFieldMapperService.java
index 4caeaef6514e5..b139fca5c2acc 100644
--- a/server/src/main/java/org/elasticsearch/indices/TimestampFieldMapperService.java
+++ b/server/src/main/java/org/elasticsearch/indices/TimestampFieldMapperService.java
@@ -42,8 +42,9 @@
 import static org.elasticsearch.core.Strings.format;
 
 /**
- * Tracks the mapping of the {@code @timestamp} field of immutable indices that expose their timestamp range in their index metadata.
- * Coordinating nodes do not have (easy) access to mappings for all indices, so we extract the type of this one field from the mapping here.
+ * Tracks the mapping of the '@timestamp' and 'event.ingested' fields of immutable indices that expose their timestamp range in their
+ * index metadata. Coordinating nodes do not have (easy) access to mappings for all indices, so we extract the type of these two fields
+ * from the mapping here, since timestamp fields can have millis or nanos level resolution.
  */
 public class TimestampFieldMapperService extends AbstractLifecycleComponent implements ClusterStateApplier {
 
@@ -53,10 +54,12 @@ public class TimestampFieldMapperService extends AbstractLifecycleComponent impl
     private final ExecutorService executor; // single thread to construct mapper services async as needed
 
     /**
-     * The type of the {@code @timestamp} field keyed by index. Futures may be completed with {@code null} to indicate that there is
-     * no usable {@code @timestamp} field.
+     * The type of the 'event.ingested' and/or '@timestamp' fields keyed by index.
+     * The inner map is keyed by field name ('@timestamp' or 'event.ingested').
+     * Futures may be completed with {@code null} to indicate that there is
+     * no usable timestamp field.
      */
-    private final Map<Index, PlainActionFuture<DateFieldMapper.DateFieldType>> fieldTypesByIndex = ConcurrentCollections.newConcurrentMap();
+    private final Map<Index, PlainActionFuture<DateFieldRangeInfo>> fieldTypesByIndex = ConcurrentCollections.newConcurrentMap();
 
     public TimestampFieldMapperService(Settings settings, ThreadPool threadPool, IndicesService indicesService) {
         this.indicesService = indicesService;
@@ -103,7 +106,7 @@ public void applyClusterState(ClusterChangedEvent event) {
 
             if (hasUsefulTimestampField(indexMetadata) && fieldTypesByIndex.containsKey(index) == false) {
                 logger.trace("computing timestamp mapping for {}", index);
-                final PlainActionFuture<DateFieldMapper.DateFieldType> future = new PlainActionFuture<>();
+                final PlainActionFuture<DateFieldRangeInfo> future = new PlainActionFuture<>();
                 fieldTypesByIndex.put(index, future);
 
                 final IndexService indexService = indicesService.indexService(index);
@@ -148,29 +151,45 @@ private static boolean hasUsefulTimestampField(IndexMetadata indexMetadata) {
             return true;
         }
 
-        final IndexLongFieldRange timestampRange = indexMetadata.getTimestampRange();
-        return timestampRange.isComplete() && timestampRange != IndexLongFieldRange.UNKNOWN;
+        IndexLongFieldRange timestampRange = indexMetadata.getTimestampRange();
+        if (timestampRange.isComplete() && timestampRange != IndexLongFieldRange.UNKNOWN) {
+            return true;
+        }
+
+        IndexLongFieldRange eventIngestedRange = indexMetadata.getEventIngestedRange();
+        return eventIngestedRange.isComplete() && eventIngestedRange != IndexLongFieldRange.UNKNOWN;
     }
 
-    private static DateFieldMapper.DateFieldType fromMapperService(MapperService mapperService) {
-        final MappedFieldType mappedFieldType = mapperService.fieldType(DataStream.TIMESTAMP_FIELD_NAME);
-        if (mappedFieldType instanceof DateFieldMapper.DateFieldType) {
-            return (DateFieldMapper.DateFieldType) mappedFieldType;
-        } else {
+    private static DateFieldRangeInfo fromMapperService(MapperService mapperService) {
+        DateFieldMapper.DateFieldType timestampFieldType = null;
+        DateFieldMapper.DateFieldType eventIngestedFieldType = null;
+
+        MappedFieldType mappedFieldType = mapperService.fieldType(DataStream.TIMESTAMP_FIELD_NAME);
+        if (mappedFieldType instanceof DateFieldMapper.DateFieldType dateFieldType) {
+            timestampFieldType = dateFieldType;
+        }
+        mappedFieldType = mapperService.fieldType(IndexMetadata.EVENT_INGESTED_FIELD_NAME);
+        if (mappedFieldType instanceof DateFieldMapper.DateFieldType dateFieldType) {
+            eventIngestedFieldType = dateFieldType;
+        }
+        if (timestampFieldType == null && eventIngestedFieldType == null) {
             return null;
         }
+        // the mapper only fills in the field types, not the actual range values
+        return new DateFieldRangeInfo(timestampFieldType, null, eventIngestedFieldType, null);
     }
 
     /**
-     * @return the field type of the {@code @timestamp} field of the given index, or {@code null} if:
+     * @return DateFieldRangeInfo holding the field types of the {@code @timestamp} and {@code event.ingested} fields of the index.
+     * or {@code null} if:
      * - the index is not found,
      * - the field is not found,
      * - the mapping is not known yet, or
-     * - the field is not a timestamp field.
+     * - the index does not have a useful timestamp field.
      */
     @Nullable
-    public DateFieldMapper.DateFieldType getTimestampFieldType(Index index) {
-        final PlainActionFuture<DateFieldMapper.DateFieldType> future = fieldTypesByIndex.get(index);
+    public DateFieldRangeInfo getTimestampFieldTypeInfo(Index index) {
+        final PlainActionFuture<DateFieldRangeInfo> future = fieldTypesByIndex.get(index);
         if (future == null || future.isDone() == false) {
             return null;
         }
@@ -181,5 +200,4 @@ public DateFieldMapper.DateFieldType getTimestampFieldType(Index index) {
             throw new UncategorizedExecutionException("An error occurred fetching timestamp field type for " + index, e);
         }
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index dd5ad26c58b12..c08e801b8fb5d 100644
--- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
@@ -73,6 +72,7 @@
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.snapshots.SnapshotShardsService;
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
index ac618ac9308c4..ff35bc75c0a9d 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
@@ -380,7 +380,7 @@ record StartRecoveryRequestToSend(StartRecoveryRequest startRecoveryRequest, Str
                 // run pre-recovery activities
                 .newForked(indexShard::preRecovery)
                 // recover the shard as far as possible based on data held locally
-                .<Long>andThen((l, v) -> {
+                .<Long>andThen(l -> {
                     logger.trace("{} preparing shard for peer recovery", recoveryTarget.shardId());
                     indexShard.prepareForIndexRecovery();
                     if (indexShard.indexSettings().getIndexMetadata().isSearchableSnapshot()) {
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index df2a9d16ebd6a..0f00f48561b75 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -700,7 +700,7 @@ void run(ActionListener<SendFileResult> listener) {
                 .newForked(this::sendShardRecoveryPlanFileInfo)
                 // instruct the target to recover files from snapshot, possibly updating the plan on failure
                 .<List<StoreFileMetadata>>andThen(
-                    (l, ignored) -> recoverSnapshotFiles(shardRecoveryPlan, l.delegateResponse((recoverSnapshotFilesListener, e) -> {
+                    l -> recoverSnapshotFiles(shardRecoveryPlan, l.delegateResponse((recoverSnapshotFilesListener, e) -> {
                         if (shardRecoveryPlan.canRecoverSnapshotFilesFromSourceNode() == false
                             && e instanceof CancellableThreads.ExecutionCancelledException == false) {
                             shardRecoveryPlan = shardRecoveryPlan.getFallbackPlan();
@@ -731,10 +731,7 @@ void run(ActionListener<SendFileResult> listener) {
                 })
                 // create a retention lease
                 .<RetentionLease>andThen(
-                    (createRetentionLeaseListener, ignored) -> createRetentionLease(
-                        shardRecoveryPlan.getStartingSeqNo(),
-                        createRetentionLeaseListener
-                    )
+                    createRetentionLeaseListener -> createRetentionLease(shardRecoveryPlan.getStartingSeqNo(), createRetentionLeaseListener)
                 )
                 // run cleanFiles, renaming temp files, removing surplus ones, creating an empty translog and so on
                 .<Void>andThen((finalRecoveryPlanListener, retentionLease) -> {
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsService.java b/server/src/main/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsService.java
index c414f7c100633..2dcedc98c9f20 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsService.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsService.java
@@ -18,24 +18,23 @@
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Version;
-import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.snapshots.get.shard.GetShardSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.shard.GetShardSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.get.shard.TransportGetShardSnapshotAction;
 import org.elasticsearch.action.support.ThreadedActionListener;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.blobstore.BlobContainer;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
 import org.elasticsearch.index.store.StoreFileMetadata;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.ShardSnapshotInfo;
@@ -94,9 +93,7 @@ public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener<Optiona
         logger.debug("Searching for peer recovery compatible snapshots in [{}]", repositories);
 
         GetShardSnapshotRequest request = GetShardSnapshotRequest.latestSnapshotInRepositories(
-            clusterService.state().getMinTransportVersion().onOrAfter(TransportVersions.SNAPSHOT_REQUEST_TIMEOUTS)
-                ? TimeValue.MINUS_ONE
-                : TimeValue.MAX_VALUE,
+            MasterNodeRequest.infiniteMasterNodeTimeout(clusterService.state().getMinTransportVersion()),
             shardId,
             repositories
         );
diff --git a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
index b8fd05f5b5224..ff566e41fa48f 100644
--- a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
+++ b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
@@ -42,6 +41,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportChannel;
diff --git a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java
index 532aca07e5513..718e8c716f39b 100644
--- a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java
+++ b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -42,6 +41,7 @@
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.store.StoreFileMetadata;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
diff --git a/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java b/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java
new file mode 100644
index 0000000000000..5c6acb78a91e3
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/inference/EmptySecretSettings.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.inference;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * This class defines an empty secret settings object. This is useful for services that do not have any secret settings.
+ */
+public record EmptySecretSettings() implements SecretSettings {
+    public static final String NAME = "empty_secret_settings";
+
+    public static final EmptySecretSettings INSTANCE = new EmptySecretSettings();
+
+    public EmptySecretSettings(StreamInput in) {
+        this();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_EIS_INTEGRATION_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {}
+}
diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java
index 62166115820f5..0000e0ddc9af9 100644
--- a/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java
+++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java
@@ -9,21 +9,22 @@
 package org.elasticsearch.inference;
 
 import org.elasticsearch.common.io.stream.NamedWriteable;
-import org.elasticsearch.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Flow;
 
-public interface InferenceServiceResults extends NamedWriteable, ToXContentFragment {
+public interface InferenceServiceResults extends NamedWriteable, ChunkedToXContent {
 
     /**
-     * Transform the result to match the format required for the TransportCoordinatedInferenceAction.
+     * <p>Transform the result to match the format required for the TransportCoordinatedInferenceAction.
      * For the inference plugin TextEmbeddingResults, the {@link #transformToLegacyFormat()} transforms the
      * results into an intermediate format only used by the plugin's return value. It doesn't align with what the
      * TransportCoordinatedInferenceAction expects. TransportCoordinatedInferenceAction expects an ml plugin
-     * TextEmbeddingResults.
+     * TextEmbeddingResults.</p>
      *
-     * For other results like SparseEmbeddingResults, this method can be a pass through to the transformToLegacyFormat.
+     * <p>For other results like SparseEmbeddingResults, this method can be a pass through to the transformToLegacyFormat.</p>
      */
     List<? extends InferenceResults> transformToCoordinationFormat();
 
@@ -37,4 +38,21 @@ public interface InferenceServiceResults extends NamedWriteable, ToXContentFragm
      * Convert the result to a map to aid with test assertions
      */
     Map<String, Object> asMap();
+
+    /**
+     * Returns {@code true} if these results are streamed as chunks, or {@code false} if these results contain the entire payload.
+     * Defaults to {@code false}.
+     */
+    default boolean isStreaming() {
+        return false;
+    }
+
+    /**
+     * When {@link #isStreaming()} is {@code true}, the InferenceAction.Results will subscribe to this publisher.
+     * Implementations should follow the {@link java.util.concurrent.Flow.Publisher} spec to stream the chunks.
+     */
+    default Flow.Publisher<ChunkedToXContent> publisher() {
+        assert isStreaming() == false : "This must be implemented when isStreaming() == true";
+        throw new UnsupportedOperationException("This must be implemented when isStreaming() == true");
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/inference/ModelConfigurations.java b/server/src/main/java/org/elasticsearch/inference/ModelConfigurations.java
index 4b6f436460fdc..ce83a7c1b6c79 100644
--- a/server/src/main/java/org/elasticsearch/inference/ModelConfigurations.java
+++ b/server/src/main/java/org/elasticsearch/inference/ModelConfigurations.java
@@ -20,7 +20,11 @@
 
 public class ModelConfigurations implements ToFilteredXContentObject, VersionedNamedWriteable {
 
-    public static final String MODEL_ID = "model_id";
+    // Due to refactoring, we now have different field names for the inference ID when it is serialized and stored to an index vs when it
+    // is returned as part of a GetInferenceModelAction
+    public static final String INDEX_ONLY_ID_FIELD_NAME = "model_id";
+    public static final String INFERENCE_ID_FIELD_NAME = "inference_id";
+    public static final String USE_ID_FOR_INDEX = "for_index";
     public static final String SERVICE = "service";
     public static final String SERVICE_SETTINGS = "service_settings";
     public static final String TASK_SETTINGS = "task_settings";
@@ -119,7 +123,12 @@ public TaskSettings getTaskSettings() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(MODEL_ID, inferenceEntityId);
+        if (params.paramAsBoolean(USE_ID_FOR_INDEX, false)) {
+            builder.field(INDEX_ONLY_ID_FIELD_NAME, inferenceEntityId);
+        } else {
+            builder.field(INDEX_ONLY_ID_FIELD_NAME, inferenceEntityId);
+            builder.field(INFERENCE_ID_FIELD_NAME, inferenceEntityId);
+        }
         builder.field(TaskType.NAME, taskType.toString());
         builder.field(SERVICE, service);
         builder.field(SERVICE_SETTINGS, serviceSettings);
@@ -131,7 +140,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     @Override
     public XContentBuilder toFilteredXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(MODEL_ID, inferenceEntityId);
+        if (params.paramAsBoolean(USE_ID_FOR_INDEX, false)) {
+            builder.field(INDEX_ONLY_ID_FIELD_NAME, inferenceEntityId);
+        } else {
+            builder.field(INDEX_ONLY_ID_FIELD_NAME, inferenceEntityId);
+            builder.field(INFERENCE_ID_FIELD_NAME, inferenceEntityId);
+        }
         builder.field(TaskType.NAME, taskType.toString());
         builder.field(SERVICE, service);
         builder.field(SERVICE_SETTINGS, serviceSettings.getFilteredXContentObject());
diff --git a/server/src/main/java/org/elasticsearch/inference/ServiceSettings.java b/server/src/main/java/org/elasticsearch/inference/ServiceSettings.java
index 34a58f83963ce..58e87105f70a3 100644
--- a/server/src/main/java/org/elasticsearch/inference/ServiceSettings.java
+++ b/server/src/main/java/org/elasticsearch/inference/ServiceSettings.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.inference;
 
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.xcontent.ToXContentObject;
 
@@ -48,5 +49,6 @@ default DenseVectorFieldMapper.ElementType elementType() {
      * be chosen when initializing a deployment within their service. In this situation, return null.
      * @return the model used to perform inference or null if the model is not defined
      */
+    @Nullable
     String modelId();
 }
diff --git a/server/src/main/java/org/elasticsearch/inference/SimilarityMeasure.java b/server/src/main/java/org/elasticsearch/inference/SimilarityMeasure.java
index ff9fedee02fac..058631044c8b1 100644
--- a/server/src/main/java/org/elasticsearch/inference/SimilarityMeasure.java
+++ b/server/src/main/java/org/elasticsearch/inference/SimilarityMeasure.java
@@ -39,8 +39,7 @@ public static SimilarityMeasure fromString(String name) {
      * @return the similarity that is known to the version passed in
      */
     public static SimilarityMeasure translateSimilarity(SimilarityMeasure similarityMeasure, TransportVersion version) {
-        if (version.before(TransportVersions.ML_INFERENCE_L2_NORM_SIMILARITY_ADDED)
-            && BEFORE_L2_NORM_ENUMS.contains(similarityMeasure) == false) {
+        if (version.before(TransportVersions.V_8_14_0) && BEFORE_L2_NORM_ENUMS.contains(similarityMeasure) == false) {
             return null;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestPipelineException.java b/server/src/main/java/org/elasticsearch/ingest/IngestPipelineException.java
new file mode 100644
index 0000000000000..a6986f18e09d3
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestPipelineException.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchWrapperException;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.ingest.CompoundProcessor.PIPELINE_ORIGIN_EXCEPTION_HEADER;
+
+/**
+ * A dedicated wrapper for exceptions encountered while executing an ingest pipeline. Unlike {@link IngestProcessorException}, this
+ * exception indicates an issue with the overall pipeline execution, either due to mid-process validation problem or other non-processor
+ * level issues with the execution. The wrapper is needed as we currently only unwrap causes for instances of
+ * {@link ElasticsearchWrapperException}.
+ */
+public class IngestPipelineException extends ElasticsearchException implements ElasticsearchWrapperException {
+
+    IngestPipelineException(final String pipeline, final Exception cause) {
+        super(cause);
+        this.addHeader(PIPELINE_ORIGIN_EXCEPTION_HEADER, List.of(pipeline));
+    }
+
+    public IngestPipelineException(final StreamInput in) throws IOException {
+        super(in);
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java
index 357fea343ea55..20f97e1871483 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java
@@ -11,6 +11,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.util.Strings;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
@@ -18,6 +19,7 @@
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.action.bulk.FailureStoreMetrics;
 import org.elasticsearch.action.bulk.TransportBulkAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
@@ -61,7 +63,8 @@
 import org.elasticsearch.node.ReportingService;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.internal.DocumentParsingProvider;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
+import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator;
+import org.elasticsearch.plugins.internal.XContentParserDecorator;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -87,6 +90,7 @@
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.function.IntConsumer;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -116,6 +120,7 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
     private volatile Map<String, PipelineHolder> pipelines = Map.of();
     private final ThreadPool threadPool;
     private final IngestMetric totalMetrics = new IngestMetric();
+    private final FailureStoreMetrics failureStoreMetrics;
     private final List<Consumer<ClusterState>> ingestClusterStateListeners = new CopyOnWriteArrayList<>();
     private volatile ClusterState state;
 
@@ -128,7 +133,12 @@ public static MatcherWatchdog createGrokThreadWatchdog(Environment env, ThreadPo
         final BiFunction<Long, Runnable, Scheduler.ScheduledCancellable> scheduler = createScheduler(threadPool);
         long intervalMillis = IngestSettings.GROK_WATCHDOG_INTERVAL.get(settings).getMillis();
         long maxExecutionTimeMillis = IngestSettings.GROK_WATCHDOG_INTERVAL.get(settings).getMillis();
-        return MatcherWatchdog.newInstance(intervalMillis, maxExecutionTimeMillis, threadPool::relativeTimeInMillis, scheduler::apply);
+        return MatcherWatchdog.newInstance(
+            intervalMillis,
+            maxExecutionTimeMillis,
+            threadPool.relativeTimeInMillisSupplier(),
+            scheduler::apply
+        );
     }
 
     /**
@@ -184,7 +194,8 @@ public IngestService(
         List<IngestPlugin> ingestPlugins,
         Client client,
         MatcherWatchdog matcherWatchdog,
-        DocumentParsingProvider documentParsingProvider
+        DocumentParsingProvider documentParsingProvider,
+        FailureStoreMetrics failureStoreMetrics
     ) {
         this.clusterService = clusterService;
         this.scriptService = scriptService;
@@ -196,7 +207,7 @@ public IngestService(
                 scriptService,
                 analysisRegistry,
                 threadPool.getThreadContext(),
-                threadPool::relativeTimeInMillis,
+                threadPool.relativeTimeInMillisSupplier(),
                 createScheduler(threadPool),
                 this,
                 client,
@@ -206,6 +217,7 @@ public IngestService(
         );
         this.threadPool = threadPool;
         this.taskQueue = clusterService.createTaskQueue("ingest-pipelines", Priority.NORMAL, PIPELINE_TASK_EXECUTOR);
+        this.failureStoreMetrics = failureStoreMetrics;
     }
 
     /**
@@ -222,6 +234,7 @@ public IngestService(
         this.taskQueue = ingestService.taskQueue;
         this.pipelines = ingestService.pipelines;
         this.state = ingestService.state;
+        this.failureStoreMetrics = ingestService.failureStoreMetrics;
     }
 
     private static Map<String, Processor.Factory> processorFactories(List<IngestPlugin> ingestPlugins, Processor.Parameters parameters) {
@@ -685,7 +698,7 @@ private static IngestPipelinesExecutionResult failAndStoreFor(String index, Exce
      * @param actionRequests The collection of requests to be processed.
      * @param onDropped A callback executed when a document is dropped by a pipeline.
      *                  Accepts the slot in the collection of requests that the document occupies.
-     * @param shouldStoreFailure A predicate executed on each ingest failure to determine if the
+     * @param resolveFailureStore A function executed on each ingest failure to determine if the
      *                           failure should be stored somewhere.
      * @param onStoreFailure A callback executed when a document fails ingest but the failure should
      *                       be persisted elsewhere. Accepts the slot in the collection of requests
@@ -703,7 +716,7 @@ public void executeBulkRequest(
         final int numberOfActionRequests,
         final Iterable<DocWriteRequest<?>> actionRequests,
         final IntConsumer onDropped,
-        final Predicate<String> shouldStoreFailure,
+        final Function<String, Boolean> resolveFailureStore,
         final TriConsumer<Integer, String, Exception> onStoreFailure,
         final BiConsumer<Integer, Exception> onFailure,
         final BiConsumer<Thread, Exception> onCompletion,
@@ -745,8 +758,10 @@ protected void doRun() {
                         }
                         final int slot = i;
                         final Releasable ref = refs.acquire();
-                        final DocumentSizeObserver documentSizeObserver = documentParsingProvider.newDocumentSizeObserver(indexRequest);
-                        final IngestDocument ingestDocument = newIngestDocument(indexRequest, documentSizeObserver);
+                        final XContentMeteringParserDecorator meteringParserDecorator = documentParsingProvider.newMeteringParserDecorator(
+                            indexRequest
+                        );
+                        final IngestDocument ingestDocument = newIngestDocument(indexRequest, meteringParserDecorator);
                         final org.elasticsearch.script.Metadata originalDocumentMetadata = ingestDocument.getMetadata().clone();
                         // the document listener gives us three-way logic: a document can fail processing (1), or it can
                         // be successfully processed. a successfully processed document can be kept (2) or dropped (3).
@@ -786,8 +801,8 @@ public void onFailure(Exception e) {
                             }
                         );
 
-                        executePipelines(pipelines, indexRequest, ingestDocument, shouldStoreFailure, documentListener);
-                        indexRequest.setNormalisedBytesParsed(documentSizeObserver.normalisedBytesParsed());
+                        executePipelines(pipelines, indexRequest, ingestDocument, resolveFailureStore, documentListener);
+                        indexRequest.setNormalisedBytesParsed(meteringParserDecorator.meteredDocumentSize().ingestedBytes());
                         assert actionRequest.index() != null;
 
                         i++;
@@ -877,7 +892,7 @@ private void executePipelines(
         final PipelineIterator pipelines,
         final IndexRequest indexRequest,
         final IngestDocument ingestDocument,
-        final Predicate<String> shouldStoreFailure,
+        final Function<String, Boolean> resolveFailureStore,
         final ActionListener<IngestPipelinesExecutionResult> listener
     ) {
         assert pipelines.hasNext();
@@ -890,9 +905,22 @@ private void executePipelines(
         ingestDocument.resetReroute();
         final String originalIndex = indexRequest.indices()[0];
         final Consumer<Exception> exceptionHandler = (Exception e) -> {
-            if (shouldStoreFailure.test(originalIndex)) {
+            String errorType = ElasticsearchException.getExceptionName(ExceptionsHelper.unwrapCause(e));
+            // If `failureStoreResolution` is true, we store the failure. If it's false, the target is a data stream,
+            // but it doesn't have the failure store enabled. If it's null, the target wasn't a data stream.
+            Boolean failureStoreResolution = resolveFailureStore.apply(originalIndex);
+            if (failureStoreResolution != null && failureStoreResolution) {
+                failureStoreMetrics.incrementFailureStore(originalIndex, errorType, FailureStoreMetrics.ErrorLocation.PIPELINE);
                 listener.onResponse(IngestPipelinesExecutionResult.failAndStoreFor(originalIndex, e));
             } else {
+                if (failureStoreResolution != null) {
+                    // If this document targeted a data stream that didn't have the failure store enabled, we increment
+                    // the rejected counter.
+                    // We also increment the total counter because this request will not reach the code that increments
+                    // the total counter for non-rejected documents.
+                    failureStoreMetrics.incrementTotal(originalIndex);
+                    failureStoreMetrics.incrementRejected(originalIndex, errorType, FailureStoreMetrics.ErrorLocation.PIPELINE, false);
+                }
                 listener.onFailure(e);
             }
         };
@@ -920,6 +948,20 @@ private void executePipelines(
                 }
 
                 if (keep == false) {
+                    // We only increment the total counter for dropped docs here, because these docs don't reach the code
+                    // that ordinarily take care of that.
+                    // We reuse `resolveFailureStore` here to determine whether the index request targets a data stream,
+                    // because we only want to track these metrics for data streams.
+                    Boolean failureStoreResolution = resolveFailureStore.apply(originalIndex);
+                    if (failureStoreResolution != null) {
+                        // Get index abstraction, resolving date math if it exists
+                        IndexAbstraction indexAbstraction = state.metadata()
+                            .getIndicesLookup()
+                            .get(IndexNameExpressionResolver.resolveDateMathExpression(originalIndex, threadPool.absoluteTimeInMillis()));
+                        DataStream dataStream = DataStream.resolveDataStream(indexAbstraction, state.metadata());
+                        String dataStreamName = dataStream != null ? dataStream.getName() : originalIndex;
+                        failureStoreMetrics.incrementTotal(dataStreamName);
+                    }
                     listener.onResponse(IngestPipelinesExecutionResult.DISCARD_RESULT);
                     return; // document dropped!
                 }
@@ -936,14 +978,17 @@ private void executePipelines(
                     // An IllegalArgumentException can be thrown when an ingest processor creates a source map that is self-referencing.
                     // In that case, we catch and wrap the exception, so we can include more details
                     exceptionHandler.accept(
-                        new IllegalArgumentException(
-                            format(
-                                "Failed to generate the source document for ingest pipeline [%s] for document [%s/%s]",
-                                pipelineId,
-                                indexRequest.index(),
-                                indexRequest.id()
-                            ),
-                            ex
+                        new IngestPipelineException(
+                            pipelineId,
+                            new IllegalArgumentException(
+                                format(
+                                    "Failed to generate the source document for ingest pipeline [%s] for document [%s/%s]",
+                                    pipelineId,
+                                    indexRequest.index(),
+                                    indexRequest.id()
+                                ),
+                                ex
+                            )
                         )
                     );
                     return; // document failed!
@@ -955,14 +1000,18 @@ private void executePipelines(
                 if (Objects.equals(originalIndex, newIndex) == false) {
                     // final pipelines cannot change the target index (either directly or by way of a reroute)
                     if (isFinalPipeline) {
+                        logger.info("Service stack: [{}]", ingestDocument.getPipelineStack());
                         exceptionHandler.accept(
-                            new IllegalStateException(
-                                format(
-                                    "final pipeline [%s] can't change the target index (from [%s] to [%s]) for document [%s]",
-                                    pipelineId,
-                                    originalIndex,
-                                    newIndex,
-                                    indexRequest.id()
+                            new IngestPipelineException(
+                                pipelineId,
+                                new IllegalStateException(
+                                    format(
+                                        "final pipeline [%s] can't change the target index (from [%s] to [%s]) for document [%s]",
+                                        pipelineId,
+                                        originalIndex,
+                                        newIndex,
+                                        indexRequest.id()
+                                    )
                                 )
                             )
                         );
@@ -975,12 +1024,15 @@ private void executePipelines(
                         List<String> indexCycle = new ArrayList<>(ingestDocument.getIndexHistory());
                         indexCycle.add(newIndex);
                         exceptionHandler.accept(
-                            new IllegalStateException(
-                                format(
-                                    "index cycle detected while processing pipeline [%s] for document [%s]: %s",
-                                    pipelineId,
-                                    indexRequest.id(),
-                                    indexCycle
+                            new IngestPipelineException(
+                                pipelineId,
+                                new IllegalStateException(
+                                    format(
+                                        "index cycle detected while processing pipeline [%s] for document [%s]: %s",
+                                        pipelineId,
+                                        indexRequest.id(),
+                                        indexCycle
+                                    )
                                 )
                             )
                         );
@@ -1001,7 +1053,7 @@ private void executePipelines(
                 }
 
                 if (newPipelines.hasNext()) {
-                    executePipelines(newPipelines, indexRequest, ingestDocument, shouldStoreFailure, listener);
+                    executePipelines(newPipelines, indexRequest, ingestDocument, resolveFailureStore, listener);
                 } else {
                     // update the index request's source and (potentially) cache the timestamp for TSDB
                     updateIndexRequestSource(indexRequest, ingestDocument);
@@ -1089,14 +1141,14 @@ static String getProcessorName(Processor processor) {
     /**
      * Builds a new ingest document from the passed-in index request.
      */
-    private static IngestDocument newIngestDocument(final IndexRequest request, DocumentSizeObserver documentSizeObserver) {
+    private static IngestDocument newIngestDocument(final IndexRequest request, XContentParserDecorator parserDecorator) {
         return new IngestDocument(
             request.index(),
             request.id(),
             request.version(),
             request.routing(),
             request.versionType(),
-            request.sourceAsMap(documentSizeObserver)
+            request.sourceAsMap(parserDecorator)
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/ingest/LogstashInternalBridge.java b/server/src/main/java/org/elasticsearch/ingest/LogstashInternalBridge.java
index 889a4ffb932f9..af0cb187ba05d 100644
--- a/server/src/main/java/org/elasticsearch/ingest/LogstashInternalBridge.java
+++ b/server/src/main/java/org/elasticsearch/ingest/LogstashInternalBridge.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 
 /**
@@ -40,9 +41,9 @@ public static void resetReroute(final IngestDocument ingestDocument) {
 
     /**
      * @param settings
-     * @return a new {@link ThreadPool} with a noop {@link MeterRegistry}
+     * @return a new {@link ThreadPool} with a noop {@link MeterRegistry} and default executors
      */
     public static ThreadPool createThreadPool(final Settings settings) {
-        return new ThreadPool(settings, MeterRegistry.NOOP);
+        return new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/injection/Injector.java b/server/src/main/java/org/elasticsearch/injection/Injector.java
new file mode 100644
index 0000000000000..03fcf18509fcc
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/Injector.java
@@ -0,0 +1,314 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.injection;
+
+import org.elasticsearch.injection.api.Inject;
+import org.elasticsearch.injection.spec.ExistingInstanceSpec;
+import org.elasticsearch.injection.spec.InjectionSpec;
+import org.elasticsearch.injection.spec.MethodHandleSpec;
+import org.elasticsearch.injection.spec.ParameterSpec;
+import org.elasticsearch.injection.step.InjectionStep;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Constructor;
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import static java.util.function.Predicate.not;
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toMap;
+
+/**
+ * The main object for dependency injection.
+ * <p>
+ * Allows the user to specify the requirements, then call {@link #inject} to create an object plus all its dependencies.
+ * <p>
+ * <em>Implementation note</em>: this class itself contains logic for <em>specifying</em> the injection requirements;
+ * the actual injection operations are performed in other classes like {@link Planner} and {@link PlanInterpreter},
+ */
+public final class Injector {
+    private static final Logger logger = LogManager.getLogger(Injector.class);
+
+    /**
+     * The specifications supplied by the user, as opposed to those inferred by the injector.
+     */
+    private final Map<Class<?>, InjectionSpec> seedSpecs;
+
+    Injector(Map<Class<?>, InjectionSpec> seedSpecs) {
+        this.seedSpecs = seedSpecs;
+    }
+
+    public static Injector create() {
+        return new Injector(new LinkedHashMap<>());
+    }
+
+    /**
+     * Instructs the injector to instantiate <code>classToProcess</code>
+     * in accordance with whatever annotations may be present on that class.
+     * <p>
+     * There are only three ways the injector can find out that it must instantiate some class:
+     * <ol>
+     *     <li>
+     *         This method
+     *     </li>
+     *     <li>
+     *         The parameter passed to {@link #inject}
+     *     </li>
+     *     <li>
+     *         A constructor parameter of some other class being instantiated,
+     *         having exactly the right class (not a supertype)
+     *     </li>
+     * </ol>
+     *
+     * @return <code>this</code>
+     */
+    public Injector addClass(Class<?> classToProcess) {
+        MethodHandleSpec methodHandleSpec = methodHandleSpecFor(classToProcess);
+        var existing = seedSpecs.put(classToProcess, methodHandleSpec);
+        if (existing != null) {
+            throw new IllegalArgumentException("class " + classToProcess.getSimpleName() + " has already been added");
+        }
+        return this;
+    }
+
+    /**
+     * Equivalent to multiple chained calls to {@link #addClass}.
+     */
+    public Injector addClasses(Collection<Class<?>> classesToProcess) {
+        classesToProcess.forEach(this::addClass);
+        return this;
+    }
+
+    /**
+     * Equivalent to {@link #addInstance addInstance(object.getClass(), object)}.
+     */
+    public <T> Injector addInstance(Object object) {
+        @SuppressWarnings("unchecked")
+        Class<T> actualClass = (Class<T>) object.getClass(); // Whatever the runtime type is, it's represented by T
+        return addInstance(actualClass, actualClass.cast(object));
+    }
+
+    /**
+     * Equivalent to multiple calls to {@link #addInstance(Object)}.
+     */
+    public Injector addInstances(Collection<?> objects) {
+        for (var x : objects) {
+            addInstance(x);
+        }
+        return this;
+    }
+
+    /**
+     * Indicates that <code>object</code> is to be injected for parameters of type <code>type</code>.
+     * The given object is treated as though it had been instantiated by the injector.
+     */
+    public <T> Injector addInstance(Class<? super T> type, T object) {
+        assert type.isInstance(object); // No unchecked casting shenanigans allowed
+        var existing = seedSpecs.put(type, new ExistingInstanceSpec(type, object));
+        if (existing != null) {
+            throw new IllegalStateException("There's already an object for " + type);
+        }
+        return this;
+    }
+
+    /**
+     * Main entry point. Causes objects to be constructed.
+     * @return {@link Map} whose keys are all the requested <code>resultTypes</code> and whose values are all the instances of those types.
+     */
+    public Map<Class<?>, Object> inject(Collection<? extends Class<?>> resultTypes) {
+        resultTypes.forEach(this::ensureClassIsSpecified);
+        PlanInterpreter i = doInjection();
+        return resultTypes.stream().collect(toMap(c -> c, i::theInstanceOf));
+    }
+
+    private <T> void ensureClassIsSpecified(Class<T> resultType) {
+        if (seedSpecs.containsKey(resultType) == false) {
+            addClass(resultType);
+        }
+    }
+
+    private PlanInterpreter doInjection() {
+        logger.debug("Starting injection");
+        Map<Class<?>, InjectionSpec> specMap = specClosure(seedSpecs);
+        Map<Class<?>, Object> existingInstances = new LinkedHashMap<>();
+        specMap.values().forEach((spec) -> {
+            if (spec instanceof ExistingInstanceSpec e) {
+                existingInstances.put(e.requestedType(), e.instance());
+            }
+        });
+        PlanInterpreter interpreter = new PlanInterpreter(existingInstances);
+        interpreter.executePlan(injectionPlan(seedSpecs.keySet(), specMap));
+        logger.debug("Done injection");
+        return interpreter;
+    }
+
+    /**
+     * Finds an {@link InjectionSpec} for every class the injector is capable of injecting.
+     * <p>
+     * We do this once the injector is fully configured, with all calls to {@link #addClass} and {@link #addInstance} finished,
+     * so that we can easily build the complete picture of how injection should occur.
+     * <p>
+     * This is not part of the planning process; it's just discovering all the things
+     * the injector needs to know about. This logic isn't concerned with ordering or dependency cycles.
+     *
+     * @param seedMap the injections the user explicitly asked for
+     * @return an {@link InjectionSpec} for every class the injector is capable of injecting.
+     */
+    private static Map<Class<?>, InjectionSpec> specClosure(Map<Class<?>, InjectionSpec> seedMap) {
+        assert seedMapIsValid(seedMap);
+
+        // For convenience, we pretend there's a gigantic method out there that takes
+        // all the seed types as parameters.
+        Queue<ParameterSpec> workQueue = seedMap.values()
+            .stream()
+            .map(InjectionSpec::requestedType)
+            .map(Injector::syntheticParameterSpec)
+            .collect(toCollection(ArrayDeque::new));
+
+        // This map doubles as a checklist of classes we're already finished processing
+        Map<Class<?>, InjectionSpec> result = new LinkedHashMap<>();
+
+        ParameterSpec p;
+        while ((p = workQueue.poll()) != null) {
+            Class<?> c = p.injectableType();
+            InjectionSpec existingResult = result.get(c);
+            if (existingResult != null) {
+                logger.trace("Spec for {} already exists", c.getSimpleName());
+                continue;
+            }
+
+            InjectionSpec spec = seedMap.get(c);
+            if (spec instanceof ExistingInstanceSpec) {
+                // simple!
+                result.put(c, spec);
+                continue;
+            }
+
+            // At this point, we know we'll need a MethodHandleSpec
+            MethodHandleSpec methodHandleSpec;
+            if (spec == null) {
+                // The user didn't specify this class; we must infer it now
+                spec = methodHandleSpec = methodHandleSpecFor(c);
+            } else if (spec instanceof MethodHandleSpec m) {
+                methodHandleSpec = m;
+            } else {
+                throw new AssertionError("Unexpected spec: " + spec);
+            }
+
+            logger.trace("Inspecting parameters for constructor of {}", c);
+            for (var ps : methodHandleSpec.parameters()) {
+                logger.trace("Enqueue {}", ps);
+                workQueue.add(ps);
+            }
+
+            registerSpec(spec, result);
+        }
+
+        if (logger.isTraceEnabled()) {
+            logger.trace("Specs: {}", result.values().stream().map(Object::toString).collect(joining("\n\t", "\n\t", "")));
+        }
+        return result;
+    }
+
+    private static MethodHandleSpec methodHandleSpecFor(Class<?> c) {
+        Constructor<?> constructor = getSuitableConstructorIfAny(c);
+        if (constructor == null) {
+            throw new IllegalStateException("No suitable constructor for " + c);
+        }
+
+        MethodHandle ctorHandle;
+        try {
+            ctorHandle = lookup().unreflectConstructor(constructor);
+        } catch (IllegalAccessException e) {
+            throw new IllegalStateException(e);
+        }
+
+        List<ParameterSpec> parameters = Stream.of(constructor.getParameters()).map(ParameterSpec::from).toList();
+
+        return new MethodHandleSpec(c, ctorHandle, parameters);
+    }
+
+    /**
+     * @return true (unless an assertion fails). Never returns false.
+     */
+    private static boolean seedMapIsValid(Map<Class<?>, InjectionSpec> seed) {
+        seed.forEach(
+            (c, s) -> { assert s.requestedType().equals(c) : "Spec must be associated with its requestedType, not " + c + ": " + s; }
+        );
+        return true;
+    }
+
+    /**
+     * For the classes we've been explicitly asked to inject,
+     * pretend there's some massive method taking all of them as parameters
+     */
+    private static ParameterSpec syntheticParameterSpec(Class<?> c) {
+        return new ParameterSpec("synthetic_" + c.getSimpleName(), c, c);
+    }
+
+    private static Constructor<?> getSuitableConstructorIfAny(Class<?> type) {
+        var constructors = Stream.of(type.getConstructors()).filter(not(Constructor::isSynthetic)).toList();
+        if (constructors.size() == 1) {
+            return constructors.get(0);
+        }
+        var injectConstructors = constructors.stream().filter(c -> c.isAnnotationPresent(Inject.class)).toList();
+        if (injectConstructors.size() == 1) {
+            return injectConstructors.get(0);
+        }
+        logger.trace("No suitable constructor for {}", type);
+        return null;
+    }
+
+    private static void registerSpec(InjectionSpec spec, Map<Class<?>, InjectionSpec> specsByClass) {
+        Class<?> requestedType = spec.requestedType();
+        var existing = specsByClass.put(requestedType, spec);
+        if (existing == null || existing.equals(spec)) {
+            logger.trace("Register spec: {}", spec);
+        } else {
+            throw new IllegalStateException("Ambiguous specifications for " + requestedType + ": " + existing + " and " + spec);
+        }
+    }
+
+    private List<InjectionStep> injectionPlan(Set<Class<?>> requiredClasses, Map<Class<?>, InjectionSpec> specsByClass) {
+        logger.trace("Constructing instantiation plan");
+        Set<Class<?>> allParameterTypes = new HashSet<>();
+        specsByClass.values().forEach(spec -> {
+            if (spec instanceof MethodHandleSpec m) {
+                m.parameters().stream().map(ParameterSpec::injectableType).forEachOrdered(allParameterTypes::add);
+            }
+        });
+
+        var plan = new Planner(specsByClass, requiredClasses, allParameterTypes).injectionPlan();
+        if (logger.isDebugEnabled()) {
+            logger.debug("Injection plan: {}", plan.stream().map(Object::toString).collect(joining("\n\t", "\n\t", "")));
+        }
+        return plan;
+    }
+
+    /**
+     * <em>Evolution note</em>: there may be cases in the where we allow the user to
+     * supply a {@link java.lang.invoke.MethodHandles.Lookup} for convenience,
+     * so that they aren't required to make things public just to participate in injection.
+     */
+    private static MethodHandles.Lookup lookup() {
+        return MethodHandles.publicLookup();
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/injection/PlanInterpreter.java b/server/src/main/java/org/elasticsearch/injection/PlanInterpreter.java
new file mode 100644
index 0000000000000..cf38dbcb24b7d
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/PlanInterpreter.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.injection;
+
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.spec.MethodHandleSpec;
+import org.elasticsearch.injection.spec.ParameterSpec;
+import org.elasticsearch.injection.step.InjectionStep;
+import org.elasticsearch.injection.step.InstantiateStep;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Performs the actual injection operations by running the {@link InjectionStep}s.
+ * <p>
+ * The intent is that this logic is as simple as possible so that we don't run complex injection
+ * logic alongside the user-supplied constructor logic. All the injector complexity is already
+ * supposed to have happened in the planning phase. In particular, no injection-related errors
+ * are supposed to be detected during execution; they should be detected during planning and validation.
+ * All exceptions thrown during execution are supposed to be caused by user-supplied code.
+ *
+ * <p>
+ * <strong>Execution model</strong>:
+ * The state of the injector during injection comprises a map from classes to objects.
+ * Before any steps execute, the map is pre-populated by object instances added via
+ * {@link Injector#addInstance(Object)}  Injector.addInstance},
+ * and then the steps begin to execute, reading and writing from this map.
+ * Some steps create objects and add them to this map; others manipulate the map itself.
+ */
+final class PlanInterpreter {
+    private static final Logger logger = LogManager.getLogger(PlanInterpreter.class);
+    private final Map<Class<?>, Object> instances = new LinkedHashMap<>();
+
+    PlanInterpreter(Map<Class<?>, Object> existingInstances) {
+        existingInstances.forEach(this::addInstance);
+    }
+
+    /**
+     * Main entry point. Contains the implementation logic for each {@link InjectionStep}.
+     */
+    void executePlan(List<InjectionStep> plan) {
+        int numConstructorCalls = 0;
+        for (InjectionStep step : plan) {
+            if (step instanceof InstantiateStep i) {
+                MethodHandleSpec spec = i.spec();
+                logger.trace("Instantiating {}", spec.requestedType().getSimpleName());
+                addInstance(spec.requestedType(), instantiate(spec));
+                ++numConstructorCalls;
+            } else {
+                // TODO: switch patterns would make this unnecessary
+                assert false : "Unexpected step type: " + step.getClass().getSimpleName();
+                throw new IllegalStateException("Unexpected step type: " + step.getClass().getSimpleName());
+            }
+        }
+        logger.debug("Instantiated {} objects", numConstructorCalls);
+    }
+
+    /**
+     * @return the list element corresponding to instances.get(type).get(0),
+     * assuming that instances.get(type) has exactly one element.
+     * @throws IllegalStateException if instances.get(type) does not have exactly one element
+     */
+    public <T> T theInstanceOf(Class<T> type) {
+        Object instance = instances.get(type);
+        if (instance == null) {
+            throw new IllegalStateException("No object of type " + type.getSimpleName());
+        }
+        return type.cast(instance);
+    }
+
+    private void addInstance(Class<?> requestedType, Object instance) {
+        Object old = instances.put(requestedType, instance);
+        if (old != null) {
+            throw new IllegalStateException("Multiple objects for " + requestedType);
+        }
+    }
+
+    /**
+     * @throws IllegalStateException if the <code>MethodHandle</code> throws.
+     */
+    @SuppressForbidden(
+        reason = "Can't call invokeExact because we don't know the method argument types statically, "
+            + "since each constructor has a different signature"
+    )
+    private Object instantiate(MethodHandleSpec spec) {
+        Object[] args = spec.parameters().stream().map(this::parameterValue).toArray();
+        try {
+            return spec.methodHandle().invokeWithArguments(args);
+        } catch (Throwable e) {
+            throw new IllegalStateException("Unexpected exception while instantiating {}" + spec, e);
+        }
+    }
+
+    private Object parameterValue(ParameterSpec parameterSpec) {
+        return theInstanceOf(parameterSpec.formalType());
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/injection/Planner.java b/server/src/main/java/org/elasticsearch/injection/Planner.java
new file mode 100644
index 0000000000000..4b6af05d57c04
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/Planner.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.injection;
+
+import org.elasticsearch.injection.spec.ExistingInstanceSpec;
+import org.elasticsearch.injection.spec.InjectionSpec;
+import org.elasticsearch.injection.spec.MethodHandleSpec;
+import org.elasticsearch.injection.step.InjectionStep;
+import org.elasticsearch.injection.step.InstantiateStep;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import static java.util.Collections.unmodifiableMap;
+import static java.util.Collections.unmodifiableSet;
+
+/**
+ * <em>Evolution note</em>: the intent is to plan one domain/subsystem at a time.
+ */
+final class Planner {
+    private static final Logger logger = LogManager.getLogger(Planner.class);
+
+    final List<InjectionStep> plan;
+    final Map<Class<?>, InjectionSpec> specsByClass;
+    final Set<Class<?>> requiredTypes; // The injector's job is to ensure there is an instance of these; this is like the "root set"
+    final Set<Class<?>> allParameterTypes; // All the injectable types in all dependencies (recursively) of all required types
+    final Set<InjectionSpec> startedPlanning;
+    final Set<InjectionSpec> finishedPlanning;
+    final Set<Class<?>> alreadyProxied;
+
+    /**
+     * @param specsByClass an {@link InjectionSpec} indicating how each class should be injected
+     * @param requiredTypes the classes of which we need instances
+     * @param allParameterTypes the classes that appear as the type of any parameter of any constructor we might call
+     */
+    Planner(Map<Class<?>, InjectionSpec> specsByClass, Set<Class<?>> requiredTypes, Set<Class<?>> allParameterTypes) {
+        this.requiredTypes = requiredTypes;
+        this.plan = new ArrayList<>();
+        this.specsByClass = unmodifiableMap(specsByClass);
+        this.allParameterTypes = unmodifiableSet(allParameterTypes);
+        this.startedPlanning = new HashSet<>();
+        this.finishedPlanning = new HashSet<>();
+        this.alreadyProxied = new HashSet<>();
+    }
+
+    /**
+     * Intended to be called once.
+     * <p>
+     * Note that not all proxies are resolved once this plan has been executed.
+     * <p>
+     *
+     * <em>Evolution note</em>: in a world with multiple domains/subsystems,
+     * it will become necessary to defer proxy resolution until after other plans
+     * have been executed, because they could create additional objects that ought
+     * to be included in the proxies created by this plan.
+     *
+     * @return the {@link InjectionStep} objects listed in execution order.
+     */
+    List<InjectionStep> injectionPlan() {
+        for (Class<?> c : requiredTypes) {
+            planForClass(c, 0);
+        }
+        return plan;
+    }
+
+    /**
+     * Recursive procedure that determines what effect <code>requestedClass</code>
+     * should have on the plan under construction.
+     *
+     * @param depth is used just for indenting the logs
+     */
+    private void planForClass(Class<?> requestedClass, int depth) {
+        InjectionSpec spec = specsByClass.get(requestedClass);
+        if (spec == null) {
+            throw new IllegalStateException("Cannot instantiate " + requestedClass + ": no specification provided");
+        }
+        planForSpec(spec, depth);
+    }
+
+    private void planForSpec(InjectionSpec spec, int depth) {
+        if (finishedPlanning.contains(spec)) {
+            logger.trace("{}Already planned {}", indent(depth), spec);
+            return;
+        }
+
+        logger.trace("{}Planning for {}", indent(depth), spec);
+        if (startedPlanning.add(spec) == false) {
+            // TODO: Better cycle detection and reporting. Use SCCs
+            throw new IllegalStateException("Cyclic dependency involving " + spec);
+        }
+
+        if (spec instanceof MethodHandleSpec m) {
+            for (var p : m.parameters()) {
+                logger.trace("{}- Recursing into {} for actual parameter {}", indent(depth), p.injectableType(), p);
+                planForClass(p.injectableType(), depth + 1);
+            }
+            addStep(new InstantiateStep(m), depth);
+        } else if (spec instanceof ExistingInstanceSpec e) {
+            logger.trace("{}- Plan {}", indent(depth), e);
+            // Nothing to do. The injector will already have the required object.
+        } else {
+            throw new AssertionError("Unexpected injection spec: " + spec);
+        }
+
+        finishedPlanning.add(spec);
+    }
+
+    private void addStep(InjectionStep newStep, int depth) {
+        logger.trace("{}- Add step {}", indent(depth), newStep);
+        plan.add(newStep);
+    }
+
+    private static Supplier<String> indent(int depth) {
+        return () -> "\t".repeat(depth);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/injection/api/Inject.java b/server/src/main/java/org/elasticsearch/injection/api/Inject.java
new file mode 100644
index 0000000000000..d5c57d1e5e2e2
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/api/Inject.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.injection.api;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.CONSTRUCTOR;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Designates a constructor to be called by the injector.
+ */
+@Target(CONSTRUCTOR)
+@Retention(RUNTIME)
+public @interface Inject {
+}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/AbstractModule.java b/server/src/main/java/org/elasticsearch/injection/guice/AbstractModule.java
similarity index 95%
rename from server/src/main/java/org/elasticsearch/common/inject/AbstractModule.java
rename to server/src/main/java/org/elasticsearch/injection/guice/AbstractModule.java
index 386c9f6cf9f97..73aaca80ccdbc 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/AbstractModule.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/AbstractModule.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.binder.AnnotatedBindingBuilder;
+import org.elasticsearch.injection.guice.binder.AnnotatedBindingBuilder;
 
 import java.util.Objects;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/AbstractProcessor.java b/server/src/main/java/org/elasticsearch/injection/guice/AbstractProcessor.java
similarity index 87%
rename from server/src/main/java/org/elasticsearch/common/inject/AbstractProcessor.java
rename to server/src/main/java/org/elasticsearch/injection/guice/AbstractProcessor.java
index 3eba654d412bd..33d8541211239 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/AbstractProcessor.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/AbstractProcessor.java
@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.spi.Element;
-import org.elasticsearch.common.inject.spi.ElementVisitor;
-import org.elasticsearch.common.inject.spi.Message;
-import org.elasticsearch.common.inject.spi.ProviderLookup;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.spi.Element;
+import org.elasticsearch.injection.guice.spi.ElementVisitor;
+import org.elasticsearch.injection.guice.spi.Message;
+import org.elasticsearch.injection.guice.spi.ProviderLookup;
 
 import java.util.Iterator;
 import java.util.List;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Binder.java b/server/src/main/java/org/elasticsearch/injection/guice/Binder.java
similarity index 96%
rename from server/src/main/java/org/elasticsearch/common/inject/Binder.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Binder.java
index 07a8979eb18a6..c34bebd10c2e1 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Binder.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Binder.java
@@ -14,11 +14,12 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.binder.AnnotatedBindingBuilder;
-import org.elasticsearch.common.inject.binder.LinkedBindingBuilder;
-import org.elasticsearch.common.inject.spi.Message;
+import org.elasticsearch.injection.guice.binder.AnnotatedBindingBuilder;
+import org.elasticsearch.injection.guice.binder.LinkedBindingBuilder;
+import org.elasticsearch.injection.guice.name.Named;
+import org.elasticsearch.injection.guice.spi.Message;
 
 /**
  * Collects configuration information (primarily <i>bindings</i>) which will be
@@ -145,7 +146,7 @@
  *         .to(BlueService.class);</pre>
  *
  * Differentiating by names is a common enough use case that we provided a
- * standard annotation, {@link org.elasticsearch.common.inject.name.Named @Named}.  Because of
+ * standard annotation, {@link Named @Named}.  Because of
  * Guice's library support, binding by name is quite easier than in the
  * arbitrary binding annotation case we just saw.  However, remember that these
  * names will live in a single flat namespace with all the other names used in
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Binding.java b/server/src/main/java/org/elasticsearch/injection/guice/Binding.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/common/inject/Binding.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Binding.java
index 9bc446a867aa7..faaf1f1ef18c1 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Binding.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Binding.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.spi.BindingTargetVisitor;
-import org.elasticsearch.common.inject.spi.Element;
+import org.elasticsearch.injection.guice.spi.BindingTargetVisitor;
+import org.elasticsearch.injection.guice.spi.Element;
 
 /**
  * A mapping from a key (type and optional annotation) to the strategy for getting instances of the
diff --git a/server/src/main/java/org/elasticsearch/common/inject/BindingAnnotation.java b/server/src/main/java/org/elasticsearch/injection/guice/BindingAnnotation.java
similarity index 96%
rename from server/src/main/java/org/elasticsearch/common/inject/BindingAnnotation.java
rename to server/src/main/java/org/elasticsearch/injection/guice/BindingAnnotation.java
index 3d18b573ca062..240c5a49f56fe 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/BindingAnnotation.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/BindingAnnotation.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/BindingProcessor.java b/server/src/main/java/org/elasticsearch/injection/guice/BindingProcessor.java
similarity index 87%
rename from server/src/main/java/org/elasticsearch/common/inject/BindingProcessor.java
rename to server/src/main/java/org/elasticsearch/injection/guice/BindingProcessor.java
index dfa4fcb16bc62..9223261ec2dd5 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/BindingProcessor.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/BindingProcessor.java
@@ -14,25 +14,25 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
-
-import org.elasticsearch.common.inject.internal.BindingImpl;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InstanceBindingImpl;
-import org.elasticsearch.common.inject.internal.InternalFactory;
-import org.elasticsearch.common.inject.internal.LinkedBindingImpl;
-import org.elasticsearch.common.inject.internal.LinkedProviderBindingImpl;
-import org.elasticsearch.common.inject.internal.ProviderInstanceBindingImpl;
-import org.elasticsearch.common.inject.internal.Scoping;
-import org.elasticsearch.common.inject.internal.UntargettedBindingImpl;
-import org.elasticsearch.common.inject.spi.BindingTargetVisitor;
-import org.elasticsearch.common.inject.spi.InjectionPoint;
-import org.elasticsearch.common.inject.spi.InstanceBinding;
-import org.elasticsearch.common.inject.spi.LinkedKeyBinding;
-import org.elasticsearch.common.inject.spi.ProviderInstanceBinding;
-import org.elasticsearch.common.inject.spi.ProviderKeyBinding;
-import org.elasticsearch.common.inject.spi.UntargettedBinding;
+package org.elasticsearch.injection.guice;
+
+import org.elasticsearch.injection.guice.internal.BindingImpl;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InstanceBindingImpl;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
+import org.elasticsearch.injection.guice.internal.LinkedBindingImpl;
+import org.elasticsearch.injection.guice.internal.LinkedProviderBindingImpl;
+import org.elasticsearch.injection.guice.internal.ProviderInstanceBindingImpl;
+import org.elasticsearch.injection.guice.internal.Scoping;
+import org.elasticsearch.injection.guice.internal.UntargettedBindingImpl;
+import org.elasticsearch.injection.guice.spi.BindingTargetVisitor;
+import org.elasticsearch.injection.guice.spi.InjectionPoint;
+import org.elasticsearch.injection.guice.spi.InstanceBinding;
+import org.elasticsearch.injection.guice.spi.LinkedKeyBinding;
+import org.elasticsearch.injection.guice.spi.ProviderInstanceBinding;
+import org.elasticsearch.injection.guice.spi.ProviderKeyBinding;
+import org.elasticsearch.injection.guice.spi.UntargettedBinding;
 
 import java.util.ArrayList;
 import java.util.List;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/BoundProviderFactory.java b/server/src/main/java/org/elasticsearch/injection/guice/BoundProviderFactory.java
similarity index 82%
rename from server/src/main/java/org/elasticsearch/common/inject/BoundProviderFactory.java
rename to server/src/main/java/org/elasticsearch/injection/guice/BoundProviderFactory.java
index 317b708f99169..3b713506cd9ef 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/BoundProviderFactory.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/BoundProviderFactory.java
@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.BindingProcessor.CreationListener;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalContext;
-import org.elasticsearch.common.inject.internal.InternalFactory;
-import org.elasticsearch.common.inject.spi.Dependency;
+import org.elasticsearch.injection.guice.BindingProcessor.CreationListener;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
+import org.elasticsearch.injection.guice.spi.Dependency;
 
 /**
  * Delegates to a custom factory which is also bound in the injector.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/ConfigurationException.java b/server/src/main/java/org/elasticsearch/injection/guice/ConfigurationException.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/common/inject/ConfigurationException.java
rename to server/src/main/java/org/elasticsearch/injection/guice/ConfigurationException.java
index f9231ba8faedd..7726414b39be8 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/ConfigurationException.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/ConfigurationException.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.spi.Message;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.spi.Message;
 
 import java.util.Collection;
 import java.util.Locale;
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/ConstantFactory.java b/server/src/main/java/org/elasticsearch/injection/guice/ConstantFactory.java
new file mode 100644
index 0000000000000..a944ad9173b4b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/ConstantFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2006 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.elasticsearch.injection.guice;
+
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
+import org.elasticsearch.injection.guice.internal.ToStringBuilder;
+import org.elasticsearch.injection.guice.spi.Dependency;
+
+/**
+ * @author crazybob@google.com (Bob Lee)
+ */
+class ConstantFactory<T> implements InternalFactory<T> {
+
+    private final Initializable<T> initializable;
+
+    ConstantFactory(Initializable<T> initializable) {
+        this.initializable = initializable;
+    }
+
+    @Override
+    public T get(Errors errors, InternalContext context, Dependency<?> dependency) throws ErrorsException {
+        return initializable.get(errors);
+    }
+
+    @Override
+    public String toString() {
+        return new ToStringBuilder(ConstantFactory.class).add("value", initializable).toString();
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/ConstructionProxy.java b/server/src/main/java/org/elasticsearch/injection/guice/ConstructionProxy.java
similarity index 91%
rename from server/src/main/java/org/elasticsearch/common/inject/ConstructionProxy.java
rename to server/src/main/java/org/elasticsearch/injection/guice/ConstructionProxy.java
index 74f2ed24801f6..f56cc24769802 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/ConstructionProxy.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/ConstructionProxy.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.spi.InjectionPoint;
+import org.elasticsearch.injection.guice.spi.InjectionPoint;
 
 import java.lang.reflect.InvocationTargetException;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/ConstructorBindingImpl.java b/server/src/main/java/org/elasticsearch/injection/guice/ConstructorBindingImpl.java
similarity index 80%
rename from server/src/main/java/org/elasticsearch/common/inject/ConstructorBindingImpl.java
rename to server/src/main/java/org/elasticsearch/injection/guice/ConstructorBindingImpl.java
index 153c9627d736e..7f567155f7b1d 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/ConstructorBindingImpl.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/ConstructorBindingImpl.java
@@ -14,18 +14,18 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.BindingImpl;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalContext;
-import org.elasticsearch.common.inject.internal.InternalFactory;
-import org.elasticsearch.common.inject.internal.Scoping;
-import org.elasticsearch.common.inject.internal.ToStringBuilder;
-import org.elasticsearch.common.inject.spi.BindingTargetVisitor;
-import org.elasticsearch.common.inject.spi.ConstructorBinding;
-import org.elasticsearch.common.inject.spi.Dependency;
+import org.elasticsearch.injection.guice.internal.BindingImpl;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
+import org.elasticsearch.injection.guice.internal.Scoping;
+import org.elasticsearch.injection.guice.internal.ToStringBuilder;
+import org.elasticsearch.injection.guice.spi.BindingTargetVisitor;
+import org.elasticsearch.injection.guice.spi.ConstructorBinding;
+import org.elasticsearch.injection.guice.spi.Dependency;
 
 class ConstructorBindingImpl<T> extends BindingImpl<T> implements ConstructorBinding<T> {
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/ConstructorInjector.java b/server/src/main/java/org/elasticsearch/injection/guice/ConstructorInjector.java
similarity index 91%
rename from server/src/main/java/org/elasticsearch/common/inject/ConstructorInjector.java
rename to server/src/main/java/org/elasticsearch/injection/guice/ConstructorInjector.java
index d38a75e0720d7..6a7176a39c59c 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/ConstructorInjector.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/ConstructorInjector.java
@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.ConstructionContext;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.ConstructionContext;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalContext;
 
 import java.lang.reflect.InvocationTargetException;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/ConstructorInjectorStore.java b/server/src/main/java/org/elasticsearch/injection/guice/ConstructorInjectorStore.java
similarity index 92%
rename from server/src/main/java/org/elasticsearch/common/inject/ConstructorInjectorStore.java
rename to server/src/main/java/org/elasticsearch/injection/guice/ConstructorInjectorStore.java
index 97a495f97cfbd..d2307bd866f24 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/ConstructorInjectorStore.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/ConstructorInjectorStore.java
@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.FailableCache;
-import org.elasticsearch.common.inject.spi.InjectionPoint;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.FailableCache;
+import org.elasticsearch.injection.guice.spi.InjectionPoint;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/ContextualCallable.java b/server/src/main/java/org/elasticsearch/injection/guice/ContextualCallable.java
similarity index 80%
rename from server/src/main/java/org/elasticsearch/common/inject/ContextualCallable.java
rename to server/src/main/java/org/elasticsearch/injection/guice/ContextualCallable.java
index b6d09e7e28bf0..2f699da8f5956 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/ContextualCallable.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/ContextualCallable.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalContext;
 
 /**
  * @author crazybob@google.com (Bob Lee)
diff --git a/server/src/main/java/org/elasticsearch/common/inject/CreationException.java b/server/src/main/java/org/elasticsearch/injection/guice/CreationException.java
similarity index 91%
rename from server/src/main/java/org/elasticsearch/common/inject/CreationException.java
rename to server/src/main/java/org/elasticsearch/injection/guice/CreationException.java
index 78f89e95e5ff7..d7ceaded19a6a 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/CreationException.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/CreationException.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.spi.Message;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.spi.Message;
 
 import java.util.Collection;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/DeferredLookups.java b/server/src/main/java/org/elasticsearch/injection/guice/DeferredLookups.java
similarity index 89%
rename from server/src/main/java/org/elasticsearch/common/inject/DeferredLookups.java
rename to server/src/main/java/org/elasticsearch/injection/guice/DeferredLookups.java
index 912c258f83692..9313c54c0ccb6 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/DeferredLookups.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/DeferredLookups.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.spi.Element;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.spi.Element;
 
 import java.util.ArrayList;
 import java.util.List;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/FactoryProxy.java b/server/src/main/java/org/elasticsearch/injection/guice/FactoryProxy.java
similarity index 81%
rename from server/src/main/java/org/elasticsearch/common/inject/FactoryProxy.java
rename to server/src/main/java/org/elasticsearch/injection/guice/FactoryProxy.java
index 7e102d2c492a8..d78ac65f166df 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/FactoryProxy.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/FactoryProxy.java
@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
-
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalContext;
-import org.elasticsearch.common.inject.internal.InternalFactory;
-import org.elasticsearch.common.inject.internal.ToStringBuilder;
-import org.elasticsearch.common.inject.spi.Dependency;
+package org.elasticsearch.injection.guice;
+
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
+import org.elasticsearch.injection.guice.internal.ToStringBuilder;
+import org.elasticsearch.injection.guice.spi.Dependency;
 
 /**
  * A placeholder which enables us to swap in the real factory once the injector is created.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Guice.java b/server/src/main/java/org/elasticsearch/injection/guice/Guice.java
similarity index 98%
rename from server/src/main/java/org/elasticsearch/common/inject/Guice.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Guice.java
index 1202707a41d60..0a77a13b77b03 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Guice.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Guice.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
 import java.util.Arrays;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/InheritingState.java b/server/src/main/java/org/elasticsearch/injection/guice/InheritingState.java
similarity index 90%
rename from server/src/main/java/org/elasticsearch/common/inject/InheritingState.java
rename to server/src/main/java/org/elasticsearch/injection/guice/InheritingState.java
index bf6dbabafb034..6eb8d639d9826 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/InheritingState.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/InheritingState.java
@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
-
-import org.elasticsearch.common.inject.internal.BindingImpl;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.InstanceBindingImpl;
-import org.elasticsearch.common.inject.internal.InternalFactory;
-import org.elasticsearch.common.inject.internal.MatcherAndConverter;
-import org.elasticsearch.common.inject.internal.SourceProvider;
+package org.elasticsearch.injection.guice;
+
+import org.elasticsearch.injection.guice.internal.BindingImpl;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.InstanceBindingImpl;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
+import org.elasticsearch.injection.guice.internal.MatcherAndConverter;
+import org.elasticsearch.injection.guice.internal.SourceProvider;
 
 import java.util.ArrayList;
 import java.util.Collections;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Initializable.java b/server/src/main/java/org/elasticsearch/injection/guice/Initializable.java
similarity index 84%
rename from server/src/main/java/org/elasticsearch/common/inject/Initializable.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Initializable.java
index 26c3da5d3fbad..6ab3573d5d8eb 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Initializable.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Initializable.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
 
 /**
  * Holds a reference that requires initialization to be performed before it can be used.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Initializables.java b/server/src/main/java/org/elasticsearch/injection/guice/Initializables.java
similarity index 91%
rename from server/src/main/java/org/elasticsearch/common/inject/Initializables.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Initializables.java
index 5f13be2d5c9bd..299adb044374e 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Initializables.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Initializables.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
+import org.elasticsearch.injection.guice.internal.Errors;
 
 /**
  * @author jessewilson@google.com (Jesse Wilson)
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Initializer.java b/server/src/main/java/org/elasticsearch/injection/guice/Initializer.java
similarity index 96%
rename from server/src/main/java/org/elasticsearch/common/inject/Initializer.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Initializer.java
index 6fe7db99ff55c..1f4e29be70caf 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Initializer.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Initializer.java
@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.spi.InjectionPoint;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.spi.InjectionPoint;
 
 import java.util.ArrayList;
 import java.util.IdentityHashMap;
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/Inject.java b/server/src/main/java/org/elasticsearch/injection/guice/Inject.java
new file mode 100644
index 0000000000000..eae40c2f9cdd6
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Inject.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2006 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.elasticsearch.injection.guice;
+
+import org.elasticsearch.injection.guice.binder.LinkedBindingBuilder;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.CONSTRUCTOR;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Annotates members of your implementation class (constructors, methods
+ * and fields) into which the {@link Injector} should inject values.
+ * The Injector fulfills injection requests for:
+ * <ul>
+ * <li>Every instance it constructs. The class being constructed must have
+ * exactly one of its constructors marked with {@code @Inject} or must have a
+ * constructor taking no parameters. The Injector then proceeds to perform
+ * method and field injections.
+ * <li>Pre-constructed instances passed to
+ * {@link LinkedBindingBuilder#toInstance(Object)} and
+ * {@link LinkedBindingBuilder#toProvider(Provider)}.
+ * In this case all constructors are, of course, ignored.
+ * </ul>
+ * <p>
+ * In all cases, a member can be injected regardless of its Java access
+ * specifier (private, default, protected, public).
+ *
+ * @author crazybob@google.com (Bob Lee)
+ */
+@Target({ METHOD, CONSTRUCTOR })
+@Retention(RUNTIME)
+@Documented
+public @interface Inject {
+
+    /**
+     * If true, and the appropriate binding is not found,
+     * the Injector will skip injection of this method or field rather than
+     * produce an error. When applied to a field, any default value already
+     * assigned to the field will remain (guice will not actively null out the
+     * field). When applied to a method, the method will only be invoked if
+     * bindings for <i>all</i> parameters are found. When applied to a
+     * constructor, an error will result upon Injector creation.
+     */
+    boolean optional() default false;
+}
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/Injector.java b/server/src/main/java/org/elasticsearch/injection/guice/Injector.java
new file mode 100644
index 0000000000000..3ba90185e6bc1
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Injector.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2006 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.elasticsearch.injection.guice;
+
+import java.util.List;
+
+/**
+ * Builds the graphs of objects that make up your application. The injector tracks the dependencies
+ * for each type and uses bindings to inject them. This is the core of Guice, although you rarely
+ * interact with it directly. This "behind-the-scenes" operation is what distinguishes dependency
+ * injection from its cousin, the service locator pattern.
+ * <p>
+ * Contains several default bindings:
+ * <ul>
+ * <li>This {@link Injector} instance itself
+ * <li>A {@code Provider<T>} for each binding of type {@code T}
+ * <li>The {@link java.util.logging.Logger} for the class being injected
+ * </ul>
+ * <p>
+ * Injectors are created using the facade class {@link Guice}.
+ *
+ * @author crazybob@google.com (Bob Lee)
+ * @author jessewilson@google.com (Jesse Wilson)
+ */
+public interface Injector {
+
+    /**
+     * Returns all explicit bindings for {@code type}.
+     * <p>
+     * This method is part of the Guice SPI and is intended for use by tools and extensions.
+     */
+    <T> List<Binding<T>> findBindingsByType(TypeLiteral<T> type);
+
+    /**
+     * Returns the provider used to obtain instances for the given injection key. When feasible, avoid
+     * using this method, in favor of having Guice inject your dependencies ahead of time.
+     *
+     * @throws ConfigurationException if this injector cannot find or create the provider.
+     * @see Binder#getProvider(Key) for an alternative that offers up front error detection
+     */
+    <T> Provider<T> getProvider(Key<T> key);
+
+    /**
+     * Returns the appropriate instance for the given injection key; equivalent to {@code
+     * getProvider(key).get()}. When feasible, avoid using this method, in favor of having Guice
+     * inject your dependencies ahead of time.
+     *
+     * @throws ConfigurationException if this injector cannot find or create the provider.
+     * @throws ProvisionException     if there was a runtime failure while providing an instance.
+     */
+    <T> T getInstance(Key<T> key);
+
+    /**
+     * Returns the appropriate instance for the given injection type; equivalent to {@code
+     * getProvider(type).get()}. When feasible, avoid using this method, in favor of having Guice
+     * inject your dependencies ahead of time.
+     *
+     * @throws ConfigurationException if this injector cannot find or create the provider.
+     * @throws ProvisionException     if there was a runtime failure while providing an instance.
+     */
+    <T> T getInstance(Class<T> type);
+}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/InjectorBuilder.java b/server/src/main/java/org/elasticsearch/injection/guice/InjectorBuilder.java
similarity index 93%
rename from server/src/main/java/org/elasticsearch/common/inject/InjectorBuilder.java
rename to server/src/main/java/org/elasticsearch/injection/guice/InjectorBuilder.java
index 4c5cb95491ebb..99d42faf6a803 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/InjectorBuilder.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/InjectorBuilder.java
@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
-
-import org.elasticsearch.common.inject.internal.BindingImpl;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalContext;
-import org.elasticsearch.common.inject.internal.Stopwatch;
-import org.elasticsearch.common.inject.spi.Dependency;
+package org.elasticsearch.injection.guice;
+
+import org.elasticsearch.injection.guice.internal.BindingImpl;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.Stopwatch;
+import org.elasticsearch.injection.guice.spi.Dependency;
 
 import java.util.List;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/InjectorImpl.java b/server/src/main/java/org/elasticsearch/injection/guice/InjectorImpl.java
similarity index 95%
rename from server/src/main/java/org/elasticsearch/common/inject/InjectorImpl.java
rename to server/src/main/java/org/elasticsearch/injection/guice/InjectorImpl.java
index 10385c54860bc..b93b32f963635 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/InjectorImpl.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/InjectorImpl.java
@@ -14,25 +14,25 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
 import org.elasticsearch.common.Classes;
-import org.elasticsearch.common.inject.internal.BindingImpl;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InstanceBindingImpl;
-import org.elasticsearch.common.inject.internal.InternalContext;
-import org.elasticsearch.common.inject.internal.InternalFactory;
-import org.elasticsearch.common.inject.internal.MatcherAndConverter;
-import org.elasticsearch.common.inject.internal.Scoping;
-import org.elasticsearch.common.inject.internal.SourceProvider;
-import org.elasticsearch.common.inject.internal.ToStringBuilder;
-import org.elasticsearch.common.inject.spi.BindingTargetVisitor;
-import org.elasticsearch.common.inject.spi.ConvertedConstantBinding;
-import org.elasticsearch.common.inject.spi.Dependency;
-import org.elasticsearch.common.inject.spi.ProviderBinding;
-import org.elasticsearch.common.inject.spi.ProviderKeyBinding;
-import org.elasticsearch.common.inject.util.Providers;
+import org.elasticsearch.injection.guice.internal.BindingImpl;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InstanceBindingImpl;
+import org.elasticsearch.injection.guice.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
+import org.elasticsearch.injection.guice.internal.MatcherAndConverter;
+import org.elasticsearch.injection.guice.internal.Scoping;
+import org.elasticsearch.injection.guice.internal.SourceProvider;
+import org.elasticsearch.injection.guice.internal.ToStringBuilder;
+import org.elasticsearch.injection.guice.spi.BindingTargetVisitor;
+import org.elasticsearch.injection.guice.spi.ConvertedConstantBinding;
+import org.elasticsearch.injection.guice.spi.Dependency;
+import org.elasticsearch.injection.guice.spi.ProviderBinding;
+import org.elasticsearch.injection.guice.spi.ProviderKeyBinding;
+import org.elasticsearch.injection.guice.util.Providers;
 
 import java.lang.reflect.GenericArrayType;
 import java.lang.reflect.Modifier;
@@ -205,7 +205,7 @@ public String toString() {
      * Converts a constant string binding to the required type.
      *
      * @return the binding if it could be resolved, or null if the binding doesn't exist
-     * @throws org.elasticsearch.common.inject.internal.ErrorsException
+     * @throws ErrorsException
      *          if there was an error resolving the binding
      */
     private <T> BindingImpl<T> convertConstantStringBinding(Key<T> key, Errors errors) throws ErrorsException {
@@ -384,7 +384,7 @@ private <T> BindingImpl<T> createJustInTimeBindingRecursive(Key<T> key, Errors e
      * <li>The constructor of the raw type. Only for unannotated keys.
      * </ol>
      *
-     * @throws org.elasticsearch.common.inject.internal.ErrorsException
+     * @throws ErrorsException
      *          if the binding cannot be created.
      */
     <T> BindingImpl<T> createJustInTimeBinding(Key<T> key, Errors errors) throws ErrorsException {
diff --git a/server/src/main/java/org/elasticsearch/common/inject/InjectorShell.java b/server/src/main/java/org/elasticsearch/injection/guice/InjectorShell.java
similarity index 89%
rename from server/src/main/java/org/elasticsearch/common/inject/InjectorShell.java
rename to server/src/main/java/org/elasticsearch/injection/guice/InjectorShell.java
index e6df9e2641cd8..497681282e5cb 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/InjectorShell.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/InjectorShell.java
@@ -14,19 +14,19 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
-
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.InternalContext;
-import org.elasticsearch.common.inject.internal.InternalFactory;
-import org.elasticsearch.common.inject.internal.ProviderInstanceBindingImpl;
-import org.elasticsearch.common.inject.internal.Scoping;
-import org.elasticsearch.common.inject.internal.SourceProvider;
-import org.elasticsearch.common.inject.internal.Stopwatch;
-import org.elasticsearch.common.inject.spi.Dependency;
-import org.elasticsearch.common.inject.spi.Element;
-import org.elasticsearch.common.inject.spi.Elements;
-import org.elasticsearch.common.inject.spi.InjectionPoint;
+package org.elasticsearch.injection.guice;
+
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
+import org.elasticsearch.injection.guice.internal.ProviderInstanceBindingImpl;
+import org.elasticsearch.injection.guice.internal.Scoping;
+import org.elasticsearch.injection.guice.internal.SourceProvider;
+import org.elasticsearch.injection.guice.internal.Stopwatch;
+import org.elasticsearch.injection.guice.spi.Dependency;
+import org.elasticsearch.injection.guice.spi.Element;
+import org.elasticsearch.injection.guice.spi.Elements;
+import org.elasticsearch.injection.guice.spi.InjectionPoint;
 
 import java.util.ArrayList;
 import java.util.List;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/InternalFactoryToProviderAdapter.java b/server/src/main/java/org/elasticsearch/injection/guice/InternalFactoryToProviderAdapter.java
similarity index 80%
rename from server/src/main/java/org/elasticsearch/common/inject/InternalFactoryToProviderAdapter.java
rename to server/src/main/java/org/elasticsearch/injection/guice/InternalFactoryToProviderAdapter.java
index 668bd41f2ba6a..4a5e49ec088f8 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/InternalFactoryToProviderAdapter.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/InternalFactoryToProviderAdapter.java
@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
-
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalContext;
-import org.elasticsearch.common.inject.internal.InternalFactory;
-import org.elasticsearch.common.inject.internal.SourceProvider;
-import org.elasticsearch.common.inject.spi.Dependency;
+package org.elasticsearch.injection.guice;
+
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
+import org.elasticsearch.injection.guice.internal.SourceProvider;
+import org.elasticsearch.injection.guice.spi.Dependency;
 
 import java.util.Objects;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Key.java b/server/src/main/java/org/elasticsearch/injection/guice/Key.java
similarity index 97%
rename from server/src/main/java/org/elasticsearch/common/inject/Key.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Key.java
index f956edaca7bb0..bbe6f97729497 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Key.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Key.java
@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Annotations;
-import org.elasticsearch.common.inject.internal.MoreTypes;
-import org.elasticsearch.common.inject.internal.ToStringBuilder;
+import org.elasticsearch.injection.guice.internal.Annotations;
+import org.elasticsearch.injection.guice.internal.MoreTypes;
+import org.elasticsearch.injection.guice.internal.ToStringBuilder;
 
 import java.lang.annotation.Annotation;
 import java.lang.reflect.Type;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/LookupProcessor.java b/server/src/main/java/org/elasticsearch/injection/guice/LookupProcessor.java
similarity index 84%
rename from server/src/main/java/org/elasticsearch/common/inject/LookupProcessor.java
rename to server/src/main/java/org/elasticsearch/injection/guice/LookupProcessor.java
index 18372f67179bf..e52dc619bf7c7 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/LookupProcessor.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/LookupProcessor.java
@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.spi.ProviderLookup;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.spi.ProviderLookup;
 
 /**
  *
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Lookups.java b/server/src/main/java/org/elasticsearch/injection/guice/Lookups.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/common/inject/Lookups.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Lookups.java
index bd98a81e5c303..a2258be4f0a23 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Lookups.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Lookups.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
 /**
  * Accessors for providers and members injectors. The returned values will not be functional until
diff --git a/server/src/main/java/org/elasticsearch/common/inject/MembersInjector.java b/server/src/main/java/org/elasticsearch/injection/guice/MembersInjector.java
similarity index 95%
rename from server/src/main/java/org/elasticsearch/common/inject/MembersInjector.java
rename to server/src/main/java/org/elasticsearch/injection/guice/MembersInjector.java
index ffaee1648ab5a..232fd06720df6 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/MembersInjector.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/MembersInjector.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
 /**
  * Injects dependencies into the fields and methods on instances of type {@code T}. Ignores the
diff --git a/server/src/main/java/org/elasticsearch/common/inject/MembersInjectorImpl.java b/server/src/main/java/org/elasticsearch/injection/guice/MembersInjectorImpl.java
similarity index 89%
rename from server/src/main/java/org/elasticsearch/common/inject/MembersInjectorImpl.java
rename to server/src/main/java/org/elasticsearch/injection/guice/MembersInjectorImpl.java
index 8c190ef301651..90bbdd9bfd327 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/MembersInjectorImpl.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/MembersInjectorImpl.java
@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalContext;
 
 import java.util.List;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/MembersInjectorStore.java b/server/src/main/java/org/elasticsearch/injection/guice/MembersInjectorStore.java
similarity index 91%
rename from server/src/main/java/org/elasticsearch/common/inject/MembersInjectorStore.java
rename to server/src/main/java/org/elasticsearch/injection/guice/MembersInjectorStore.java
index 925739af25742..f0b0ea7f56e71 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/MembersInjectorStore.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/MembersInjectorStore.java
@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.FailableCache;
-import org.elasticsearch.common.inject.spi.InjectionPoint;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.FailableCache;
+import org.elasticsearch.injection.guice.spi.InjectionPoint;
 
 import java.util.ArrayList;
 import java.util.Collections;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/MessageProcessor.java b/server/src/main/java/org/elasticsearch/injection/guice/MessageProcessor.java
similarity index 90%
rename from server/src/main/java/org/elasticsearch/common/inject/MessageProcessor.java
rename to server/src/main/java/org/elasticsearch/injection/guice/MessageProcessor.java
index 8a60d4c2c5012..9459add1e3ff9 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/MessageProcessor.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/MessageProcessor.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.spi.Message;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.spi.Message;
 
 /**
  * Handles {@link Binder#addError} commands.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Module.java b/server/src/main/java/org/elasticsearch/injection/guice/Module.java
similarity index 96%
rename from server/src/main/java/org/elasticsearch/common/inject/Module.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Module.java
index 38eddcdb200b7..078ed035c3a72 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Module.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Module.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
 /**
  * A module contributes configuration information, typically interface
diff --git a/server/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java b/server/src/main/java/org/elasticsearch/injection/guice/ModulesBuilder.java
similarity index 97%
rename from server/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java
rename to server/src/main/java/org/elasticsearch/injection/guice/ModulesBuilder.java
index 60e6fa5fff22a..6cb9d55579fe2 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/ModulesBuilder.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
 import java.util.ArrayList;
 import java.util.Collections;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/PrivateBinder.java b/server/src/main/java/org/elasticsearch/injection/guice/PrivateBinder.java
similarity index 95%
rename from server/src/main/java/org/elasticsearch/common/inject/PrivateBinder.java
rename to server/src/main/java/org/elasticsearch/injection/guice/PrivateBinder.java
index fd80e6271b2cf..d6cfca2424815 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/PrivateBinder.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/PrivateBinder.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
 /**
  * Returns a binder whose configuration information is hidden from its environment by default.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Provider.java b/server/src/main/java/org/elasticsearch/injection/guice/Provider.java
similarity index 98%
rename from server/src/main/java/org/elasticsearch/common/inject/Provider.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Provider.java
index 6531897ba1fd7..692617239ea74 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Provider.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Provider.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
 /**
  * An object capable of providing instances of type {@code T}. Providers are used in numerous ways
diff --git a/server/src/main/java/org/elasticsearch/common/inject/ProviderToInternalFactoryAdapter.java b/server/src/main/java/org/elasticsearch/injection/guice/ProviderToInternalFactoryAdapter.java
similarity index 86%
rename from server/src/main/java/org/elasticsearch/common/inject/ProviderToInternalFactoryAdapter.java
rename to server/src/main/java/org/elasticsearch/injection/guice/ProviderToInternalFactoryAdapter.java
index 67f262cf36dee..9755ce251490e 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/ProviderToInternalFactoryAdapter.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/ProviderToInternalFactoryAdapter.java
@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalFactory;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
 
 /**
  * @author crazybob@google.com (Bob Lee)
diff --git a/server/src/main/java/org/elasticsearch/common/inject/ProvisionException.java b/server/src/main/java/org/elasticsearch/injection/guice/ProvisionException.java
similarity index 91%
rename from server/src/main/java/org/elasticsearch/common/inject/ProvisionException.java
rename to server/src/main/java/org/elasticsearch/injection/guice/ProvisionException.java
index 1d4ab8ccbe1d8..ad14189db38e9 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/ProvisionException.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/ProvisionException.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.spi.Message;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.spi.Message;
 
 import java.util.Collection;
 import java.util.Set;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Scope.java b/server/src/main/java/org/elasticsearch/injection/guice/Scope.java
similarity index 98%
rename from server/src/main/java/org/elasticsearch/common/inject/Scope.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Scope.java
index 03ca3aff8bd72..681fc17bc6353 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Scope.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Scope.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
 /**
  * A scope is a level of visibility that instances provided by Guice may have.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Scopes.java b/server/src/main/java/org/elasticsearch/injection/guice/Scopes.java
similarity index 95%
rename from server/src/main/java/org/elasticsearch/common/inject/Scopes.java
rename to server/src/main/java/org/elasticsearch/injection/guice/Scopes.java
index 60f36fd879aa2..d5b61407b4975 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Scopes.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/Scopes.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.InternalFactory;
-import org.elasticsearch.common.inject.internal.Scoping;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
+import org.elasticsearch.injection.guice.internal.Scoping;
 
 import java.util.Locale;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/SingleMethodInjector.java b/server/src/main/java/org/elasticsearch/injection/guice/SingleMethodInjector.java
similarity index 87%
rename from server/src/main/java/org/elasticsearch/common/inject/SingleMethodInjector.java
rename to server/src/main/java/org/elasticsearch/injection/guice/SingleMethodInjector.java
index d36bc1e623a99..580ce216fc32e 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/SingleMethodInjector.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/SingleMethodInjector.java
@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalContext;
-import org.elasticsearch.common.inject.spi.InjectionPoint;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalContext;
+import org.elasticsearch.injection.guice.spi.InjectionPoint;
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/SingleParameterInjector.java b/server/src/main/java/org/elasticsearch/injection/guice/SingleParameterInjector.java
similarity index 86%
rename from server/src/main/java/org/elasticsearch/common/inject/SingleParameterInjector.java
rename to server/src/main/java/org/elasticsearch/injection/guice/SingleParameterInjector.java
index 6ec48fc7b7a0d..468db1d7211b3 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/SingleParameterInjector.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/SingleParameterInjector.java
@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.InternalContext;
-import org.elasticsearch.common.inject.internal.InternalFactory;
-import org.elasticsearch.common.inject.spi.Dependency;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.InternalFactory;
+import org.elasticsearch.injection.guice.spi.Dependency;
 
 /**
  * Resolves a single parameter, to be used in a constructor or method invocation.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/State.java b/server/src/main/java/org/elasticsearch/injection/guice/State.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/common/inject/State.java
rename to server/src/main/java/org/elasticsearch/injection/guice/State.java
index e8b9404447f4f..915711c98011e 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/State.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/State.java
@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.BindingImpl;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.MatcherAndConverter;
+import org.elasticsearch.injection.guice.internal.BindingImpl;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.MatcherAndConverter;
 
 import java.util.Map;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/TypeConverterBindingProcessor.java b/server/src/main/java/org/elasticsearch/injection/guice/TypeConverterBindingProcessor.java
similarity index 91%
rename from server/src/main/java/org/elasticsearch/common/inject/TypeConverterBindingProcessor.java
rename to server/src/main/java/org/elasticsearch/injection/guice/TypeConverterBindingProcessor.java
index 0e4b3b3275e2e..c87a44d36a244 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/TypeConverterBindingProcessor.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/TypeConverterBindingProcessor.java
@@ -14,16 +14,16 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
-
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.MatcherAndConverter;
-import org.elasticsearch.common.inject.internal.SourceProvider;
-import org.elasticsearch.common.inject.internal.Strings;
-import org.elasticsearch.common.inject.matcher.AbstractMatcher;
-import org.elasticsearch.common.inject.matcher.Matcher;
-import org.elasticsearch.common.inject.matcher.Matchers;
-import org.elasticsearch.common.inject.spi.TypeConverter;
+package org.elasticsearch.injection.guice;
+
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.MatcherAndConverter;
+import org.elasticsearch.injection.guice.internal.SourceProvider;
+import org.elasticsearch.injection.guice.internal.Strings;
+import org.elasticsearch.injection.guice.matcher.AbstractMatcher;
+import org.elasticsearch.injection.guice.matcher.Matcher;
+import org.elasticsearch.injection.guice.matcher.Matchers;
+import org.elasticsearch.injection.guice.spi.TypeConverter;
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/TypeLiteral.java b/server/src/main/java/org/elasticsearch/injection/guice/TypeLiteral.java
similarity index 96%
rename from server/src/main/java/org/elasticsearch/common/inject/TypeLiteral.java
rename to server/src/main/java/org/elasticsearch/injection/guice/TypeLiteral.java
index d39c4e44d2ff9..930a9482f0ae5 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/TypeLiteral.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/TypeLiteral.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.internal.MoreTypes;
-import org.elasticsearch.common.inject.util.Types;
+import org.elasticsearch.injection.guice.internal.MoreTypes;
+import org.elasticsearch.injection.guice.util.Types;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.GenericArrayType;
@@ -31,8 +31,6 @@
 import java.util.List;
 import java.util.Objects;
 
-import static org.elasticsearch.common.inject.internal.MoreTypes.canonicalize;
-
 /**
  * Represents a generic type {@code T}. Java doesn't yet provide a way to
  * represent generic types, so this class does. Forces clients to create a
@@ -89,7 +87,7 @@ protected TypeLiteral() {
      */
     @SuppressWarnings("unchecked")
     TypeLiteral(Type type) {
-        this.type = canonicalize(Objects.requireNonNull(type, "type"));
+        this.type = MoreTypes.canonicalize(Objects.requireNonNull(type, "type"));
         this.rawType = (Class<? super T>) MoreTypes.getRawType(this.type);
         this.hashCode = MoreTypes.hashCode(this.type);
     }
@@ -104,7 +102,7 @@ static Type getSuperclassTypeParameter(Class<?> subclass) {
             throw new RuntimeException("Missing type parameter.");
         }
         ParameterizedType parameterized = (ParameterizedType) superclass;
-        return canonicalize(parameterized.getActualTypeArguments()[0]);
+        return MoreTypes.canonicalize(parameterized.getActualTypeArguments()[0]);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/common/inject/WeakKeySet.java b/server/src/main/java/org/elasticsearch/injection/guice/WeakKeySet.java
similarity index 97%
rename from server/src/main/java/org/elasticsearch/common/inject/WeakKeySet.java
rename to server/src/main/java/org/elasticsearch/injection/guice/WeakKeySet.java
index c72cc25313466..a621be1190ad6 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/WeakKeySet.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/WeakKeySet.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
 import java.util.HashSet;
 import java.util.Set;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/binder/AnnotatedBindingBuilder.java b/server/src/main/java/org/elasticsearch/injection/guice/binder/AnnotatedBindingBuilder.java
similarity index 83%
rename from server/src/main/java/org/elasticsearch/common/inject/binder/AnnotatedBindingBuilder.java
rename to server/src/main/java/org/elasticsearch/injection/guice/binder/AnnotatedBindingBuilder.java
index 3e0c288edd21a..ed761664a8522 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/binder/AnnotatedBindingBuilder.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/binder/AnnotatedBindingBuilder.java
@@ -14,10 +14,12 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.binder;
+package org.elasticsearch.injection.guice.binder;
+
+import org.elasticsearch.injection.guice.Binder;
 
 /**
- * See the EDSL examples at {@link org.elasticsearch.common.inject.Binder}.
+ * See the EDSL examples at {@link Binder}.
  *
  * @author crazybob@google.com (Bob Lee)
  */
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/binder/LinkedBindingBuilder.java b/server/src/main/java/org/elasticsearch/injection/guice/binder/LinkedBindingBuilder.java
new file mode 100644
index 0000000000000..d0f605e80fb25
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/binder/LinkedBindingBuilder.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2006 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.elasticsearch.injection.guice.binder;
+
+import org.elasticsearch.injection.guice.Binder;
+import org.elasticsearch.injection.guice.Provider;
+
+/**
+ * See the EDSL examples at {@link Binder}.
+ *
+ * @author crazybob@google.com (Bob Lee)
+ */
+public interface LinkedBindingBuilder<T> extends ScopedBindingBuilder {
+
+    /**
+     * See the EDSL examples at {@link Binder}.
+     */
+    ScopedBindingBuilder to(Class<? extends T> implementation);
+
+    /**
+     * See the EDSL examples at {@link Binder}.
+     */
+    void toInstance(T instance);
+
+    /**
+     * See the EDSL examples at {@link Binder}.
+     */
+    ScopedBindingBuilder toProvider(Provider<? extends T> provider);
+
+}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/binder/ScopedBindingBuilder.java b/server/src/main/java/org/elasticsearch/injection/guice/binder/ScopedBindingBuilder.java
similarity index 75%
rename from server/src/main/java/org/elasticsearch/common/inject/binder/ScopedBindingBuilder.java
rename to server/src/main/java/org/elasticsearch/injection/guice/binder/ScopedBindingBuilder.java
index dad91c3fb8878..533d2837f9e06 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/binder/ScopedBindingBuilder.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/binder/ScopedBindingBuilder.java
@@ -14,20 +14,23 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.binder;
+package org.elasticsearch.injection.guice.binder;
+
+import org.elasticsearch.injection.guice.Binder;
+import org.elasticsearch.injection.guice.Injector;
 
 /**
- * See the EDSL examples at {@link org.elasticsearch.common.inject.Binder}.
+ * See the EDSL examples at {@link Binder}.
  *
  * @author crazybob@google.com (Bob Lee)
  */
 public interface ScopedBindingBuilder {
 
     /**
-     * Instructs the {@link org.elasticsearch.common.inject.Injector} to eagerly initialize this
+     * Instructs the {@link Injector} to eagerly initialize this
      * singleton-scoped binding upon creation. Useful for application
      * initialization logic.  See the EDSL examples at
-     * {@link org.elasticsearch.common.inject.Binder}.
+     * {@link Binder}.
      */
     void asEagerSingleton();
 }
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/binder/package-info.java b/server/src/main/java/org/elasticsearch/injection/guice/binder/package-info.java
new file mode 100644
index 0000000000000..ae713ad733a0c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/binder/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2006 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Interfaces which make up {@link org.elasticsearch.injection.guice.Binder}'s
+ * expression language.
+ */
+package org.elasticsearch.injection.guice.binder;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/AbstractBindingBuilder.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/AbstractBindingBuilder.java
similarity index 91%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/AbstractBindingBuilder.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/AbstractBindingBuilder.java
index 345c92ce8d354..28053c5f1d557 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/AbstractBindingBuilder.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/AbstractBindingBuilder.java
@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.spi.Element;
-import org.elasticsearch.common.inject.spi.InstanceBinding;
+import org.elasticsearch.injection.guice.Binder;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.spi.Element;
+import org.elasticsearch.injection.guice.spi.InstanceBinding;
 
 import java.util.List;
 
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/internal/Annotations.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/Annotations.java
new file mode 100644
index 0000000000000..99ecc086e1df1
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/Annotations.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2006 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.elasticsearch.injection.guice.internal;
+
+import org.elasticsearch.injection.guice.BindingAnnotation;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.TypeLiteral;
+
+import java.lang.annotation.Annotation;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.reflect.Member;
+
+/**
+ * Annotation utilities.
+ *
+ * @author crazybob@google.com (Bob Lee)
+ */
+public class Annotations {
+
+    /**
+     * Returns true if the given annotation is retained at runtime.
+     */
+    public static boolean isRetainedAtRuntime(Class<? extends Annotation> annotationType) {
+        Retention retention = annotationType.getAnnotation(Retention.class);
+        return retention != null && retention.value() == RetentionPolicy.RUNTIME;
+    }
+
+    /**
+     * Gets a key for the given type, member and annotations.
+     */
+    public static Key<?> getKey(TypeLiteral<?> type, Member member, Annotation[] annotations, Errors errors) throws ErrorsException {
+        int numErrorsBefore = errors.size();
+        Annotation found = findBindingAnnotation(errors, member, annotations);
+        errors.throwIfNewErrors(numErrorsBefore);
+        return found == null ? Key.get(type) : Key.get(type, found);
+    }
+
+    /**
+     * Returns the binding annotation on {@code member}, or null if there isn't one.
+     */
+    public static Annotation findBindingAnnotation(Errors errors, Member member, Annotation[] annotations) {
+        Annotation found = null;
+
+        for (Annotation annotation : annotations) {
+            if (annotation.annotationType().getAnnotation(BindingAnnotation.class) != null) {
+                if (found != null) {
+                    errors.duplicateBindingAnnotations(member, found.annotationType(), annotation.annotationType());
+                } else {
+                    found = annotation;
+                }
+            }
+        }
+
+        return found;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/BindingBuilder.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/BindingBuilder.java
similarity index 86%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/BindingBuilder.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/BindingBuilder.java
index fd40879025c65..73f6470a1e52d 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/BindingBuilder.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/BindingBuilder.java
@@ -14,16 +14,16 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.inject.ConfigurationException;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Provider;
-import org.elasticsearch.common.inject.binder.AnnotatedBindingBuilder;
-import org.elasticsearch.common.inject.spi.Element;
-import org.elasticsearch.common.inject.spi.InjectionPoint;
-import org.elasticsearch.common.inject.spi.Message;
+import org.elasticsearch.injection.guice.Binder;
+import org.elasticsearch.injection.guice.ConfigurationException;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.Provider;
+import org.elasticsearch.injection.guice.binder.AnnotatedBindingBuilder;
+import org.elasticsearch.injection.guice.spi.Element;
+import org.elasticsearch.injection.guice.spi.InjectionPoint;
+import org.elasticsearch.injection.guice.spi.Message;
 
 import java.util.List;
 import java.util.Objects;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/BindingImpl.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/BindingImpl.java
similarity index 88%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/BindingImpl.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/BindingImpl.java
index b68600ef5381f..5b11e69eee95e 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/BindingImpl.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/BindingImpl.java
@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
-
-import org.elasticsearch.common.inject.Binding;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Provider;
-import org.elasticsearch.common.inject.spi.ElementVisitor;
-import org.elasticsearch.common.inject.spi.InstanceBinding;
+package org.elasticsearch.injection.guice.internal;
+
+import org.elasticsearch.injection.guice.Binding;
+import org.elasticsearch.injection.guice.Injector;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.Provider;
+import org.elasticsearch.injection.guice.spi.ElementVisitor;
+import org.elasticsearch.injection.guice.spi.InstanceBinding;
 
 /**
  * @author crazybob@google.com (Bob Lee)
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/ConstructionContext.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/ConstructionContext.java
similarity index 98%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/ConstructionContext.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/ConstructionContext.java
index 0f22c5fe2584c..22b91f0ef3759 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/ConstructionContext.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/ConstructionContext.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
 import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.InvocationTargetException;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/Errors.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/Errors.java
similarity index 97%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/Errors.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/Errors.java
index cb78cf34e7d8a..40ea83cc25068 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/Errors.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/Errors.java
@@ -14,18 +14,18 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
 import org.apache.lucene.util.CollectionUtil;
-import org.elasticsearch.common.inject.ConfigurationException;
-import org.elasticsearch.common.inject.CreationException;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.ProvisionException;
-import org.elasticsearch.common.inject.TypeLiteral;
-import org.elasticsearch.common.inject.spi.Dependency;
-import org.elasticsearch.common.inject.spi.InjectionPoint;
-import org.elasticsearch.common.inject.spi.Message;
+import org.elasticsearch.injection.guice.ConfigurationException;
+import org.elasticsearch.injection.guice.CreationException;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.ProvisionException;
+import org.elasticsearch.injection.guice.TypeLiteral;
+import org.elasticsearch.injection.guice.spi.Dependency;
+import org.elasticsearch.injection.guice.spi.InjectionPoint;
+import org.elasticsearch.injection.guice.spi.Message;
 
 import java.io.PrintWriter;
 import java.io.StringWriter;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/ErrorsException.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/ErrorsException.java
similarity index 95%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/ErrorsException.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/ErrorsException.java
index eac46f5db64e2..9645d4d7c5c66 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/ErrorsException.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/ErrorsException.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
 /**
  * Indicates that a result could not be returned while preparing or resolving a binding. The caller
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/FailableCache.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/FailableCache.java
similarity index 97%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/FailableCache.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/FailableCache.java
index e71bc2e5785b7..b68987014330f 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/FailableCache.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/FailableCache.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
 import java.util.concurrent.ConcurrentHashMap;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/InstanceBindingImpl.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/InstanceBindingImpl.java
similarity index 84%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/InstanceBindingImpl.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/InstanceBindingImpl.java
index eae400dbbe052..dc545e2774550 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/InstanceBindingImpl.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/InstanceBindingImpl.java
@@ -14,15 +14,15 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Provider;
-import org.elasticsearch.common.inject.spi.BindingTargetVisitor;
-import org.elasticsearch.common.inject.spi.InjectionPoint;
-import org.elasticsearch.common.inject.spi.InstanceBinding;
-import org.elasticsearch.common.inject.util.Providers;
+import org.elasticsearch.injection.guice.Injector;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.Provider;
+import org.elasticsearch.injection.guice.spi.BindingTargetVisitor;
+import org.elasticsearch.injection.guice.spi.InjectionPoint;
+import org.elasticsearch.injection.guice.spi.InstanceBinding;
+import org.elasticsearch.injection.guice.util.Providers;
 
 import java.util.Set;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/InternalContext.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/InternalContext.java
similarity index 93%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/InternalContext.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/InternalContext.java
index c25333c30873c..12020011b1ac1 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/InternalContext.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/InternalContext.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.spi.Dependency;
+import org.elasticsearch.injection.guice.spi.Dependency;
 
 import java.util.HashMap;
 import java.util.Map;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/InternalFactory.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/InternalFactory.java
similarity index 89%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/InternalFactory.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/InternalFactory.java
index ec9e5eb17edc0..faee1d8ea0671 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/InternalFactory.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/InternalFactory.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.spi.Dependency;
+import org.elasticsearch.injection.guice.spi.Dependency;
 
 /**
  * Creates objects which will be injected.
@@ -53,7 +53,7 @@ public String toString() {
      *
      * @param context of this injection
      * @return instance to be injected
-     * @throws org.elasticsearch.common.inject.internal.ErrorsException
+     * @throws ErrorsException
      *          if a value cannot be provided
      */
     T get(Errors errors, InternalContext context, Dependency<?> dependency) throws ErrorsException;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/LinkedBindingImpl.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/LinkedBindingImpl.java
similarity index 87%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/LinkedBindingImpl.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/LinkedBindingImpl.java
index 79e2a4f34c7ec..e225924aa2f17 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/LinkedBindingImpl.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/LinkedBindingImpl.java
@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.spi.BindingTargetVisitor;
-import org.elasticsearch.common.inject.spi.LinkedKeyBinding;
+import org.elasticsearch.injection.guice.Injector;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.spi.BindingTargetVisitor;
+import org.elasticsearch.injection.guice.spi.LinkedKeyBinding;
 
 public final class LinkedBindingImpl<T> extends BindingImpl<T> implements LinkedKeyBinding<T> {
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/LinkedProviderBindingImpl.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/LinkedProviderBindingImpl.java
similarity index 86%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/LinkedProviderBindingImpl.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/LinkedProviderBindingImpl.java
index 7f9e36be4e49d..93d87629c9e3c 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/LinkedProviderBindingImpl.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/LinkedProviderBindingImpl.java
@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Provider;
-import org.elasticsearch.common.inject.spi.BindingTargetVisitor;
-import org.elasticsearch.common.inject.spi.ProviderKeyBinding;
+import org.elasticsearch.injection.guice.Injector;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.Provider;
+import org.elasticsearch.injection.guice.spi.BindingTargetVisitor;
+import org.elasticsearch.injection.guice.spi.ProviderKeyBinding;
 
 public final class LinkedProviderBindingImpl<T> extends BindingImpl<T> implements ProviderKeyBinding<T> {
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/MatcherAndConverter.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/MatcherAndConverter.java
similarity index 87%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/MatcherAndConverter.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/MatcherAndConverter.java
index 9103f16888f03..445a0a7976b2f 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/MatcherAndConverter.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/MatcherAndConverter.java
@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.TypeLiteral;
-import org.elasticsearch.common.inject.matcher.Matcher;
-import org.elasticsearch.common.inject.spi.TypeConverter;
+import org.elasticsearch.injection.guice.TypeLiteral;
+import org.elasticsearch.injection.guice.matcher.Matcher;
+import org.elasticsearch.injection.guice.spi.TypeConverter;
 
 import java.util.Objects;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/MoreTypes.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/MoreTypes.java
similarity index 98%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/MoreTypes.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/MoreTypes.java
index 0981a0dbe8678..ba214bc0f8d27 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/MoreTypes.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/MoreTypes.java
@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.ConfigurationException;
-import org.elasticsearch.common.inject.TypeLiteral;
-import org.elasticsearch.common.inject.spi.Message;
+import org.elasticsearch.injection.guice.ConfigurationException;
+import org.elasticsearch.injection.guice.TypeLiteral;
+import org.elasticsearch.injection.guice.spi.Message;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/Nullability.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/Nullability.java
similarity index 96%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/Nullability.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/Nullability.java
index b8184eae28ec9..b8294a70538d3 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/Nullability.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/Nullability.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
 import java.lang.annotation.Annotation;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/ProviderInstanceBindingImpl.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/ProviderInstanceBindingImpl.java
similarity index 86%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/ProviderInstanceBindingImpl.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/ProviderInstanceBindingImpl.java
index ae8823e2f7246..1ddc94c1d3c07 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/ProviderInstanceBindingImpl.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/ProviderInstanceBindingImpl.java
@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Provider;
-import org.elasticsearch.common.inject.spi.BindingTargetVisitor;
-import org.elasticsearch.common.inject.spi.InjectionPoint;
-import org.elasticsearch.common.inject.spi.ProviderInstanceBinding;
+import org.elasticsearch.injection.guice.Injector;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.Provider;
+import org.elasticsearch.injection.guice.spi.BindingTargetVisitor;
+import org.elasticsearch.injection.guice.spi.InjectionPoint;
+import org.elasticsearch.injection.guice.spi.ProviderInstanceBinding;
 
 import java.util.Set;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/Scoping.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/Scoping.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/Scoping.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/Scoping.java
index d0544aff20e39..fcb03f34f4204 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/Scoping.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/Scoping.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.Scope;
-import org.elasticsearch.common.inject.Scopes;
+import org.elasticsearch.injection.guice.Scope;
+import org.elasticsearch.injection.guice.Scopes;
 
 /**
  * References a scope, either directly (as a scope instance), or indirectly (as a scope annotation).
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/SourceProvider.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/SourceProvider.java
similarity index 97%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/SourceProvider.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/SourceProvider.java
index bca35dba6bdd9..ac5debab9bfd5 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/SourceProvider.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/SourceProvider.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
 import java.util.HashSet;
 import java.util.Set;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/StackTraceElements.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/StackTraceElements.java
similarity index 96%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/StackTraceElements.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/StackTraceElements.java
index 63739a9d6c949..9961e6026fa43 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/StackTraceElements.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/StackTraceElements.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Member;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/Stopwatch.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/Stopwatch.java
similarity index 96%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/Stopwatch.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/Stopwatch.java
index e271a22fe0780..48380dae08d25 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/Stopwatch.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/Stopwatch.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
 import org.elasticsearch.core.TimeValue;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/Strings.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/Strings.java
similarity index 97%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/Strings.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/Strings.java
index c2cd0dd458546..07a0f39b8a0c9 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/Strings.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/Strings.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
 /**
  * String utilities.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/ToStringBuilder.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/ToStringBuilder.java
similarity index 96%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/ToStringBuilder.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/ToStringBuilder.java
index 7229886b44a3c..ac3636dfc6d97 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/ToStringBuilder.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/ToStringBuilder.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
 import java.util.LinkedHashMap;
 import java.util.Map;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/internal/UntargettedBindingImpl.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/UntargettedBindingImpl.java
similarity index 84%
rename from server/src/main/java/org/elasticsearch/common/inject/internal/UntargettedBindingImpl.java
rename to server/src/main/java/org/elasticsearch/injection/guice/internal/UntargettedBindingImpl.java
index d0ef8affa0bfb..4c14cd48d9ed7 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/internal/UntargettedBindingImpl.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/UntargettedBindingImpl.java
@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.internal;
+package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.spi.BindingTargetVisitor;
-import org.elasticsearch.common.inject.spi.UntargettedBinding;
+import org.elasticsearch.injection.guice.Injector;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.spi.BindingTargetVisitor;
+import org.elasticsearch.injection.guice.spi.UntargettedBinding;
 
 public class UntargettedBindingImpl<T> extends BindingImpl<T> implements UntargettedBinding<T> {
 
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/internal/package-info.java b/server/src/main/java/org/elasticsearch/injection/guice/internal/package-info.java
new file mode 100644
index 0000000000000..da05fd263d18c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/internal/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2006 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <i>Guice</i> (sounds like like "juice")
+ */
+
+package org.elasticsearch.injection.guice.internal;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/matcher/AbstractMatcher.java b/server/src/main/java/org/elasticsearch/injection/guice/matcher/AbstractMatcher.java
similarity index 93%
rename from server/src/main/java/org/elasticsearch/common/inject/matcher/AbstractMatcher.java
rename to server/src/main/java/org/elasticsearch/injection/guice/matcher/AbstractMatcher.java
index 97d93d920e5f5..440a7dfd90fa4 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/matcher/AbstractMatcher.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/matcher/AbstractMatcher.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.matcher;
+package org.elasticsearch.injection.guice.matcher;
 
 /**
  * Implements {@code and()} and {@code or()}.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/matcher/Matcher.java b/server/src/main/java/org/elasticsearch/injection/guice/matcher/Matcher.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/common/inject/matcher/Matcher.java
rename to server/src/main/java/org/elasticsearch/injection/guice/matcher/Matcher.java
index d89e98bca35ec..e66719a85a8ac 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/matcher/Matcher.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/matcher/Matcher.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.matcher;
+package org.elasticsearch.injection.guice.matcher;
 
 /**
  * Returns {@code true} or {@code false} for a given input.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java b/server/src/main/java/org/elasticsearch/injection/guice/matcher/Matchers.java
similarity index 98%
rename from server/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java
rename to server/src/main/java/org/elasticsearch/injection/guice/matcher/Matchers.java
index dac1cdef9f055..b936e4952086f 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/matcher/Matchers.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.matcher;
+package org.elasticsearch.injection.guice.matcher;
 
 import java.util.Objects;
 
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/matcher/package-info.java b/server/src/main/java/org/elasticsearch/injection/guice/matcher/package-info.java
new file mode 100644
index 0000000000000..2a08200f1f8a7
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/matcher/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2006 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Used for matching things. Primarily used to pick out methods to which to
+ * apply interceptors.
+ */
+package org.elasticsearch.injection.guice.matcher;
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/multibindings/Element.java b/server/src/main/java/org/elasticsearch/injection/guice/multibindings/Element.java
new file mode 100644
index 0000000000000..9f788b7c7a43a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/multibindings/Element.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.elasticsearch.injection.guice.multibindings;
+
+import org.elasticsearch.injection.guice.BindingAnnotation;
+
+import java.lang.annotation.Retention;
+
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * An internal binding annotation applied to each element in a multibinding.
+ * All elements are assigned a globally-unique id to allow different modules
+ * to contribute multibindings independently.
+ *
+ * @author jessewilson@google.com (Jesse Wilson)
+ */
+@Retention(RUNTIME)
+@BindingAnnotation
+@interface Element {
+    int uniqueId();
+}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/multibindings/MapBinder.java b/server/src/main/java/org/elasticsearch/injection/guice/multibindings/MapBinder.java
similarity index 93%
rename from server/src/main/java/org/elasticsearch/common/inject/multibindings/MapBinder.java
rename to server/src/main/java/org/elasticsearch/injection/guice/multibindings/MapBinder.java
index 5ba9bb9cdead0..77c0fa7ba87cc 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/multibindings/MapBinder.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/multibindings/MapBinder.java
@@ -14,18 +14,17 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.multibindings;
-
-import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.Provider;
-import org.elasticsearch.common.inject.TypeLiteral;
-import org.elasticsearch.common.inject.binder.LinkedBindingBuilder;
-import org.elasticsearch.common.inject.multibindings.Multibinder.RealMultibinder;
-import org.elasticsearch.common.inject.spi.ProviderWithDependencies;
-import org.elasticsearch.common.inject.util.Types;
+package org.elasticsearch.injection.guice.multibindings;
+
+import org.elasticsearch.injection.guice.Binder;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.Module;
+import org.elasticsearch.injection.guice.Provider;
+import org.elasticsearch.injection.guice.TypeLiteral;
+import org.elasticsearch.injection.guice.binder.LinkedBindingBuilder;
+import org.elasticsearch.injection.guice.spi.ProviderWithDependencies;
+import org.elasticsearch.injection.guice.util.Types;
 
 import java.util.Collections;
 import java.util.LinkedHashMap;
@@ -33,8 +32,8 @@
 import java.util.Map.Entry;
 import java.util.Set;
 
-import static org.elasticsearch.common.inject.util.Types.newParameterizedType;
-import static org.elasticsearch.common.inject.util.Types.newParameterizedTypeWithOwner;
+import static org.elasticsearch.injection.guice.util.Types.newParameterizedType;
+import static org.elasticsearch.injection.guice.util.Types.newParameterizedTypeWithOwner;
 
 /**
  * An API to bind multiple map entries separately, only to later inject them as
@@ -183,7 +182,7 @@ public static final class RealMapBinder<K, V> extends MapBinder<K, V> implements
         private final TypeLiteral<V> valueType;
         private final Key<Map<K, V>> mapKey;
         private final Key<Map<K, Provider<V>>> providerMapKey;
-        private final RealMultibinder<Map.Entry<K, Provider<V>>> entrySetBinder;
+        private final Multibinder.RealMultibinder<Entry<K, Provider<V>>> entrySetBinder;
 
         /* the target injector's binder. non-null until initialization, null afterwards */
         private Binder binder;
@@ -198,7 +197,7 @@ private RealMapBinder(
             this.valueType = valueType;
             this.mapKey = mapKey;
             this.providerMapKey = providerMapKey;
-            this.entrySetBinder = (RealMultibinder<Entry<K, Provider<V>>>) entrySetBinder;
+            this.entrySetBinder = (Multibinder.RealMultibinder<Entry<K, Provider<V>>>) entrySetBinder;
             this.binder = binder;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/multibindings/Multibinder.java b/server/src/main/java/org/elasticsearch/injection/guice/multibindings/Multibinder.java
similarity index 91%
rename from server/src/main/java/org/elasticsearch/common/inject/multibindings/Multibinder.java
rename to server/src/main/java/org/elasticsearch/injection/guice/multibindings/Multibinder.java
index dc44f68f2a3d0..c82f5bb1f388c 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/multibindings/Multibinder.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/multibindings/Multibinder.java
@@ -14,21 +14,21 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.multibindings;
-
-import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.inject.Binding;
-import org.elasticsearch.common.inject.ConfigurationException;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.Provider;
-import org.elasticsearch.common.inject.TypeLiteral;
-import org.elasticsearch.common.inject.binder.LinkedBindingBuilder;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.spi.Message;
-import org.elasticsearch.common.inject.util.Types;
+package org.elasticsearch.injection.guice.multibindings;
+
+import org.elasticsearch.injection.guice.Binder;
+import org.elasticsearch.injection.guice.Binding;
+import org.elasticsearch.injection.guice.ConfigurationException;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.injection.guice.Injector;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.Module;
+import org.elasticsearch.injection.guice.Provider;
+import org.elasticsearch.injection.guice.TypeLiteral;
+import org.elasticsearch.injection.guice.binder.LinkedBindingBuilder;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.spi.Message;
+import org.elasticsearch.injection.guice.util.Types;
 
 import java.lang.reflect.Type;
 import java.util.ArrayList;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/multibindings/RealElement.java b/server/src/main/java/org/elasticsearch/injection/guice/multibindings/RealElement.java
similarity index 96%
rename from server/src/main/java/org/elasticsearch/common/inject/multibindings/RealElement.java
rename to server/src/main/java/org/elasticsearch/injection/guice/multibindings/RealElement.java
index a1c403744a23d..6119110184806 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/multibindings/RealElement.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/multibindings/RealElement.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.multibindings;
+package org.elasticsearch.injection.guice.multibindings;
 
 import java.lang.annotation.Annotation;
 import java.util.concurrent.atomic.AtomicInteger;
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/multibindings/package-info.java b/server/src/main/java/org/elasticsearch/injection/guice/multibindings/package-info.java
new file mode 100644
index 0000000000000..4abc7eff8baa2
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/multibindings/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Extension for binding multiple instances in a collection; this extension requires {@code
+ * guice-multibindings-2.0.jar}.
+ */
+package org.elasticsearch.injection.guice.multibindings;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/name/Named.java b/server/src/main/java/org/elasticsearch/injection/guice/name/Named.java
similarity index 90%
rename from server/src/main/java/org/elasticsearch/common/inject/name/Named.java
rename to server/src/main/java/org/elasticsearch/injection/guice/name/Named.java
index 5483ea61eb52a..6f51035cfa5e5 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/name/Named.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/name/Named.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.name;
+package org.elasticsearch.injection.guice.name;
 
-import org.elasticsearch.common.inject.BindingAnnotation;
+import org.elasticsearch.injection.guice.BindingAnnotation;
 
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/name/package-info.java b/server/src/main/java/org/elasticsearch/injection/guice/name/package-info.java
new file mode 100644
index 0000000000000..39515b057434c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/name/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2006 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Support for binding to string-based names.
+ */
+package org.elasticsearch.injection.guice.name;
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/package-info.java b/server/src/main/java/org/elasticsearch/injection/guice/package-info.java
new file mode 100644
index 0000000000000..e7278755e0557
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/package-info.java
@@ -0,0 +1,49 @@
+/* @notice
+ * Copyright (C) 2006 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2020 Elasticsearch B.V.
+ */
+
+/**
+ * <i>Google Guice</i> (pronounced "juice") is an ultra-lightweight dependency
+ * injection framework.  Please refer to the Guice
+ * <a href="http://docs.google.com/Doc?id=dd2fhx4z_5df5hw8">User's Guide</a>
+ * for a gentle introduction.
+ *
+ * <p>The principal public APIs in this package are:
+ *
+ * <dl>
+ * <dt>{@link org.elasticsearch.injection.guice.Inject}
+ * <dd>The annotation you will use in your implementation classes to tell Guice
+ *     where and how it should send in ("inject") the objects you depend on
+ *     (your "dependencies").
+ *
+ * <dt>{@link org.elasticsearch.injection.guice.Module}
+ * <dd>The interface you will implement in order to specify "bindings" --
+ *     instructions for how Guice should handle injection -- for a particular
+ *     set of interfaces.
+ *
+ * <dt>{@link org.elasticsearch.injection.guice.Binder}
+ * <dd>The object that Guice passes into your {@link org.elasticsearch.injection.guice.Module}
+ *     to collect these bindings.
+ *
+ * <dt>{@link org.elasticsearch.injection.guice.Provider}
+ * <dd>The interface you will implement when you need to customize exactly how
+ *     Guice creates instances for a particular binding.
+ *
+ * </dl>
+ *
+ */
+package org.elasticsearch.injection.guice;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/BindingTargetVisitor.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/BindingTargetVisitor.java
similarity index 98%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/BindingTargetVisitor.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/BindingTargetVisitor.java
index ea451b1289453..7264a6dc051f8 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/BindingTargetVisitor.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/BindingTargetVisitor.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
 /**
  * Visits each of the strategies used to find an instance to satisfy an injection.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/ConstructorBinding.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/ConstructorBinding.java
similarity index 89%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/ConstructorBinding.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/ConstructorBinding.java
index 3c9b044658a80..6dbf3c138b276 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/ConstructorBinding.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/ConstructorBinding.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Binding;
+import org.elasticsearch.injection.guice.Binding;
 
 /**
  * A binding to the constructor of a concrete clss. To resolve injections, an instance is
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/ConvertedConstantBinding.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/ConvertedConstantBinding.java
similarity index 90%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/ConvertedConstantBinding.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/ConvertedConstantBinding.java
index 24ec0d3addb45..ea2a0e3a223d6 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/ConvertedConstantBinding.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/ConvertedConstantBinding.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Binding;
+import org.elasticsearch.injection.guice.Binding;
 
 /**
  * A binding created from converting a bound instance to a new type. The source binding has the same
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/Dependency.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/Dependency.java
similarity index 97%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/Dependency.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/Dependency.java
index 55e0bde8be376..3d3f3550d2bfe 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/Dependency.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/Dependency.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Key;
+import org.elasticsearch.injection.guice.Key;
 
 import java.util.Objects;
 
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/spi/Element.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/Element.java
new file mode 100644
index 0000000000000..c8d0361c921b8
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/Element.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.elasticsearch.injection.guice.spi;
+
+import org.elasticsearch.injection.guice.Module;
+
+/**
+ * A core component of a module or injector.
+ * <p>
+ * The elements of a module can be inspected, validated and rewritten. Use {@link
+ * Elements#getElements(Module[]) Elements.getElements()} to read the elements
+ * from a module to rewrite them.
+ * This can be used for static analysis and generation of Guice modules.
+ *
+ * @author jessewilson@google.com (Jesse Wilson)
+ * @author crazybob@google.com (Bob Lee)
+ * @since 2.0
+ */
+public interface Element {
+
+    /**
+     * Returns an arbitrary object containing information about the "place" where this element was
+     * configured. Used by Guice in the production of descriptive error messages.
+     * <p>
+     * Tools might specially handle types they know about; {@code StackTraceElement} is a good
+     * example. Tools should simply call {@code toString()} on the source object if the type is
+     * unfamiliar.
+     */
+    Object getSource();
+
+    /**
+     * Accepts an element visitor. Invokes the visitor method specific to this element's type.
+     *
+     * @param visitor to call back on
+     */
+    <T> T acceptVisitor(ElementVisitor<T> visitor);
+
+}
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/ElementVisitor.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/ElementVisitor.java
similarity index 92%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/ElementVisitor.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/ElementVisitor.java
index 094e869d8caa8..ed96023d7ad9b 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/ElementVisitor.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/ElementVisitor.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Binding;
+import org.elasticsearch.injection.guice.Binding;
 
 /**
  * Visit elements.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/Elements.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/Elements.java
similarity index 89%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/Elements.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/Elements.java
index 0115abcfa3f03..2ad4333c61d31 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/Elements.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/Elements.java
@@ -14,22 +14,22 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.PrivateBinder;
-import org.elasticsearch.common.inject.Provider;
-import org.elasticsearch.common.inject.TypeLiteral;
-import org.elasticsearch.common.inject.binder.AnnotatedBindingBuilder;
-import org.elasticsearch.common.inject.internal.AbstractBindingBuilder;
-import org.elasticsearch.common.inject.internal.BindingBuilder;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.SourceProvider;
+import org.elasticsearch.injection.guice.AbstractModule;
+import org.elasticsearch.injection.guice.Binder;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.Module;
+import org.elasticsearch.injection.guice.PrivateBinder;
+import org.elasticsearch.injection.guice.Provider;
+import org.elasticsearch.injection.guice.TypeLiteral;
+import org.elasticsearch.injection.guice.binder.AnnotatedBindingBuilder;
+import org.elasticsearch.injection.guice.internal.AbstractBindingBuilder;
+import org.elasticsearch.injection.guice.internal.BindingBuilder;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.SourceProvider;
 
 import java.util.ArrayList;
 import java.util.Arrays;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/InjectionPoint.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/InjectionPoint.java
similarity index 93%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/InjectionPoint.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/InjectionPoint.java
index 945dfca96072e..e566ef7deea90 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/InjectionPoint.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/InjectionPoint.java
@@ -14,17 +14,17 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
-
-import org.elasticsearch.common.inject.ConfigurationException;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.TypeLiteral;
-import org.elasticsearch.common.inject.internal.Annotations;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.internal.MoreTypes;
-import org.elasticsearch.common.inject.internal.Nullability;
+package org.elasticsearch.injection.guice.spi;
+
+import org.elasticsearch.injection.guice.ConfigurationException;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.TypeLiteral;
+import org.elasticsearch.injection.guice.internal.Annotations;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.ErrorsException;
+import org.elasticsearch.injection.guice.internal.MoreTypes;
+import org.elasticsearch.injection.guice.internal.Nullability;
 
 import java.lang.annotation.Annotation;
 import java.lang.reflect.AnnotatedElement;
@@ -43,7 +43,6 @@
 import java.util.Set;
 
 import static java.util.Collections.unmodifiableSet;
-import static org.elasticsearch.common.inject.internal.MoreTypes.getRawType;
 
 /**
  * A constructor, field or method that can receive injections. Typically this is a member with the
@@ -154,7 +153,7 @@ public String toString() {
      *                                parameter with multiple binding annotations.
      */
     public static InjectionPoint forConstructorOf(TypeLiteral<?> type) {
-        Class<?> rawType = getRawType(type.getType());
+        Class<?> rawType = MoreTypes.getRawType(type.getType());
         Errors errors = new Errors(rawType);
 
         Constructor<?> injectableConstructor = null;
@@ -284,7 +283,7 @@ private static void addInjectorsForMembers(
         Collection<InjectionPoint> injectionPoints,
         Errors errors
     ) {
-        for (Method member : getRawType(typeLiteral.getType()).getMethods()) {
+        for (Method member : MoreTypes.getRawType(typeLiteral.getType()).getMethods()) {
             if (isStatic(member) != statics) {
                 continue;
             }
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/InstanceBinding.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/InstanceBinding.java
similarity index 92%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/InstanceBinding.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/InstanceBinding.java
index ae1a837a4a4fd..fa02c29475e78 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/InstanceBinding.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/InstanceBinding.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Binding;
+import org.elasticsearch.injection.guice.Binding;
 
 import java.util.Set;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/LinkedKeyBinding.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/LinkedKeyBinding.java
similarity index 86%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/LinkedKeyBinding.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/LinkedKeyBinding.java
index 2b0a817a489b7..a27b0b0e5555f 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/LinkedKeyBinding.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/LinkedKeyBinding.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Binding;
-import org.elasticsearch.common.inject.Key;
+import org.elasticsearch.injection.guice.Binding;
+import org.elasticsearch.injection.guice.Key;
 
 /**
  * A binding to a linked key. The other key's binding is used to resolve injections.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/Message.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/Message.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/Message.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/Message.java
index aa9b636650ce3..d234c2eadb2ce 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/Message.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/Message.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.SourceProvider;
+import org.elasticsearch.injection.guice.internal.Errors;
+import org.elasticsearch.injection.guice.internal.SourceProvider;
 
 import java.util.Collections;
 import java.util.List;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/ProviderBinding.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderBinding.java
similarity index 86%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/ProviderBinding.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderBinding.java
index 993de584505a1..9d35e03d0f803 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/ProviderBinding.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderBinding.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Binding;
-import org.elasticsearch.common.inject.Provider;
+import org.elasticsearch.injection.guice.Binding;
+import org.elasticsearch.injection.guice.Provider;
 
 /**
  * A binding to a {@link Provider} that delegates to the binding for the provided type. This binding
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/ProviderInstanceBinding.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderInstanceBinding.java
similarity index 89%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/ProviderInstanceBinding.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderInstanceBinding.java
index 85e11629df722..ed81d5355f048 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/ProviderInstanceBinding.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderInstanceBinding.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Binding;
-import org.elasticsearch.common.inject.Provider;
+import org.elasticsearch.injection.guice.Binding;
+import org.elasticsearch.injection.guice.Provider;
 
 import java.util.Set;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/ProviderKeyBinding.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderKeyBinding.java
similarity index 84%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/ProviderKeyBinding.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderKeyBinding.java
index 04b2d3cb9f21e..eca059ed57c6c 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/ProviderKeyBinding.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderKeyBinding.java
@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Binding;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Provider;
+import org.elasticsearch.injection.guice.Binding;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.Provider;
 
 /**
  * A binding to a provider key. To resolve injections, the provider key is first resolved, then that
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/ProviderLookup.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderLookup.java
similarity index 95%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/ProviderLookup.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderLookup.java
index a85937389c6f8..17a68cce80741 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/ProviderLookup.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderLookup.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Provider;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.Provider;
 
 import java.util.Objects;
 
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/ProviderWithDependencies.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderWithDependencies.java
similarity index 90%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/ProviderWithDependencies.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderWithDependencies.java
index b99fb48960414..c0e91499a6428 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/ProviderWithDependencies.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/ProviderWithDependencies.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Provider;
+import org.elasticsearch.injection.guice.Provider;
 
 /**
  * A provider with dependencies on other injected types. If a {@link Provider} has dependencies that
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/TypeConverter.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/TypeConverter.java
similarity index 90%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/TypeConverter.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/TypeConverter.java
index 27539d9f67461..d06f2b855afb6 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/TypeConverter.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/TypeConverter.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.TypeLiteral;
+import org.elasticsearch.injection.guice.TypeLiteral;
 
 /**
  * Converts constant string values to a different type.
diff --git a/server/src/main/java/org/elasticsearch/common/inject/spi/UntargettedBinding.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/UntargettedBinding.java
similarity index 89%
rename from server/src/main/java/org/elasticsearch/common/inject/spi/UntargettedBinding.java
rename to server/src/main/java/org/elasticsearch/injection/guice/spi/UntargettedBinding.java
index b69391b6d844b..b3c76b7b4b987 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/spi/UntargettedBinding.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/UntargettedBinding.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.spi;
+package org.elasticsearch.injection.guice.spi;
 
-import org.elasticsearch.common.inject.Binding;
+import org.elasticsearch.injection.guice.Binding;
 
 /**
  * An untargetted binding. This binding indicates that the injector should use its implicit binding
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/spi/package-info.java b/server/src/main/java/org/elasticsearch/injection/guice/spi/package-info.java
new file mode 100644
index 0000000000000..2c9edd871baac
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/spi/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2006 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Guice service provider interface
+ */
+
+package org.elasticsearch.injection.guice.spi;
diff --git a/server/src/main/java/org/elasticsearch/common/inject/util/Providers.java b/server/src/main/java/org/elasticsearch/injection/guice/util/Providers.java
similarity index 94%
rename from server/src/main/java/org/elasticsearch/common/inject/util/Providers.java
rename to server/src/main/java/org/elasticsearch/injection/guice/util/Providers.java
index 760c5bd7f9a94..ab1d4b25c68e7 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/util/Providers.java
+++ b/server/src/main/java/org/elasticsearch/injection/guice/util/Providers.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package org.elasticsearch.common.inject.util;
+package org.elasticsearch.injection.guice.util;
 
-import org.elasticsearch.common.inject.Provider;
+import org.elasticsearch.injection.guice.Provider;
 
 /**
  * Static utility methods for creating and working with instances of
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/util/Types.java b/server/src/main/java/org/elasticsearch/injection/guice/util/Types.java
new file mode 100644
index 0000000000000..a40d8e46d17d5
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/util/Types.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright (C) 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.elasticsearch.injection.guice.util;
+
+import org.elasticsearch.injection.guice.Provider;
+import org.elasticsearch.injection.guice.internal.MoreTypes;
+
+import java.lang.reflect.GenericArrayType;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.lang.reflect.WildcardType;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Static methods for working with types.
+ *
+ * @author crazybob@google.com (Bob Lee)
+ * @since 2.0
+ */
+public final class Types {
+    private Types() {}
+
+    /**
+     * Returns a new parameterized type, applying {@code typeArguments} to
+     * {@code rawType}. The returned type does not have an owner type.
+     *
+     * @return a parameterized type.
+     */
+    public static ParameterizedType newParameterizedType(Type rawType, Type... typeArguments) {
+        return newParameterizedTypeWithOwner(null, rawType, typeArguments);
+    }
+
+    /**
+     * Returns a new parameterized type, applying {@code typeArguments} to
+     * {@code rawType} and enclosed by {@code ownerType}.
+     *
+     * @return a parameterized type.
+     */
+    public static ParameterizedType newParameterizedTypeWithOwner(Type ownerType, Type rawType, Type... typeArguments) {
+        return new MoreTypes.ParameterizedTypeImpl(ownerType, rawType, typeArguments);
+    }
+
+    /**
+     * Returns an array type whose elements are all instances of
+     * {@code componentType}.
+     *
+     * @return a generic array type.
+     */
+    public static GenericArrayType arrayOf(Type componentType) {
+        return new MoreTypes.GenericArrayTypeImpl(componentType);
+    }
+
+    /**
+     * Returns a type that represents an unknown type that extends {@code bound}.
+     * For example, if {@code bound} is {@code CharSequence.class}, this returns
+     * {@code ? extends CharSequence}. If {@code bound} is {@code Object.class},
+     * this returns {@code ?}, which is shorthand for {@code ? extends Object}.
+     */
+    public static WildcardType subtypeOf(Type bound) {
+        return new MoreTypes.WildcardTypeImpl(new Type[] { bound }, MoreTypes.EMPTY_TYPE_ARRAY);
+    }
+
+    /**
+     * Returns a type that represents an unknown supertype of {@code bound}. For
+     * example, if {@code bound} is {@code String.class}, this returns {@code ?
+     * super String}.
+     */
+    public static WildcardType supertypeOf(Type bound) {
+        return new MoreTypes.WildcardTypeImpl(new Type[] { Object.class }, new Type[] { bound });
+    }
+
+    /**
+     * Returns a type modelling a {@link Set} whose elements are of type
+     * {@code elementType}.
+     *
+     * @return a parameterized type.
+     */
+    public static ParameterizedType setOf(Type elementType) {
+        return newParameterizedType(Set.class, elementType);
+    }
+
+    /**
+     * Returns a type modelling a {@link Map} whose keys are of type
+     * {@code keyType} and whose values are of type {@code valueType}.
+     *
+     * @return a parameterized type.
+     */
+    public static ParameterizedType mapOf(Type keyType, Type valueType) {
+        return newParameterizedType(Map.class, keyType, valueType);
+    }
+
+    // for other custom collections types, use newParameterizedType()
+
+    /**
+     * Returns a type modelling a {@link Provider} that provides elements of type
+     * {@code elementType}.
+     *
+     * @return a parameterized type.
+     */
+    public static ParameterizedType providerOf(Type providedType) {
+        return newParameterizedType(Provider.class, providedType);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/injection/guice/util/package-info.java b/server/src/main/java/org/elasticsearch/injection/guice/util/package-info.java
new file mode 100644
index 0000000000000..589aa6021e5bc
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/guice/util/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2009 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Helper methods for working with Guice.
+ */
+package org.elasticsearch.injection.guice.util;
diff --git a/server/src/main/java/org/elasticsearch/injection/package-info.java b/server/src/main/java/org/elasticsearch/injection/package-info.java
new file mode 100644
index 0000000000000..01dd1e878651c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/package-info.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * Our dependency injection technologies: our bespoke injector, plus our legacy vendored version of Google Guice.
+ * <h2>Usage</h2>
+ * The new injector is {@link org.elasticsearch.injection.Injector}.
+ * You create an instance using {@link org.elasticsearch.injection.Injector#create()},
+ * call various methods like {@link org.elasticsearch.injection.Injector#addClass} to configure it,
+ * then call {@link org.elasticsearch.injection.Injector#inject} to cause the constructors to be called.
+ *
+ * <h2>Operation</h2>
+ * Injection proceeds in three phases:
+ * <ol>
+ *     <li>
+ *         <em>Configuration</em>: the {@link org.elasticsearch.injection.Injector} captures the user's
+ *         intent in the form of {@link org.elasticsearch.injection.spec.InjectionSpec} objects,
+ *         one for each class.
+ *     </li>
+ *     <li>
+ *         <em>Planning</em>: the {@link org.elasticsearch.injection.Planner} analyzes the
+ *         {@link org.elasticsearch.injection.spec.InjectionSpec} objects, validates them,
+ *         and generates a <em>plan</em> in the form of a list of {@link org.elasticsearch.injection.step.InjectionStep} objects.
+ *     </li>
+ *     <li>
+ *         <em>Execution</em>: the {@link org.elasticsearch.injection.PlanInterpreter} runs
+ *         the steps in the plan, in sequence, to actually instantiate the objects and pass them
+ *         to each others' constructors.
+ *     </li>
+ * </ol>
+ *
+ * <h2>Google Guice</h2>
+ * The older injector, based on Google Guice, is in the {@code guice} package.
+ * The new injector is unrelated to Guice, and is intended to replace Guice eventually.
+ */
+package org.elasticsearch.injection;
diff --git a/server/src/main/java/org/elasticsearch/injection/spec/ExistingInstanceSpec.java b/server/src/main/java/org/elasticsearch/injection/spec/ExistingInstanceSpec.java
new file mode 100644
index 0000000000000..f443e045442c9
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/spec/ExistingInstanceSpec.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.injection.spec;
+
+public record ExistingInstanceSpec(Class<?> requestedType, Object instance) implements InjectionSpec {
+    @Override
+    public String toString() {
+        // Don't call instance.toString; who knows what that will return
+        return "ExistingInstanceSpec[" + "requestedType=" + requestedType + ']';
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/injection/spec/InjectionSpec.java b/server/src/main/java/org/elasticsearch/injection/spec/InjectionSpec.java
new file mode 100644
index 0000000000000..552d2c2ba9ebb
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/spec/InjectionSpec.java
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.injection.spec;
+
+public sealed interface InjectionSpec permits MethodHandleSpec, ExistingInstanceSpec {
+    Class<?> requestedType();
+}
diff --git a/server/src/main/java/org/elasticsearch/injection/spec/MethodHandleSpec.java b/server/src/main/java/org/elasticsearch/injection/spec/MethodHandleSpec.java
new file mode 100644
index 0000000000000..06c4cd0faac63
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/spec/MethodHandleSpec.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.injection.spec;
+
+import java.lang.invoke.MethodHandle;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Indicates that a type should be instantiated by calling the given {@link java.lang.invoke.MethodHandle}.
+ * <p>
+ * <em>Design note</em>: the intent is that the semantics are fully specified by this record,
+ * and no additional reflection logic is required to determine how the object should be injected.
+ * Roughly speaking: all the reflection should be finished, and the results should be stored in this object.
+ */
+public record MethodHandleSpec(Class<?> requestedType, MethodHandle methodHandle, List<ParameterSpec> parameters) implements InjectionSpec {
+    public MethodHandleSpec {
+        assert Objects.equals(methodHandle.type().parameterList(), parameters.stream().map(ParameterSpec::formalType).toList())
+            : "MethodHandle parameter types must match the supplied parameter info; "
+                + methodHandle.type().parameterList()
+                + " vs "
+                + parameters;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/injection/spec/ParameterSpec.java b/server/src/main/java/org/elasticsearch/injection/spec/ParameterSpec.java
new file mode 100644
index 0000000000000..da15bd024fbf4
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/spec/ParameterSpec.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.injection.spec;
+
+import java.lang.reflect.Parameter;
+
+/**
+ * Captures the pertinent info required to inject one of the arguments of a constructor.
+ * @param name is for troubleshooting; it's not strictly needed
+ * @param formalType is the declared class of the parameter
+ * @param injectableType is the target type of the injection dependency
+ */
+public record ParameterSpec(String name, Class<?> formalType, Class<?> injectableType) {
+    public static ParameterSpec from(Parameter parameter) {
+        // We currently have no cases where the formal and injectable types are different.
+        return new ParameterSpec(parameter.getName(), parameter.getType(), parameter.getType());
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/injection/spec/package-info.java b/server/src/main/java/org/elasticsearch/injection/spec/package-info.java
new file mode 100644
index 0000000000000..26cb1e8ff8543
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/spec/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * Objects that describe the means by which an object instance is created for (or associated with) some given type.
+ * <p>
+ * The hierarchy is rooted at {@link org.elasticsearch.injection.spec.InjectionSpec}.
+ * <p>
+ * Differs from {@link org.elasticsearch.injection.step.InjectionStep InjectionStep} in that:
+ *
+ * <ul>
+ *     <li>
+ *         this describes the requirements, while <code>InjectionStep</code> describes the solution
+ *     </li>
+ *     <li>
+ *         this is declarative, while <code>InjectionStep</code> is imperative
+ *     </li>
+ * </ul>
+ */
+package org.elasticsearch.injection.spec;
diff --git a/server/src/main/java/org/elasticsearch/injection/step/InjectionStep.java b/server/src/main/java/org/elasticsearch/injection/step/InjectionStep.java
new file mode 100644
index 0000000000000..6e27f45b4f4df
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/step/InjectionStep.java
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.injection.step;
+
+public sealed interface InjectionStep permits InstantiateStep {}
diff --git a/server/src/main/java/org/elasticsearch/injection/step/InstantiateStep.java b/server/src/main/java/org/elasticsearch/injection/step/InstantiateStep.java
new file mode 100644
index 0000000000000..2342978dcfdb0
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/step/InstantiateStep.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.injection.step;
+
+import org.elasticsearch.injection.spec.MethodHandleSpec;
+
+/**
+ * Constructs a new object by invoking a {@link java.lang.invoke.MethodHandle}
+ * as specified by a given {@link MethodHandleSpec}.
+ */
+public record InstantiateStep(MethodHandleSpec spec) implements InjectionStep {}
diff --git a/server/src/main/java/org/elasticsearch/injection/step/package-info.java b/server/src/main/java/org/elasticsearch/injection/step/package-info.java
new file mode 100644
index 0000000000000..c0a3e05cb53f6
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/injection/step/package-info.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * Objects that describe one operation to be performed by the <code>PlanInterpreter</code>.
+ * Injection is achieved by executing the steps in order.
+ * <p>
+ * See <code>PlanInterpreter</code> for more details on the execution model.
+ */
+package org.elasticsearch.injection.step;
diff --git a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java
index 07eec973c77e0..304a48335fd11 100644
--- a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
+import org.elasticsearch.search.retriever.rankdoc.RankDocsQuery;
 import org.elasticsearch.search.runtime.AbstractScriptFieldQuery;
 import org.elasticsearch.search.vectors.KnnScoreDocQuery;
 
@@ -255,10 +256,10 @@ public void visitLeaf(Query leafQuery) {
                     hasUnknownLeaf[0] = true;
                 }
                 /**
-                 * KnnScoreDocQuery requires the same reader that built the docs
+                 * KnnScoreDocQuery and RankDocsQuery requires the same reader that built the docs
                  * When using {@link HighlightFlag#WEIGHT_MATCHES} different readers are used and isn't supported by this query
                  */
-                if (leafQuery instanceof KnnScoreDocQuery) {
+                if (leafQuery instanceof KnnScoreDocQuery || leafQuery instanceof RankDocsQuery) {
                     hasUnknownLeaf[0] = true;
                 }
                 super.visitLeaf(query);
diff --git a/server/src/main/java/org/elasticsearch/lucene/spatial/ShapeDocValuesQuery.java b/server/src/main/java/org/elasticsearch/lucene/spatial/ShapeDocValuesQuery.java
index 6804901d9511e..f79d5303ab65a 100644
--- a/server/src/main/java/org/elasticsearch/lucene/spatial/ShapeDocValuesQuery.java
+++ b/server/src/main/java/org/elasticsearch/lucene/spatial/ShapeDocValuesQuery.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.ConstantScoreScorer;
 import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryVisitor;
@@ -109,11 +110,7 @@ private ConstantScoreWeight getStandardWeight(ScoreMode scoreMode, float boost)
 
             @Override
             public Scorer scorer(LeafReaderContext context) throws IOException {
-                final ScorerSupplier scorerSupplier = scorerSupplier(context);
-                if (scorerSupplier == null) {
-                    return null;
-                }
-                return scorerSupplier.get(Long.MAX_VALUE);
+                return scorerSupplier(context).get(Long.MAX_VALUE);
             }
 
             @Override
@@ -127,7 +124,7 @@ public Scorer get(long leadCost) throws IOException {
                         // binary doc values allocate an array upfront, lets only allocate it if we are going to use it
                         final BinaryDocValues values = context.reader().getBinaryDocValues(field);
                         if (values == null) {
-                            return null;
+                            return new ConstantScoreScorer(weight, 0f, scoreMode, DocIdSetIterator.empty());
                         }
                         final GeometryDocValueReader reader = new GeometryDocValueReader();
                         final Component2DVisitor visitor = Component2DVisitor.getVisitor(component2D, relation, encoder);
@@ -171,11 +168,7 @@ private ConstantScoreWeight getContainsWeight(ScoreMode scoreMode, float boost)
 
             @Override
             public Scorer scorer(LeafReaderContext context) throws IOException {
-                final ScorerSupplier scorerSupplier = scorerSupplier(context);
-                if (scorerSupplier == null) {
-                    return null;
-                }
-                return scorerSupplier.get(Long.MAX_VALUE);
+                return scorerSupplier(context).get(Long.MAX_VALUE);
             }
 
             @Override
@@ -189,7 +182,7 @@ public Scorer get(long leadCost) throws IOException {
                         // binary doc values allocate an array upfront, lets only allocate it if we are going to use it
                         final BinaryDocValues values = context.reader().getBinaryDocValues(field);
                         if (values == null) {
-                            return null;
+                            return new ConstantScoreScorer(weight, 0f, scoreMode, DocIdSetIterator.empty());
                         }
                         final Component2DVisitor[] visitors = new Component2DVisitor[components2D.size()];
                         for (int i = 0; i < components2D.size(); i++) {
diff --git a/server/src/main/java/org/elasticsearch/lucene/spatial/XYQueriesUtils.java b/server/src/main/java/org/elasticsearch/lucene/spatial/XYQueriesUtils.java
new file mode 100644
index 0000000000000..23aaf2ab16722
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/lucene/spatial/XYQueriesUtils.java
@@ -0,0 +1,305 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.lucene.spatial;
+
+import org.apache.lucene.document.XYDocValuesField;
+import org.apache.lucene.document.XYPointField;
+import org.apache.lucene.document.XYShape;
+import org.apache.lucene.geo.Component2D;
+import org.apache.lucene.geo.XYGeometry;
+import org.apache.lucene.geo.XYPoint;
+import org.apache.lucene.geo.XYRectangle;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.FieldExistsQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.geo.LuceneGeometriesUtils;
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.geometry.Geometry;
+
+import java.util.Arrays;
+
+/** Utility methods that generate a lucene query for a spatial query over a cartesian field.* */
+public class XYQueriesUtils {
+
+    /** Generates a lucene query for a field that has been previously indexed using {@link XYPoint}.It expects
+     * either {code indexed} or {@code has docValues} to be true or or both to be true.
+     *
+     *  Note that lucene only supports intersects spatial relation so we build other relations
+     *  using just that one.
+     * */
+    public static Query toXYPointQuery(Geometry geometry, String fieldName, ShapeRelation relation, boolean indexed, boolean hasDocValues) {
+        assert indexed || hasDocValues;
+        final XYGeometry[] luceneGeometries = LuceneGeometriesUtils.toXYGeometry(geometry, t -> {});
+        // XYPointField only supports intersects query so we build all the relationships using that logic.
+        // it is not very efficient but it works.
+        return switch (relation) {
+            case INTERSECTS -> buildIntersectsQuery(fieldName, indexed, hasDocValues, luceneGeometries);
+            case DISJOINT -> buildDisjointQuery(fieldName, indexed, hasDocValues, luceneGeometries);
+            case CONTAINS -> buildContainsQuery(fieldName, indexed, hasDocValues, luceneGeometries);
+            case WITHIN -> buildWithinQuery(fieldName, indexed, hasDocValues, luceneGeometries);
+        };
+    }
+
+    private static Query buildIntersectsQuery(String fieldName, boolean isIndexed, boolean hasDocValues, XYGeometry... luceneGeometries) {
+        // This is supported natively in lucene
+        Query query;
+        if (isIndexed) {
+            query = XYPointField.newGeometryQuery(fieldName, luceneGeometries);
+            if (hasDocValues) {
+                final Query queryDocValues = XYDocValuesField.newSlowGeometryQuery(fieldName, luceneGeometries);
+                query = new IndexOrDocValuesQuery(query, queryDocValues);
+            }
+        } else {
+            query = XYDocValuesField.newSlowGeometryQuery(fieldName, luceneGeometries);
+        }
+        return query;
+    }
+
+    private static Query buildDisjointQuery(String fieldName, boolean isIndexed, boolean hasDocValues, XYGeometry... luceneGeometries) {
+        // first collect all the documents that contain a shape
+        final BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        if (hasDocValues) {
+            builder.add(new FieldExistsQuery(fieldName), BooleanClause.Occur.FILTER);
+        } else {
+            builder.add(
+                buildIntersectsQuery(
+                    fieldName,
+                    isIndexed,
+                    hasDocValues,
+                    new XYRectangle(-Float.MAX_VALUE, Float.MAX_VALUE, -Float.MAX_VALUE, Float.MAX_VALUE)
+                ),
+                BooleanClause.Occur.FILTER
+            );
+        }
+        // then remove all intersecting documents
+        builder.add(buildIntersectsQuery(fieldName, isIndexed, hasDocValues, luceneGeometries), BooleanClause.Occur.MUST_NOT);
+        return builder.build();
+    }
+
+    private static Query buildContainsQuery(String fieldName, boolean isIndexed, boolean hasDocValues, XYGeometry... luceneGeometries) {
+        // for non-point data the result is always false
+        if (allPoints(luceneGeometries) == false) {
+            return new MatchNoDocsQuery();
+        }
+        // for a unique point, it behaves like intersect
+        if (luceneGeometries.length == 1) {
+            return buildIntersectsQuery(fieldName, isIndexed, hasDocValues, luceneGeometries);
+        }
+        // for a multi point, all points needs to be in the document
+        final BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        for (XYGeometry geometry : luceneGeometries) {
+            builder.add(buildIntersectsQuery(fieldName, isIndexed, hasDocValues, geometry), BooleanClause.Occur.FILTER);
+        }
+        return builder.build();
+    }
+
+    private static Query buildWithinQuery(String fieldName, boolean isIndexed, boolean hasDocValues, XYGeometry... luceneGeometries) {
+        final BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        // collect all the intersecting documents
+        builder.add(buildIntersectsQuery(fieldName, isIndexed, hasDocValues, luceneGeometries), BooleanClause.Occur.FILTER);
+        // This is the tricky part as we need to remove all documents that they have at least one disjoint point.
+        // In order to do that, we introduce a InverseXYGeometry which return all documents that have at least one disjoint point
+        // with the original geometry.
+        builder.add(
+            buildIntersectsQuery(fieldName, isIndexed, hasDocValues, new InverseXYGeometry(luceneGeometries)),
+            BooleanClause.Occur.MUST_NOT
+        );
+        return builder.build();
+    }
+
+    private static boolean allPoints(XYGeometry[] geometries) {
+        return Arrays.stream(geometries).allMatch(g -> g instanceof XYPoint);
+    }
+
+    private static class InverseXYGeometry extends XYGeometry {
+        private final XYGeometry[] geometries;
+
+        InverseXYGeometry(XYGeometry... geometries) {
+            this.geometries = geometries;
+        }
+
+        @Override
+        protected Component2D toComponent2D() {
+            final Component2D component2D = XYGeometry.create(geometries);
+            return new Component2D() {
+                @Override
+                public double getMinX() {
+                    return -Float.MAX_VALUE;
+                }
+
+                @Override
+                public double getMaxX() {
+                    return Float.MAX_VALUE;
+                }
+
+                @Override
+                public double getMinY() {
+                    return -Float.MAX_VALUE;
+                }
+
+                @Override
+                public double getMaxY() {
+                    return Float.MAX_VALUE;
+                }
+
+                @Override
+                public boolean contains(double x, double y) {
+                    return component2D.contains(x, y) == false;
+                }
+
+                @Override
+                public PointValues.Relation relate(double minX, double maxX, double minY, double maxY) {
+                    PointValues.Relation relation = component2D.relate(minX, maxX, minY, maxY);
+                    return switch (relation) {
+                        case CELL_INSIDE_QUERY -> PointValues.Relation.CELL_OUTSIDE_QUERY;
+                        case CELL_OUTSIDE_QUERY -> PointValues.Relation.CELL_INSIDE_QUERY;
+                        case CELL_CROSSES_QUERY -> PointValues.Relation.CELL_CROSSES_QUERY;
+                    };
+                }
+
+                @Override
+                public boolean intersectsLine(
+                    double minX,
+                    double maxX,
+                    double minY,
+                    double maxY,
+                    double aX,
+                    double aY,
+                    double bX,
+                    double bY
+                ) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public boolean intersectsTriangle(
+                    double minX,
+                    double maxX,
+                    double minY,
+                    double maxY,
+                    double aX,
+                    double aY,
+                    double bX,
+                    double bY,
+                    double cX,
+                    double cY
+                ) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public boolean containsLine(
+                    double minX,
+                    double maxX,
+                    double minY,
+                    double maxY,
+                    double aX,
+                    double aY,
+                    double bX,
+                    double bY
+                ) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public boolean containsTriangle(
+                    double minX,
+                    double maxX,
+                    double minY,
+                    double maxY,
+                    double aX,
+                    double aY,
+                    double bX,
+                    double bY,
+                    double cX,
+                    double cY
+                ) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public WithinRelation withinPoint(double x, double y) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public WithinRelation withinLine(
+                    double minX,
+                    double maxX,
+                    double minY,
+                    double maxY,
+                    double aX,
+                    double aY,
+                    boolean ab,
+                    double bX,
+                    double bY
+                ) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public WithinRelation withinTriangle(
+                    double minX,
+                    double maxX,
+                    double minY,
+                    double maxY,
+                    double aX,
+                    double aY,
+                    boolean ab,
+                    double bX,
+                    double bY,
+                    boolean bc,
+                    double cX,
+                    double cY,
+                    boolean ca
+                ) {
+                    throw new UnsupportedOperationException();
+                }
+            };
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            InverseXYGeometry that = (InverseXYGeometry) o;
+            return Arrays.equals(geometries, that.geometries);
+        }
+
+        @Override
+        public int hashCode() {
+            return Arrays.hashCode(geometries);
+        }
+    }
+
+    /** Generates a lucene query for a field that has been previously indexed using {@link XYShape}.It expects
+     * either {code indexed} or {@code has docValues} to be true or both to be true. */
+    public static Query toXYShapeQuery(Geometry geometry, String fieldName, ShapeRelation relation, boolean indexed, boolean hasDocValues) {
+        assert indexed || hasDocValues;
+        if (geometry == null || geometry.isEmpty()) {
+            return new MatchNoDocsQuery();
+        }
+        final XYGeometry[] luceneGeometries = LuceneGeometriesUtils.toXYGeometry(geometry, t -> {});
+        Query query;
+        if (indexed) {
+            query = XYShape.newGeometryQuery(fieldName, relation.getLuceneRelation(), luceneGeometries);
+            if (hasDocValues) {
+                final Query queryDocValues = new CartesianShapeDocValuesQuery(fieldName, relation.getLuceneRelation(), luceneGeometries);
+                query = new IndexOrDocValuesQuery(query, queryDocValues);
+            }
+        } else {
+            query = new CartesianShapeDocValuesQuery(fieldName, relation.getLuceneRelation(), luceneGeometries);
+        }
+        return query;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/monitor/fs/FsHealthService.java b/server/src/main/java/org/elasticsearch/monitor/fs/FsHealthService.java
index de26d23a149e3..56ece73f2d681 100644
--- a/server/src/main/java/org/elasticsearch/monitor/fs/FsHealthService.java
+++ b/server/src/main/java/org/elasticsearch/monitor/fs/FsHealthService.java
@@ -87,7 +87,7 @@ public FsHealthService(Settings settings, ClusterSettings clusterSettings, Threa
         this.enabled = ENABLED_SETTING.get(settings);
         this.refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
         this.slowPathLoggingThreshold = SLOW_PATH_LOGGING_THRESHOLD_SETTING.get(settings);
-        this.currentTimeMillisSupplier = threadPool::relativeTimeInMillis;
+        this.currentTimeMillisSupplier = threadPool.relativeTimeInMillisSupplier();
         this.nodeEnv = nodeEnv;
         clusterSettings.addSettingsUpdateConsumer(SLOW_PATH_LOGGING_THRESHOLD_SETTING, this::setSlowPathLoggingThreshold);
         clusterSettings.addSettingsUpdateConsumer(ENABLED_SETTING, this::setEnabled);
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index 56f2c6c8d8c57..3302114b078a8 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -13,6 +13,9 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.action.search.TransportSearchAction;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.RefCountingListener;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.bootstrap.BootstrapContext;
 import org.elasticsearch.client.internal.Client;
@@ -32,7 +35,6 @@
 import org.elasticsearch.common.StopWatch;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.NodeAndClusterIdStateListener;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -41,7 +43,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.core.Assertions;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.PathUtils;
@@ -61,6 +62,7 @@
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
 import org.elasticsearch.indices.store.IndicesStore;
+import org.elasticsearch.injection.guice.Injector;
 import org.elasticsearch.monitor.fs.FsHealthService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.monitor.metrics.NodeMetrics;
@@ -101,11 +103,14 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 import javax.net.ssl.SNIHostName;
 
@@ -591,52 +596,63 @@ public synchronized void close() throws IOException {
      * Invokes hooks to prepare this node to be closed. This should be called when Elasticsearch receives a request to shut down
      * gracefully from the underlying operating system, before system resources are closed. This method will block
      * until the node is ready to shut down.
-     *
+     * <p>
      * Note that this class is part of infrastructure to react to signals from the operating system - most graceful shutdown
      * logic should use Node Shutdown, see {@link org.elasticsearch.cluster.metadata.NodesShutdownMetadata}.
      */
     public void prepareForClose() {
-        HttpServerTransport httpServerTransport = injector.getInstance(HttpServerTransport.class);
-        Map<String, Runnable> stoppers = new HashMap<>();
-        TimeValue maxTimeout = MAXIMUM_SHUTDOWN_TIMEOUT_SETTING.get(this.settings());
-        stoppers.put("http-server-transport-stop", httpServerTransport::close);
-        stoppers.put("async-search-stop", () -> this.awaitSearchTasksComplete(maxTimeout));
-        if (terminationHandler != null) {
-            stoppers.put("termination-handler-stop", terminationHandler::handleTermination);
+        final var maxTimeout = MAXIMUM_SHUTDOWN_TIMEOUT_SETTING.get(this.settings());
+
+        record Stopper(String name, SubscribableListener<Void> listener) {
+            boolean isIncomplete() {
+                return listener().isDone() == false;
+            }
         }
 
-        Map<String, CompletableFuture<Void>> futures = new HashMap<>(stoppers.size());
-        for (var stopperEntry : stoppers.entrySet()) {
-            var future = new CompletableFuture<Void>();
-            new Thread(() -> {
-                try {
-                    stopperEntry.getValue().run();
-                } catch (Exception ex) {
-                    logger.warn("unexpected exception in shutdown task [" + stopperEntry.getKey() + "]", ex);
-                } finally {
-                    future.complete(null);
-                }
-            }, stopperEntry.getKey()).start();
-            futures.put(stopperEntry.getKey(), future);
+        final var stoppers = new ArrayList<Stopper>();
+        final var allStoppersFuture = new PlainActionFuture<Void>();
+        try (var listeners = new RefCountingListener(allStoppersFuture)) {
+            final BiConsumer<String, Runnable> stopperRunner = (name, action) -> {
+                final var stopper = new Stopper(name, new SubscribableListener<>());
+                stoppers.add(stopper);
+                stopper.listener().addListener(listeners.acquire());
+                new Thread(() -> {
+                    try {
+                        action.run();
+                    } catch (Exception ex) {
+                        logger.warn("unexpected exception in shutdown task [" + stopper.name() + "]", ex);
+                    } finally {
+                        stopper.listener().onResponse(null);
+                    }
+                }, stopper.name()).start();
+            };
+
+            stopperRunner.accept("http-server-transport-stop", injector.getInstance(HttpServerTransport.class)::close);
+            stopperRunner.accept("async-search-stop", () -> awaitSearchTasksComplete(maxTimeout));
+            if (terminationHandler != null) {
+                stopperRunner.accept("termination-handler-stop", terminationHandler::handleTermination);
+            }
         }
 
-        @SuppressWarnings(value = "rawtypes") // Can't make an array of parameterized types, but it complains if you leave the type out
-        CompletableFuture<Void> allStoppers = CompletableFuture.allOf(futures.values().toArray(new CompletableFuture[stoppers.size()]));
+        final Supplier<String> incompleteStoppersDescriber = () -> stoppers.stream()
+            .filter(Stopper::isIncomplete)
+            .map(Stopper::name)
+            .collect(Collectors.joining(", ", "[", "]"));
 
         try {
             if (TimeValue.ZERO.equals(maxTimeout)) {
-                FutureUtils.get(allStoppers);
+                allStoppersFuture.get();
             } else {
-                FutureUtils.get(allStoppers, maxTimeout.millis(), TimeUnit.MILLISECONDS);
+                allStoppersFuture.get(maxTimeout.millis(), TimeUnit.MILLISECONDS);
             }
-
-        } catch (ElasticsearchTimeoutException t) {
-            var unfinishedTasks = futures.entrySet()
-                .stream()
-                .filter(entry -> entry.getValue().isDone() == false)
-                .map(Map.Entry::getKey)
-                .toList();
-            logger.warn("timed out while waiting for graceful shutdown tasks: " + unfinishedTasks);
+        } catch (ExecutionException e) {
+            assert false : e; // listeners are never completed exceptionally
+            logger.warn("failed during graceful shutdown tasks", e);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            logger.warn("interrupted while waiting for graceful shutdown tasks: " + incompleteStoppersDescriber.get(), e);
+        } catch (TimeoutException e) {
+            logger.warn("timed out while waiting for graceful shutdown tasks: " + incompleteStoppersDescriber.get());
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
index 83e23d9a60b58..eb9ef08b329ab 100644
--- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
+++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.admin.cluster.repositories.reservedstate.ReservedRepositoryAction;
 import org.elasticsearch.action.admin.indices.template.reservedstate.ReservedComposableIndexTemplateAction;
+import org.elasticsearch.action.bulk.FailureStoreMetrics;
 import org.elasticsearch.action.datastreams.autosharding.DataStreamAutoShardingService;
 import org.elasticsearch.action.ingest.ReservedPipelineAction;
 import org.elasticsearch.action.search.SearchExecutionStatsCollector;
@@ -42,7 +43,7 @@
 import org.elasticsearch.cluster.coordination.StableMasterHealthIndicatorService;
 import org.elasticsearch.cluster.features.NodeFeaturesFixupListener;
 import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.IndexMetadataVerifier;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService;
@@ -64,10 +65,6 @@
 import org.elasticsearch.cluster.version.CompatibilityVersions;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -83,6 +80,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -134,6 +132,10 @@
 import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService;
 import org.elasticsearch.indices.recovery.plan.ShardSnapshotsService;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.injection.guice.Injector;
+import org.elasticsearch.injection.guice.Key;
+import org.elasticsearch.injection.guice.Module;
+import org.elasticsearch.injection.guice.ModulesBuilder;
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.monitor.fs.FsHealthService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
@@ -198,8 +200,10 @@
 import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.telemetry.tracing.Tracer;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.threadpool.internal.BuiltInExecutorBuilders;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.upgrades.SystemIndexMigrationExecutor;
@@ -213,6 +217,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.IdentityHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
@@ -225,6 +230,9 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static java.lang.invoke.MethodHandles.lookup;
+import static java.util.Collections.newSetFromMap;
+import static java.util.function.Predicate.not;
 import static org.elasticsearch.core.Types.forciblyCast;
 
 /**
@@ -479,6 +487,7 @@ private ThreadPool createThreadPool(Settings settings, MeterRegistry meterRegist
         ThreadPool threadPool = new ThreadPool(
             settings,
             meterRegistry,
+            pluginsService.loadSingletonServiceProvider(BuiltInExecutorBuilders.class, DefaultBuiltInExecutorBuilders::new),
             pluginsService.flatMap(p -> p.getExecutorBuilders(settings)).toArray(ExecutorBuilder<?>[]::new)
         );
         resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS));
@@ -588,25 +597,27 @@ private ScriptService createScriptService(SettingsModule settingsModule, ThreadP
         return scriptService;
     }
 
-    private DataStreamGlobalRetentionResolver createDataStreamServicesAndGlobalRetentionResolver(
+    private DataStreamGlobalRetentionSettings createDataStreamServicesAndGlobalRetentionResolver(
+        Settings settings,
         ThreadPool threadPool,
         ClusterService clusterService,
         IndicesService indicesService,
         MetadataCreateIndexService metadataCreateIndexService
     ) {
-        DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver = new DataStreamGlobalRetentionResolver(
+        DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings = DataStreamGlobalRetentionSettings.create(
+            clusterService.getClusterSettings(),
             DataStreamFactoryRetention.load(pluginsService, clusterService.getClusterSettings())
         );
-        modules.bindToInstance(DataStreamGlobalRetentionResolver.class, dataStreamGlobalRetentionResolver);
+        modules.bindToInstance(DataStreamGlobalRetentionSettings.class, dataStreamGlobalRetentionSettings);
         modules.bindToInstance(
             MetadataCreateDataStreamService.class,
             new MetadataCreateDataStreamService(threadPool, clusterService, metadataCreateIndexService)
         );
         modules.bindToInstance(
             MetadataDataStreamsService.class,
-            new MetadataDataStreamsService(clusterService, indicesService, dataStreamGlobalRetentionResolver)
+            new MetadataDataStreamsService(clusterService, indicesService, dataStreamGlobalRetentionSettings)
         );
-        return dataStreamGlobalRetentionResolver;
+        return dataStreamGlobalRetentionSettings;
     }
 
     private UpdateHelper createUpdateHelper(DocumentParsingProvider documentParsingProvider, ScriptService scriptService) {
@@ -657,6 +668,7 @@ private void construct(
 
         modules.bindToInstance(DocumentParsingProvider.class, documentParsingProvider);
 
+        FailureStoreMetrics failureStoreMetrics = new FailureStoreMetrics(telemetryProvider.getMeterRegistry());
         final IngestService ingestService = new IngestService(
             clusterService,
             threadPool,
@@ -666,7 +678,8 @@ private void construct(
             pluginsService.filterPlugins(IngestPlugin.class).toList(),
             client,
             IngestService.createGrokThreadWatchdog(environment, threadPool),
-            documentParsingProvider
+            documentParsingProvider,
+            failureStoreMetrics
         );
 
         SystemIndices systemIndices = createSystemIndices(settings);
@@ -728,7 +741,7 @@ private void construct(
                 clusterService::state,
                 clusterService.getClusterSettings(),
                 client,
-                threadPool::relativeTimeInMillis,
+                threadPool.relativeTimeInMillisSupplier(),
                 rerouteService
             )::onNewInfo
         );
@@ -815,34 +828,14 @@ private void construct(
             threadPool
         );
 
-        final DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver = createDataStreamServicesAndGlobalRetentionResolver(
+        final DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings = createDataStreamServicesAndGlobalRetentionResolver(
+            settings,
             threadPool,
             clusterService,
             indicesService,
             metadataCreateIndexService
         );
 
-        record PluginServiceInstances(
-            Client client,
-            ClusterService clusterService,
-            RerouteService rerouteService,
-            ThreadPool threadPool,
-            ResourceWatcherService resourceWatcherService,
-            ScriptService scriptService,
-            NamedXContentRegistry xContentRegistry,
-            Environment environment,
-            NodeEnvironment nodeEnvironment,
-            NamedWriteableRegistry namedWriteableRegistry,
-            IndexNameExpressionResolver indexNameExpressionResolver,
-            RepositoriesService repositoriesService,
-            TelemetryProvider telemetryProvider,
-            AllocationService allocationService,
-            IndicesService indicesService,
-            FeatureService featureService,
-            SystemIndices systemIndices,
-            DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver,
-            DocumentParsingProvider documentParsingProvider
-        ) implements Plugin.PluginServices {}
         PluginServiceInstances pluginServices = new PluginServiceInstances(
             client,
             clusterService,
@@ -861,11 +854,34 @@ record PluginServiceInstances(
             indicesService,
             featureService,
             systemIndices,
-            dataStreamGlobalRetentionResolver,
+            dataStreamGlobalRetentionSettings,
             documentParsingProvider
         );
 
-        Collection<?> pluginComponents = pluginsService.flatMap(p -> p.createComponents(pluginServices)).toList();
+        Collection<?> pluginComponents = pluginsService.flatMap(plugin -> {
+            Collection<?> allItems = plugin.createComponents(pluginServices);
+            List<?> componentObjects = allItems.stream().filter(not(x -> x instanceof Class<?>)).toList();
+            List<? extends Class<?>> classes = allItems.stream().filter(x -> x instanceof Class<?>).map(x -> (Class<?>) x).toList();
+
+            // Then, injection
+            Collection<?> componentsFromInjector;
+            if (classes.isEmpty()) {
+                componentsFromInjector = Set.of();
+            } else {
+                logger.debug("Using injector to instantiate classes for {}: {}", plugin.getClass().getSimpleName(), classes);
+                var injector = org.elasticsearch.injection.Injector.create();
+                injector.addInstances(componentObjects);
+                addRecordContents(injector, pluginServices);
+                var resultMap = injector.inject(classes);
+                // For now, assume we want all components added to the Guice injector
+                var distinctObjects = newSetFromMap(new IdentityHashMap<>());
+                distinctObjects.addAll(resultMap.values());
+                componentsFromInjector = distinctObjects;
+            }
+
+            // Return both
+            return Stream.of(componentObjects, componentsFromInjector).flatMap(Collection::stream).toList();
+        }).toList();
 
         var terminationHandlers = pluginsService.loadServiceProviders(TerminationHandlerProvider.class)
             .stream()
@@ -895,7 +911,7 @@ record PluginServiceInstances(
                 systemIndices,
                 indexSettingProviders,
                 metadataCreateIndexService,
-                dataStreamGlobalRetentionResolver
+                dataStreamGlobalRetentionSettings
             ),
             pluginsService.loadSingletonServiceProvider(RestExtension.class, RestExtension::allowAll)
         );
@@ -1151,6 +1167,7 @@ record PluginServiceInstances(
             b.bind(FileSettingsService.class).toInstance(fileSettingsService);
             b.bind(CompatibilityVersions.class).toInstance(compatibilityVersions);
             b.bind(DataStreamAutoShardingService.class).toInstance(dataStreamAutoShardingService);
+            b.bind(FailureStoreMetrics.class).toInstance(failureStoreMetrics);
         });
 
         if (ReadinessService.enabled(environment)) {
@@ -1165,6 +1182,24 @@ record PluginServiceInstances(
         postInjection(clusterModule, actionModule, clusterService, transportService, featureService);
     }
 
+    /**
+     * For each "component" (getter) <em>c</em> of a {@link Record},
+     * calls {@link org.elasticsearch.injection.Injector#addInstance(Object) Injector.addInstance}
+     * to register the value with the component's declared type.
+     */
+    @SuppressForbidden(reason = "Can't call invokeExact because we don't know the exact Record subtype statically")
+    private static <T> void addRecordContents(org.elasticsearch.injection.Injector injector, Record r) {
+        for (var c : r.getClass().getRecordComponents()) {
+            try {
+                @SuppressWarnings("unchecked")
+                Class<T> type = (Class<T>) c.getType(); // T represents the declared type of the record component, whatever it is
+                injector.addInstance(type, type.cast(lookup().unreflect(c.getAccessor()).invoke(r)));
+            } catch (Throwable e) {
+                throw new IllegalStateException("Unable to read record component " + c, e);
+            }
+        }
+    }
+
     private ClusterService createClusterService(SettingsModule settingsModule, ThreadPool threadPool, TaskManager taskManager) {
         ClusterService clusterService = new ClusterService(
             settingsModule.getSettings(),
@@ -1447,7 +1482,12 @@ private Module loadPersistedClusterStateService(
             ClusterCoordinationPlugin.PersistedClusterStateServiceFactory.class
         ).map(f -> f.newPersistedClusterStateService(nodeEnvironment, xContentRegistry, clusterSettings, threadPool, compatibilityVersions))
             .orElseGet(
-                () -> new PersistedClusterStateService(nodeEnvironment, xContentRegistry, clusterSettings, threadPool::relativeTimeInMillis)
+                () -> new PersistedClusterStateService(
+                    nodeEnvironment,
+                    xContentRegistry,
+                    clusterSettings,
+                    threadPool.relativeTimeInMillisSupplier()
+                )
             );
 
         return b -> b.bind(PersistedClusterStateService.class).toInstance(service);
@@ -1460,7 +1500,7 @@ private List<ReservedClusterStateHandler<?>> buildReservedStateHandlers(
         SystemIndices systemIndices,
         IndexSettingProviders indexSettingProviders,
         MetadataCreateIndexService metadataCreateIndexService,
-        DataStreamGlobalRetentionResolver globalRetentionResolver
+        DataStreamGlobalRetentionSettings globalRetentionSettings
     ) {
         List<ReservedClusterStateHandler<?>> reservedStateHandlers = new ArrayList<>();
 
@@ -1475,7 +1515,7 @@ private List<ReservedClusterStateHandler<?>> buildReservedStateHandlers(
             xContentRegistry,
             systemIndices,
             indexSettingProviders,
-            globalRetentionResolver
+            globalRetentionSettings
         );
         reservedStateHandlers.add(new ReservedComposableIndexTemplateAction(templateService, settingsModule.getIndexScopedSettings()));
 
@@ -1580,4 +1620,5 @@ private Module loadPersistentTasksService(
             b.bind(PersistentTasksClusterService.class).toInstance(persistentTasksClusterService);
         };
     }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/node/PluginServiceInstances.java b/server/src/main/java/org/elasticsearch/node/PluginServiceInstances.java
new file mode 100644
index 0000000000000..7c8775502fd64
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/node/PluginServiceInstances.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.node;
+
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.RerouteService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.internal.DocumentParsingProvider;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.telemetry.TelemetryProvider;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+
+public record PluginServiceInstances(
+    Client client,
+    ClusterService clusterService,
+    RerouteService rerouteService,
+    ThreadPool threadPool,
+    ResourceWatcherService resourceWatcherService,
+    ScriptService scriptService,
+    NamedXContentRegistry xContentRegistry,
+    Environment environment,
+    NodeEnvironment nodeEnvironment,
+    NamedWriteableRegistry namedWriteableRegistry,
+    IndexNameExpressionResolver indexNameExpressionResolver,
+    RepositoriesService repositoriesService,
+    TelemetryProvider telemetryProvider,
+    AllocationService allocationService,
+    IndicesService indicesService,
+    FeatureService featureService,
+    SystemIndices systemIndices,
+    DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings,
+    DocumentParsingProvider documentParsingProvider
+) implements Plugin.PluginServices {}
diff --git a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java
index 7ab682d3143e7..621bcd213ce6f 100644
--- a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java
@@ -18,9 +18,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java
index 26cf0658f60b9..e53f26eef467c 100644
--- a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java
@@ -18,9 +18,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java
index ce0e46e7b0425..29af1c5cbb197 100644
--- a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java
@@ -18,10 +18,10 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java
index 6ecefa1bbf847..6e4ed6bc626ed 100644
--- a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java
+++ b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java
@@ -18,9 +18,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java
index 316dd37c2b029..a8bfda54b0646 100644
--- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java
@@ -10,7 +10,7 @@
 
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.routing.RerouteService;
@@ -156,10 +156,10 @@ public interface PluginServices {
         SystemIndices systemIndices();
 
         /**
-         * A service that resolves the data stream global retention that applies to
+         * A service that holds the data stream global retention settings that applies to
          * data streams managed by the data stream lifecycle.
          */
-        DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver();
+        DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings();
 
         /**
          * A provider of utilities to observe and report parsing of documents
diff --git a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java b/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java
index d29b893447be0..adfef008f2096 100644
--- a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java
+++ b/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java
@@ -39,7 +39,7 @@ default DocumentSizeAccumulator createDocumentSizeAccumulator() {
     /**
      * @return an observer
      */
-    default <T> DocumentSizeObserver newDocumentSizeObserver(DocWriteRequest<T> request) {
-        return DocumentSizeObserver.EMPTY_INSTANCE;
+    default <T> XContentMeteringParserDecorator newMeteringParserDecorator(DocWriteRequest<T> request) {
+        return XContentMeteringParserDecorator.NOOP;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentSizeAccumulator.java b/server/src/main/java/org/elasticsearch/plugins/internal/DocumentSizeAccumulator.java
index 27bce3c637c65..6f351a3762217 100644
--- a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentSizeAccumulator.java
+++ b/server/src/main/java/org/elasticsearch/plugins/internal/DocumentSizeAccumulator.java
@@ -13,7 +13,7 @@
 import java.util.Map;
 
 /**
- * An interface to allow accumulating results of document parsing (collected with {@link DocumentSizeObserver})
+ * An interface to allow accumulating results of document parsing (collected with {@link XContentParserDecorator})
  */
 public interface DocumentSizeAccumulator {
     DocumentSizeAccumulator EMPTY_INSTANCE = new DocumentSizeAccumulator() {
diff --git a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentSizeObserver.java b/server/src/main/java/org/elasticsearch/plugins/internal/DocumentSizeObserver.java
deleted file mode 100644
index 386a90b65b60f..0000000000000
--- a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentSizeObserver.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.plugins.internal;
-
-import org.elasticsearch.xcontent.XContentParser;
-
-/**
- * An interface to allow wrapping an XContentParser and observe the events emitted while parsing
- * A default implementation returns a noop DocumentSizeObserver
- */
-public interface DocumentSizeObserver {
-    /**
-     * a default noop implementation
-     */
-    DocumentSizeObserver EMPTY_INSTANCE = new DocumentSizeObserver() {
-        @Override
-        public XContentParser wrapParser(XContentParser xContentParser) {
-            return xContentParser;
-        }
-
-        @Override
-        public long normalisedBytesParsed() {
-            return 0;
-        }
-
-    };
-
-    /**
-     * Decorates a provided xContentParser with additional logic (gather some state).
-     *
-     * @param xContentParser to be decorated
-     * @return a decorator xContentParser
-     */
-    XContentParser wrapParser(XContentParser xContentParser);
-
-    /**
-     * Returns the state gathered during parsing
-     *
-     * @return a number representing a state parsed
-     */
-    long normalisedBytesParsed();
-
-    /**
-     * Indicates if an observer was used on an update request with script
-     *
-     * @return true if update was done by script, false otherwise
-     */
-    default boolean isUpdateByScript() {
-        return false;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecorator.java b/server/src/main/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecorator.java
new file mode 100644
index 0000000000000..4fce2a20415a3
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecorator.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.internal;
+
+import org.elasticsearch.index.mapper.ParsedDocument.DocumentSize;
+import org.elasticsearch.xcontent.XContentParser;
+
+public interface XContentMeteringParserDecorator extends XContentParserDecorator {
+    /**
+     * a default noop implementation
+     */
+    XContentMeteringParserDecorator NOOP = new XContentMeteringParserDecorator() {
+        @Override
+        public DocumentSize meteredDocumentSize() {
+            return DocumentSize.UNKNOWN;
+        }
+
+        @Override
+        public XContentParser decorate(XContentParser xContentParser) {
+            return xContentParser;
+        }
+    };
+
+    DocumentSize meteredDocumentSize();
+}
diff --git a/server/src/main/java/org/elasticsearch/plugins/internal/XContentParserDecorator.java b/server/src/main/java/org/elasticsearch/plugins/internal/XContentParserDecorator.java
new file mode 100644
index 0000000000000..fc34a486d4aa7
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/plugins/internal/XContentParserDecorator.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.internal;
+
+import org.elasticsearch.xcontent.XContentParser;
+
+public interface XContentParserDecorator {
+    XContentParserDecorator NOOP = parser -> parser;
+
+    XContentParser decorate(XContentParser xContentParser);
+}
diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
index 37f1850c1fb2d..67d59924652db 100644
--- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.snapshots.SnapshotDeleteListener;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 
@@ -85,9 +84,10 @@ public void deleteSnapshots(
         Collection<SnapshotId> snapshotIds,
         long repositoryDataGeneration,
         IndexVersion minimumNodeVersion,
-        SnapshotDeleteListener listener
+        ActionListener<RepositoryData> repositoryDataUpdateListener,
+        Runnable onCompletion
     ) {
-        in.deleteSnapshots(snapshotIds, repositoryDataGeneration, minimumNodeVersion, listener);
+        in.deleteSnapshots(snapshotIds, repositoryDataGeneration, minimumNodeVersion, repositoryDataUpdateListener, onCompletion);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java b/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java
index 948ae747e11a9..2aba6fbbebce2 100644
--- a/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.snapshots.SnapshotDeleteListener;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 
@@ -92,9 +91,10 @@ public void deleteSnapshots(
         Collection<SnapshotId> snapshotIds,
         long repositoryDataGeneration,
         IndexVersion minimumNodeVersion,
-        SnapshotDeleteListener listener
+        ActionListener<RepositoryData> repositoryDataUpdateListener,
+        Runnable onCompletion
     ) {
-        listener.onFailure(createCreationException());
+        repositoryDataUpdateListener.onFailure(createCreationException());
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesFeatures.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesFeatures.java
new file mode 100644
index 0000000000000..141dac0c5c430
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesFeatures.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories;
+
+import org.elasticsearch.features.FeatureSpecification;
+import org.elasticsearch.features.NodeFeature;
+
+import java.util.Set;
+
+public class RepositoriesFeatures implements FeatureSpecification {
+    public static final NodeFeature SUPPORTS_REPOSITORIES_USAGE_STATS = new NodeFeature("repositories.supports_usage_stats");
+
+    @Override
+    public Set<NodeFeature> getFeatures() {
+        return Set.of(SUPPORTS_REPOSITORIES_USAGE_STATS);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java
index 85f06580cee79..02cc1a7458712 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java
@@ -22,7 +22,7 @@ public record RepositoriesMetrics(
     LongCounter unsuccessfulOperationCounter,
     LongHistogram exceptionHistogram,
     LongHistogram throttleHistogram,
-    LongHistogram httpRequestTimeInMicroHistogram
+    LongHistogram httpRequestTimeInMillisHistogram
 ) {
 
     public static RepositoriesMetrics NOOP = new RepositoriesMetrics(MeterRegistry.NOOP);
@@ -36,7 +36,7 @@ public record RepositoriesMetrics(
     public static final String METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL = "es.repositories.operations.unsuccessful.total";
     public static final String METRIC_EXCEPTIONS_HISTOGRAM = "es.repositories.exceptions.histogram";
     public static final String METRIC_THROTTLES_HISTOGRAM = "es.repositories.throttles.histogram";
-    public static final String HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM = "es.repositories.requests.http_request_time.histogram";
+    public static final String HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM = "es.repositories.requests.http_request_time.histogram";
 
     public RepositoriesMetrics(MeterRegistry meterRegistry) {
         this(
@@ -54,9 +54,9 @@ public RepositoriesMetrics(MeterRegistry meterRegistry) {
             meterRegistry.registerLongHistogram(METRIC_EXCEPTIONS_HISTOGRAM, "repository request exception histogram", "unit"),
             meterRegistry.registerLongHistogram(METRIC_THROTTLES_HISTOGRAM, "repository request throttle histogram", "unit"),
             meterRegistry.registerLongHistogram(
-                HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM,
-                "HttpRequestTime in microseconds expressed as as a histogram",
-                "micros"
+                HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM,
+                "HttpRequestTime in milliseconds expressed as as a histogram",
+                "ms"
             )
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index c63be88215655..732a18dffe233 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
+import org.elasticsearch.action.admin.cluster.stats.RepositoryUsageStats;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.node.NodeClient;
@@ -137,7 +138,7 @@ public RepositoriesService(
         this.repositoriesStatsArchive = new RepositoriesStatsArchive(
             REPOSITORIES_STATS_ARCHIVE_RETENTION_PERIOD.get(settings),
             REPOSITORIES_STATS_ARCHIVE_MAX_ARCHIVED_STATS.get(settings),
-            threadPool::relativeTimeInMillis
+            threadPool.relativeTimeInMillisSupplier()
         );
         this.preRestoreChecks = preRestoreChecks;
     }
@@ -166,7 +167,7 @@ record RegisterRepositoryTaskResult(AcknowledgedResponse ackResponse, boolean ch
             // When publication has completed (and all acks received or timed out) then verify the repository.
             // (if acks timed out then acknowledgementStep completes before the master processes this cluster state, hence why we have
             // to wait for the publication to be complete too)
-            .<RegisterRepositoryTaskResult>andThen((clusterUpdateStep, ignored) -> {
+            .<RegisterRepositoryTaskResult>andThen(clusterUpdateStep -> {
                 final ListenableFuture<AcknowledgedResponse> acknowledgementStep = new ListenableFuture<>();
                 final ListenableFuture<Boolean> publicationStep = new ListenableFuture<>(); // Boolean==changed.
                 submitUnbatchedTask(
@@ -221,7 +222,7 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState)
                         })
                         // When verification has completed, get the repository data for the first time
                         .<RepositoryData>andThen(
-                            (getRepositoryDataStep, ignored) -> threadPool.generic()
+                            getRepositoryDataStep -> threadPool.generic()
                                 .execute(
                                     ActionRunnable.wrap(
                                         getRepositoryDataStep,
@@ -944,15 +945,33 @@ public List<BiConsumer<Snapshot, IndexVersion>> getPreRestoreVersionChecks() {
         return preRestoreChecks;
     }
 
-    @Override
-    protected void doStart() {
+    public static String COUNT_USAGE_STATS_NAME = "count";
 
+    public RepositoryUsageStats getUsageStats() {
+        if (repositories.isEmpty()) {
+            return RepositoryUsageStats.EMPTY;
+        }
+        final var statsByType = new HashMap<String, Map<String, Long>>();
+        for (final var repository : repositories.values()) {
+            final var repositoryType = repository.getMetadata().type();
+            final var typeStats = statsByType.computeIfAbsent(repositoryType, ignored -> new HashMap<>());
+            typeStats.compute(COUNT_USAGE_STATS_NAME, (k, count) -> (count == null ? 0L : count) + 1);
+            final var repositoryUsageTags = repository.getUsageFeatures();
+            assert repositoryUsageTags.contains(COUNT_USAGE_STATS_NAME) == false : repositoryUsageTags;
+            for (final var repositoryUsageTag : repositoryUsageTags) {
+                typeStats.compute(repositoryUsageTag, (k, count) -> (count == null ? 0L : count) + 1);
+            }
+        }
+        return new RepositoryUsageStats(
+            statsByType.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> Map.copyOf(e.getValue())))
+        );
     }
 
     @Override
-    protected void doStop() {
+    protected void doStart() {}
 
-    }
+    @Override
+    protected void doStop() {}
 
     @Override
     protected void doClose() throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java
index 06a53053bca88..09f4782b6e5fa 100644
--- a/server/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.snapshots.SnapshotDeleteListener;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -161,13 +160,19 @@ public void onFailure(Exception e) {
      * @param repositoryDataGeneration     the generation of the {@link RepositoryData} in the repository at the start of the deletion
      * @param minimumNodeVersion           the minimum {@link IndexVersion} across the nodes in the cluster, with which the repository
      *                                     format must remain compatible
-     * @param listener                     completion listener, see {@link SnapshotDeleteListener}.
+     * @param repositoryDataUpdateListener listener completed when the {@link RepositoryData} is updated, or when the process fails
+     *                                     without changing the repository contents - in either case, it is now safe for the next operation
+     *                                     on this repository to proceed.
+     * @param onCompletion                 action executed on completion of the cleanup actions that follow a successful
+     *                                     {@link RepositoryData} update; not called if {@code repositoryDataUpdateListener} completes
+     *                                     exceptionally.
      */
     void deleteSnapshots(
         Collection<SnapshotId> snapshotIds,
         long repositoryDataGeneration,
         IndexVersion minimumNodeVersion,
-        SnapshotDeleteListener listener
+        ActionListener<RepositoryData> repositoryDataUpdateListener,
+        Runnable onCompletion
     );
 
     /**
@@ -307,6 +312,14 @@ void cloneShardSnapshot(
      */
     void awaitIdle();
 
+    /**
+     * @return a set of the names of the features that this repository instance uses, for reporting in the cluster stats for telemetry
+     *         collection.
+     */
+    default Set<String> getUsageFeatures() {
+        return Set.of();
+    }
+
     static boolean assertSnapshotMetaThread() {
         return ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SNAPSHOT_META);
     }
diff --git a/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java b/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java
index 7821c865e166c..853de48a483a1 100644
--- a/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.snapshots.SnapshotDeleteListener;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 
@@ -90,9 +89,10 @@ public void deleteSnapshots(
         Collection<SnapshotId> snapshotIds,
         long repositoryDataGeneration,
         IndexVersion minimumNodeVersion,
-        SnapshotDeleteListener listener
+        ActionListener<RepositoryData> repositoryDataUpdateListener,
+        Runnable onCompletion
     ) {
-        listener.onFailure(createUnknownTypeException());
+        repositoryDataUpdateListener.onFailure(createUnknownTypeException());
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
index 6750cced06191..14595d79bd585 100644
--- a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryCoordinationAction.java b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryCoordinationAction.java
index b892ff93c7a9c..f6c2934af2c05 100644
--- a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryCoordinationAction.java
+++ b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryCoordinationAction.java
@@ -17,9 +17,10 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.VerifyNodeRepositoryAction.Request;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportException;
@@ -68,7 +69,7 @@ public LocalAction(
             ClusterService clusterService,
             NodeClient client
         ) {
-            super(NAME, actionFilters, transportService.getTaskManager());
+            super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
             this.transportService = transportService;
             this.clusterService = clusterService;
             this.client = client;
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 96fcf0512cbff..cc56e940530e8 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -123,7 +123,6 @@
 import org.elasticsearch.repositories.SnapshotShardContext;
 import org.elasticsearch.snapshots.AbortedSnapshotException;
 import org.elasticsearch.snapshots.PausedSnapshotException;
-import org.elasticsearch.snapshots.SnapshotDeleteListener;
 import org.elasticsearch.snapshots.SnapshotException;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
@@ -847,8 +846,8 @@ private RepositoryData safeRepositoryData(long repositoryDataGeneration, Map<Str
      * protect by adding some delays in between operations so that things have a chance to settle down. This method is the hook that allows
      * the delete process to add this protection when necessary.
      */
-    protected SnapshotDeleteListener wrapWithWeakConsistencyProtection(SnapshotDeleteListener snapshotDeleteListener) {
-        return snapshotDeleteListener;
+    protected ActionListener<RepositoryData> wrapWithWeakConsistencyProtection(ActionListener<RepositoryData> listener) {
+        return listener;
     }
 
     @Override
@@ -856,19 +855,15 @@ public void deleteSnapshots(
         Collection<SnapshotId> snapshotIds,
         long repositoryDataGeneration,
         IndexVersion minimumNodeVersion,
-        SnapshotDeleteListener listener
+        ActionListener<RepositoryData> repositoryDataUpdateListener,
+        Runnable onCompletion
     ) {
-        createSnapshotsDeletion(snapshotIds, repositoryDataGeneration, minimumNodeVersion, new ActionListener<>() {
-            @Override
-            public void onResponse(SnapshotsDeletion snapshotsDeletion) {
-                snapshotsDeletion.runDelete(listener);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                listener.onFailure(e);
-            }
-        });
+        createSnapshotsDeletion(
+            snapshotIds,
+            repositoryDataGeneration,
+            minimumNodeVersion,
+            repositoryDataUpdateListener.delegateFailureAndWrap((l, snapshotsDeletion) -> snapshotsDeletion.runDelete(l, onCompletion))
+        );
     }
 
     /**
@@ -933,7 +928,7 @@ private void createSnapshotsDeletion(
      * </ul>
      * <p>
      *     Until the {@link RepositoryData} is updated there should be no other activities in the repository, and in particular the root
-     *     blob must not change until it is updated by this deletion and {@link SnapshotDeleteListener#onRepositoryDataWritten} is called.
+     *     blob must not change until it is updated by this deletion and the {@code repositoryDataUpdateListener} is completed.
      * </p>
      */
     class SnapshotsDeletion {
@@ -1027,46 +1022,35 @@ class SnapshotsDeletion {
         // ---------------------------------------------------------------------------------------------------------------------------------
         // The overall flow of execution
 
-        void runDelete(SnapshotDeleteListener listener) {
-            final var releasingListener = new SnapshotDeleteListener() {
-                @Override
-                public void onDone() {
-                    try {
-                        shardBlobsToDelete.close();
-                    } finally {
-                        listener.onDone();
-                    }
-                }
-
-                @Override
-                public void onRepositoryDataWritten(RepositoryData repositoryData) {
-                    listener.onRepositoryDataWritten(repositoryData);
+        void runDelete(ActionListener<RepositoryData> repositoryDataUpdateListener, Runnable onCompletion) {
+            final var releasingListener = repositoryDataUpdateListener.delegateResponse((l, e) -> {
+                try {
+                    shardBlobsToDelete.close();
+                } finally {
+                    l.onFailure(e);
                 }
-
-                @Override
-                public void onFailure(Exception e) {
-                    try {
-                        shardBlobsToDelete.close();
-                    } finally {
-                        listener.onFailure(e);
-                    }
-
+            });
+            final Runnable releasingOnCompletion = () -> {
+                try {
+                    shardBlobsToDelete.close();
+                } finally {
+                    onCompletion.run();
                 }
             };
             if (useShardGenerations) {
-                runWithUniqueShardMetadataNaming(releasingListener);
+                runWithUniqueShardMetadataNaming(releasingListener, releasingOnCompletion);
             } else {
-                runWithLegacyNumericShardMetadataNaming(wrapWithWeakConsistencyProtection(releasingListener));
+                runWithLegacyNumericShardMetadataNaming(wrapWithWeakConsistencyProtection(releasingListener), releasingOnCompletion);
             }
         }
 
-        private void runWithUniqueShardMetadataNaming(SnapshotDeleteListener listener) {
+        private void runWithUniqueShardMetadataNaming(ActionListener<RepositoryData> repositoryDataUpdateListener, Runnable onCompletion) {
             SubscribableListener
 
                 // First write the new shard state metadata (without the removed snapshots) and compute deletion targets
                 .newForked(this::writeUpdatedShardMetadataAndComputeDeletes)
 
-                .<RepositoryData>andThen((l, ignored) -> {
+                .<RepositoryData>andThen(l -> {
                     // Once we have put the new shard-level metadata into place, we can update the repository metadata as follows:
                     // 1. Remove the snapshots from the list of existing snapshots
                     // 2. Update the index shard generations of all updated shard folders
@@ -1082,30 +1066,29 @@ private void runWithUniqueShardMetadataNaming(SnapshotDeleteListener listener) {
                     );
                 })
 
-                .addListener(
-                    ActionListener.wrap(
-                        // Once we have updated the repository, run the clean-ups
-                        newRepositoryData -> {
-                            listener.onRepositoryDataWritten(newRepositoryData);
-                            // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion
-                            try (var refs = new RefCountingRunnable(listener::onDone)) {
-                                cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener());
-                                cleanupUnlinkedShardLevelBlobs(refs.acquireListener());
-                            }
-                        },
-                        listener::onFailure
-                    )
-                );
+                .<RepositoryData>andThen((l, newRepositoryData) -> {
+                    l.onResponse(newRepositoryData);
+                    // Once we have updated the repository, run the unreferenced blobs cleanup in parallel to shard-level snapshot deletion
+                    try (var refs = new RefCountingRunnable(onCompletion)) {
+                        cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener());
+                        cleanupUnlinkedShardLevelBlobs(refs.acquireListener());
+                    }
+                })
+
+                .addListener(repositoryDataUpdateListener);
         }
 
-        private void runWithLegacyNumericShardMetadataNaming(SnapshotDeleteListener listener) {
+        private void runWithLegacyNumericShardMetadataNaming(
+            ActionListener<RepositoryData> repositoryDataUpdateListener,
+            Runnable onCompletion
+        ) {
             // Write the new repository data first (with the removed snapshot), using no shard generations
             updateRepositoryData(
                 originalRepositoryData.removeSnapshots(snapshotIds, ShardGenerations.EMPTY),
-                ActionListener.wrap(newRepositoryData -> {
+                repositoryDataUpdateListener.delegateFailure((delegate, newRepositoryData) -> {
                     try (var refs = new RefCountingRunnable(() -> {
-                        listener.onRepositoryDataWritten(newRepositoryData);
-                        listener.onDone();
+                        delegate.onResponse(newRepositoryData);
+                        onCompletion.run();
                     })) {
                         // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion
                         cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener());
@@ -1120,7 +1103,7 @@ private void runWithLegacyNumericShardMetadataNaming(SnapshotDeleteListener list
                             )
                         );
                     }
-                }, listener::onFailure)
+                })
             );
         }
 
@@ -3960,4 +3943,29 @@ public String getAnalysisFailureExtraDetail() {
             ReferenceDocs.SNAPSHOT_REPOSITORY_ANALYSIS
         );
     }
+
+    public static final String READ_ONLY_USAGE_STATS_NAME = "read_only";
+    public static final String READ_WRITE_USAGE_STATS_NAME = "read_write";
+
+    @Override
+    public final Set<String> getUsageFeatures() {
+        final var extraUsageFeatures = getExtraUsageFeatures();
+        assert extraUsageFeatures.contains(READ_ONLY_USAGE_STATS_NAME) == false : extraUsageFeatures;
+        assert extraUsageFeatures.contains(READ_WRITE_USAGE_STATS_NAME) == false : extraUsageFeatures;
+        return Set.copyOf(
+            Stream.concat(Stream.of(isReadOnly() ? READ_ONLY_USAGE_STATS_NAME : READ_WRITE_USAGE_STATS_NAME), extraUsageFeatures.stream())
+                .toList()
+        );
+    }
+
+    /**
+     * All blob-store repositories include the counts of read-only and read-write repositories in their telemetry. This method returns other
+     * features of the repositories in use.
+     *
+     * @return a set of the names of the extra features that this repository instance uses, for reporting in the cluster stats for telemetry
+     *         collection.
+     */
+    protected Set<String> getExtraUsageFeatures() {
+        return Set.of();
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunner.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunner.java
index bd4084f1a4015..38875e94f419c 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunner.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunner.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.PrioritizedThrottledTaskRunner;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.repositories.SnapshotShardContext;
 
 import java.io.IOException;
@@ -38,23 +39,41 @@ public class ShardSnapshotTaskRunner {
     private final CheckedBiConsumer<SnapshotShardContext, FileInfo, IOException> fileSnapshotter;
 
     abstract static class SnapshotTask extends AbstractRunnable implements Comparable<SnapshotTask> {
-
-        private static final Comparator<SnapshotTask> COMPARATOR = Comparator.comparingLong(
-            (SnapshotTask t) -> t.context().snapshotStartTime()
-        ).thenComparing(t -> t.context().snapshotId().getUUID()).thenComparingInt(SnapshotTask::priority);
-
         protected final SnapshotShardContext context;
 
         SnapshotTask(SnapshotShardContext context) {
             this.context = context;
         }
 
+        public final SnapshotShardContext context() {
+            return context;
+        }
+
+        private long getSnapshotStartTime() {
+            return context().snapshotStartTime();
+        }
+
+        private String snapshotUUID() {
+            return context().snapshotId().getUUID();
+        }
+
         public abstract int priority();
 
-        public SnapshotShardContext context() {
-            return context;
+        @SuppressWarnings("resource")
+        private ShardId shardId() {
+            return context().store().shardId();
         }
 
+        private static final Comparator<SnapshotTask> COMPARATOR = Comparator
+            // Prefer work from the oldest running snapshot ...
+            .comparingLong(SnapshotTask::getSnapshotStartTime)
+            // ... tiebreaking on UUID just in case two of them start at the same millisecond ...
+            .thenComparing(SnapshotTask::snapshotUUID)
+            // ... then prefer starting new shard snapshots over uploading files for ongoing ones (for fast noop shard snapshots) ...
+            .thenComparingInt(SnapshotTask::priority)
+            // ... then break ties by shard ID to limit WIP and avoid PAUSED_FOR_NODE_REMOVAL discarding more work than it needs to
+            .thenComparing(SnapshotTask::shardId);
+
         @Override
         public final int compareTo(SnapshotTask other) {
             return COMPARATOR.compare(this, other);
diff --git a/server/src/main/java/org/elasticsearch/rest/ChunkedZipResponse.java b/server/src/main/java/org/elasticsearch/rest/ChunkedZipResponse.java
new file mode 100644
index 0000000000000..2d4ada2d3341a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/ChunkedZipResponse.java
@@ -0,0 +1,553 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.rest;
+
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.ReleasableBytesReference;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.io.stream.BytesStream;
+import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
+import org.elasticsearch.common.recycler.Recycler;
+import org.elasticsearch.core.AbstractRefCounted;
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.RefCounted;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.transport.Transports;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Queue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * A REST response with {@code Content-type: application/zip} to which the caller can write entries in an asynchronous and streaming
+ * fashion.
+ * <p>
+ * Callers obtain a listener for individual entries using {@link #newEntryListener} and complete these listeners to submit the corresponding
+ * entries for transmission. Internally, the output entries are held in a queue in the order in which the entry listeners are completed.
+ * If the queue becomes empty then the response transmission is paused until the next entry becomes available.
+ * <p>
+ * The internal queue is unbounded. It is the caller's responsibility to ensure that the response does not consume an excess of resources
+ * while it's being sent.
+ * <p>
+ * The caller must eventually call {@link ChunkedZipResponse#close} to finish the transmission of the response.
+ * <p>
+ * Note that individual entries can also pause themselves mid-transmission, since listeners returned by {@link #newEntryListener} accept a
+ * pauseable {@link ChunkedRestResponseBodyPart}. Zip files do not have any mechanism which supports the multiplexing of outputs, so if the
+ * entry at the head of the queue is paused then that will hold up the transmission of all subsequent entries too.
+ */
+public final class ChunkedZipResponse implements Releasable {
+
+    public static final String ZIP_CONTENT_TYPE = "application/zip";
+
+    /**
+     * The underlying stream that collects the raw bytes to be transmitted. Mutable, because we collect the contents of each chunk in a
+     * distinct stream that is held in this field while that chunk is under construction.
+     */
+    @Nullable // if there's no chunk under construction
+    private BytesStream targetStream;
+
+    private final ZipOutputStream zipOutputStream = new ZipOutputStream(new OutputStream() {
+        @Override
+        public void write(int b) throws IOException {
+            assert targetStream != null;
+            targetStream.write(b);
+        }
+
+        @Override
+        public void write(byte[] b, int off, int len) throws IOException {
+            assert targetStream != null;
+            targetStream.write(b, off, len);
+        }
+    }, StandardCharsets.UTF_8);
+
+    private final String filename;
+    private final RestChannel restChannel;
+
+    /**
+     * A listener for the first part (i.e. sequence of chunks of zipped data) of the next entry to become available for transmission after a
+     * pause. Completed with the newly-created unique active {@link AvailableChunksZipResponseBodyPart} within {@link #enqueueEntry}, and
+     * subscribed to via {@link AvailableChunksZipResponseBodyPart#getNextPart} when the current {@link AvailableChunksZipResponseBodyPart}
+     * becomes inactive because of a transmission pause.
+     */
+    @Nullable // if the first part hasn't been sent yet
+    private SubscribableListener<ChunkedRestResponseBodyPart> nextAvailableChunksListener;
+
+    /**
+     * A resource to be released when the transmission of the current entry is complete. Note that we may complete the transmission of
+     * multiple entries at the same time, if they are all processed by one call to {@link AvailableChunksZipResponseBodyPart#encodeChunk}
+     * and transmitted together.
+     */
+    @Nullable // if not currently sending an entry
+    private Releasable currentEntryReleasable;
+
+    /**
+     * @param filename     The name of the zip file, which appears in the {@code Content-Disposition} HTTP header of the response, and also
+     *                     is used as a directory prefix for all entries.
+     * @param restChannel  The {@link RestChannel} on which to send the response.
+     * @param onCompletion A resource which is released when the transmission is complete.
+     */
+    public ChunkedZipResponse(String filename, RestChannel restChannel, Releasable onCompletion) {
+        this.filename = filename;
+        this.restChannel = restChannel;
+        this.listenersRefs = AbstractRefCounted.of(() -> enqueueEntry(null, NO_MORE_ENTRIES, onCompletion));
+        this.rootListenerRef = Releasables.releaseOnce(listenersRefs::decRef);
+    }
+
+    private final RefCounted listenersRefs;
+    private final Releasable rootListenerRef;
+
+    /**
+     * Close this {@link ChunkedZipResponse}. Once closed, when there are no more pending listeners the zip file footer is sent.
+     */
+    @Override
+    public void close() {
+        rootListenerRef.close();
+    }
+
+    /**
+     * Create a listener which, when completed, will write the result {@link ChunkedRestResponseBodyPart}, and any following parts, as an
+     * entry in the response stream with the given name. If the listener is completed successfully with {@code null}, or exceptionally, then
+     * no entry is sent. When all listeners created by this method have been completed, the zip file footer is sent.
+     * <p>
+     * This method may be called as long as this {@link ChunkedZipResponse} is not closed, or there is at least one other incomplete entry
+     * listener.
+     *
+     * @param entryName  The name of the entry in the response zip file.
+     * @param releasable A resource which is released when the entry has been completely processed, i.e. when
+     *                   <ul>
+     *                   <li>the sequence of {@link ChunkedRestResponseBodyPart} instances have been fully sent, or</li>
+     *                   <li>the listener was completed with {@code null}, or an exception, indicating that no entry is to be sent, or</li>
+     *                   <li>the overall response was cancelled before completion and all resources related to the partial transmission of
+     *                   this entry have been released.</li>
+     *                   </ul>
+     */
+    public ActionListener<ChunkedRestResponseBodyPart> newEntryListener(String entryName, Releasable releasable) {
+        if (listenersRefs.tryIncRef()) {
+            final var zipEntry = new ZipEntry(filename + "/" + entryName);
+            return ActionListener.assertOnce(ActionListener.releaseAfter(new ActionListener<>() {
+                @Override
+                public void onResponse(ChunkedRestResponseBodyPart chunkedRestResponseBodyPart) {
+                    if (chunkedRestResponseBodyPart == null) {
+                        Releasables.closeExpectNoException(releasable);
+                    } else {
+                        enqueueEntry(zipEntry, chunkedRestResponseBodyPart, releasable);
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    Releasables.closeExpectNoException(releasable);
+                }
+
+                @Override
+                public String toString() {
+                    return "ZipEntry[" + zipEntry.getName() + "]";
+                }
+            }, listenersRefs::decRef));
+        } else {
+            assert false : "already closed";
+            throw new AlreadyClosedException("response already closed");
+        }
+    }
+
+    /**
+     * A zip file entry which is ready for transmission, to be stored in {@link #entryQueue}.
+     *
+     * @param zipEntry      The entry metadata, to be written in its header.
+     * @param firstBodyPart The first part of the entry body. Subsequent parts, if present, come from
+     *                      {@link ChunkedRestResponseBodyPart#getNextPart}.
+     * @param releasable    A resource to release when this entry has been fully transmitted, or is no longer required because the
+     *                      transmission was cancelled.
+     */
+    private record ChunkedZipEntry(ZipEntry zipEntry, ChunkedRestResponseBodyPart firstBodyPart, Releasable releasable) {}
+
+    /**
+     * Queue of entries that are ready for transmission.
+     */
+    private final Queue<ChunkedZipEntry> entryQueue = new LinkedBlockingQueue<>();
+
+    /**
+     * Upper bound on the number of entries in the queue, atomically modified to ensure there's only one thread processing queue entries at
+     * once.
+     */
+    private final AtomicInteger queueLength = new AtomicInteger();
+
+    /**
+     * Ref-counting for access to the queue, to avoid clearing the queue on abort concurrently with an entry being sent.
+     */
+    private final RefCounted queueRefs = AbstractRefCounted.of(this::drainQueue);
+
+    /**
+     * Flag to indicate if the request has been aborted, at which point we should stop enqueueing more entries and promptly clean up the
+     * ones being sent. It's safe to ignore this, but without it in theory a constant stream of calls to {@link #enqueueEntry} could prevent
+     * {@link #drainQueue} from running for arbitrarily long.
+     */
+    private final AtomicBoolean isRestResponseFinished = new AtomicBoolean();
+
+    private boolean tryAcquireQueueRef() {
+        return isRestResponseFinished.get() == false && queueRefs.tryIncRef();
+    }
+
+    /**
+     * Called when an entry is ready for its transmission to start. Adds the entry to {@link #entryQueue} and spawns a new
+     * {@link AvailableChunksZipResponseBodyPart} if none is currently active.
+     *
+     * @param zipEntry      The entry metadata.
+     * @param firstBodyPart The first part of the entry. Entries may comprise multiple parts, with transmission pauses in between.
+     * @param releasable    Released when the entry has been fully transmitted.
+     */
+    private void enqueueEntry(ZipEntry zipEntry, ChunkedRestResponseBodyPart firstBodyPart, Releasable releasable) {
+        if (tryAcquireQueueRef()) {
+            try {
+                entryQueue.add(new ChunkedZipEntry(zipEntry, firstBodyPart, releasable));
+                if (queueLength.getAndIncrement() == 0) {
+                    // There is no active AvailableChunksZipResponseBodyPart, but there is now an entry in the queue, so we must create a
+                    // AvailableChunksZipResponseBodyPart to process it (along with any other entries that are concurrently added to the
+                    // queue). It's safe to mutate releasable and continuationListener here because they are only otherwise accessed by an
+                    // active AvailableChunksZipResponseBodyPart (which does not exist) or when all queueRefs have been released (which they
+                    // have not here).
+                    final var nextEntry = entryQueue.poll();
+                    assert nextEntry != null;
+                    final var availableChunks = new AvailableChunksZipResponseBodyPart(nextEntry.zipEntry(), nextEntry.firstBodyPart());
+                    assert currentEntryReleasable == null;
+                    currentEntryReleasable = nextEntry.releasable();
+                    final var currentAvailableChunksListener = nextAvailableChunksListener;
+                    nextAvailableChunksListener = new SubscribableListener<>();
+                    if (currentAvailableChunksListener == null) {
+                        // We are not resuming after a pause, this is the first entry to be sent, so we start the response transmission.
+                        final var restResponse = RestResponse.chunked(RestStatus.OK, availableChunks, this::restResponseFinished);
+                        restResponse.addHeader("content-disposition", Strings.format("attachment; filename=\"%s.zip\"", filename));
+                        restChannel.sendResponse(restResponse);
+                    } else {
+                        // We are resuming transmission after a pause, so just carry on sending the response body.
+                        currentAvailableChunksListener.onResponse(availableChunks);
+                    }
+                }
+            } finally {
+                queueRefs.decRef();
+            }
+        } else {
+            Releasables.closeExpectNoException(releasable);
+        }
+    }
+
+    private void restResponseFinished() {
+        assert Transports.assertTransportThread();
+        if (isRestResponseFinished.compareAndSet(false, true)) {
+            queueRefs.decRef();
+        }
+    }
+
+    private void drainQueue() {
+        assert isRestResponseFinished.get();
+        assert queueRefs.hasReferences() == false;
+        final var taskCount = queueLength.get() + 1;
+        final var releasables = new ArrayList<Releasable>(taskCount);
+        try {
+            releasables.add(currentEntryReleasable);
+            currentEntryReleasable = null;
+            ChunkedZipEntry entry;
+            while ((entry = entryQueue.poll()) != null) {
+                releasables.add(entry.releasable());
+            }
+            assert entryQueue.isEmpty() : entryQueue.size(); // no concurrent adds
+            assert releasables.size() == taskCount || releasables.size() == taskCount - 1 : taskCount + " vs " + releasables.size();
+        } finally {
+            Releasables.closeExpectNoException(Releasables.wrap(releasables));
+        }
+    }
+
+    /**
+     * A {@link ChunkedRestResponseBodyPart} which will yield all currently-available chunks by consuming entries from {@link #entryQueue}.
+     * There is only ever at most one active instance of this class at any time, in the sense that one such instance becoming inactive
+     * <i>happens-before</i> the creation of the next instance. One of these parts may send chunks for more than one entry.
+     */
+    private final class AvailableChunksZipResponseBodyPart implements ChunkedRestResponseBodyPart {
+
+        /**
+         * The next {@link ZipEntry} header whose transmission to start.
+         */
+        @Nullable // if no entry is available, or we've already sent the header for the current entry and are now sending its body.
+        private ZipEntry zipEntry;
+
+        /**
+         * The body part which is currently being transmitted, or {@link #NO_MORE_ENTRIES} if we're transmitting the zip file footer.
+         */
+        private ChunkedRestResponseBodyPart bodyPart;
+
+        /**
+         * True when we have run out of compressed chunks ready for immediate transmission, so the response is paused, but we expect to send
+         * more data later.
+         */
+        private boolean isResponsePaused;
+
+        /**
+         * True when we have sent the zip file footer, or the response was cancelled.
+         */
+        private boolean isResponseComplete;
+
+        /**
+         * A listener which is created when there are no more available chunks, so transmission is paused, subscribed to in
+         * {@link #getNextPart}, and then completed with the next body part (sequence of zipped chunks, i.e. a new (unique) active
+         * {@link AvailableChunksZipResponseBodyPart}).
+         */
+        private SubscribableListener<ChunkedRestResponseBodyPart> getNextPartListener;
+
+        /**
+         * A cache for an empty list to be used to collect the {@code Releasable} instances to be released when the next chunk has been
+         * fully transmitted. It's a list because a call to {@link #encodeChunk} may yield a chunk that completes several entries, each of
+         * which has its own resources to release. We cache this value across chunks because most chunks won't release anything, so we can
+         * keep the empty list around for later to save on allocations.
+         */
+        private ArrayList<Releasable> nextReleasablesCache = new ArrayList<>();
+
+        AvailableChunksZipResponseBodyPart(ZipEntry zipEntry, ChunkedRestResponseBodyPart bodyPart) {
+            this.zipEntry = zipEntry;
+            this.bodyPart = bodyPart;
+        }
+
+        /**
+         * @return whether this part of the compressed response is complete
+         */
+        @Override
+        public boolean isPartComplete() {
+            return isResponsePaused || isResponseComplete;
+        }
+
+        @Override
+        public boolean isLastPart() {
+            return isResponseComplete;
+        }
+
+        @Override
+        public void getNextPart(ActionListener<ChunkedRestResponseBodyPart> listener) {
+            assert getNextPartListener != null;
+            getNextPartListener.addListener(listener);
+        }
+
+        /**
+         * Transfer {@link #currentEntryReleasable} into the supplied collection (i.e. add it to {@code releasables} and then clear
+         * {@link #currentEntryReleasable}). Called when the last chunk of the last part of the current entry is serialized, so that we can
+         * start serializing chunks of the next entry straight away whilst delaying the release of the current entry's resources until the
+         * transmission of the chunk that is currently under construction.
+         */
+        private void transferCurrentEntryReleasable(ArrayList<Releasable> releasables) {
+            assert queueRefs.hasReferences();
+
+            if (currentEntryReleasable == null) {
+                return;
+            }
+
+            if (releasables == nextReleasablesCache) {
+                // adding the first value, so we must line up a new cached value for the next caller
+                nextReleasablesCache = new ArrayList<>();
+            }
+
+            releasables.add(currentEntryReleasable);
+            currentEntryReleasable = null;
+        }
+
+        @Override
+        public ReleasableBytesReference encodeChunk(int sizeHint, Recycler<BytesRef> recycler) throws IOException {
+            assert Transports.isTransportThread(Thread.currentThread());
+
+            final ArrayList<Releasable> releasables = nextReleasablesCache;
+            assert releasables.isEmpty();
+            try {
+                if (tryAcquireQueueRef()) {
+                    try {
+                        assert queueLength.get() > 0;
+                        // This is the current unique active AvailableChunksZipResponseBodyPart (i.e. queueLength is strictly positive and
+                        // we hold a queueRef), so any concurrent calls to enqueueEntry() at this point will just add to the queue and won't
+                        // spawn a new AvailableChunksZipResponseBodyPart or mutate any fields.
+
+                        final RecyclerBytesStreamOutput chunkStream = new RecyclerBytesStreamOutput(recycler);
+                        assert targetStream == null;
+                        targetStream = chunkStream;
+
+                        do {
+                            writeNextBytes(sizeHint, recycler, releasables);
+                        } while (isResponseComplete == false && isResponsePaused == false && chunkStream.size() < sizeHint);
+
+                        assert (releasables == nextReleasablesCache) == releasables.isEmpty();
+                        assert nextReleasablesCache.isEmpty();
+
+                        final Releasable chunkStreamReleasable = () -> Releasables.closeExpectNoException(chunkStream);
+                        final var result = new ReleasableBytesReference(
+                            chunkStream.bytes(),
+                            releasables.isEmpty()
+                                ? chunkStreamReleasable
+                                : Releasables.wrap(Iterators.concat(Iterators.single(chunkStreamReleasable), releasables.iterator()))
+                        );
+
+                        targetStream = null;
+                        return result;
+                    } finally {
+                        queueRefs.decRef();
+                    }
+                } else {
+                    // request aborted, nothing more to send (queue is being cleared by queueRefs#closeInternal)
+                    isResponseComplete = true;
+                    return new ReleasableBytesReference(BytesArray.EMPTY, () -> {});
+                }
+            } catch (Exception e) {
+                logger.error("failure encoding chunk", e);
+                throw e;
+            } finally {
+                if (targetStream != null) {
+                    assert false : "failure encoding chunk";
+                    IOUtils.closeWhileHandlingException(targetStream, Releasables.wrap(releasables));
+                    targetStream = null;
+                }
+            }
+        }
+
+        private void writeNextBytes(int sizeHint, Recycler<BytesRef> recycler, ArrayList<Releasable> releasables) throws IOException {
+            try {
+                if (bodyPart == NO_MORE_ENTRIES) {
+                    // When the last ref from listenersRefs is completed we enqueue a final sentinel entry to trigger the transmission of
+                    // the zip file footer, which happens here:
+                    finishResponse(releasables);
+                    return;
+                }
+
+                if (zipEntry != null) {
+                    // This is the start of a new entry, so write the entry header:
+                    zipOutputStream.putNextEntry(zipEntry);
+                    zipEntry = null;
+                }
+
+                // Write the next chunk of the current entry to the zip stream
+                if (bodyPart.isPartComplete() == false) {
+                    try (var innerChunk = bodyPart.encodeChunk(sizeHint, recycler)) {
+                        final var iterator = innerChunk.iterator();
+                        BytesRef bytesRef;
+                        while ((bytesRef = iterator.next()) != null) {
+                            zipOutputStream.write(bytesRef.bytes, bytesRef.offset, bytesRef.length);
+                        }
+                    }
+                }
+                if (bodyPart.isPartComplete()) {
+                    // Complete the current part: if the current entry is incomplete then set up a listener for its next part, otherwise
+                    // move on to the next available entry and start sending its content.
+                    finishCurrentPart(releasables);
+                }
+            } finally {
+                // Flush any buffered data (but not the compressor) to chunkStream so that its size is accurate.
+                zipOutputStream.flush();
+            }
+        }
+
+        private void finishCurrentPart(ArrayList<Releasable> releasables) throws IOException {
+            if (bodyPart.isLastPart()) {
+                zipOutputStream.closeEntry();
+                transferCurrentEntryReleasable(releasables);
+                final var newQueueLength = queueLength.decrementAndGet();
+                if (newQueueLength == 0) {
+                    // The current entry is complete, but the next entry isn't available yet, so we pause transmission. This means we are no
+                    // longer an active AvailableChunksZipResponseBodyPart, so any concurrent calls to enqueueEntry() at this point will now
+                    // spawn a new AvailableChunksZipResponseBodyPart to take our place.
+                    isResponsePaused = true;
+                    assert getNextPartListener == null;
+                    assert nextAvailableChunksListener != null;
+                    // Calling our getNextPart() will eventually yield the next body part supplied to enqueueEntry():
+                    getNextPartListener = nextAvailableChunksListener;
+                } else {
+                    // The current entry is complete, and the first part of the next entry is already available, so we start sending its
+                    // chunks too. This means we're still the unique active AvailableChunksZipResponseBodyPart. We re-use this
+                    // AvailableChunksZipResponseBodyPart instance rather than creating a new one to avoid unnecessary allocations.
+                    final var nextEntry = entryQueue.poll();
+                    assert nextEntry != null;
+                    zipEntry = nextEntry.zipEntry();
+                    bodyPart = nextEntry.firstBodyPart();
+                    currentEntryReleasable = nextEntry.releasable();
+                }
+            } else {
+                // The current entry has more parts to come, but we have reached the end of the current part, so we assume that the next
+                // part is not yet available and therefore must pause transmission. This means we are no longer an active
+                // AvailableChunksZipResponseBodyPart, but also another call to enqueueEntry() won't create a new
+                // AvailableChunksZipResponseBodyPart because the current entry is still counted in queueLength:
+                assert queueLength.get() > 0;
+                // Instead, we create a new active AvailableChunksZipResponseBodyPart when the next part of the current entry becomes
+                // available. It doesn't affect correctness if the next part is already available, it's just a little less efficient to make
+                // a new AvailableChunksZipResponseBodyPart in that case. That's ok, entries can coalesce all the available parts together
+                // themselves if efficiency really matters.
+                isResponsePaused = true;
+                assert getNextPartListener == null;
+                // Calling our getNextPart() will eventually yield the next body part from the current entry:
+                getNextPartListener = SubscribableListener.newForked(
+                    l -> bodyPart.getNextPart(l.map(p -> new AvailableChunksZipResponseBodyPart(null, p)))
+                );
+            }
+        }
+
+        private void finishResponse(ArrayList<Releasable> releasables) throws IOException {
+            assert zipEntry == null;
+            assert entryQueue.isEmpty() : entryQueue.size();
+            zipOutputStream.finish();
+            isResponseComplete = true;
+            transferCurrentEntryReleasable(releasables);
+            assert getNextPartListener == null;
+        }
+
+        @Override
+        public String getResponseContentTypeString() {
+            return ZIP_CONTENT_TYPE;
+        }
+    }
+
+    /**
+     * Sentinel body part indicating the end of the zip file.
+     */
+    private static final ChunkedRestResponseBodyPart NO_MORE_ENTRIES = new ChunkedRestResponseBodyPart() {
+        @Override
+        public boolean isPartComplete() {
+            assert false : "never called";
+            return true;
+        }
+
+        @Override
+        public boolean isLastPart() {
+            assert false : "never called";
+            return true;
+        }
+
+        @Override
+        public void getNextPart(ActionListener<ChunkedRestResponseBodyPart> listener) {
+            assert false : "never called";
+            listener.onFailure(new IllegalStateException("impossible"));
+        }
+
+        @Override
+        public ReleasableBytesReference encodeChunk(int sizeHint, Recycler<BytesRef> recycler) {
+            assert false : "never called";
+            return ReleasableBytesReference.empty();
+        }
+
+        @Override
+        public String getResponseContentTypeString() {
+            assert false : "never called";
+            return ZIP_CONTENT_TYPE;
+        }
+    };
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/FilterRestHandler.java b/server/src/main/java/org/elasticsearch/rest/FilterRestHandler.java
index a3b3c9d7b17f1..e07d8aad04609 100644
--- a/server/src/main/java/org/elasticsearch/rest/FilterRestHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/FilterRestHandler.java
@@ -48,8 +48,8 @@ public boolean allowsUnsafeBuffers() {
     }
 
     @Override
-    public boolean supportsContentStream() {
-        return delegate.supportsContentStream();
+    public boolean supportsBulkContent() {
+        return delegate.supportsBulkContent();
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index 3f9c0dbaa11d6..8592888d2dd03 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -432,7 +432,7 @@ private void dispatchRequest(
             }
             final XContentType xContentType = request.getXContentType();
             // TODO consider refactoring to handler.supportsContentStream(xContentType). It is only used with JSON and SMILE
-            if (handler.supportsContentStream()
+            if (handler.supportsBulkContent()
                 && XContentType.JSON != xContentType.canonical()
                 && XContentType.SMILE != xContentType.canonical()) {
                 channel.sendResponse(
diff --git a/server/src/main/java/org/elasticsearch/rest/RestFeatures.java b/server/src/main/java/org/elasticsearch/rest/RestFeatures.java
index 78d7754dec22e..84df4fde32f4c 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestFeatures.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestFeatures.java
@@ -22,7 +22,11 @@
 public class RestFeatures implements FeatureSpecification {
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(RestNodesCapabilitiesAction.CAPABILITIES_ACTION, UNIFIED_HIGHLIGHTER_MATCHED_FIELDS);
+        return Set.of(
+            RestNodesCapabilitiesAction.CAPABILITIES_ACTION,
+            RestNodesCapabilitiesAction.LOCAL_ONLY_CAPABILITIES,
+            UNIFIED_HIGHLIGHTER_MATCHED_FIELDS
+        );
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java
index 11208a24ceb10..474bf02b4b92b 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java
@@ -39,11 +39,11 @@ default boolean canTripCircuitBreaker() {
     }
 
     /**
-     * Indicates if the RestHandler supports content as a stream. A stream would be multiple objects delineated by
-     * {@link XContent#streamSeparator()}. If a handler returns true this will affect the types of content that can be sent to
-     * this endpoint.
+     * Indicates if the RestHandler supports bulk content. A bulk request contains multiple objects
+     * delineated by {@link XContent#bulkSeparator()}. If a handler returns true this will affect
+     * the types of content that can be sent to this endpoint.
      */
-    default boolean supportsContentStream() {
+    default boolean supportsBulkContent() {
         return false;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/StreamingXContentResponse.java b/server/src/main/java/org/elasticsearch/rest/StreamingXContentResponse.java
new file mode 100644
index 0000000000000..9f20416ff8b06
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/StreamingXContentResponse.java
@@ -0,0 +1,435 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.rest;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.ReleasableBytesReference;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.io.stream.BytesStream;
+import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
+import org.elasticsearch.common.recycler.Recycler;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.core.AbstractRefCounted;
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.RefCounted;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.Streams;
+import org.elasticsearch.transport.Transports;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Queue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A REST response with an XContent body to which the caller can write fragments of content in an asynchronous and streaming fashion.
+ * <p>
+ * Callers submit individual fragments of content using {@link #writeFragment}. Internally, the output entries are held in a queue.
+ * If the queue becomes empty then the response transmission is paused until the next entry becomes available.
+ * <p>
+ * The internal queue is unbounded. It is the caller's responsibility to ensure that the response does not consume an excess of resources
+ * while it's being sent.
+ * <p>
+ * The caller must eventually call {@link StreamingXContentResponse#close} to finish the transmission of the response.
+ */
+public final class StreamingXContentResponse implements Releasable {
+
+    /**
+     * The underlying stream that collects the raw bytes to be transmitted. Mutable, because we collect the contents of each chunk in a
+     * distinct stream that is held in this field while that chunk is under construction.
+     */
+    @Nullable // if there's no chunk under construction
+    private BytesStream targetStream;
+
+    private final XContentBuilder xContentBuilder;
+
+    private final RestChannel restChannel;
+    private final ToXContent.Params params;
+    private final Releasable onCompletion;
+
+    /**
+     * A listener for the next fragment to become available for transmission after a pause. Completed with the newly-created unique active
+     * {@link AvailableFragmentsResponseBodyPart} within {@link #writeFragment}, and  subscribed to via
+     * {@link AvailableFragmentsResponseBodyPart#getNextPart} when the current {@link AvailableFragmentsResponseBodyPart}
+     * becomes inactive because of a transmission pause.
+     */
+    @Nullable // if the first fragment hasn't been sent yet
+    private SubscribableListener<ChunkedRestResponseBodyPart> nextAvailableFragmentListener;
+
+    /**
+     * A resource to be released when the transmission of the current fragment is complete. Note that we may complete the transmission of
+     * multiple fragments at the same time, if they are all processed by one call to {@link AvailableFragmentsResponseBodyPart#encodeChunk}
+     * and transmitted together.
+     */
+    @Nullable // if not currently sending a fragment
+    private Releasable currentFragmentReleasable;
+
+    /**
+     * @param restChannel  The {@link RestChannel} on which to send the response.
+     * @param params       The {@link ToXContent.Params} to control the serialization.
+     * @param onCompletion A resource which is released when the transmission is complete.
+     */
+    public StreamingXContentResponse(RestChannel restChannel, ToXContent.Params params, Releasable onCompletion) throws IOException {
+        this.restChannel = restChannel;
+        this.params = params;
+        this.onCompletion = onCompletion;
+        this.xContentBuilder = restChannel.newBuilder(
+            restChannel.request().getXContentType(),
+            null,
+            true,
+            Streams.noCloseStream(new OutputStream() {
+                @Override
+                public void write(int b) throws IOException {
+                    assert targetStream != null;
+                    targetStream.write(b);
+                }
+
+                @Override
+                public void write(byte[] b, int off, int len) throws IOException {
+                    assert targetStream != null;
+                    targetStream.write(b, off, len);
+                }
+            })
+        );
+    }
+
+    /**
+     * Close this {@link StreamingXContentResponse}, indicating that there will be no more fragments to send.
+     */
+    @Override
+    public void close() {
+        writeFragment(p -> NO_MORE_FRAGMENTS, () -> {
+            if (isRestResponseFinished.compareAndSet(false, true)) {
+                queueRefs.decRef();
+            }
+        });
+    }
+
+    private Iterator<? extends ToXContent> getChunksIterator(StreamingFragment fragment) {
+        return xContentBuilder.getRestApiVersion() == RestApiVersion.V_7
+            ? fragment.fragment().toXContentChunkedV7(params)
+            : fragment.fragment().toXContentChunked(params);
+    }
+
+    /**
+     * Enqueue the given fragment for transmission.
+     * @param fragment   The fragment to send.
+     * @param releasable A resource which is released when the fragment has been completely processed, i.e. when
+     *                   <ul>
+     *                   <li>it has been fully sent, or</li>
+     *                   <li>the overall response was cancelled before completion and all resources related to the partial transmission of
+     *                   this fragment have been released.</li>
+     *                   </ul>
+     */
+    public void writeFragment(ChunkedToXContent fragment, Releasable releasable) {
+        if (tryAcquireQueueRef()) {
+            try {
+                fragmentQueue.add(new StreamingFragment(fragment, releasable));
+                if (queueLength.getAndIncrement() == 0) {
+                    // There is no active AvailableChunksZipResponseBodyPart, but there is now an entry in the queue, so we must create a
+                    // AvailableChunksZipResponseBodyPart to process it (along with any other entries that are concurrently added to the
+                    // queue). It's safe to mutate releasable and continuationListener here because they are only otherwise accessed by an
+                    // active AvailableChunksZipResponseBodyPart (which does not exist) or when all queueRefs have been released (which they
+                    // have not here).
+                    final var nextFragment = fragmentQueue.poll();
+                    assert nextFragment != null;
+                    final var availableFragments = new AvailableFragmentsResponseBodyPart(getChunksIterator(nextFragment));
+                    assert currentFragmentReleasable == null;
+                    currentFragmentReleasable = nextFragment.releasable();
+                    final var currentAvailableFragmentListener = nextAvailableFragmentListener;
+                    nextAvailableFragmentListener = new SubscribableListener<>();
+                    if (currentAvailableFragmentListener == null) {
+                        // We are not resuming after a pause, this is the first fragment to be sent, so we start the response transmission.
+                        restChannel.sendResponse(RestResponse.chunked(RestStatus.OK, availableFragments, this::restResponseFinished));
+                    } else {
+                        // We are resuming transmission after a pause, so just carry on sending the response body.
+                        assert currentAvailableFragmentListener.isDone() == false;
+                        currentAvailableFragmentListener.onResponse(availableFragments);
+                    }
+                }
+            } finally {
+                queueRefs.decRef();
+            }
+        } else {
+            Releasables.closeExpectNoException(releasable);
+        }
+    }
+
+    /**
+     * A fragment which is ready for transmission, to be stored in {@link #fragmentQueue}.
+     *
+     * @param fragment      The fragment of XContent to send.
+     * @param releasable    A resource to release when this fragment has been fully transmitted, or is no longer required because the
+     *                      transmission was cancelled.
+     */
+    private record StreamingFragment(ChunkedToXContent fragment, Releasable releasable) {}
+
+    /**
+     * Queue of fragments that are ready for transmission.
+     */
+    private final Queue<StreamingFragment> fragmentQueue = new LinkedBlockingQueue<>();
+
+    /**
+     * Upper bound on the number of fragments in the queue, atomically modified to ensure there's only one thread processing the queue
+     * at once.
+     */
+    private final AtomicInteger queueLength = new AtomicInteger();
+
+    /**
+     * Ref-counting for access to the queue, to avoid clearing the queue on abort concurrently with a fragment being sent.
+     */
+    private final RefCounted queueRefs = AbstractRefCounted.of(this::drainQueue);
+
+    /**
+     * Flag to indicate if the request has been aborted, at which point we should stop enqueueing more fragments and promptly clean up the
+     * ones being sent. It's safe to ignore this, but without it in theory a constant stream of calls to {@link #writeFragment} could
+     * prevent {@link #drainQueue} from running for arbitrarily long.
+     */
+    private final AtomicBoolean isRestResponseFinished = new AtomicBoolean();
+
+    private boolean tryAcquireQueueRef() {
+        return isRestResponseFinished.get() == false && queueRefs.tryIncRef();
+    }
+
+    private void restResponseFinished() {
+        assert Transports.assertTransportThread();
+        if (isRestResponseFinished.compareAndSet(false, true)) {
+            queueRefs.decRef();
+        }
+    }
+
+    private void drainQueue() {
+        assert isRestResponseFinished.get();
+        assert queueRefs.hasReferences() == false;
+        final var taskCount = queueLength.get() + 2 /* currentFragmentReleasable and onCompletion */ ;
+        final var releasables = new ArrayList<Releasable>(taskCount);
+        try {
+            releasables.add(currentFragmentReleasable);
+            currentFragmentReleasable = null;
+            StreamingFragment fragment;
+            while ((fragment = fragmentQueue.poll()) != null) {
+                releasables.add(fragment.releasable());
+            }
+            assert fragmentQueue.isEmpty() : fragmentQueue.size(); // no concurrent adds
+            assert releasables.size() == taskCount - 1 || releasables.size() == taskCount - 2 : taskCount + " vs " + releasables.size();
+        } finally {
+            releasables.add(onCompletion);
+            Releasables.closeExpectNoException(Releasables.wrap(releasables));
+        }
+    }
+
+    /**
+     * A {@link ChunkedRestResponseBodyPart} which will yield all currently-available fragments by consuming from {@link #fragmentQueue}.
+     * There is only ever at most one active instance of this class at any time, in the sense that one such instance becoming inactive
+     * <i>happens-before</i> the creation of the next instance. One of these parts may send chunks for more than one fragment.
+     */
+    private final class AvailableFragmentsResponseBodyPart implements ChunkedRestResponseBodyPart {
+
+        /**
+         * An iterator over the chunks of the fragment currently being transmitted.
+         */
+        private Iterator<? extends ToXContent> fragmentChunksIterator;
+
+        /**
+         * True when we have run out of chunks ready for immediate transmission, so the response is paused, but we expect to send more data
+         * later.
+         */
+        private boolean isResponsePaused;
+
+        /**
+         * True when we have sent the last chunk of the last fragment, or the response was cancelled.
+         */
+        private boolean isResponseComplete;
+
+        /**
+         * A listener which is created when there are no more available fragments, so transmission is paused, subscribed to in
+         * {@link #getNextPart}, and then completed with the next body part (sequence of fragments, i.e. a new (unique) active
+         * {@link AvailableFragmentsResponseBodyPart}).
+         */
+        private SubscribableListener<ChunkedRestResponseBodyPart> getNextPartListener;
+
+        /**
+         * A cache for an empty list to be used to collect the {@code Releasable} instances to be released when the next chunk has been
+         * fully transmitted. It's a list because a call to {@link #encodeChunk} may yield a chunk that completes several fragments, each of
+         * which has its own resources to release. We cache this value across chunks because most chunks won't release anything, so we can
+         * keep the empty list around for later to save on allocations.
+         */
+        private ArrayList<Releasable> nextReleasablesCache = new ArrayList<>();
+
+        AvailableFragmentsResponseBodyPart(Iterator<? extends ToXContent> fragmentChunksIterator) {
+            this.fragmentChunksIterator = fragmentChunksIterator;
+        }
+
+        /**
+         * @return whether this part of the response is complete
+         */
+        @Override
+        public boolean isPartComplete() {
+            return isResponsePaused || isResponseComplete;
+        }
+
+        @Override
+        public boolean isLastPart() {
+            return isResponseComplete;
+        }
+
+        @Override
+        public void getNextPart(ActionListener<ChunkedRestResponseBodyPart> listener) {
+            assert getNextPartListener != null;
+            getNextPartListener.addListener(listener);
+        }
+
+        /**
+         * Transfer {@link #currentFragmentReleasable} into the supplied collection (i.e. add it to {@code releasables} and then clear
+         * {@link #currentFragmentReleasable}). Called when the last chunk of the current fragment is serialized, so that we
+         * can start serializing chunks of the next fragment straight away whilst delaying the release of the current fragment's resources
+         * until the transmission of the chunk that is currently under construction.
+         */
+        private void transferCurrentFragmentReleasable(ArrayList<Releasable> releasables) {
+            assert queueRefs.hasReferences();
+
+            if (currentFragmentReleasable == null) {
+                return;
+            }
+
+            if (releasables == nextReleasablesCache) {
+                // adding the first value, so we must line up a new cached value for the next caller
+                nextReleasablesCache = new ArrayList<>();
+            }
+
+            releasables.add(currentFragmentReleasable);
+            currentFragmentReleasable = null;
+        }
+
+        @Override
+        public ReleasableBytesReference encodeChunk(int sizeHint, Recycler<BytesRef> recycler) throws IOException {
+            assert Transports.isTransportThread(Thread.currentThread());
+
+            final ArrayList<Releasable> releasables = nextReleasablesCache;
+            assert releasables.isEmpty();
+            try {
+                if (tryAcquireQueueRef()) {
+                    try {
+                        assert queueLength.get() > 0;
+                        // This is the current unique active AvailableFragmentsResponseBodyPart (i.e. queueLength is strictly positive and
+                        // we hold a queueRef), so any concurrent calls to writeFragment() at this point will just add to the queue and
+                        // won't spawn a new AvailableFragmentsResponseBodyPart or mutate any fields.
+
+                        final RecyclerBytesStreamOutput chunkStream = new RecyclerBytesStreamOutput(recycler);
+                        assert targetStream == null;
+                        targetStream = chunkStream;
+
+                        do {
+                            if (fragmentChunksIterator.hasNext()) {
+                                fragmentChunksIterator.next().toXContent(xContentBuilder, params);
+                            } else {
+                                completeCurrentFragment(releasables);
+                            }
+                        } while (isResponseComplete == false && isResponsePaused == false && chunkStream.size() < sizeHint);
+
+                        assert (releasables == nextReleasablesCache) == releasables.isEmpty();
+                        assert nextReleasablesCache.isEmpty();
+
+                        final Releasable chunkStreamReleasable = () -> Releasables.closeExpectNoException(chunkStream);
+                        final var result = new ReleasableBytesReference(
+                            chunkStream.bytes(),
+                            releasables.isEmpty()
+                                ? chunkStreamReleasable
+                                : Releasables.wrap(Iterators.concat(Iterators.single(chunkStreamReleasable), releasables.iterator()))
+                        );
+                        targetStream = null;
+                        return result;
+                    } finally {
+                        queueRefs.decRef();
+                    }
+                } else {
+                    // request aborted, nothing more to send (queue is being cleared by queueRefs#closeInternal)
+                    isResponseComplete = true;
+                    return new ReleasableBytesReference(BytesArray.EMPTY, () -> {});
+                }
+            } catch (Exception e) {
+                logger.error("failure encoding chunk", e);
+                throw e;
+            } finally {
+                if (targetStream != null) {
+                    assert false : "failure encoding chunk";
+                    IOUtils.closeWhileHandlingException(targetStream, Releasables.wrap(releasables));
+                    targetStream = null;
+                }
+            }
+        }
+
+        private void completeCurrentFragment(ArrayList<Releasable> releasables) throws IOException {
+            transferCurrentFragmentReleasable(releasables);
+            final var localNextAvailableFragmentListener = nextAvailableFragmentListener; // read before queue len decr
+            final var newQueueLength = queueLength.decrementAndGet();
+            if (fragmentChunksIterator == NO_MORE_FRAGMENTS) {
+                // The current fragment is the last-fragment sentinel, so we stop processing the queue completely. Note
+                // that closing the XContentBuilder here ensures that the response is well-formed - it's up to the
+                // caller to ensure this, even if errors occur.
+                xContentBuilder.close();
+                isResponseComplete = true;
+            } else if (newQueueLength == 0) {
+                // The current fragment is complete, but the next fragment isn't available yet, so we pause
+                // transmission. This means we are no longer an active AvailableFragmentsResponseBodyPart, so any
+                // concurrent calls to writeFragment() at this point will now spawn a new
+                // AvailableFragmentsResponseBodyPart to take our place.
+                xContentBuilder.flush();
+                isResponsePaused = true;
+                assert getNextPartListener == null;
+                assert localNextAvailableFragmentListener != null;
+                // Calling our getNextPart() will eventually yield the next fragment supplied to writeFragment():
+                getNextPartListener = localNextAvailableFragmentListener;
+            } else {
+                // The current fragment is complete, and the next fragment is already available, so we start sending its
+                // chunks too. This means we're still the unique active AvailableFragmentsResponseBodyPart. We re-use
+                // this AvailableFragmentsResponseBodyPart instance rather than creating a new one to avoid unnecessary
+                // allocations.
+
+                final var nextFragment = fragmentQueue.poll();
+                assert nextFragment != null;
+                currentFragmentReleasable = nextFragment.releasable();
+                fragmentChunksIterator = getChunksIterator(nextFragment);
+            }
+        }
+
+        @Override
+        public String getResponseContentTypeString() {
+            return xContentBuilder.getResponseContentTypeString();
+        }
+    }
+
+    /**
+     * Sentinel fragment indicating the end of the response.
+     */
+    private static final Iterator<? extends ToXContent> NO_MORE_FRAGMENTS = new Iterator<>() {
+        @Override
+        public boolean hasNext() {
+            return false;
+        }
+
+        @Override
+        public ToXContent next() {
+            assert false : "not called";
+            return ToXContent.EMPTY;
+        }
+    };
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java
index 5140fe8b99136..68bb85756b8dc 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java
@@ -9,11 +9,13 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
+import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
@@ -41,12 +43,18 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-        final ClusterSearchShardsRequest clusterSearchShardsRequest = new ClusterSearchShardsRequest(indices);
+        final ClusterSearchShardsRequest clusterSearchShardsRequest = new ClusterSearchShardsRequest(
+            RestUtils.getMasterNodeTimeout(request),
+            Strings.splitStringByCommaToArray(request.param("index"))
+        );
         clusterSearchShardsRequest.local(request.paramAsBoolean("local", clusterSearchShardsRequest.local()));
         clusterSearchShardsRequest.routing(request.param("routing"));
         clusterSearchShardsRequest.preference(request.param("preference"));
         clusterSearchShardsRequest.indicesOptions(IndicesOptions.fromRequest(request, clusterSearchShardsRequest.indicesOptions()));
-        return channel -> client.admin().cluster().searchShards(clusterSearchShardsRequest, new RestToXContentListener<>(channel));
+        return channel -> client.execute(
+            TransportClusterSearchShardsAction.TYPE,
+            clusterSearchShardsRequest,
+            new RestToXContentListener<>(channel)
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
index b3fb5ccfdfddf..cb9be980f35ce 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
@@ -117,7 +117,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             new RestChunkedToXContentListener<RestClusterStateResponse>(
                 channel,
                 new ToXContent.DelegatingMapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API), request)
-            ).map(response -> new RestClusterStateResponse(clusterStateRequest, response, threadPool::relativeTimeInMillis))
+            ).map(response -> new RestClusterStateResponse(clusterStateRequest, response, threadPool.relativeTimeInMillisSupplier()))
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java
index a245d66d658b4..36de14cec2aa2 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -37,11 +38,15 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        String id = request.param("id");
-        DeleteStoredScriptRequest deleteStoredScriptRequest = new DeleteStoredScriptRequest(id);
-        deleteStoredScriptRequest.ackTimeout(getAckTimeout(request));
-        deleteStoredScriptRequest.masterNodeTimeout(getMasterNodeTimeout(request));
-
-        return channel -> client.admin().cluster().deleteStoredScript(deleteStoredScriptRequest, new RestToXContentListener<>(channel));
+        final var deleteStoredScriptRequest = new DeleteStoredScriptRequest(
+            getMasterNodeTimeout(request),
+            getAckTimeout(request),
+            request.param("id")
+        );
+        return channel -> client.execute(
+            TransportDeleteStoredScriptAction.TYPE,
+            deleteStoredScriptRequest,
+            new RestToXContentListener<>(channel)
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java
index f827b07ebe96c..2fe75a55fcca4 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.rest.action.admin.cluster;
 
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
 import org.elasticsearch.client.internal.node.NodeClient;
@@ -37,11 +38,11 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException {
-        String id = request.param("id");
-        GetStoredScriptRequest getRequest = new GetStoredScriptRequest(id);
-        getRequest.masterNodeTimeout(getMasterNodeTimeout(request));
-        return channel -> client.admin()
-            .cluster()
-            .getStoredScript(getRequest, new RestToXContentListener<>(channel, GetStoredScriptResponse::status));
+        final var getRequest = new GetStoredScriptRequest(getMasterNodeTimeout(request), request.param("id"));
+        return channel -> client.execute(
+            GetStoredScriptAction.INSTANCE,
+            getRequest,
+            new RestToXContentListener<>(channel, GetStoredScriptResponse::status)
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java
index 2bf389c2c2849..3ec2561cd5a65 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java
@@ -30,6 +30,7 @@
 public class RestNodesCapabilitiesAction extends BaseRestHandler {
 
     public static final NodeFeature CAPABILITIES_ACTION = new NodeFeature("rest.capabilities_action");
+    public static final NodeFeature LOCAL_ONLY_CAPABILITIES = new NodeFeature("rest.local_only_capabilities");
 
     @Override
     public List<Route> routes() {
@@ -38,7 +39,7 @@ public List<Route> routes() {
 
     @Override
     public Set<String> supportedQueryParameters() {
-        return Set.of("timeout", "method", "path", "parameters", "capabilities");
+        return Set.of("timeout", "method", "path", "parameters", "capabilities", "local_only");
     }
 
     @Override
@@ -48,7 +49,11 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        NodesCapabilitiesRequest r = new NodesCapabilitiesRequest().timeout(getTimeout(request))
+        NodesCapabilitiesRequest requestNodes = request.paramAsBoolean("local_only", false)
+            ? new NodesCapabilitiesRequest(client.getLocalNodeId())
+            : new NodesCapabilitiesRequest();
+
+        NodesCapabilitiesRequest r = requestNodes.timeout(getTimeout(request))
             .method(RestRequest.Method.valueOf(request.param("method", "GET")))
             .path(URLDecoder.decode(request.param("path"), StandardCharsets.UTF_8))
             .parameters(request.paramAsStringArray("parameters", Strings.EMPTY_ARRAY))
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
index 9cd793a89a57c..caceb86a36b9f 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
@@ -8,15 +8,14 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.script.StoredScriptSource;
-import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.List;
@@ -46,15 +45,17 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        String id = request.param("id");
-        String context = request.param("context");
-        BytesReference content = request.requiredContent();
-        XContentType xContentType = request.getXContentType();
-        StoredScriptSource source = StoredScriptSource.parse(content, xContentType);
-
-        PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, context, content, request.getXContentType(), source);
-        putRequest.masterNodeTimeout(getMasterNodeTimeout(request));
-        putRequest.ackTimeout(getAckTimeout(request));
-        return channel -> client.admin().cluster().putStoredScript(putRequest, new RestToXContentListener<>(channel));
+        final var content = request.requiredContent();
+        final var xContentType = request.getXContentType();
+        final var putRequest = new PutStoredScriptRequest(
+            getMasterNodeTimeout(request),
+            getAckTimeout(request),
+            request.param("id"),
+            request.param("context"),
+            content,
+            request.getXContentType(),
+            StoredScriptSource.parse(content, xContentType)
+        );
+        return channel -> client.execute(TransportPutStoredScriptAction.TYPE, putRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java
index 849e2d68cb2dc..876edad49a7dc 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java
@@ -76,7 +76,7 @@ protected RestChannelConsumer doCatRequest(final RestRequest request, NodeClient
                 getComposableTemplatesRequest,
                 getComposableTemplatesStep.delegateResponse((l, e) -> {
                     if (ExceptionsHelper.unwrapCause(e) instanceof ResourceNotFoundException) {
-                        l.onResponse(new GetComposableIndexTemplateAction.Response(Map.of(), null));
+                        l.onResponse(new GetComposableIndexTemplateAction.Response(Map.of()));
                     } else {
                         l.onFailure(e);
                     }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
index 0bb97b1f51ff5..d213d4410c07c 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
@@ -101,7 +101,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
     }
 
     @Override
-    public boolean supportsContentStream() {
+    public boolean supportsBulkContent() {
         return true;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index 0a7a4a9701a90..d45cbaecb5a62 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -209,7 +209,7 @@ public static void parseMultiLineRequest(
     }
 
     @Override
-    public boolean supportsContentStream() {
+    public boolean supportsBulkContent() {
         return true;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/script/ScoreScript.java b/server/src/main/java/org/elasticsearch/script/ScoreScript.java
index 503bd11fb434a..61d225c069c68 100644
--- a/server/src/main/java/org/elasticsearch/script/ScoreScript.java
+++ b/server/src/main/java/org/elasticsearch/script/ScoreScript.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.script;
 
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Scorable;
 import org.elasticsearch.common.logging.DeprecationCategory;
@@ -26,7 +27,6 @@
  * A script used for adjusting the score on a per document basis.
  */
 public abstract class ScoreScript extends DocBasedScript {
-
     /** A helper to take in an explanation from a script and turn it into an {@link org.apache.lucene.search.Explanation}  */
     public static class ExplanationHolder {
         private String description;
@@ -82,6 +82,8 @@ public Explanation get(double score, Explanation subQueryExplanation) {
     private int shardId = -1;
     private String indexName = null;
 
+    private ScriptTermStats termStats = null;
+
     public ScoreScript(Map<String, Object> params, SearchLookup searchLookup, DocReader docReader) {
         // searchLookup parameter is ignored but part of the ScriptFactory contract. It is part of that contract because it's required
         // for expressions. Expressions should eventually be transitioned to using DocReader.
@@ -90,13 +92,13 @@ public ScoreScript(Map<String, Object> params, SearchLookup searchLookup, DocRea
         if (docReader == null) {
             assert params == null;
             this.params = null;
-            ;
             this.docBase = 0;
         } else {
             params = new HashMap<>(params);
             params.putAll(docReader.docAsMap());
             this.params = new DynamicMap(params, PARAMS_FUNCTIONS);
-            this.docBase = ((DocValuesDocReader) docReader).getLeafReaderContext().docBase;
+            LeafReaderContext leafReaderContext = ((DocValuesDocReader) docReader).getLeafReaderContext();
+            this.docBase = leafReaderContext.docBase;
         }
     }
 
@@ -189,14 +191,33 @@ public void _setIndexName(String indexName) {
         this.indexName = indexName;
     }
 
+    /**
+     * Starting a name with underscore, so that the user cannot access this function directly through a script.
+     */
+    public void _setTermStats(ScriptTermStats termStats) {
+        this.termStats = termStats;
+    }
+
+    /**
+     * Accessed as _termStats in the painless script.
+     */
+    public ScriptTermStats get_termStats() {
+        assert termStats != null : "termStats is not available";
+        return termStats;
+    }
+
     /** A factory to construct {@link ScoreScript} instances. */
     public interface LeafFactory {
-
         /**
          * Return {@code true} if the script needs {@code _score} calculated, or {@code false} otherwise.
          */
         boolean needs_score();
 
+        /**
+         * Return {@code true} if the script needs {@code _termStats} calculated, or {@code false} otherwise.
+         */
+        boolean needs_termStats();
+
         ScoreScript newInstance(DocReader reader) throws IOException;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java b/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java
index d4d78bf08844b..2522ecee83223 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java
@@ -16,6 +16,6 @@
 public final class ScriptFeatures implements FeatureSpecification {
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(VectorScoreScriptUtils.HAMMING_DISTANCE_FUNCTION);
+        return Set.of(VectorScoreScriptUtils.HAMMING_DISTANCE_FUNCTION, ScriptTermStats.TERM_STAT_FEATURE);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java
new file mode 100644
index 0000000000000..0863e649487b9
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java
@@ -0,0 +1,234 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.TermStates;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.IndexSearcher;
+import org.elasticsearch.common.util.CachedSupplier;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.search.internal.ContextIndexSearcher;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Set;
+import java.util.function.IntSupplier;
+import java.util.function.Supplier;
+
+/**
+ * Access the term statistics of the children query of a script_score query.
+ */
+public class ScriptTermStats {
+
+    public static final NodeFeature TERM_STAT_FEATURE = new NodeFeature("script.term_stats");
+
+    private final IntSupplier docIdSupplier;
+    private final Term[] terms;
+    private final IndexSearcher searcher;
+    private final LeafReaderContext leafReaderContext;
+    private final StatsSummary statsSummary = new StatsSummary();
+    private final Supplier<TermStates[]> termContextsSupplier;
+    private final Supplier<PostingsEnum[]> postingsSupplier;
+    private final Supplier<StatsSummary> docFreqSupplier;
+    private final Supplier<StatsSummary> totalTermFreqSupplier;
+
+    public ScriptTermStats(IndexSearcher searcher, LeafReaderContext leafReaderContext, IntSupplier docIdSupplier, Set<Term> terms) {
+        this.searcher = searcher;
+        this.leafReaderContext = leafReaderContext;
+        this.docIdSupplier = docIdSupplier;
+        this.terms = terms.toArray(new Term[0]);
+        this.termContextsSupplier = CachedSupplier.wrap(this::loadTermContexts);
+        this.postingsSupplier = CachedSupplier.wrap(this::loadPostings);
+        this.docFreqSupplier = CachedSupplier.wrap(this::loadDocFreq);
+        this.totalTermFreqSupplier = CachedSupplier.wrap(this::loadTotalTermFreq);
+    }
+
+    /**
+     * Number of unique terms in the query.
+     *
+     * @return the number of unique terms
+     */
+    public int uniqueTermsCount() {
+        return terms.length;
+    }
+
+    /**
+     * Number of terms that are matched im the query.
+     *
+     * @return the number of matched terms
+     */
+    public int matchedTermsCount() {
+        final int docId = docIdSupplier.getAsInt();
+        int matchedTerms = 0;
+
+        try {
+            for (PostingsEnum postingsEnum : postingsSupplier.get()) {
+                if (postingsEnum != null && postingsEnum.advance(docId) == docId && postingsEnum.freq() > 0) {
+                    matchedTerms++;
+                }
+            }
+            return matchedTerms;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    /**
+     * Collect docFreq (number of documents a term occurs in) for the terms of the query and returns statistics for them.
+     *
+     * @return statistics on docFreq for the terms of the query.
+     */
+    public StatsSummary docFreq() {
+        return docFreqSupplier.get();
+    }
+
+    private StatsSummary loadDocFreq() {
+        StatsSummary docFreqStats = new StatsSummary();
+        TermStates[] termContexts = termContextsSupplier.get();
+
+        try {
+            for (int i = 0; i < termContexts.length; i++) {
+                if (searcher instanceof ContextIndexSearcher contextIndexSearcher) {
+                    docFreqStats.accept(contextIndexSearcher.docFreq(terms[i], termContexts[i].docFreq()));
+                } else {
+                    docFreqStats.accept(termContexts[i].docFreq());
+                }
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
+        return docFreqStats;
+    }
+
+    /**
+     * Collect totalTermFreq (total number of occurrence of a term in the index) for the terms of the query and returns statistics for them.
+     *
+     * @return statistics on totalTermFreq for the terms of the query.
+     */
+    public StatsSummary totalTermFreq() {
+        return this.totalTermFreqSupplier.get();
+    }
+
+    private StatsSummary loadTotalTermFreq() {
+        StatsSummary totalTermFreqStats = new StatsSummary();
+        TermStates[] termContexts = termContextsSupplier.get();
+
+        try {
+            for (int i = 0; i < termContexts.length; i++) {
+                if (searcher instanceof ContextIndexSearcher contextIndexSearcher) {
+                    totalTermFreqStats.accept(contextIndexSearcher.totalTermFreq(terms[i], termContexts[i].totalTermFreq()));
+                } else {
+                    totalTermFreqStats.accept(termContexts[i].totalTermFreq());
+                }
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
+        return totalTermFreqStats;
+    }
+
+    /**
+     * Collect totalFreq (number of occurrence of a term in the current doc for the terms of the query and returns statistics for them.
+     *
+     * @return statistics on totalTermFreq for the terms of the query in the current dac
+     */
+    public StatsSummary termFreq() {
+        statsSummary.reset();
+        final int docId = docIdSupplier.getAsInt();
+
+        try {
+            for (PostingsEnum postingsEnum : postingsSupplier.get()) {
+                if (postingsEnum == null || postingsEnum.advance(docId) != docId) {
+                    statsSummary.accept(0);
+                } else {
+                    statsSummary.accept(postingsEnum.freq());
+                }
+            }
+
+            return statsSummary;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    /**
+     * Collect termPositions (positions of a term in the current document) for the terms of the query and returns statistics for them.
+     *
+     * @return statistics on termPositions for the terms of the query in the current dac
+     */
+    public StatsSummary termPositions() {
+        try {
+            statsSummary.reset();
+            int docId = docIdSupplier.getAsInt();
+
+            for (PostingsEnum postingsEnum : postingsSupplier.get()) {
+                if (postingsEnum == null || postingsEnum.advance(docId) != docId) {
+                    continue;
+                }
+                for (int i = 0; i < postingsEnum.freq(); i++) {
+                    statsSummary.accept(postingsEnum.nextPosition() + 1);
+                }
+            }
+
+            return statsSummary;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private TermStates[] loadTermContexts() {
+        try {
+            TermStates[] termContexts = new TermStates[terms.length];
+
+            for (int i = 0; i < terms.length; i++) {
+                termContexts[i] = TermStates.build(searcher, terms[i], true);
+            }
+
+            return termContexts;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private PostingsEnum[] loadPostings() {
+        try {
+            PostingsEnum[] postings = new PostingsEnum[terms.length];
+            TermStates[] contexts = termContextsSupplier.get();
+
+            for (int i = 0; i < terms.length; i++) {
+                TermStates termStates = contexts[i];
+                if (termStates.docFreq() == 0) {
+                    postings[i] = null;
+                    continue;
+                }
+
+                TermState state = termStates.get(leafReaderContext);
+                if (state == null) {
+                    postings[i] = null;
+                    continue;
+                }
+
+                TermsEnum termsEnum = leafReaderContext.reader().terms(terms[i].field()).iterator();
+                termsEnum.seekExact(terms[i].bytes(), state);
+
+                postings[i] = termsEnum.postings(null, PostingsEnum.ALL);
+            }
+
+            return postings;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/script/StatsSummary.java b/server/src/main/java/org/elasticsearch/script/StatsSummary.java
new file mode 100644
index 0000000000000..d59a7af7e1fa9
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/script/StatsSummary.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.Strings;
+
+import java.util.Objects;
+import java.util.function.DoubleConsumer;
+
+/**
+ * The {@link StatsSummary} class accumulates statistical data for a sequence of double values.
+ *
+ * <p>This class provides statistics such as count, sum, minimum, maximum, and arithmetic mean
+ * of the recorded values.
+ */
+public class StatsSummary implements DoubleConsumer {
+
+    private long count = 0;
+    private double sum = 0d;
+    private Double min;
+    private Double max;
+
+    public StatsSummary() {}
+
+    StatsSummary(long count, double sum, double min, double max) {
+        this.count = count;
+        this.sum = sum;
+        this.min = min;
+        this.max = max;
+    }
+
+    @Override
+    public void accept(double value) {
+        count++;
+        sum += value;
+        min = min == null ? value : (value < min ? value : min);
+        max = max == null ? value : (value > max ? value : max);
+    }
+
+    /**
+     * Returns the min for recorded value.
+     */
+    public double getMin() {
+        return min == null ? 0.0 : min;
+    }
+
+    /**
+     * Returns the max for recorded values.
+     */
+    public double getMax() {
+        return max == null ? 0.0 : max;
+    }
+
+    /**
+     * Returns the arithmetic mean for recorded values.
+     */
+    public double getAverage() {
+        return count == 0.0 ? 0.0 : sum / count;
+    }
+
+    /**
+     * Returns the sum of all recorded values.
+     */
+    public double getSum() {
+        return sum;
+    }
+
+    /**
+     * Returns the number of recorded values.
+     */
+    public long getCount() {
+        return count;
+    }
+
+    /**
+     * Resets the accumulator, clearing all accumulated statistics.
+     * After calling this method, the accumulator will be in its initial state.
+     */
+    public void reset() {
+        count = 0;
+        sum = 0d;
+        min = null;
+        max = null;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(count, sum, min, max);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        StatsSummary other = (StatsSummary) obj;
+
+        return Objects.equals(count, other.count)
+            && Objects.equals(sum, other.sum)
+            && Objects.equals(min, other.min)
+            && Objects.equals(max, other.max);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.format(
+            "%s{count=%d, sum=%f, min=%f, average=%f, max=%f}",
+            this.getClass().getSimpleName(),
+            getCount(),
+            getSum(),
+            getMin(),
+            getAverage(),
+            getMax()
+        );
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index bac5fe8c1d1ac..33c64f3eb6350 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -237,6 +237,8 @@
 import org.elasticsearch.search.retriever.RetrieverBuilder;
 import org.elasticsearch.search.retriever.RetrieverParserContext;
 import org.elasticsearch.search.retriever.StandardRetrieverBuilder;
+import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder;
+import org.elasticsearch.search.retriever.rankdoc.RankDocsSortBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
@@ -851,10 +853,12 @@ private void registerRescorer(RescorerSpec<?> spec) {
     }
 
     private void registerRankers() {
+        namedWriteables.add(new NamedWriteableRegistry.Entry(RankDoc.class, RankDoc.NAME, RankDoc::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(RankDoc.class, RankFeatureDoc.NAME, RankFeatureDoc::new));
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(RankShardResult.class, RankFeatureShardResult.NAME, RankFeatureShardResult::new)
         );
+        namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, RankDocsQueryBuilder.NAME, RankDocsQueryBuilder::new));
     }
 
     private void registerSorts() {
@@ -862,6 +866,7 @@ private void registerSorts() {
         namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScoreSortBuilder.NAME, ScoreSortBuilder::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScriptSortBuilder.NAME, ScriptSortBuilder::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, FieldSortBuilder.NAME, FieldSortBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, RankDocsSortBuilder.NAME, RankDocsSortBuilder::new));
     }
 
     private static <T> void registerFromPlugin(List<SearchPlugin> plugins, Function<SearchPlugin, List<T>> producer, Consumer<T> consumer) {
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 979a59b4d0b94..67d5d6337d77c 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -84,6 +84,7 @@
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.AggregationContext.ProductionAggregationContext;
+import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.builder.SubSearchSourceBuilder;
 import org.elasticsearch.search.collapse.CollapseContext;
@@ -1130,7 +1131,7 @@ private DefaultSearchContext createSearchContext(
                 reader,
                 request,
                 shardTarget,
-                threadPool::relativeTimeInMillis,
+                threadPool.relativeTimeInMillisSupplier(),
                 timeout,
                 fetchPhase,
                 lowLevelCancellation,
@@ -1578,6 +1579,10 @@ public ResponseCollectorService getResponseCollectorService() {
         return this.responseCollectorService;
     }
 
+    public long getDefaultKeepAliveInMillis() {
+        return defaultKeepAlive;
+    }
+
     /**
      * Used to indicate which result object should be instantiated when creating a search context
      */
@@ -1821,8 +1826,8 @@ private void rewriteAndFetchShardRequest(IndexShard shard, ShardSearchRequest re
     /**
      * Returns a new {@link QueryRewriteContext} with the given {@code now} provider
      */
-    public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis, ResolvedIndices resolvedIndices) {
-        return indicesService.getRewriteContext(nowInMillis, resolvedIndices);
+    public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis, ResolvedIndices resolvedIndices, PointInTimeBuilder pit) {
+        return indicesService.getRewriteContext(nowInMillis, resolvedIndices, pit);
     }
 
     public CoordinatorRewriteContextProvider getCoordinatorRewriteContextProvider(LongSupplier nowInMillis) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
index 297bb81b27b25..4f234c33b13a6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
@@ -226,34 +226,11 @@ public static InternalAggregations topLevelReduce(List<InternalAggregations> agg
         }
         if (context.isFinalReduce()) {
             List<InternalAggregation> reducedInternalAggs = reduced.getInternalAggregations();
-            List<InternalAggregation> internalAggregations = null;
-            for (int i = 0; i < reducedInternalAggs.size(); i++) {
-                InternalAggregation agg = reducedInternalAggs.get(i);
-                InternalAggregation internalAggregation = agg.reducePipelines(
-                    agg,
-                    context,
-                    context.pipelineTreeRoot().subTree(agg.getName())
-                );
-                if (internalAggregation.equals(agg) == false) {
-                    if (internalAggregations == null) {
-                        internalAggregations = new ArrayList<>(reducedInternalAggs);
-                    }
-                    internalAggregations.set(i, internalAggregation);
-                }
-            }
+            reducedInternalAggs = reducedInternalAggs.stream()
+                .map(agg -> agg.reducePipelines(agg, context, context.pipelineTreeRoot().subTree(agg.getName())))
+                .collect(Collectors.toCollection(ArrayList::new));
 
-            var pipelineAggregators = context.pipelineTreeRoot().aggregators();
-            if (pipelineAggregators.isEmpty()) {
-                if (internalAggregations == null) {
-                    return reduced;
-                }
-                return from(internalAggregations);
-            }
-            if (internalAggregations != null) {
-                reducedInternalAggs = internalAggregations;
-            }
-            reducedInternalAggs = new ArrayList<>(reducedInternalAggs);
-            for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
+            for (PipelineAggregator pipelineAggregator : context.pipelineTreeRoot().aggregators()) {
                 SiblingPipelineAggregator sib = (SiblingPipelineAggregator) pipelineAggregator;
                 InternalAggregation newAgg = sib.doReduce(from(reducedInternalAggs), context);
                 reducedInternalAggs.add(newAgg);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
index e046b5fc9244c..de19c26daff92 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
@@ -207,31 +207,16 @@ public void forEachBucket(Consumer<InternalAggregations> consumer) {
     }
 
     private List<B> reducePipelineBuckets(AggregationReduceContext reduceContext, PipelineTree pipelineTree) {
-        List<B> reducedBuckets = null;
-        var buckets = getBuckets();
-        for (int bucketIndex = 0; bucketIndex < buckets.size(); bucketIndex++) {
-            B bucket = buckets.get(bucketIndex);
-            List<InternalAggregation> aggs = null;
-            int aggIndex = 0;
-            for (InternalAggregation agg : bucket.getAggregations()) {
+        List<B> reducedBuckets = new ArrayList<>();
+        for (B bucket : getBuckets()) {
+            List<InternalAggregation> aggs = new ArrayList<>();
+            for (Aggregation agg : bucket.getAggregations()) {
                 PipelineTree subTree = pipelineTree.subTree(agg.getName());
-                var reduced = agg.reducePipelines(agg, reduceContext, subTree);
-                if (reduced.equals(agg) == false) {
-                    if (aggs == null) {
-                        aggs = bucket.getAggregations().copyResults();
-                    }
-                    aggs.set(aggIndex, reduced);
-                }
-                aggIndex++;
-            }
-            if (aggs != null) {
-                if (reducedBuckets == null) {
-                    reducedBuckets = new ArrayList<>(buckets);
-                }
-                reducedBuckets.set(bucketIndex, createBucket(InternalAggregations.from(aggs), bucket));
+                aggs.add(((InternalAggregation) agg).reducePipelines((InternalAggregation) agg, reduceContext, subTree));
             }
+            reducedBuckets.add(createBucket(InternalAggregations.from(aggs), bucket));
         }
-        return reducedBuckets == null ? buckets : reducedBuckets;
+        return reducedBuckets;
     }
 
     public abstract static class InternalBucket implements Bucket, Writeable {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java
index 04028de5656ca..62b7a0747ca00 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java
@@ -84,9 +84,6 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I
             double key = roundKey * interval + offset;
             return new InternalHistogram.Bucket(key, docCount, keyed, formatter, subAggregationResults);
         }, (owningBucketOrd, buckets) -> {
-            if (buckets.isEmpty()) {
-                return buildEmptyAggregation();
-            }
             // the contract of the histogram aggregation is that shards must return buckets ordered by key in ascending order
             CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator());
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
index cb01aa5a31a9a..2c57bd4b38a04 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
@@ -340,9 +340,6 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I
         return buildAggregationsForVariableBuckets(owningBucketOrds, bucketOrds, (bucketValue, docCount, subAggregationResults) -> {
             return new InternalDateHistogram.Bucket(bucketValue, docCount, keyed, formatter, subAggregationResults);
         }, (owningBucketOrd, buckets) -> {
-            if (buckets.isEmpty()) {
-                return buildEmptyAggregation();
-            }
             // the contract of the histogram aggregation is that shards must return buckets ordered by key in ascending order
             CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator());
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
index 4cfa7f449cf57..c0c7f68802762 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
@@ -238,7 +238,7 @@ public InternalDateHistogram(StreamInput in) throws IOException {
         }
         buckets = in.readCollectionAsList(stream -> Bucket.readFrom(stream, keyed, format));
         // we changed the order format in 8.13 for partial reduce, therefore we need to order them to perform merge sort
-        if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.HISTOGRAM_AGGS_KEY_SORTED)) {
+        if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_14_0)) {
             // list is mutable by #readCollectionAsList contract
             buckets.sort(Comparator.comparingLong(b -> b.key));
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
index 2404de76fdd35..8d7a7f4b760cc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
@@ -221,7 +221,7 @@ public InternalHistogram(StreamInput in) throws IOException {
         keyed = in.readBoolean();
         buckets = in.readCollectionAsList(stream -> Bucket.readFrom(stream, keyed, format));
         // we changed the order format in 8.13 for partial reduce, therefore we need to order them to perform merge sort
-        if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.HISTOGRAM_AGGS_KEY_SORTED)) {
+        if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_14_0)) {
             // list is mutable by #readCollectionAsList contract
             buckets.sort(Comparator.comparingDouble(b -> b.key));
         }
@@ -259,17 +259,11 @@ BucketOrder getOrder() {
 
     @Override
     public InternalHistogram create(List<Bucket> buckets) {
-        if (this.buckets.equals(buckets)) {
-            return this;
-        }
         return new InternalHistogram(name, buckets, order, minDocCount, emptyBucketInfo, format, keyed, metadata);
     }
 
     @Override
     public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
-        if (prototype.aggregations.equals(aggregations)) {
-            return prototype;
-        }
         return new Bucket(prototype.key, prototype.docCount, prototype.keyed, prototype.format, aggregations);
     }
 
@@ -456,9 +450,6 @@ public InternalAggregation get() {
                         CollectionUtil.introSort(reducedBuckets, order.comparator());
                     }
                 }
-                if (reducedBuckets.equals(buckets)) {
-                    return InternalHistogram.this;
-                }
                 return new InternalHistogram(getName(), reducedBuckets, order, minDocCount, emptyBucketInfo, format, keyed, getMetadata());
             }
         };
@@ -504,9 +495,14 @@ public Number getKey(MultiBucketsAggregation.Bucket bucket) {
     }
 
     @Override
-    @SuppressWarnings({ "rawtypes", "unchecked" })
     public InternalAggregation createAggregation(List<MultiBucketsAggregation.Bucket> buckets) {
-        return new InternalHistogram(name, (List) buckets, order, minDocCount, emptyBucketInfo, format, keyed, getMetadata());
+        // convert buckets to the right type
+        List<Bucket> buckets2 = new ArrayList<>(buckets.size());
+        for (Object b : buckets) {
+            buckets2.add((Bucket) b);
+        }
+        buckets2 = Collections.unmodifiableList(buckets2);
+        return new InternalHistogram(name, buckets2, order, minDocCount, emptyBucketInfo, format, keyed, getMetadata());
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java
index 675b5d218c882..01a178e83db77 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogram.java
@@ -246,7 +246,7 @@ public InternalVariableWidthHistogram(StreamInput in) throws IOException {
         buckets = in.readCollectionAsList(stream -> Bucket.readFrom(stream, format));
         targetNumBuckets = in.readVInt();
         // we changed the order format in 8.13 for partial reduce, therefore we need to order them to perform merge sort
-        if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.HISTOGRAM_AGGS_KEY_SORTED)) {
+        if (in.getTransportVersion().between(TransportVersions.V_8_13_0, TransportVersions.V_8_14_0)) {
             // list is mutable by #readCollectionAsList contract
             buckets.sort(Comparator.comparingDouble(b -> b.centroid));
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
index 2b5bcd9931f6e..528c37de7a4a8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
@@ -72,8 +72,8 @@ private static Bucket createFromStream(StreamInput in, DocValueFormat format, bo
             String key = in.getTransportVersion().equals(TransportVersions.V_8_0_0) ? in.readString()
                 : in.getTransportVersion().onOrAfter(TransportVersions.V_7_17_1) ? in.readOptionalString()
                 : in.readString();
-            BytesRef from = in.readBoolean() ? in.readBytesRef() : null;
-            BytesRef to = in.readBoolean() ? in.readBytesRef() : null;
+            BytesRef from = in.readOptional(StreamInput::readBytesRef);
+            BytesRef to = in.readOptional(StreamInput::readBytesRef);
             long docCount = in.readLong();
             InternalAggregations aggregations = InternalAggregations.readFrom(in);
 
@@ -89,14 +89,8 @@ public void writeTo(StreamOutput out) throws IOException {
             } else {
                 out.writeString(key == null ? generateKey(from, to, format) : key);
             }
-            out.writeBoolean(from != null);
-            if (from != null) {
-                out.writeBytesRef(from);
-            }
-            out.writeBoolean(to != null);
-            if (to != null) {
-                out.writeBytesRef(to);
-            }
+            out.writeOptional(StreamOutput::writeBytesRef, from);
+            out.writeOptional(StreamOutput::writeBytesRef, to);
             out.writeLong(docCount);
             aggregations.writeTo(out);
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java
index 68a1a22369d2a..9fb94f77be020 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/InternalRandomSampler.java
@@ -55,7 +55,7 @@ public InternalRandomSampler(StreamInput in) throws IOException {
         super(in);
         this.seed = in.readInt();
         this.probability = in.readDouble();
-        if (in.getTransportVersion().onOrAfter(TransportVersions.RANDOM_AGG_SHARD_SEED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             this.shardSeed = in.readOptionalInt();
         } else {
             this.shardSeed = null;
@@ -67,7 +67,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         super.doWriteTo(out);
         out.writeInt(seed);
         out.writeDouble(probability);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.RANDOM_AGG_SHARD_SEED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeOptionalInt(shardSeed);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java
index 9bd9ab45b633a..14aef1187b99b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java
@@ -79,7 +79,7 @@ public RandomSamplerAggregationBuilder(StreamInput in) throws IOException {
         super(in);
         this.p = in.readDouble();
         this.seed = in.readInt();
-        if (in.getTransportVersion().onOrAfter(TransportVersions.RANDOM_AGG_SHARD_SEED)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             this.shardSeed = in.readOptionalInt();
         }
     }
@@ -99,7 +99,7 @@ protected RandomSamplerAggregationBuilder(
     protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeDouble(p);
         out.writeInt(seed);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.RANDOM_AGG_SHARD_SEED)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeOptionalInt(shardSeed);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
index bf923339c73f5..1c7ca7441ad80 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
@@ -37,8 +37,6 @@
 import java.util.Objects;
 import java.util.function.ToLongFunction;
 
-import static org.elasticsearch.TransportVersions.AGGS_EXCLUDED_DELETED_DOCS;
-
 public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<TermsAggregationBuilder> {
     public static final int KEY_ORDER_CONCURRENCY_THRESHOLD = 50;
 
@@ -199,7 +197,7 @@ public TermsAggregationBuilder(StreamInput in) throws IOException {
         includeExclude = in.readOptionalWriteable(IncludeExclude::new);
         order = InternalOrder.Streams.readOrder(in);
         showTermDocCountError = in.readBoolean();
-        if (in.getTransportVersion().onOrAfter(AGGS_EXCLUDED_DELETED_DOCS)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             excludeDeletedDocs = in.readBoolean();
         }
     }
@@ -217,7 +215,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(includeExclude);
         order.writeTo(out);
         out.writeBoolean(showTermDocCountError);
-        if (out.getTransportVersion().onOrAfter(AGGS_EXCLUDED_DELETED_DOCS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeBoolean(excludeDeletedDocs);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java
index 9d2079b9f1e96..8875ccc93dd4c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java
@@ -56,6 +56,7 @@ public static boolean hasMetric(String name) {
     protected final double max;
     protected final double sum;
 
+    @SuppressWarnings("this-escape")
     public InternalStats(
         String name,
         long count,
diff --git a/server/src/main/java/org/elasticsearch/search/builder/PointInTimeBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/PointInTimeBuilder.java
index 1966f7eaa1e69..db79feb986b84 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/PointInTimeBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/PointInTimeBuilder.java
@@ -136,6 +136,13 @@ public TimeValue getKeepAlive() {
         return keepAlive;
     }
 
+    /**
+     * Returns {@code true} if the point in time is explicitly released when returning the response.
+     */
+    public boolean singleSession() {
+        return keepAlive != null && TimeValue.MINUS_ONE.equals(keepAlive);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index a80378db99348..909b6a7882a34 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -2194,14 +2194,24 @@ public boolean supportsParallelCollection(ToLongFunction<String> fieldCardinalit
     }
 
     private void validate() throws ValidationException {
-        var exceptions = validate(null, false);
+        var exceptions = validate(null, false, false);
         if (exceptions != null) {
             throw exceptions;
         }
     }
 
-    public ActionRequestValidationException validate(ActionRequestValidationException validationException, boolean isScroll) {
+    public ActionRequestValidationException validate(
+        ActionRequestValidationException validationException,
+        boolean isScroll,
+        boolean allowPartialSearchResults
+    ) {
         if (retriever() != null) {
+            if (allowPartialSearchResults && retriever().isCompound()) {
+                validationException = addValidationError(
+                    "cannot specify a compound retriever and [allow_partial_search_results]",
+                    validationException
+                );
+            }
             List<String> specified = new ArrayList<>();
             if (subSearches().isEmpty() == false) {
                 specified.add(QUERY_FIELD.getPreferredName());
@@ -2218,15 +2228,15 @@ public ActionRequestValidationException validate(ActionRequestValidationExceptio
             if (sorts() != null) {
                 specified.add(SORT_FIELD.getPreferredName());
             }
-            if (rescores() != null) {
-                specified.add(RESCORE_FIELD.getPreferredName());
-            }
             if (minScore() != null) {
                 specified.add(MIN_SCORE_FIELD.getPreferredName());
             }
             if (rankBuilder() != null) {
                 specified.add(RANK_FIELD.getPreferredName());
             }
+            if (rescores() != null) {
+                specified.add(RESCORE_FIELD.getPreferredName());
+            }
             if (specified.isEmpty() == false) {
                 validationException = addValidationError(
                     "cannot specify [" + RETRIEVER.getPreferredName() + "] and " + specified,
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
index fe430aeec4723..7620da2c227cb 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
@@ -141,7 +141,7 @@ protected AbstractHighlighterBuilder(StreamInput in) throws IOException {
         postTags(in.readOptionalStringArray());
         fragmentSize(in.readOptionalVInt());
         numOfFragments(in.readOptionalVInt());
-        if (in.getTransportVersion().onOrAfter(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             encoder(in.readOptionalString());
         }
         highlighterType(in.readOptionalString());
@@ -180,7 +180,7 @@ public final void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalStringArray(postTags);
         out.writeOptionalVInt(fragmentSize);
         out.writeOptionalVInt(numOfFragments);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeOptionalString(encoder);
         }
         out.writeOptionalString(highlighterType);
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java
index 05767d8fc7dbf..7059d54f3fa8a 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java
@@ -121,7 +121,7 @@ public HighlightBuilder(HighlightBuilder template, QueryBuilder highlightQuery,
      */
     public HighlightBuilder(StreamInput in) throws IOException {
         super(in);
-        if (in.getTransportVersion().before(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) {
+        if (in.getTransportVersion().before(TransportVersions.V_8_14_0)) {
             encoder(in.readOptionalString());
         }
         useExplicitFieldOrder(in.readBoolean());
@@ -131,7 +131,7 @@ public HighlightBuilder(StreamInput in) throws IOException {
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
-        if (out.getTransportVersion().before(TransportVersions.HIGHLIGHTERS_TAGS_ON_FIELD_LEVEL)) {
+        if (out.getTransportVersion().before(TransportVersions.V_8_14_0)) {
             out.writeOptionalString(encoder);
         }
         out.writeBoolean(useExplicitFieldOrder);
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index cba2cf761e6f3..b23988af92606 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -495,13 +495,11 @@ static void intersectScorerAndBitSet(Scorer scorer, BitSet acceptDocs, LeafColle
 
     @Override
     public TermStatistics termStatistics(Term term, int docFreq, long totalTermFreq) throws IOException {
-        if (aggregatedDfs == null) {
-            // we are either executing the dfs phase or the search_type doesn't include the dfs phase.
-            return super.termStatistics(term, docFreq, totalTermFreq);
-        }
-        TermStatistics termStatistics = aggregatedDfs.termStatistics().get(term);
+        TermStatistics termStatistics = termStatisticsFromDfs(term);
+
         if (termStatistics == null) {
-            // we don't have stats for this - this might be a must_not clauses etc. that doesn't allow extract terms on the query
+            // we don't have stats for this - dfs might be disabled pr this might be a must_not clauses etc.
+            // that doesn't allow extract terms on the query
             return super.termStatistics(term, docFreq, totalTermFreq);
         }
         return termStatistics;
@@ -521,6 +519,33 @@ public CollectionStatistics collectionStatistics(String field) throws IOExceptio
         return collectionStatistics;
     }
 
+    public long docFreq(Term term, long docFreq) throws IOException {
+        TermStatistics termStatistics = termStatisticsFromDfs(term);
+
+        if (termStatistics == null) {
+            return docFreq;
+        }
+        return termStatistics.docFreq();
+    }
+
+    public long totalTermFreq(Term term, long totalTermFreq) throws IOException {
+        TermStatistics termStatistics = termStatisticsFromDfs(term);
+
+        if (termStatistics == null) {
+            return totalTermFreq;
+        }
+        return termStatistics.docFreq();
+    }
+
+    private TermStatistics termStatisticsFromDfs(Term term) {
+        if (aggregatedDfs == null) {
+            // we are either executing the dfs phase or the search_type doesn't include the dfs phase.
+            return null;
+        }
+
+        return aggregatedDfs.termStatistics().get(term);
+    }
+
     public DirectoryReader getDirectoryReader() {
         final IndexReader reader = getIndexReader();
         assert reader instanceof DirectoryReader : "expected an instance of DirectoryReader, got " + reader.getClass();
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java
index 7516ab93f75a5..6f38669edf716 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java
@@ -8,12 +8,13 @@
 
 package org.elasticsearch.search.lookup;
 
-import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader;
 import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
 
 import java.io.IOException;
+import java.util.Map;
 
 // NB This is written under the assumption that individual segments are accessed by a single
 // thread, even if separate segments may be searched concurrently.  If we ever implement
@@ -21,7 +22,7 @@
 class StoredFieldSourceProvider implements SourceProvider {
 
     private final StoredFieldLoader storedFieldLoader;
-    private volatile LeafStoredFieldSourceProvider[] leaves;
+    private final Map<Object, LeafStoredFieldSourceProvider> leaves = ConcurrentCollections.newConcurrentMap();
 
     StoredFieldSourceProvider(StoredFieldLoader storedFieldLoader) {
         this.storedFieldLoader = storedFieldLoader;
@@ -29,32 +30,14 @@ class StoredFieldSourceProvider implements SourceProvider {
 
     @Override
     public Source getSource(LeafReaderContext ctx, int doc) throws IOException {
-        LeafStoredFieldSourceProvider[] leaves = getLeavesUnderLock(findParentContext(ctx));
-        if (leaves[ctx.ord] == null) {
-            // individual segments are currently only accessed on one thread so there's no need
-            // for locking here.
-            leaves[ctx.ord] = new LeafStoredFieldSourceProvider(storedFieldLoader.getLoader(ctx, null));
+        final Object id = ctx.id();
+        var provider = leaves.get(id);
+        if (provider == null) {
+            provider = new LeafStoredFieldSourceProvider(storedFieldLoader.getLoader(ctx, null));
+            var existing = leaves.put(id, provider);
+            assert existing == null : "unexpected source provider [" + existing + "]";
         }
-        return leaves[ctx.ord].getSource(doc);
-    }
-
-    private static IndexReaderContext findParentContext(LeafReaderContext ctx) {
-        if (ctx.parent != null) {
-            return ctx.parent;
-        }
-        assert ctx.isTopLevel;
-        return ctx;
-    }
-
-    private LeafStoredFieldSourceProvider[] getLeavesUnderLock(IndexReaderContext parentCtx) {
-        if (leaves == null) {
-            synchronized (this) {
-                if (leaves == null) {
-                    leaves = new LeafStoredFieldSourceProvider[parentCtx.leaves().size()];
-                }
-            }
-        }
-        return leaves;
+        return provider.getSource(doc);
     }
 
     private static class LeafStoredFieldSourceProvider {
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java
index bccfc22dc7e95..a4549f0814a06 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java
@@ -8,13 +8,14 @@
 
 package org.elasticsearch.search.lookup;
 
-import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader;
 import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
 import org.elasticsearch.index.mapper.SourceLoader;
 
 import java.io.IOException;
+import java.util.Map;
 
 // NB This is written under the assumption that individual segments are accessed by a single
 // thread, even if separate segments may be searched concurrently.  If we ever implement
@@ -22,7 +23,7 @@
 class SyntheticSourceProvider implements SourceProvider {
 
     private final SourceLoader sourceLoader;
-    private volatile SyntheticSourceLeafLoader[] leafLoaders;
+    private final Map<Object, SyntheticSourceLeafLoader> leaves = ConcurrentCollections.newConcurrentMap();
 
     SyntheticSourceProvider(SourceLoader sourceLoader) {
         this.sourceLoader = sourceLoader;
@@ -30,31 +31,14 @@ class SyntheticSourceProvider implements SourceProvider {
 
     @Override
     public Source getSource(LeafReaderContext ctx, int doc) throws IOException {
-        maybeInit(ctx);
-        if (leafLoaders[ctx.ord] == null) {
-            // individual segments are currently only accessed on one thread so there's no need
-            // for locking here.
-            leafLoaders[ctx.ord] = new SyntheticSourceLeafLoader(ctx);
+        final Object id = ctx.id();
+        var provider = leaves.get(id);
+        if (provider == null) {
+            provider = new SyntheticSourceLeafLoader(ctx);
+            var existing = leaves.put(id, provider);
+            assert existing == null : "unexpected source provider [" + existing + "]";
         }
-        return leafLoaders[ctx.ord].getSource(doc);
-    }
-
-    private void maybeInit(LeafReaderContext ctx) {
-        if (leafLoaders == null) {
-            synchronized (this) {
-                if (leafLoaders == null) {
-                    leafLoaders = new SyntheticSourceLeafLoader[findParentContext(ctx).leaves().size()];
-                }
-            }
-        }
-    }
-
-    private IndexReaderContext findParentContext(LeafReaderContext ctx) {
-        if (ctx.parent != null) {
-            return ctx.parent;
-        }
-        assert ctx.isTopLevel;
-        return ctx;
+        return provider.getSource(doc);
     }
 
     private class SyntheticSourceLeafLoader {
diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java b/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java
index 50b3ddc0f370a..02c03223738b5 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java
@@ -8,29 +8,37 @@
 
 package org.elasticsearch.search.rank;
 
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xcontent.ToXContentFragment;
+import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Objects;
 
 /**
  * {@code RankDoc} is the base class for all ranked results.
- * Subclasses should extend this with additional information
- * required for their global ranking method.
+ * Subclasses should extend this with additional information required for their global ranking method.
  */
-public abstract class RankDoc extends ScoreDoc implements NamedWriteable {
+public class RankDoc extends ScoreDoc implements NamedWriteable, ToXContentFragment {
+
+    public static final String NAME = "rank_doc";
 
     public static final int NO_RANK = -1;
 
     /**
-     * If this document has been ranked, this is its final
-     * rrf ranking from all the result sets.
+     * If this document has been ranked, this is its final rrf ranking from all the result sets.
      */
     public int rank = NO_RANK;
 
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
     public record RankKey(int doc, int shardIndex) {}
 
     public RankDoc(int doc, float score, int shardIndex) {
@@ -51,7 +59,26 @@ public final void writeTo(StreamOutput out) throws IOException {
         doWriteTo(out);
     }
 
-    protected abstract void doWriteTo(StreamOutput out) throws IOException;
+    protected void doWriteTo(StreamOutput out) throws IOException {};
+
+    /**
+     * Explain the ranking of this document.
+     */
+    public Explanation explain() {
+        return Explanation.match(rank, "doc [" + doc + "] with an original score of [" + score + "] is at rank [" + rank + "].");
+    }
+
+    @Override
+    public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("_rank", rank);
+        builder.field("_doc", doc);
+        builder.field("_shard", shardIndex);
+        builder.field("_score", score);
+        doToXContent(builder, params);
+        return builder;
+    }
+
+    protected void doToXContent(XContentBuilder builder, Params params) throws IOException {}
 
     @Override
     public final boolean equals(Object o) {
@@ -61,17 +88,21 @@ public final boolean equals(Object o) {
         return doc == rd.doc && score == rd.score && shardIndex == rd.shardIndex && rank == rd.rank && doEquals(rd);
     }
 
-    protected abstract boolean doEquals(RankDoc rd);
+    protected boolean doEquals(RankDoc rd) {
+        return true;
+    }
 
     @Override
     public final int hashCode() {
         return Objects.hash(doc, score, shardIndex, doHashCode());
     }
 
-    protected abstract int doHashCode();
+    protected int doHashCode() {
+        return 0;
+    }
 
     @Override
     public String toString() {
-        return "RankDoc{" + "score=" + score + ", doc=" + doc + ", shardIndex=" + shardIndex + '}';
+        return "RankDoc{" + "_rank=" + rank + ", _doc=" + doc + ", _shard=" + shardIndex + ", _score=" + score + '}';
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankCoordinatorContext.java b/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankCoordinatorContext.java
index 1be8544758a8f..4d1c9c6785453 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankCoordinatorContext.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankCoordinatorContext.java
@@ -17,7 +17,7 @@
 /**
  * {@link QueryPhaseRankCoordinatorContext} is running on the coordinator node and is
  * responsible for combining the query phase results from the shards and rank them accordingly.
- * The output is a `window_size` ranked list of ordered results from all shards.
+ * The output is a `rank_window_size` ranked list of ordered results from all shards.
  * Note: Currently this can use only sort by score; sort by field is not supported.
  */
 public abstract class QueryPhaseRankCoordinatorContext {
diff --git a/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankShardContext.java b/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankShardContext.java
index f562977afb857..fa413485797e8 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankShardContext.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/context/QueryPhaseRankShardContext.java
@@ -15,7 +15,7 @@
 import java.util.List;
 
 /**
- * {@link QueryPhaseRankShardContext} is used to generate the top {@code window_size}
+ * {@link QueryPhaseRankShardContext} is used to generate the top {@code rank_window_size}
  * results on each shard. It specifies the queries to run during {@code QueryPhase} and is responsible for combining all query scores and
  * order all results through the {@link QueryPhaseRankShardContext#combineQueryPhaseResults} method.
  */
diff --git a/server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankCoordinatorContext.java b/server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankCoordinatorContext.java
index 02834f03f54ab..9faa5e4e4450c 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankCoordinatorContext.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankCoordinatorContext.java
@@ -74,16 +74,12 @@ public void computeRankScoresForGlobalResults(
         RankFeatureDoc[] featureDocs = extractFeatureDocs(rankSearchResults);
 
         // generate the final `topResults` results, and pass them to fetch phase through the `rankListener`
-        if (featureDocs.length == 0) {
-            rankListener.onResponse(new RankFeatureDoc[0]);
-        } else {
-            computeScores(featureDocs, rankListener.delegateFailureAndWrap((listener, scores) -> {
-                for (int i = 0; i < featureDocs.length; i++) {
-                    featureDocs[i].score = scores[i];
-                }
-                listener.onResponse(featureDocs);
-            }));
-        }
+        computeScores(featureDocs, rankListener.delegateFailureAndWrap((listener, scores) -> {
+            for (int i = 0; i < featureDocs.length; i++) {
+                featureDocs[i].score = scores[i];
+            }
+            listener.onResponse(featureDocs);
+        }));
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java
index d8b4ec10410f1..8b0cc33844f82 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java
@@ -8,9 +8,11 @@
 
 package org.elasticsearch.search.rank.feature;
 
+import org.apache.lucene.search.Explanation;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -22,7 +24,7 @@ public class RankFeatureDoc extends RankDoc {
 
     public static final String NAME = "rank_feature_doc";
 
-    // todo: update to support more than 1 fields; and not restrict to string data
+    // TODO: update to support more than 1 fields; and not restrict to string data
     public String featureData;
 
     public RankFeatureDoc(int doc, float score, int shardIndex) {
@@ -34,6 +36,11 @@ public RankFeatureDoc(StreamInput in) throws IOException {
         featureData = in.readOptionalString();
     }
 
+    @Override
+    public Explanation explain() {
+        throw new UnsupportedOperationException("explain is not supported for {" + getClass() + "}");
+    }
+
     public void featureData(String featureData) {
         this.featureData = featureData;
     }
@@ -58,4 +65,9 @@ protected int doHashCode() {
     public String getWriteableName() {
         return NAME;
     }
+
+    @Override
+    protected void doToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("featureData", featureData);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureResult.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureResult.java
index 1e16d18cda367..204add3a37c6b 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureResult.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureResult.java
@@ -27,12 +27,14 @@ public class RankFeatureResult extends SearchPhaseResult {
 
     public RankFeatureResult() {}
 
+    @SuppressWarnings("this-escape")
     public RankFeatureResult(ShardSearchContextId id, SearchShardTarget shardTarget, ShardSearchRequest request) {
         this.contextId = id;
         setSearchShardTarget(shardTarget);
         setShardSearchRequest(request);
     }
 
+    @SuppressWarnings("this-escape")
     public RankFeatureResult(StreamInput in) throws IOException {
         super(in);
         contextId = new ShardSearchContextId(in);
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java
index b369324b3ee52..f2a2a606c0e01 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java
@@ -10,7 +10,11 @@
 
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder;
+import org.elasticsearch.search.vectors.ExactKnnQueryBuilder;
 import org.elasticsearch.search.vectors.KnnSearchBuilder;
 import org.elasticsearch.search.vectors.QueryVectorBuilder;
 import org.elasticsearch.search.vectors.VectorData;
@@ -120,6 +124,16 @@ public String getName() {
         return NAME;
     }
 
+    @Override
+    public QueryBuilder topDocsQuery() {
+        assert rankDocs != null : "{rankDocs} should have been materialized at this point";
+
+        BoolQueryBuilder knnTopResultsQuery = new BoolQueryBuilder().filter(new RankDocsQueryBuilder(rankDocs))
+            .should(new ExactKnnQueryBuilder(VectorData.fromFloats(queryVector), field, similarity));
+        preFilterQueryBuilders.forEach(knnTopResultsQuery::filter);
+        return knnTopResultsQuery;
+    }
+
     @Override
     public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
         KnnSearchBuilder knnSearchBuilder = new KnnSearchBuilder(
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java
new file mode 100644
index 0000000000000..d1f6a41dc789f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.retriever;
+
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.DisMaxQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder;
+import org.elasticsearch.search.retriever.rankdoc.RankDocsSortBuilder;
+import org.elasticsearch.search.sort.ScoreSortBuilder;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+/**
+ * An {@link RetrieverBuilder} that is used to retrieve documents based on the rank of the documents.
+ */
+public class RankDocsRetrieverBuilder extends RetrieverBuilder {
+
+    public static final String NAME = "rank_docs_retriever";
+    private final int rankWindowSize;
+    final List<RetrieverBuilder> sources;
+    final Supplier<RankDoc[]> rankDocs;
+
+    public RankDocsRetrieverBuilder(
+        int rankWindowSize,
+        List<RetrieverBuilder> sources,
+        Supplier<RankDoc[]> rankDocs,
+        List<QueryBuilder> preFilterQueryBuilders
+    ) {
+        this.rankWindowSize = rankWindowSize;
+        this.rankDocs = rankDocs;
+        this.sources = sources;
+        this.preFilterQueryBuilders = preFilterQueryBuilders;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    private boolean sourceShouldRewrite(QueryRewriteContext ctx) throws IOException {
+        for (var source : sources) {
+            if (source.isCompound()) {
+                return true;
+            }
+            var newSource = source.rewrite(ctx);
+            if (newSource != source) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException {
+        assert false == sourceShouldRewrite(ctx) : "retriever sources should be rewritten first";
+        var rewrittenFilters = rewritePreFilters(ctx);
+        if (rewrittenFilters != preFilterQueryBuilders) {
+            return new RankDocsRetrieverBuilder(rankWindowSize, sources, rankDocs, rewrittenFilters);
+        }
+        return this;
+    }
+
+    @Override
+    public QueryBuilder topDocsQuery() {
+        // this is used to fetch all documents form the parent retrievers (i.e. sources)
+        // so that we can use all the matched documents to compute aggregations, nested hits etc
+        DisMaxQueryBuilder disMax = new DisMaxQueryBuilder().tieBreaker(0f);
+        for (var retriever : sources) {
+            var query = retriever.topDocsQuery();
+            if (query != null) {
+                if (retriever.retrieverName() != null) {
+                    query.queryName(retriever.retrieverName());
+                }
+                disMax.add(query);
+            }
+        }
+        // ignore prefilters of this level, they are already propagated to children
+        return disMax;
+    }
+
+    @Override
+    public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
+        // here we force a custom sort based on the rank of the documents
+        // TODO: should we adjust to account for other fields sort options just for the top ranked docs?
+        if (searchSourceBuilder.rescores() == null || searchSourceBuilder.rescores().isEmpty()) {
+            searchSourceBuilder.sort(Arrays.asList(new RankDocsSortBuilder(rankDocs.get()), new ScoreSortBuilder()));
+        }
+        if (searchSourceBuilder.explain() != null && searchSourceBuilder.explain()) {
+            searchSourceBuilder.trackScores(true);
+        }
+        BoolQueryBuilder boolQuery = new BoolQueryBuilder();
+        RankDocsQueryBuilder rankQuery = new RankDocsQueryBuilder(rankDocs.get());
+        // if we have aggregations we need to compute them based on all doc matches, not just the top hits
+        // so we just post-filter the top hits based on the rank queries we have
+        if (searchSourceBuilder.aggregations() != null) {
+            boolQuery.should(rankQuery);
+            // compute a disjunction of all the query sources that were executed to compute the top rank docs
+            QueryBuilder disjunctionOfSources = topDocsQuery();
+            if (disjunctionOfSources != null) {
+                boolQuery.should(disjunctionOfSources);
+            }
+            // post filter the results so that the top docs are still the same
+            searchSourceBuilder.postFilter(rankQuery);
+        } else {
+            boolQuery.must(rankQuery);
+        }
+        // add any prefilters present in the retriever
+        for (var preFilterQueryBuilder : preFilterQueryBuilders) {
+            boolQuery.filter(preFilterQueryBuilder);
+        }
+        searchSourceBuilder.query(boolQuery);
+    }
+
+    @Override
+    protected boolean doEquals(Object o) {
+        RankDocsRetrieverBuilder other = (RankDocsRetrieverBuilder) o;
+        return Arrays.equals(rankDocs.get(), other.rankDocs.get())
+            && sources.equals(other.sources)
+            && rankWindowSize == other.rankWindowSize;
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(super.hashCode(), Arrays.hashCode(rankDocs.get()), sources, rankWindowSize);
+    }
+
+    @Override
+    protected void doToXContent(XContentBuilder builder, Params params) throws IOException {
+        throw new UnsupportedOperationException("toXContent() is not supported for " + this.getClass());
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
index 3a9979030683a..0857ef21adaaf 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.Rewriteable;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.xcontent.AbstractObjectParser;
 import org.elasticsearch.xcontent.FilterXContentParserWrapper;
 import org.elasticsearch.xcontent.NamedObjectNotFoundException;
@@ -191,6 +192,34 @@ public boolean isCompound() {
         return false;
     }
 
+    protected RankDoc[] rankDocs = null;
+
+    public RetrieverBuilder() {}
+
+    protected final List<QueryBuilder> rewritePreFilters(QueryRewriteContext ctx) throws IOException {
+        List<QueryBuilder> newFilters = new ArrayList<>(preFilterQueryBuilders.size());
+        boolean changed = false;
+        for (var filter : preFilterQueryBuilders) {
+            var newFilter = filter.rewrite(ctx);
+            changed |= filter != newFilter;
+            newFilters.add(newFilter);
+        }
+        if (changed) {
+            return newFilters;
+        }
+        return preFilterQueryBuilders;
+    }
+
+    /**
+     * This function is called by compound {@link RetrieverBuilder} to return the original query that
+     * was used by this retriever to compute its top documents.
+     */
+    public abstract QueryBuilder topDocsQuery();
+
+    public void setRankDocs(RankDoc[] rankDocs) {
+        this.rankDocs = rankDocs;
+    }
+
     /**
      * Gets the filters for this retriever.
      */
@@ -254,5 +283,9 @@ public String toString() {
         return Strings.toString(this, true, true);
     }
 
+    public String retrieverName() {
+        return retrieverName;
+    }
+
     // ---- END FOR TESTING ----
 }
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java
index 4694780770617..682d456295ba9 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java
@@ -106,6 +106,17 @@ public static StandardRetrieverBuilder fromXContent(XContentParser parser, Retri
     Float minScore;
     CollapseBuilder collapseBuilder;
 
+    @Override
+    public QueryBuilder topDocsQuery() {
+        // TODO: for compound retrievers this will have to be reworked as queries like knn could be executed twice
+        if (preFilterQueryBuilders.isEmpty()) {
+            return queryBuilder;
+        }
+        var ret = new BoolQueryBuilder().filter(queryBuilder);
+        preFilterQueryBuilders.stream().forEach(ret::filter);
+        return ret;
+    }
+
     @Override
     public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
         if (preFilterQueryBuilders.isEmpty() == false) {
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java
new file mode 100644
index 0000000000000..77da1cc80bc97
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java
@@ -0,0 +1,199 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.retriever.rankdoc;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
+import org.elasticsearch.search.rank.RankDoc;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
+/**
+ * A {@code RankDocsQuery} returns the top k documents in the order specified by the global doc IDs.
+ * This is used by retrievers that compute a score for a large document set, and need access to just the top results,
+ * after performing any reranking or filtering.
+ */
+public class RankDocsQuery extends Query {
+
+    private final RankDoc[] docs;
+    private final int[] segmentStarts;
+    private final Object contextIdentity;
+
+    /**
+     * Creates a {@code RankDocsQuery} based on the provided docs.
+     *
+     * @param docs the global doc IDs of documents that match, in ascending order
+     * @param segmentStarts the indexes in docs and scores corresponding to the first matching
+     *     document in each segment. If a segment has no matching documents, it should be assigned
+     *     the index of the next segment that does. There should be a final entry that is always
+     *     docs.length-1.
+     * @param contextIdentity an object identifying the reader context that was used to build this
+     *     query
+     */
+    RankDocsQuery(RankDoc[] docs, int[] segmentStarts, Object contextIdentity) {
+        this.docs = docs;
+        this.segmentStarts = segmentStarts;
+        this.contextIdentity = contextIdentity;
+    }
+
+    @Override
+    public Query rewrite(IndexSearcher searcher) throws IOException {
+        if (docs.length == 0) {
+            return new MatchNoDocsQuery();
+        }
+        return this;
+    }
+
+    RankDoc[] rankDocs() {
+        return docs;
+    }
+
+    @Override
+    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+        if (searcher.getIndexReader().getContext().id() != contextIdentity) {
+            throw new IllegalStateException("This RankDocsDocQuery was created by a different reader");
+        }
+        return new Weight(this) {
+
+            @Override
+            public int count(LeafReaderContext context) {
+                return segmentStarts[context.ord + 1] - segmentStarts[context.ord];
+            }
+
+            @Override
+            public Explanation explain(LeafReaderContext context, int doc) {
+                int found = Arrays.binarySearch(docs, doc + context.docBase, (a, b) -> Integer.compare(((RankDoc) a).doc, (int) b));
+                if (found < 0) {
+                    return Explanation.noMatch("doc not found in top " + docs.length + " rank docs");
+                }
+                return docs[found].explain();
+            }
+
+            @Override
+            public Scorer scorer(LeafReaderContext context) {
+                // Segment starts indicate how many docs are in the segment,
+                // upper equalling lower indicates no documents for this segment
+                if (segmentStarts[context.ord] == segmentStarts[context.ord + 1]) {
+                    return null;
+                }
+                return new Scorer(this) {
+                    final int lower = segmentStarts[context.ord];
+                    final int upper = segmentStarts[context.ord + 1];
+                    int upTo = -1;
+                    float score;
+
+                    @Override
+                    public DocIdSetIterator iterator() {
+                        return new DocIdSetIterator() {
+                            @Override
+                            public int docID() {
+                                return currentDocId();
+                            }
+
+                            @Override
+                            public int nextDoc() {
+                                if (upTo == -1) {
+                                    upTo = lower;
+                                } else {
+                                    ++upTo;
+                                }
+                                return currentDocId();
+                            }
+
+                            @Override
+                            public int advance(int target) throws IOException {
+                                return slowAdvance(target);
+                            }
+
+                            @Override
+                            public long cost() {
+                                return upper - lower;
+                            }
+                        };
+                    }
+
+                    @Override
+                    public float getMaxScore(int docId) {
+                        if (docId != NO_MORE_DOCS) {
+                            docId += context.docBase;
+                        }
+                        float maxScore = 0;
+                        for (int idx = Math.max(lower, upTo); idx < upper && docs[idx].doc <= docId; idx++) {
+                            maxScore = Math.max(maxScore, docs[idx].score);
+                        }
+                        return maxScore;
+                    }
+
+                    @Override
+                    public float score() {
+                        return docs[upTo].score;
+                    }
+
+                    @Override
+                    public int docID() {
+                        return currentDocId();
+                    }
+
+                    private int currentDocId() {
+                        if (upTo == -1) {
+                            return -1;
+                        }
+                        if (upTo >= upper) {
+                            return NO_MORE_DOCS;
+                        }
+                        return docs[upTo].doc - context.docBase;
+                    }
+
+                };
+            }
+
+            @Override
+            public boolean isCacheable(LeafReaderContext ctx) {
+                return true;
+            }
+        };
+    }
+
+    @Override
+    public String toString(String field) {
+        return this.getClass().getSimpleName() + "{rank_docs:" + Arrays.toString(docs) + "}";
+    }
+
+    @Override
+    public void visit(QueryVisitor visitor) {
+        visitor.visitLeaf(this);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (sameClassAs(obj) == false) {
+            return false;
+        }
+        return Arrays.equals(docs, ((RankDocsQuery) obj).docs)
+            && Arrays.equals(segmentStarts, ((RankDocsQuery) obj).segmentStarts)
+            && contextIdentity == ((RankDocsQuery) obj).contextIdentity;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(classHash(), Arrays.hashCode(docs), Arrays.hashCode(segmentStarts), contextIdentity);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilder.java
new file mode 100644
index 0000000000000..ff2085bc8903f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilder.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.retriever.rankdoc;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Comparator;
+
+public class RankDocsQueryBuilder extends AbstractQueryBuilder<RankDocsQueryBuilder> {
+
+    public static final String NAME = "rank_docs_query";
+
+    private final RankDoc[] rankDocs;
+
+    public RankDocsQueryBuilder(RankDoc[] rankDocs) {
+        this.rankDocs = rankDocs;
+    }
+
+    public RankDocsQueryBuilder(StreamInput in) throws IOException {
+        super(in);
+        this.rankDocs = in.readArray(c -> c.readNamedWriteable(RankDoc.class), RankDoc[]::new);
+    }
+
+    RankDoc[] rankDocs() {
+        return rankDocs;
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeArray(StreamOutput::writeNamedWriteable, rankDocs);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    protected Query doToQuery(SearchExecutionContext context) throws IOException {
+        RankDoc[] shardRankDocs = Arrays.stream(rankDocs)
+            .filter(r -> r.shardIndex == context.getShardRequestIndex())
+            .sorted(Comparator.comparingInt(r -> r.doc))
+            .toArray(RankDoc[]::new);
+        IndexReader reader = context.getIndexReader();
+        int[] segmentStarts = findSegmentStarts(reader, shardRankDocs);
+        return new RankDocsQuery(shardRankDocs, segmentStarts, reader.getContext().id());
+    }
+
+    private static int[] findSegmentStarts(IndexReader reader, RankDoc[] docs) {
+        int[] starts = new int[reader.leaves().size() + 1];
+        starts[starts.length - 1] = docs.length;
+        if (starts.length == 2) {
+            return starts;
+        }
+        int resultIndex = 0;
+        for (int i = 1; i < starts.length - 1; i++) {
+            int upper = reader.leaves().get(i).docBase;
+            resultIndex = Arrays.binarySearch(docs, resultIndex, docs.length, upper, (a, b) -> Integer.compare(((RankDoc) a).doc, (int) b));
+            if (resultIndex < 0) {
+                resultIndex = -1 - resultIndex;
+            }
+            starts[i] = resultIndex;
+        }
+        return starts;
+    }
+
+    @Override
+    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(NAME);
+        builder.startArray("rank_docs");
+        for (RankDoc doc : rankDocs) {
+            builder.startObject();
+            doc.toXContent(builder, params);
+            builder.endObject();
+        }
+        builder.endArray();
+        builder.endObject();
+    }
+
+    @Override
+    protected boolean doEquals(RankDocsQueryBuilder other) {
+        return Arrays.equals(rankDocs, other.rankDocs);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Arrays.hashCode(rankDocs);
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.RANK_DOCS_RETRIEVER;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilder.java
new file mode 100644
index 0000000000000..0122e6ee9ea12
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilder.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.retriever.rankdoc;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.search.sort.BucketedSort;
+import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.sort.SortFieldAndFormat;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * Builds a {@code RankDocsSortField} that sorts documents by their rank as computed through the {@code RankDocsRetrieverBuilder}.
+ */
+public class RankDocsSortBuilder extends SortBuilder<RankDocsSortBuilder> {
+    public static final String NAME = "rank_docs_sort";
+
+    private RankDoc[] rankDocs;
+
+    public RankDocsSortBuilder(RankDoc[] rankDocs) {
+        this.rankDocs = rankDocs;
+    }
+
+    public RankDocsSortBuilder(StreamInput in) throws IOException {
+        this.rankDocs = in.readArray(c -> c.readNamedWriteable(RankDoc.class), RankDoc[]::new);
+    }
+
+    public RankDocsSortBuilder(RankDocsSortBuilder original) {
+        this.rankDocs = original.rankDocs;
+    }
+
+    public RankDocsSortBuilder rankDocs(RankDoc[] rankDocs) {
+        this.rankDocs = rankDocs;
+        return this;
+    }
+
+    public RankDoc[] rankDocs() {
+        return this.rankDocs;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeArray(StreamOutput::writeNamedWriteable, rankDocs);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public SortBuilder<?> rewrite(QueryRewriteContext ctx) throws IOException {
+        return this;
+    }
+
+    @Override
+    protected SortFieldAndFormat build(SearchExecutionContext context) throws IOException {
+        RankDoc[] shardRankDocs = Arrays.stream(rankDocs)
+            .filter(r -> r.shardIndex == context.getShardRequestIndex())
+            .toArray(RankDoc[]::new);
+        return new SortFieldAndFormat(new RankDocsSortField(shardRankDocs), DocValueFormat.RAW);
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.RANK_DOCS_RETRIEVER;
+    }
+
+    @Override
+    public BucketedSort buildBucketedSort(SearchExecutionContext context, BigArrays bigArrays, int bucketSize, BucketedSort.ExtraData extra)
+        throws IOException {
+        throw new UnsupportedOperationException("buildBucketedSort() is not supported for " + this.getClass());
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        throw new UnsupportedOperationException("toXContent() is not supported for " + this.getClass());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        RankDocsSortBuilder that = (RankDocsSortBuilder) obj;
+        return Arrays.equals(rankDocs, that.rankDocs) && this.order.equals(that.order);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(Arrays.hashCode(this.rankDocs), this.order);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortField.java b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortField.java
new file mode 100644
index 0000000000000..3cd29d352028b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortField.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.retriever.rankdoc;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.LeafFieldComparator;
+import org.apache.lucene.search.Pruning;
+import org.apache.lucene.search.Scorable;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.comparators.NumericComparator;
+import org.apache.lucene.util.hnsw.IntToIntFunction;
+import org.elasticsearch.search.rank.RankDoc;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * A {@link SortField} that sorts documents by their rank as computed through the {@code RankDocsRetrieverBuilder}.
+ * This is used when we want to score and rank the documents irrespective of their original scores,
+ * but based on the provided rank they were assigned, e.g. through an RRF retriever.
+ **/
+public class RankDocsSortField extends SortField {
+
+    public static final String NAME = "_rank";
+
+    public RankDocsSortField(RankDoc[] rankDocs) {
+        super(NAME, new FieldComparatorSource() {
+            @Override
+            public FieldComparator<?> newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) {
+                return new RankDocsComparator(numHits, rankDocs);
+            }
+        });
+    }
+
+    private static class RankDocsComparator extends NumericComparator<Integer> {
+        private final int[] values;
+        private final Map<Integer, Integer> rankDocMap;
+        private int topValue;
+        private int bottom;
+
+        private RankDocsComparator(int numHits, RankDoc[] rankDocs) {
+            super(NAME, Integer.MAX_VALUE, false, Pruning.NONE, Integer.BYTES);
+            this.values = new int[numHits];
+            this.rankDocMap = Arrays.stream(rankDocs).collect(Collectors.toMap(k -> k.doc, v -> v.rank));
+        }
+
+        @Override
+        public int compare(int slot1, int slot2) {
+            return Integer.compare(values[slot1], values[slot2]);
+        }
+
+        @Override
+        public Integer value(int slot) {
+            return Integer.valueOf(values[slot]);
+        }
+
+        @Override
+        public void setTopValue(Integer value) {
+            topValue = value;
+        }
+
+        @Override
+        public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
+            IntToIntFunction docToRank = doc -> rankDocMap.getOrDefault(context.docBase + doc, Integer.MAX_VALUE);
+            return new LeafFieldComparator() {
+                @Override
+                public void setBottom(int slot) throws IOException {
+                    bottom = values[slot];
+                }
+
+                @Override
+                public int compareBottom(int doc) {
+                    return Integer.compare(bottom, docToRank.apply(doc));
+                }
+
+                @Override
+                public int compareTop(int doc) {
+                    return Integer.compare(topValue, docToRank.apply(doc));
+                }
+
+                @Override
+                public void copy(int slot, int doc) {
+                    values[slot] = docToRank.apply(doc);
+                }
+
+                @Override
+                public void setScorer(Scorable scorer) {}
+            };
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java
index 1f05b215699b1..4f36ddbedf23b 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilder.java
@@ -32,6 +32,7 @@ public class ExactKnnQueryBuilder extends AbstractQueryBuilder<ExactKnnQueryBuil
     public static final String NAME = "exact_knn";
     private final String field;
     private final VectorData query;
+    private final Float vectorSimilarity;
 
     /**
      * Creates a query builder.
@@ -39,29 +40,26 @@ public class ExactKnnQueryBuilder extends AbstractQueryBuilder<ExactKnnQueryBuil
      * @param query    the query vector
      * @param field    the field that was used for the kNN query
      */
-    public ExactKnnQueryBuilder(float[] query, String field) {
-        this(VectorData.fromFloats(query), field);
-    }
-
-    /**
-     * Creates a query builder.
-     *
-     * @param query    the query vector
-     * @param field    the field that was used for the kNN query
-     */
-    public ExactKnnQueryBuilder(VectorData query, String field) {
+    public ExactKnnQueryBuilder(VectorData query, String field, Float vectorSimilarity) {
         this.query = query;
         this.field = field;
+        this.vectorSimilarity = vectorSimilarity;
     }
 
     public ExactKnnQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        if (in.getTransportVersion().onOrAfter(TransportVersions.KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             this.query = in.readOptionalWriteable(VectorData::new);
         } else {
             this.query = VectorData.fromFloats(in.readFloatArray());
         }
         this.field = in.readString();
+        if (in.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS)
+            || in.getTransportVersion().isPatchFrom(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15)) {
+            this.vectorSimilarity = in.readOptionalFloat();
+        } else {
+            this.vectorSimilarity = null;
+        }
     }
 
     String getField() {
@@ -72,6 +70,10 @@ VectorData getQuery() {
         return query;
     }
 
+    Float vectorSimilarity() {
+        return vectorSimilarity;
+    }
+
     @Override
     public String getWriteableName() {
         return NAME;
@@ -79,12 +81,16 @@ public String getWriteableName() {
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
-        if (out.getTransportVersion().onOrAfter(TransportVersions.KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeOptionalWriteable(query);
         } else {
             out.writeFloatArray(query.asFloatVector());
         }
         out.writeString(field);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS)
+            || out.getTransportVersion().isPatchFrom(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15)) {
+            out.writeOptionalFloat(vectorSimilarity);
+        }
     }
 
     @Override
@@ -92,6 +98,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.startObject(NAME);
         builder.field("query", query);
         builder.field("field", field);
+        if (vectorSimilarity != null) {
+            builder.field("similarity", vectorSimilarity);
+        }
         boostAndQueryNameToXContent(builder);
         builder.endObject();
     }
@@ -108,17 +117,17 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException {
             );
         }
         final DenseVectorFieldMapper.DenseVectorFieldType vectorFieldType = (DenseVectorFieldMapper.DenseVectorFieldType) fieldType;
-        return vectorFieldType.createExactKnnQuery(query);
+        return vectorFieldType.createExactKnnQuery(query, vectorSimilarity);
     }
 
     @Override
     protected boolean doEquals(ExactKnnQueryBuilder other) {
-        return field.equals(other.field) && Objects.equals(query, other.query);
+        return field.equals(other.field) && Objects.equals(query, other.query) && Objects.equals(vectorSimilarity, other.vectorSimilarity);
     }
 
     @Override
     protected int doHashCode() {
-        return Objects.hash(field, Objects.hashCode(query));
+        return Objects.hash(field, Objects.hashCode(query), vectorSimilarity);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java
index 4be1d7a613a76..f6cef8a06106b 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java
@@ -37,6 +37,7 @@ public class KnnScoreDocQueryBuilder extends AbstractQueryBuilder<KnnScoreDocQue
     private final ScoreDoc[] scoreDocs;
     private final String fieldName;
     private final VectorData queryVector;
+    private final Float vectorSimilarity;
 
     /**
      * Creates a query builder.
@@ -44,10 +45,11 @@ public class KnnScoreDocQueryBuilder extends AbstractQueryBuilder<KnnScoreDocQue
      * @param scoreDocs the docs and scores this query should match. The array must be
      *                  sorted in order of ascending doc IDs.
      */
-    public KnnScoreDocQueryBuilder(ScoreDoc[] scoreDocs, String fieldName, VectorData queryVector) {
+    public KnnScoreDocQueryBuilder(ScoreDoc[] scoreDocs, String fieldName, VectorData queryVector, Float vectorSimilarity) {
         this.scoreDocs = scoreDocs;
         this.fieldName = fieldName;
         this.queryVector = queryVector;
+        this.vectorSimilarity = vectorSimilarity;
     }
 
     public KnnScoreDocQueryBuilder(StreamInput in) throws IOException {
@@ -56,7 +58,7 @@ public KnnScoreDocQueryBuilder(StreamInput in) throws IOException {
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
             this.fieldName = in.readOptionalString();
             if (in.readBoolean()) {
-                if (in.getTransportVersion().onOrAfter(TransportVersions.KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING)) {
+                if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                     this.queryVector = in.readOptionalWriteable(VectorData::new);
                 } else {
                     this.queryVector = VectorData.fromFloats(in.readFloatArray());
@@ -68,6 +70,12 @@ public KnnScoreDocQueryBuilder(StreamInput in) throws IOException {
             this.fieldName = null;
             this.queryVector = null;
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS)
+            || in.getTransportVersion().isPatchFrom(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15)) {
+            this.vectorSimilarity = in.readOptionalFloat();
+        } else {
+            this.vectorSimilarity = null;
+        }
     }
 
     @Override
@@ -87,6 +95,10 @@ VectorData queryVector() {
         return queryVector;
     }
 
+    Float vectorSimilarity() {
+        return vectorSimilarity;
+    }
+
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeArray(Lucene::writeScoreDoc, scoreDocs);
@@ -94,7 +106,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
             out.writeOptionalString(fieldName);
             if (queryVector != null) {
                 out.writeBoolean(true);
-                if (out.getTransportVersion().onOrAfter(TransportVersions.KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                     out.writeOptionalWriteable(queryVector);
                 } else {
                     out.writeFloatArray(queryVector.asFloatVector());
@@ -103,6 +115,10 @@ protected void doWriteTo(StreamOutput out) throws IOException {
                 out.writeBoolean(false);
             }
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS)
+            || out.getTransportVersion().isPatchFrom(TransportVersions.FIX_VECTOR_SIMILARITY_INNER_HITS_BACKPORT_8_15)) {
+            out.writeOptionalFloat(vectorSimilarity);
+        }
     }
 
     @Override
@@ -119,6 +135,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         if (queryVector != null) {
             builder.field("query", queryVector);
         }
+        if (vectorSimilarity != null) {
+            builder.field("similarity", vectorSimilarity);
+        }
         boostAndQueryNameToXContent(builder);
         builder.endObject();
     }
@@ -144,7 +163,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
             return new MatchNoneQueryBuilder("The \"" + getName() + "\" query was rewritten to a \"match_none\" query.");
         }
         if (queryRewriteContext.convertToInnerHitsRewriteContext() != null && queryVector != null && fieldName != null) {
-            return new ExactKnnQueryBuilder(queryVector, fieldName);
+            return new ExactKnnQueryBuilder(queryVector, fieldName, vectorSimilarity);
         }
         return super.doRewrite(queryRewriteContext);
     }
@@ -183,7 +202,9 @@ protected boolean doEquals(KnnScoreDocQueryBuilder other) {
                 return false;
             }
         }
-        return Objects.equals(fieldName, other.fieldName) && Objects.equals(queryVector, other.queryVector);
+        return Objects.equals(fieldName, other.fieldName)
+            && Objects.equals(queryVector, other.queryVector)
+            && Objects.equals(vectorSimilarity, other.vectorSimilarity);
     }
 
     @Override
@@ -193,7 +214,7 @@ protected int doHashCode() {
             int hashCode = Objects.hash(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex);
             result = 31 * result + hashCode;
         }
-        return Objects.hash(result, fieldName, Objects.hashCode(queryVector));
+        return Objects.hash(result, fieldName, vectorSimilarity, Objects.hashCode(queryVector));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
index 348a65d0c4960..b74498b89dc75 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
@@ -253,7 +253,7 @@ public KnnSearchBuilder(StreamInput in) throws IOException {
         this.field = in.readString();
         this.k = in.readVInt();
         this.numCands = in.readVInt();
-        if (in.getTransportVersion().onOrAfter(TransportVersions.KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             this.queryVector = in.readOptionalWriteable(VectorData::new);
         } else {
             this.queryVector = VectorData.fromFloats(in.readFloatArray());
@@ -407,6 +407,10 @@ public KnnVectorQueryBuilder toQueryBuilder() {
             .addFilterQueries(filterQueries);
     }
 
+    public Float getSimilarity() {
+        return similarity;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -489,7 +493,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(field);
         out.writeVInt(k);
         out.writeVInt(numCands);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeOptionalWriteable(queryVector);
         } else {
             out.writeFloatArray(queryVector.asFloatVector());
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
index f1b1c24c50788..9d0d5ea9e0e6f 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
@@ -201,7 +201,7 @@ public KnnVectorQueryBuilder(StreamInput in) throws IOException {
         } else {
             this.numCands = in.readVInt();
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             this.queryVector = in.readOptionalWriteable(VectorData::new);
         } else {
             if (in.getTransportVersion().before(TransportVersions.V_8_7_0)
@@ -221,7 +221,7 @@ public KnnVectorQueryBuilder(StreamInput in) throws IOException {
         } else {
             this.vectorSimilarity = null;
         }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.KNN_QUERY_VECTOR_BUILDER)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             this.queryVectorBuilder = in.readOptionalNamedWriteable(QueryVectorBuilder.class);
         } else {
             this.queryVectorBuilder = null;
@@ -296,7 +296,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
                 out.writeVInt(numCands);
             }
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.KNN_EXPLICIT_BYTE_QUERY_VECTOR_PARSING)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeOptionalWriteable(queryVector);
         } else {
             if (out.getTransportVersion().before(TransportVersions.V_8_7_0)
@@ -314,7 +314,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) {
             out.writeOptionalFloat(vectorSimilarity);
         }
-        if (out.getTransportVersion().before(TransportVersions.KNN_QUERY_VECTOR_BUILDER) && queryVectorBuilder != null) {
+        if (out.getTransportVersion().before(TransportVersions.V_8_14_0) && queryVectorBuilder != null) {
             throw new IllegalArgumentException(
                 format(
                     "cannot serialize [%s] to older node of version [%s]",
@@ -323,7 +323,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
                 )
             );
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.KNN_QUERY_VECTOR_BUILDER)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeOptionalNamedWriteable(queryVectorBuilder);
         }
     }
@@ -401,7 +401,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext ctx) throws IOException {
             ).queryName(queryName).addFilterQueries(filterQueries);
         }
         if (ctx.convertToInnerHitsRewriteContext() != null) {
-            return new ExactKnnQueryBuilder(queryVector, fieldName).boost(boost).queryName(queryName);
+            return new ExactKnnQueryBuilder(queryVector, fieldName, vectorSimilarity).boost(boost).queryName(queryName);
         }
         boolean changed = false;
         List<QueryBuilder> rewrittenQueries = new ArrayList<>(filterQueries.size());
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RegisteredPolicySnapshots.java b/server/src/main/java/org/elasticsearch/snapshots/RegisteredPolicySnapshots.java
new file mode 100644
index 0000000000000..9eecf6e47aa90
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/snapshots/RegisteredPolicySnapshots.java
@@ -0,0 +1,288 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.cluster.SimpleDiffable;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.snapshots.SnapshotsService.POLICY_ID_METADATA_FIELD;
+
+/**
+ * {@link RegisteredPolicySnapshots} records a set of snapshot IDs along with their SLM policy name. It is used to infer
+ * the failure of snapshots which did not record their failure in SnapshotLifecycleStats. The set is stored in the
+ * cluster state as custom metadata. When a snapshot is started by SLM, it is added to this set. Upon completion,
+ * is it removed. If a snapshot does not record its failure in SnapshotLifecycleStats, likely due to a master shutdown,
+ * it will not be removed from the registered set. A subsequent snapshot will then find that a registered snapshot
+ * is no longer running and will infer that it failed, updating SnapshotLifecycleStats accordingly.
+ */
+public class RegisteredPolicySnapshots implements Metadata.Custom {
+
+    public static final String TYPE = "registered_snapshots";
+    private static final ParseField SNAPSHOTS = new ParseField("snapshots");
+    public static final RegisteredPolicySnapshots EMPTY = new RegisteredPolicySnapshots(List.of());
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<RegisteredPolicySnapshots, Void> PARSER = new ConstructingObjectParser<>(
+        TYPE,
+        a -> new RegisteredPolicySnapshots((List<PolicySnapshot>) a[0])
+    );
+
+    static {
+        PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> PolicySnapshot.parse(p), SNAPSHOTS);
+    }
+
+    private final List<PolicySnapshot> snapshots;
+
+    public RegisteredPolicySnapshots(List<PolicySnapshot> snapshots) {
+        this.snapshots = Collections.unmodifiableList(snapshots);
+    }
+
+    public RegisteredPolicySnapshots(StreamInput in) throws IOException {
+        this.snapshots = in.readCollectionAsImmutableList(PolicySnapshot::new);
+    }
+
+    public List<PolicySnapshot> getSnapshots() {
+        return snapshots;
+    }
+
+    public boolean contains(SnapshotId snapshotId) {
+        return snapshots.stream().map(PolicySnapshot::getSnapshotId).anyMatch(snapshotId::equals);
+    }
+
+    public List<SnapshotId> getSnapshotsByPolicy(String policy) {
+        return snapshots.stream().filter(s -> s.getPolicy().equals(policy)).map(PolicySnapshot::getSnapshotId).toList();
+    }
+
+    @Override
+    public EnumSet<Metadata.XContentContext> context() {
+        return Metadata.ALL_CONTEXTS;
+    }
+
+    @Override
+    public Diff<Metadata.Custom> diff(Metadata.Custom previousState) {
+        return new RegisteredSnapshotsDiff((RegisteredPolicySnapshots) previousState, this);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.REGISTER_SLM_STATS;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeCollection(snapshots);
+    }
+
+    @Override
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(Iterators.single((builder, params) -> {
+            builder.field(SNAPSHOTS.getPreferredName(), snapshots);
+            return builder;
+        }));
+    }
+
+    public static RegisteredPolicySnapshots parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public String toString() {
+        return "RegisteredSnapshots{" + "snapshots=" + snapshots + '}';
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(snapshots);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        RegisteredPolicySnapshots other = (RegisteredPolicySnapshots) obj;
+        return Objects.equals(snapshots, other.snapshots);
+    }
+
+    public static class RegisteredSnapshotsDiff implements NamedDiff<Metadata.Custom> {
+        final List<PolicySnapshot> snapshots;
+
+        RegisteredSnapshotsDiff(RegisteredPolicySnapshots before, RegisteredPolicySnapshots after) {
+            this.snapshots = after.snapshots;
+        }
+
+        public RegisteredSnapshotsDiff(StreamInput in) throws IOException {
+            this.snapshots = new RegisteredPolicySnapshots(in).snapshots;
+        }
+
+        @Override
+        public Metadata.Custom apply(Metadata.Custom part) {
+            return new RegisteredPolicySnapshots(snapshots);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeCollection(snapshots);
+        }
+
+        @Override
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersions.REGISTER_SLM_STATS;
+        }
+    }
+
+    public Builder builder() {
+        return new Builder(this);
+    }
+
+    private static String getPolicyFromMetadata(Map<String, Object> userMetadata) {
+        if (userMetadata != null && userMetadata.get(POLICY_ID_METADATA_FIELD) instanceof String p) {
+            return p;
+        }
+        return null;
+    }
+
+    public static class Builder {
+        final List<PolicySnapshot> snapshots;
+
+        Builder(RegisteredPolicySnapshots registeredPolicySnapshots) {
+            this.snapshots = new ArrayList<>(registeredPolicySnapshots.snapshots);
+        }
+
+        /**
+         * Add the snapshotId to the registered set if its metadata contains a policyId, meaning that it was initiated by SLM.
+         * @param userMetadata metadata provided by the user in the CreateSnapshotRequest
+         *                     If the request is from SLM it will contain a key "policy" with an SLM policy name as the value.
+         * @param snapshotId the snapshotId to potentially add to the registered set
+         */
+        void maybeAdd(Map<String, Object> userMetadata, SnapshotId snapshotId) {
+            final String policy = getPolicyFromMetadata(userMetadata);
+            if (policy != null) {
+                snapshots.add(new PolicySnapshot(policy, snapshotId));
+            }
+        }
+
+        RegisteredPolicySnapshots build() {
+            return new RegisteredPolicySnapshots(snapshots);
+        }
+    }
+
+    public static class PolicySnapshot implements SimpleDiffable<PolicySnapshot>, Writeable, ToXContentObject {
+        private final String policy;
+        private final SnapshotId snapshotId;
+
+        private static final ParseField POLICY = new ParseField("policy");
+        private static final ParseField SNAPSHOT_ID = new ParseField("snapshot_id");
+
+        @SuppressWarnings("unchecked")
+        private static final ConstructingObjectParser<PolicySnapshot, String> PARSER = new ConstructingObjectParser<>(
+            "snapshot",
+            true,
+            (a, id) -> new PolicySnapshot((String) a[0], (SnapshotId) a[1])
+        );
+
+        static {
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), POLICY);
+            PARSER.declareObject(ConstructingObjectParser.constructorArg(), SnapshotId::parse, SNAPSHOT_ID);
+        }
+
+        public PolicySnapshot(String policy, SnapshotId snapshotId) {
+            this.policy = policy;
+            this.snapshotId = snapshotId;
+        }
+
+        public PolicySnapshot(StreamInput in) throws IOException {
+            this.policy = in.readString();
+            this.snapshotId = new SnapshotId(in);
+        }
+
+        public String getPolicy() {
+            return policy;
+        }
+
+        public SnapshotId getSnapshotId() {
+            return snapshotId;
+        }
+
+        public static PolicySnapshot parse(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(this.policy);
+            snapshotId.writeTo(out);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(POLICY.getPreferredName(), this.policy);
+            builder.field(SNAPSHOT_ID.getPreferredName(), this.snapshotId);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(policy, snapshotId);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || obj.getClass() != getClass()) {
+                return false;
+            }
+            PolicySnapshot other = (PolicySnapshot) obj;
+            return Objects.equals(policy, other.policy) && Objects.equals(snapshotId, other.snapshotId);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 25796606f2b1b..d8987495f9035 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.RefCountingRunnable;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateApplier;
@@ -56,8 +57,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
@@ -93,8 +92,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -188,6 +188,8 @@ public final class RestoreService implements ClusterStateApplier {
 
     private final ThreadPool threadPool;
 
+    private final Executor snapshotMetaExecutor;
+
     private volatile boolean refreshRepositoryUuidOnRestore;
 
     public RestoreService(
@@ -216,6 +218,7 @@ public RestoreService(
         this.indicesService = indicesService;
         this.fileSettingsService = fileSettingsService;
         this.threadPool = threadPool;
+        this.snapshotMetaExecutor = threadPool.executor(ThreadPool.Names.SNAPSHOT_META);
         this.refreshRepositoryUuidOnRestore = REFRESH_REPO_UUID_ON_RESTORE_SETTING.get(clusterService.getSettings());
         clusterService.getClusterSettings()
             .addSettingsUpdateConsumer(REFRESH_REPO_UUID_ON_RESTORE_SETTING, this::setRefreshRepositoryUuidOnRestore);
@@ -244,59 +247,67 @@ public void restoreSnapshot(
         final ActionListener<RestoreCompletionResponse> listener,
         final BiConsumer<ClusterState, Metadata.Builder> updater
     ) {
-        try {
-            // Try and fill in any missing repository UUIDs in case they're needed during the restore
-            final var repositoryUuidRefreshStep = new ListenableFuture<Void>();
-            refreshRepositoryUuids(refreshRepositoryUuidOnRestore, repositoriesService, () -> repositoryUuidRefreshStep.onResponse(null));
-
-            // Read snapshot info and metadata from the repository
-            final String repositoryName = request.repository();
-            Repository repository = repositoriesService.repository(repositoryName);
-            final ListenableFuture<RepositoryData> repositoryDataListener = new ListenableFuture<>();
-            repository.getRepositoryData(
-                EsExecutors.DIRECT_EXECUTOR_SERVICE, // TODO contemplate threading here, do we need to fork, see #101445?
-                repositoryDataListener
-            );
+        assert Repository.assertSnapshotMetaThread();
 
-            repositoryDataListener.addListener(
-                listener.delegateFailureAndWrap(
-                    (delegate, repositoryData) -> repositoryUuidRefreshStep.addListener(
-                        delegate.delegateFailureAndWrap((subDelegate, ignored) -> {
-                            final String snapshotName = request.snapshot();
-                            final Optional<SnapshotId> matchingSnapshotId = repositoryData.getSnapshotIds()
-                                .stream()
-                                .filter(s -> snapshotName.equals(s.getName()))
-                                .findFirst();
-                            if (matchingSnapshotId.isPresent() == false) {
-                                throw new SnapshotRestoreException(repositoryName, snapshotName, "snapshot does not exist");
-                            }
+        // Try and fill in any missing repository UUIDs in case they're needed during the restore
+        final var repositoryUuidRefreshStep = SubscribableListener.newForked(
+            l -> refreshRepositoryUuids(refreshRepositoryUuidOnRestore, repositoriesService, () -> l.onResponse(null), snapshotMetaExecutor)
+        );
 
-                            final SnapshotId snapshotId = matchingSnapshotId.get();
-                            if (request.snapshotUuid() != null && request.snapshotUuid().equals(snapshotId.getUUID()) == false) {
-                                throw new SnapshotRestoreException(
-                                    repositoryName,
-                                    snapshotName,
-                                    "snapshot UUID mismatch: expected ["
-                                        + request.snapshotUuid()
-                                        + "] but got ["
-                                        + snapshotId.getUUID()
-                                        + "]"
-                                );
-                            }
-                            repository.getSnapshotInfo(
-                                snapshotId,
-                                subDelegate.delegateFailureAndWrap(
-                                    (l, snapshotInfo) -> startRestore(snapshotInfo, repository, request, repositoryData, updater, l)
-                                )
-                            );
-                        })
-                    )
+        // AtomicReference just so we have somewhere to hold these objects, there's no interesting concurrency here
+        final AtomicReference<Repository> repositoryRef = new AtomicReference<>();
+        final AtomicReference<RepositoryData> repositoryDataRef = new AtomicReference<>();
+
+        SubscribableListener
+
+            .<Void>newForked(repositorySetListener -> {
+                // do this within newForked for exception handling
+                repositoryRef.set(repositoriesService.repository(request.repository()));
+                repositorySetListener.onResponse(null);
+            })
+
+            .<RepositoryData>andThen(
+                repositoryDataListener -> repositoryRef.get().getRepositoryData(snapshotMetaExecutor, repositoryDataListener)
+            )
+            .andThenAccept(repositoryDataRef::set)
+            .andThen(repositoryUuidRefreshStep::addListener)
+
+            .<SnapshotInfo>andThen(snapshotInfoListener -> {
+                assert Repository.assertSnapshotMetaThread();
+                final String snapshotName = request.snapshot();
+                final SnapshotId snapshotId = repositoryDataRef.get()
+                    .getSnapshotIds()
+                    .stream()
+                    .filter(s -> snapshotName.equals(s.getName()))
+                    .findFirst()
+                    .orElseThrow(() -> new SnapshotRestoreException(request.repository(), snapshotName, "snapshot does not exist"));
+
+                if (request.snapshotUuid() != null && request.snapshotUuid().equals(snapshotId.getUUID()) == false) {
+                    throw new SnapshotRestoreException(
+                        request.repository(),
+                        snapshotName,
+                        "snapshot UUID mismatch: expected [" + request.snapshotUuid() + "] but got [" + snapshotId.getUUID() + "]"
+                    );
+                }
+
+                repositoryRef.get().getSnapshotInfo(snapshotId, snapshotInfoListener);
+            })
+
+            .<RestoreCompletionResponse>andThen(
+                (responseListener, snapshotInfo) -> startRestore(
+                    snapshotInfo,
+                    repositoryRef.get(),
+                    request,
+                    repositoryDataRef.get(),
+                    updater,
+                    responseListener
                 )
-            );
-        } catch (Exception e) {
-            logger.warn(() -> "[" + request.repository() + ":" + request.snapshot() + "] failed to restore snapshot", e);
-            listener.onFailure(e);
-        }
+            )
+
+            .addListener(listener.delegateResponse((delegate, e) -> {
+                logger.warn(() -> "[" + request.repository() + ":" + request.snapshot() + "] failed to restore snapshot", e);
+                delegate.onFailure(e);
+            }));
     }
 
     /**
@@ -511,12 +522,18 @@ private void setRefreshRepositoryUuidOnRestore(boolean refreshRepositoryUuidOnRe
      * Best-effort attempt to make sure that we know all the repository UUIDs. Calls {@link Repository#getRepositoryData} on every
      * {@link BlobStoreRepository} with a missing UUID.
      *
-     * @param enabled If {@code false} this method completes the listener immediately
+     * @param enabled             If {@code false} this method completes the listener immediately
      * @param repositoriesService Supplies the repositories to check
-     * @param onCompletion Action that is executed when all repositories have been refreshed.
+     * @param onCompletion        Action that is executed when all repositories have been refreshed.
+     * @param responseExecutor    Executor on which to execute {@code onCompletion} if not using the calling thread.
      */
     // Exposed for tests
-    static void refreshRepositoryUuids(boolean enabled, RepositoriesService repositoriesService, Runnable onCompletion) {
+    static void refreshRepositoryUuids(
+        boolean enabled,
+        RepositoriesService repositoriesService,
+        Runnable onCompletion,
+        Executor responseExecutor
+    ) {
         try (var refs = new RefCountingRunnable(onCompletion)) {
             if (enabled == false) {
                 logger.debug("repository UUID refresh is disabled");
@@ -530,20 +547,17 @@ static void refreshRepositoryUuids(boolean enabled, RepositoriesService reposito
                 if (repository instanceof BlobStoreRepository && repository.getMetadata().uuid().equals(RepositoryData.MISSING_UUID)) {
                     final var repositoryName = repository.getMetadata().name();
                     logger.info("refreshing repository UUID for repository [{}]", repositoryName);
-                    repository.getRepositoryData(
-                        EsExecutors.DIRECT_EXECUTOR_SERVICE, // TODO contemplate threading here, do we need to fork, see #101445?
-                        ActionListener.releaseAfter(new ActionListener<>() {
-                            @Override
-                            public void onResponse(RepositoryData repositoryData) {
-                                logger.debug(() -> format("repository UUID [%s] refresh completed", repositoryName));
-                            }
+                    repository.getRepositoryData(responseExecutor, ActionListener.releaseAfter(new ActionListener<>() {
+                        @Override
+                        public void onResponse(RepositoryData repositoryData) {
+                            logger.debug(() -> format("repository UUID [%s] refresh completed", repositoryName));
+                        }
 
-                            @Override
-                            public void onFailure(Exception e) {
-                                logger.debug(() -> format("repository UUID [%s] refresh failed", repositoryName), e);
-                            }
-                        }, refs.acquire())
-                    );
+                        @Override
+                        public void onFailure(Exception e) {
+                            logger.debug(() -> format("repository UUID [%s] refresh failed", repositoryName), e);
+                        }
+                    }, refs.acquire()));
                 }
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotDeleteListener.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotDeleteListener.java
deleted file mode 100644
index 324ad736d7248..0000000000000
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotDeleteListener.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.snapshots;
-
-import org.elasticsearch.repositories.RepositoryData;
-
-public interface SnapshotDeleteListener {
-
-    /**
-     * Invoked once the snapshots have been fully deleted from the repository, including all async cleanup operations, indicating that
-     * listeners waiting for the end of the deletion can now be notified.
-     */
-    void onDone();
-
-    /**
-     * Invoked once the updated {@link RepositoryData} has been written to the repository and it is safe for the next repository operation
-     * to proceed.
-     *
-     * @param repositoryData updated repository data
-     */
-    void onRepositoryDataWritten(RepositoryData repositoryData);
-
-    /**
-     * Invoked if writing updated {@link RepositoryData} to the repository failed. Once {@link #onRepositoryDataWritten(RepositoryData)} has
-     * been invoked this method will never be invoked.
-     *
-     * @param e exception during metadata steps of snapshot delete
-     */
-    void onFailure(Exception e);
-}
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotId.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
index 8a67572040116..2863a6f713510 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
@@ -11,8 +11,11 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -22,8 +25,19 @@
  */
 public final class SnapshotId implements Comparable<SnapshotId>, Writeable, ToXContentObject {
 
-    private static final String NAME = "name";
-    private static final String UUID = "uuid";
+    private static final ParseField NAME = new ParseField("name");
+    private static final ParseField UUID = new ParseField("uuid");
+
+    public static final ConstructingObjectParser<SnapshotId, String> PARSER = new ConstructingObjectParser<>(
+        "snapshot_id",
+        true,
+        a -> new SnapshotId((String) a[0], (String) a[1])
+    );
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), UUID);
+    }
 
     private final String name;
     private final String uuid;
@@ -54,6 +68,10 @@ public SnapshotId(final StreamInput in) throws IOException {
         hashCode = computeHashCode();
     }
 
+    public static SnapshotId parse(XContentParser parser, String text) {
+        return PARSER.apply(parser, text);
+    }
+
     /**
      * Returns snapshot name
      *
@@ -112,8 +130,8 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(NAME, name);
-        builder.field(UUID, uuid);
+        builder.field(NAME.getPreferredName(), name);
+        builder.field(UUID.getPreferredName(), uuid);
         builder.endObject();
         return builder;
     }
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 75b5a4e6a2ea6..ed88b7272245f 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.action.support.RefCountingRunnable;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
@@ -277,9 +278,7 @@ public void createSnapshot(final CreateSnapshotRequest request, final ActionList
         final String repositoryName = request.repository();
         final String snapshotName = IndexNameExpressionResolver.resolveDateMathExpression(request.snapshot());
         validate(repositoryName, snapshotName);
-        // TODO: create snapshot UUID in CreateSnapshotRequest and make this operation idempotent to cleanly deal with transport layer
-        // retries
-        final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); // new UUID for the snapshot
+        final SnapshotId snapshotId = new SnapshotId(snapshotName, request.uuid());
         Repository repository = repositoriesService.repository(request.repository());
         if (repository.isReadOnly()) {
             listener.onFailure(new RepositoryException(repository.getMetadata().name(), "cannot create snapshot in a readonly repository"));
@@ -2493,19 +2492,11 @@ private void deleteSnapshotsFromRepository(
                 );
                 return;
             }
+            final SubscribableListener<Void> doneFuture = new SubscribableListener<>();
             repositoriesService.repository(deleteEntry.repository())
-                .deleteSnapshots(snapshotIds, repositoryData.getGenId(), minNodeVersion, new SnapshotDeleteListener() {
-
-                    private final ListenableFuture<Void> doneFuture = new ListenableFuture<>();
-
-                    @Override
-                    public void onDone() {
-                        logger.info("snapshots {} deleted", snapshotIds);
-                        doneFuture.onResponse(null);
-                    }
-
+                .deleteSnapshots(snapshotIds, repositoryData.getGenId(), minNodeVersion, new ActionListener<>() {
                     @Override
-                    public void onRepositoryDataWritten(RepositoryData updatedRepoData) {
+                    public void onResponse(RepositoryData updatedRepoData) {
                         removeSnapshotDeletionFromClusterState(
                             deleteEntry,
                             updatedRepoData,
@@ -2551,6 +2542,9 @@ protected void handleListeners(List<ActionListener<Void>> deleteListeners) {
                             }
                         );
                     }
+                }, () -> {
+                    logger.info("snapshots {} deleted", snapshotIds);
+                    doneFuture.onResponse(null);
                 });
         }
     }
@@ -3956,9 +3950,13 @@ public ClusterState execute(BatchExecutionContext<SnapshotTask> batchExecutionCo
             );
             final SnapshotsInProgress initialSnapshots = SnapshotsInProgress.get(state);
             SnapshotsInProgress snapshotsInProgress = shardsUpdateContext.computeUpdatedState();
+            final RegisteredPolicySnapshots.Builder registeredPolicySnapshots = state.metadata()
+                .custom(RegisteredPolicySnapshots.TYPE, RegisteredPolicySnapshots.EMPTY)
+                .builder();
             for (final var taskContext : batchExecutionContext.taskContexts()) {
                 if (taskContext.getTask() instanceof CreateSnapshotTask task) {
                     try {
+                        registeredPolicySnapshots.maybeAdd(task.createSnapshotRequest.userMetadata(), task.snapshot.getSnapshotId());
                         final var repoMeta = RepositoriesMetadata.get(state).repository(task.snapshot.getRepository());
                         if (Objects.equals(task.initialRepositoryMetadata, repoMeta)) {
                             snapshotsInProgress = createSnapshot(task, taskContext, state, snapshotsInProgress);
@@ -3983,7 +3981,11 @@ public ClusterState execute(BatchExecutionContext<SnapshotTask> batchExecutionCo
             if (snapshotsInProgress == initialSnapshots) {
                 return state;
             }
-            return ClusterState.builder(state).putCustom(SnapshotsInProgress.TYPE, snapshotsInProgress).build();
+
+            return ClusterState.builder(state)
+                .putCustom(SnapshotsInProgress.TYPE, snapshotsInProgress)
+                .metadata(Metadata.builder(state.metadata()).putCustom(RegisteredPolicySnapshots.TYPE, registeredPolicySnapshots.build()))
+                .build();
         }
 
         private SnapshotsInProgress createSnapshot(
diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
index 8632cd6cfea77..adcc91cd35089 100644
--- a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
+++ b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
@@ -19,12 +19,12 @@
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.client.internal.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/elasticsearch/telemetry/metric/MeterRegistry.java b/server/src/main/java/org/elasticsearch/telemetry/metric/MeterRegistry.java
index 0f690558361e4..12c62859fd372 100644
--- a/server/src/main/java/org/elasticsearch/telemetry/metric/MeterRegistry.java
+++ b/server/src/main/java/org/elasticsearch/telemetry/metric/MeterRegistry.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.telemetry.metric;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.function.Supplier;
 
 /**
@@ -15,6 +17,7 @@
  * only be registered once.
  * TODO(stu): describe name, unit and description
  */
+
 public interface MeterRegistry {
     /**
      * Register a {@link DoubleCounter}.  The returned object may be reused.
@@ -57,7 +60,20 @@ public interface MeterRegistry {
      *                 Must not throw an exception and must be safe to call from different threads.
      * @return the registered meter.
      */
-    DoubleGauge registerDoubleGauge(String name, String description, String unit, Supplier<DoubleWithAttributes> observer);
+    default DoubleGauge registerDoubleGauge(String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+        return registerDoublesGauge(name, description, unit, () -> Collections.singleton(observer.get()));
+    }
+
+    /**
+     * Register a {@link DoubleGauge}.  The returned object may be reused.
+     * @param name name of the gauge
+     * @param description description of purpose
+     * @param unit the unit (bytes, sec, hour)
+     * @param observer callback to use. This is called once during reporting period.
+     *                 Must not throw an exception and must be safe to call from different threads.
+     * @return the registered meter.
+     */
+    DoubleGauge registerDoublesGauge(String name, String description, String unit, Supplier<Collection<DoubleWithAttributes>> observer);
 
     /**
      * Retrieved a previously registered {@link DoubleGauge}.
@@ -98,7 +114,23 @@ public interface MeterRegistry {
      * @param unit the unit (bytes, sec, hour)
      * @param observer a callback to provide a metric value upon observation (metric interval)
      */
-    LongAsyncCounter registerLongAsyncCounter(String name, String description, String unit, Supplier<LongWithAttributes> observer);
+    default LongAsyncCounter registerLongAsyncCounter(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+        return registerLongsAsyncCounter(name, description, unit, () -> Collections.singleton(observer.get()));
+    }
+
+    /**
+     * Register a {@link LongAsyncCounter} with an asynchronous callback.  The returned object may be reused.
+     * @param name name of the counter
+     * @param description description of purpose
+     * @param unit the unit (bytes, sec, hour)
+     * @param observer a callback to provide a metric values upon observation (metric interval)
+     */
+    LongAsyncCounter registerLongsAsyncCounter(
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<LongWithAttributes>> observer
+    );
 
     /**
      * Retrieved a previously registered {@link LongAsyncCounter}.
@@ -114,7 +146,28 @@ public interface MeterRegistry {
      * @param unit the unit (bytes, sec, hour)
      * @param observer a callback to provide a metric value upon observation (metric interval)
      */
-    DoubleAsyncCounter registerDoubleAsyncCounter(String name, String description, String unit, Supplier<DoubleWithAttributes> observer);
+    default DoubleAsyncCounter registerDoubleAsyncCounter(
+        String name,
+        String description,
+        String unit,
+        Supplier<DoubleWithAttributes> observer
+    ) {
+        return registerDoublesAsyncCounter(name, description, unit, () -> Collections.singleton(observer.get()));
+    }
+
+    /**
+     * Register a {@link DoubleAsyncCounter} with an asynchronous callback.  The returned object may be reused.
+     * @param name name of the counter
+     * @param description description of purpose
+     * @param unit the unit (bytes, sec, hour)
+     * @param observer a callback to provide a metric values upon observation (metric interval)
+     */
+    DoubleAsyncCounter registerDoublesAsyncCounter(
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<DoubleWithAttributes>> observer
+    );
 
     /**
      * Retrieved a previously registered {@link DoubleAsyncCounter}.
@@ -155,7 +208,20 @@ public interface MeterRegistry {
      *                 Must not throw an exception and must be safe to call from different threads.
      * @return the registered meter.
      */
-    LongGauge registerLongGauge(String name, String description, String unit, Supplier<LongWithAttributes> observer);
+    default LongGauge registerLongGauge(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+        return registerLongsGauge(name, description, unit, () -> Collections.singleton(observer.get()));
+    }
+
+    /**
+     * Register a {@link LongGauge}.  The returned object may be reused.
+     * @param name name of the gauge
+     * @param description description of purpose
+     * @param unit the unit (bytes, sec, hour)
+     * @param observer callback to use. This is called once during reporting period.
+     *                 Must not throw an exception and must be safe to call from different threads.
+     * @return the registered meter.
+     */
+    LongGauge registerLongsGauge(String name, String description, String unit, Supplier<Collection<LongWithAttributes>> observer);
 
     /**
      * Retrieved a previously registered {@link LongGauge}.
@@ -204,7 +270,12 @@ public DoubleUpDownCounter getDoubleUpDownCounter(String name) {
         }
 
         @Override
-        public DoubleGauge registerDoubleGauge(String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+        public DoubleGauge registerDoublesGauge(
+            String name,
+            String description,
+            String unit,
+            Supplier<Collection<DoubleWithAttributes>> observer
+        ) {
             return DoubleGauge.NOOP;
         }
 
@@ -229,11 +300,11 @@ public LongCounter registerLongCounter(String name, String description, String u
         }
 
         @Override
-        public LongAsyncCounter registerLongAsyncCounter(
+        public LongAsyncCounter registerLongsAsyncCounter(
             String name,
             String description,
             String unit,
-            Supplier<LongWithAttributes> observer
+            Supplier<Collection<LongWithAttributes>> observer
         ) {
             return LongAsyncCounter.NOOP;
         }
@@ -244,11 +315,11 @@ public LongAsyncCounter getLongAsyncCounter(String name) {
         }
 
         @Override
-        public DoubleAsyncCounter registerDoubleAsyncCounter(
+        public DoubleAsyncCounter registerDoublesAsyncCounter(
             String name,
             String description,
             String unit,
-            Supplier<DoubleWithAttributes> observer
+            Supplier<Collection<DoubleWithAttributes>> observer
         ) {
             return DoubleAsyncCounter.NOOP;
         }
@@ -274,7 +345,12 @@ public LongUpDownCounter getLongUpDownCounter(String name) {
         }
 
         @Override
-        public LongGauge registerLongGauge(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+        public LongGauge registerLongsGauge(
+            String name,
+            String description,
+            String unit,
+            Supplier<Collection<LongWithAttributes>> observer
+        ) {
             return LongGauge.NOOP;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java b/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java
new file mode 100644
index 0000000000000..a4046f2f1594c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java
@@ -0,0 +1,216 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.threadpool;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.threadpool.internal.BuiltInExecutorBuilders;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.unmodifiableMap;
+import static org.elasticsearch.threadpool.ThreadPool.indexAutoscalingEWMA;
+import static org.elasticsearch.threadpool.ThreadPool.searchAutoscalingEWMA;
+
+public class DefaultBuiltInExecutorBuilders implements BuiltInExecutorBuilders {
+    @Override
+    @SuppressWarnings("rawtypes")
+    public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocatedProcessors) {
+        final int halfProc = ThreadPool.halfAllocatedProcessors(allocatedProcessors);
+        final int halfProcMaxAt5 = ThreadPool.halfAllocatedProcessorsMaxFive(allocatedProcessors);
+        final int halfProcMaxAt10 = ThreadPool.halfAllocatedProcessorsMaxTen(allocatedProcessors);
+        final int genericThreadPoolMax = ThreadPool.boundedBy(4 * allocatedProcessors, 128, 512);
+
+        Map<String, ExecutorBuilder> result = new HashMap<>();
+        result.put(
+            ThreadPool.Names.GENERIC,
+            new ScalingExecutorBuilder(ThreadPool.Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30), false)
+        );
+        result.put(
+            ThreadPool.Names.WRITE,
+            new FixedExecutorBuilder(
+                settings,
+                ThreadPool.Names.WRITE,
+                allocatedProcessors,
+                10000,
+                new EsExecutors.TaskTrackingConfig(true, indexAutoscalingEWMA)
+            )
+        );
+        int searchOrGetThreadPoolSize = ThreadPool.searchOrGetThreadPoolSize(allocatedProcessors);
+        result.put(
+            ThreadPool.Names.GET,
+            new FixedExecutorBuilder(
+                settings,
+                ThreadPool.Names.GET,
+                searchOrGetThreadPoolSize,
+                1000,
+                EsExecutors.TaskTrackingConfig.DO_NOT_TRACK
+            )
+        );
+        result.put(
+            ThreadPool.Names.ANALYZE,
+            new FixedExecutorBuilder(settings, ThreadPool.Names.ANALYZE, 1, 16, EsExecutors.TaskTrackingConfig.DO_NOT_TRACK)
+        );
+        result.put(
+            ThreadPool.Names.SEARCH,
+            new FixedExecutorBuilder(
+                settings,
+                ThreadPool.Names.SEARCH,
+                searchOrGetThreadPoolSize,
+                1000,
+                new EsExecutors.TaskTrackingConfig(true, searchAutoscalingEWMA)
+            )
+        );
+        result.put(
+            ThreadPool.Names.SEARCH_WORKER,
+            new FixedExecutorBuilder(
+                settings,
+                ThreadPool.Names.SEARCH_WORKER,
+                searchOrGetThreadPoolSize,
+                -1,
+                EsExecutors.TaskTrackingConfig.DEFAULT
+            )
+        );
+        result.put(
+            ThreadPool.Names.SEARCH_COORDINATION,
+            new FixedExecutorBuilder(
+                settings,
+                ThreadPool.Names.SEARCH_COORDINATION,
+                halfProc,
+                1000,
+                new EsExecutors.TaskTrackingConfig(true, searchAutoscalingEWMA)
+            )
+        );
+        result.put(
+            ThreadPool.Names.AUTO_COMPLETE,
+            new FixedExecutorBuilder(
+                settings,
+                ThreadPool.Names.AUTO_COMPLETE,
+                Math.max(allocatedProcessors / 4, 1),
+                100,
+                EsExecutors.TaskTrackingConfig.DEFAULT
+            )
+        );
+        result.put(
+            ThreadPool.Names.SEARCH_THROTTLED,
+            new FixedExecutorBuilder(settings, ThreadPool.Names.SEARCH_THROTTLED, 1, 100, EsExecutors.TaskTrackingConfig.DEFAULT)
+        );
+        result.put(
+            ThreadPool.Names.MANAGEMENT,
+            new ScalingExecutorBuilder(
+                ThreadPool.Names.MANAGEMENT,
+                1,
+                ThreadPool.boundedBy(allocatedProcessors, 1, 5),
+                TimeValue.timeValueMinutes(5),
+                false
+            )
+        );
+        result.put(
+            ThreadPool.Names.FLUSH,
+            new ScalingExecutorBuilder(ThreadPool.Names.FLUSH, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5), false)
+        );
+        // TODO: remove (or refine) this temporary stateless custom refresh pool sizing once ES-7631 is solved.
+        final int refreshThreads = DiscoveryNode.isStateless(settings) ? allocatedProcessors : halfProcMaxAt10;
+        result.put(
+            ThreadPool.Names.REFRESH,
+            new ScalingExecutorBuilder(ThreadPool.Names.REFRESH, 1, refreshThreads, TimeValue.timeValueMinutes(5), false)
+        );
+        result.put(
+            ThreadPool.Names.WARMER,
+            new ScalingExecutorBuilder(ThreadPool.Names.WARMER, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5), false)
+        );
+        final int maxSnapshotCores = ThreadPool.getMaxSnapshotThreadPoolSize(allocatedProcessors);
+        result.put(
+            ThreadPool.Names.SNAPSHOT,
+            new ScalingExecutorBuilder(ThreadPool.Names.SNAPSHOT, 1, maxSnapshotCores, TimeValue.timeValueMinutes(5), false)
+        );
+        result.put(
+            ThreadPool.Names.SNAPSHOT_META,
+            new ScalingExecutorBuilder(
+                ThreadPool.Names.SNAPSHOT_META,
+                1,
+                Math.min(allocatedProcessors * 3, 50),
+                TimeValue.timeValueSeconds(30L),
+                false
+            )
+        );
+        result.put(
+            ThreadPool.Names.FETCH_SHARD_STARTED,
+            new ScalingExecutorBuilder(
+                ThreadPool.Names.FETCH_SHARD_STARTED,
+                1,
+                2 * allocatedProcessors,
+                TimeValue.timeValueMinutes(5),
+                false
+            )
+        );
+        result.put(
+            ThreadPool.Names.FORCE_MERGE,
+            new FixedExecutorBuilder(
+                settings,
+                ThreadPool.Names.FORCE_MERGE,
+                ThreadPool.oneEighthAllocatedProcessors(allocatedProcessors),
+                -1,
+                EsExecutors.TaskTrackingConfig.DO_NOT_TRACK
+            )
+        );
+        result.put(
+            ThreadPool.Names.CLUSTER_COORDINATION,
+            new FixedExecutorBuilder(settings, ThreadPool.Names.CLUSTER_COORDINATION, 1, -1, EsExecutors.TaskTrackingConfig.DO_NOT_TRACK)
+        );
+        result.put(
+            ThreadPool.Names.FETCH_SHARD_STORE,
+            new ScalingExecutorBuilder(ThreadPool.Names.FETCH_SHARD_STORE, 1, 2 * allocatedProcessors, TimeValue.timeValueMinutes(5), false)
+        );
+        result.put(
+            ThreadPool.Names.SYSTEM_READ,
+            new FixedExecutorBuilder(
+                settings,
+                ThreadPool.Names.SYSTEM_READ,
+                halfProcMaxAt5,
+                2000,
+                EsExecutors.TaskTrackingConfig.DO_NOT_TRACK
+            )
+        );
+        result.put(
+            ThreadPool.Names.SYSTEM_WRITE,
+            new FixedExecutorBuilder(
+                settings,
+                ThreadPool.Names.SYSTEM_WRITE,
+                halfProcMaxAt5,
+                1000,
+                new EsExecutors.TaskTrackingConfig(true, indexAutoscalingEWMA)
+            )
+        );
+        result.put(
+            ThreadPool.Names.SYSTEM_CRITICAL_READ,
+            new FixedExecutorBuilder(
+                settings,
+                ThreadPool.Names.SYSTEM_CRITICAL_READ,
+                halfProcMaxAt5,
+                2000,
+                EsExecutors.TaskTrackingConfig.DO_NOT_TRACK
+            )
+        );
+        result.put(
+            ThreadPool.Names.SYSTEM_CRITICAL_WRITE,
+            new FixedExecutorBuilder(
+                settings,
+                ThreadPool.Names.SYSTEM_CRITICAL_WRITE,
+                halfProcMaxAt5,
+                1500,
+                new EsExecutors.TaskTrackingConfig(true, indexAutoscalingEWMA)
+            )
+        );
+        return unmodifiableMap(result);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 88c507404e76b..10b92f8c6dace 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -10,7 +10,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -20,7 +19,6 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.SizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.util.concurrent.EsExecutors.TaskTrackingConfig;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionHandler;
 import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
@@ -35,6 +33,7 @@
 import org.elasticsearch.telemetry.metric.LongGauge;
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
+import org.elasticsearch.threadpool.internal.BuiltInExecutorBuilders;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -55,6 +54,7 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
+import java.util.function.LongSupplier;
 import java.util.stream.Collectors;
 
 import static java.util.Map.entry;
@@ -178,12 +178,21 @@ public static ThreadPoolType fromType(String type) {
 
     public static final double searchAutoscalingEWMA = 0.1;
 
+    // This value is chosen such that a sudden increase in the task durations would need to persist roughly for 120 samples
+    // for the EWMA value to be mostly representative of the increased task durations. Mostly representative means that the
+    // EWMA value is at least within 90% of the new increased task duration. This value also determines the impact of a single
+    // long-running task on the moving average and limits it roughly to 2% of the (long) task duration, e.g. if the current
+    // moving average is 100ms, and we get one task which takes 20s the new EWMA will be ~500ms.
+    public static final double indexAutoscalingEWMA = 0.02;
+
     private final Map<String, ExecutorHolder> executors;
 
     private final ThreadPoolInfo threadPoolInfo;
 
     private final CachedTimeThread cachedTimeThread;
 
+    private final LongSupplier relativeTimeInMillisSupplier;
+
     private final ThreadContext threadContext;
 
     @SuppressWarnings("rawtypes")
@@ -226,125 +235,22 @@ public Collection<ExecutorBuilder> builders() {
      *
      * @param settings
      * @param meterRegistry
+     * @param builtInExecutorBuilders used to construct builders for the built-in thread pools
      * @param customBuilders a list of additional thread pool builders that were defined elsewhere (like a Plugin).
      */
     @SuppressWarnings({ "rawtypes", "unchecked" })
-    public ThreadPool(final Settings settings, MeterRegistry meterRegistry, final ExecutorBuilder<?>... customBuilders) {
+    public ThreadPool(
+        final Settings settings,
+        MeterRegistry meterRegistry,
+        BuiltInExecutorBuilders builtInExecutorBuilders,
+        final ExecutorBuilder<?>... customBuilders
+    ) {
         assert Node.NODE_NAME_SETTING.exists(settings);
 
-        final Map<String, ExecutorBuilder> builders = new HashMap<>();
-        final int allocatedProcessors = EsExecutors.allocatedProcessors(settings);
-        final int halfProc = halfAllocatedProcessors(allocatedProcessors);
-        final int halfProcMaxAt5 = halfAllocatedProcessorsMaxFive(allocatedProcessors);
-        final int halfProcMaxAt10 = halfAllocatedProcessorsMaxTen(allocatedProcessors);
-        final int genericThreadPoolMax = boundedBy(4 * allocatedProcessors, 128, 512);
         final Map<String, ArrayList<Instrument>> instruments = new HashMap<>();
+        final int allocatedProcessors = EsExecutors.allocatedProcessors(settings);
 
-        builders.put(
-            Names.GENERIC,
-            new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30), false)
-        );
-        builders.put(
-            Names.WRITE,
-            new FixedExecutorBuilder(settings, Names.WRITE, allocatedProcessors, 10000, new TaskTrackingConfig(true, 0.1))
-        );
-        int searchOrGetThreadPoolSize = searchOrGetThreadPoolSize(allocatedProcessors);
-        builders.put(
-            Names.GET,
-            new FixedExecutorBuilder(settings, Names.GET, searchOrGetThreadPoolSize, 1000, TaskTrackingConfig.DO_NOT_TRACK)
-        );
-        builders.put(Names.ANALYZE, new FixedExecutorBuilder(settings, Names.ANALYZE, 1, 16, TaskTrackingConfig.DO_NOT_TRACK));
-        builders.put(
-            Names.SEARCH,
-            new FixedExecutorBuilder(
-                settings,
-                Names.SEARCH,
-                searchOrGetThreadPoolSize,
-                1000,
-                new TaskTrackingConfig(true, searchAutoscalingEWMA)
-            )
-        );
-        builders.put(
-            Names.SEARCH_WORKER,
-            new FixedExecutorBuilder(settings, Names.SEARCH_WORKER, searchOrGetThreadPoolSize, -1, TaskTrackingConfig.DEFAULT)
-        );
-        builders.put(
-            Names.SEARCH_COORDINATION,
-            new FixedExecutorBuilder(
-                settings,
-                Names.SEARCH_COORDINATION,
-                halfProc,
-                1000,
-                new TaskTrackingConfig(true, searchAutoscalingEWMA)
-            )
-        );
-        builders.put(
-            Names.AUTO_COMPLETE,
-            new FixedExecutorBuilder(settings, Names.AUTO_COMPLETE, Math.max(allocatedProcessors / 4, 1), 100, TaskTrackingConfig.DEFAULT)
-        );
-        builders.put(
-            Names.SEARCH_THROTTLED,
-            new FixedExecutorBuilder(settings, Names.SEARCH_THROTTLED, 1, 100, TaskTrackingConfig.DEFAULT)
-        );
-        builders.put(
-            Names.MANAGEMENT,
-            new ScalingExecutorBuilder(Names.MANAGEMENT, 1, boundedBy(allocatedProcessors, 1, 5), TimeValue.timeValueMinutes(5), false)
-        );
-        builders.put(Names.FLUSH, new ScalingExecutorBuilder(Names.FLUSH, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5), false));
-        // TODO: remove (or refine) this temporary stateless custom refresh pool sizing once ES-7631 is solved.
-        final int refreshThreads = DiscoveryNode.isStateless(settings) ? allocatedProcessors : halfProcMaxAt10;
-        builders.put(Names.REFRESH, new ScalingExecutorBuilder(Names.REFRESH, 1, refreshThreads, TimeValue.timeValueMinutes(5), false));
-        builders.put(Names.WARMER, new ScalingExecutorBuilder(Names.WARMER, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5), false));
-        final int maxSnapshotCores = getMaxSnapshotThreadPoolSize(allocatedProcessors);
-        builders.put(Names.SNAPSHOT, new ScalingExecutorBuilder(Names.SNAPSHOT, 1, maxSnapshotCores, TimeValue.timeValueMinutes(5), false));
-        builders.put(
-            Names.SNAPSHOT_META,
-            new ScalingExecutorBuilder(
-                Names.SNAPSHOT_META,
-                1,
-                Math.min(allocatedProcessors * 3, 50),
-                TimeValue.timeValueSeconds(30L),
-                false
-            )
-        );
-        builders.put(
-            Names.FETCH_SHARD_STARTED,
-            new ScalingExecutorBuilder(Names.FETCH_SHARD_STARTED, 1, 2 * allocatedProcessors, TimeValue.timeValueMinutes(5), false)
-        );
-        builders.put(
-            Names.FORCE_MERGE,
-            new FixedExecutorBuilder(
-                settings,
-                Names.FORCE_MERGE,
-                oneEighthAllocatedProcessors(allocatedProcessors),
-                -1,
-                TaskTrackingConfig.DO_NOT_TRACK
-            )
-        );
-        builders.put(
-            Names.CLUSTER_COORDINATION,
-            new FixedExecutorBuilder(settings, Names.CLUSTER_COORDINATION, 1, -1, TaskTrackingConfig.DO_NOT_TRACK)
-        );
-        builders.put(
-            Names.FETCH_SHARD_STORE,
-            new ScalingExecutorBuilder(Names.FETCH_SHARD_STORE, 1, 2 * allocatedProcessors, TimeValue.timeValueMinutes(5), false)
-        );
-        builders.put(
-            Names.SYSTEM_READ,
-            new FixedExecutorBuilder(settings, Names.SYSTEM_READ, halfProcMaxAt5, 2000, TaskTrackingConfig.DO_NOT_TRACK)
-        );
-        builders.put(
-            Names.SYSTEM_WRITE,
-            new FixedExecutorBuilder(settings, Names.SYSTEM_WRITE, halfProcMaxAt5, 1000, new TaskTrackingConfig(true, 0.1))
-        );
-        builders.put(
-            Names.SYSTEM_CRITICAL_READ,
-            new FixedExecutorBuilder(settings, Names.SYSTEM_CRITICAL_READ, halfProcMaxAt5, 2000, TaskTrackingConfig.DO_NOT_TRACK)
-        );
-        builders.put(
-            Names.SYSTEM_CRITICAL_WRITE,
-            new FixedExecutorBuilder(settings, Names.SYSTEM_CRITICAL_WRITE, halfProcMaxAt5, 1500, new TaskTrackingConfig(true, 0.1))
-        );
+        final Map<String, ExecutorBuilder> builders = new HashMap<>(builtInExecutorBuilders.getBuilders(settings, allocatedProcessors));
 
         for (final ExecutorBuilder<?> builder : customBuilders) {
             if (builders.containsKey(builder.name())) {
@@ -380,6 +286,7 @@ public ThreadPool(final Settings settings, MeterRegistry meterRegistry, final Ex
             LATE_TIME_INTERVAL_WARN_THRESHOLD_SETTING.get(settings).millis()
         );
         this.cachedTimeThread.start();
+        this.relativeTimeInMillisSupplier = new RelativeTimeInMillisSupplier(cachedTimeThread);
     }
 
     private static ArrayList<Instrument> setupMetrics(MeterRegistry meterRegistry, String name, ExecutorHolder holder) {
@@ -440,6 +347,7 @@ protected ThreadPool() {
         this.builders = Map.of();
         this.executors = Map.of();
         this.cachedTimeThread = null;
+        this.relativeTimeInMillisSupplier = this::relativeTimeInMillis;
         this.threadPoolInfo = new ThreadPoolInfo(List.of());
         this.slowSchedulerWarnThresholdNanos = 0L;
         this.threadContext = new ThreadContext(Settings.EMPTY);
@@ -453,7 +361,14 @@ protected ThreadPool() {
      * timestamp, see {@link #absoluteTimeInMillis()}.
      */
     public long relativeTimeInMillis() {
-        return TimeValue.nsecToMSec(relativeTimeInNanos());
+        return cachedTimeThread.relativeTimeInMillis();
+    }
+
+    /**
+     * Effectively the same as {@code this::relativeTimeInMillis}, except that it returns a constant to save on allocation.
+     */
+    public LongSupplier relativeTimeInMillisSupplier() {
+        return relativeTimeInMillisSupplier;
     }
 
     /**
@@ -819,12 +734,14 @@ static class CachedTimeThread extends Thread {
 
         volatile boolean running = true;
         volatile long relativeNanos;
+        volatile long relativeMillis;
         volatile long absoluteMillis;
 
         CachedTimeThread(String name, long intervalMillis, long thresholdMillis) {
             super(name);
             this.interval = intervalMillis;
             this.relativeNanos = System.nanoTime();
+            this.relativeMillis = TimeValue.nsecToMSec(this.relativeNanos);
             this.absoluteMillis = System.currentTimeMillis();
             this.timeChangeChecker = new TimeChangeChecker(thresholdMillis, absoluteMillis, relativeNanos);
             setDaemon(true);
@@ -844,6 +761,20 @@ long relativeTimeInNanos() {
             return System.nanoTime();
         }
 
+        /**
+         * Return the current time used for relative calculations. This is {@link System#nanoTime()} converted into milliseconds.
+         * <p>
+         * If {@link ThreadPool#ESTIMATED_TIME_INTERVAL_SETTING} is set to 0
+         * then the cache is disabled and the method calls {@link System#nanoTime()}
+         * whenever called. Typically used for testing.
+         */
+        long relativeTimeInMillis() {
+            if (0 < interval) {
+                return relativeMillis;
+            }
+            return TimeValue.nsecToMSec(System.nanoTime());
+        }
+
         /**
          * Return the current epoch time, used to find absolute time. This is
          * a cached version of {@link System#currentTimeMillis()}.
@@ -863,6 +794,7 @@ long absoluteTimeInMillis() {
         public void run() {
             while (running && 0 < interval) {
                 relativeNanos = System.nanoTime();
+                relativeMillis = TimeValue.nsecToMSec(this.relativeNanos);
                 absoluteMillis = System.currentTimeMillis();
                 timeChangeChecker.check(absoluteMillis, relativeNanos);
                 try {
@@ -877,6 +809,24 @@ public void run() {
         }
     }
 
+    private static class RelativeTimeInMillisSupplier implements LongSupplier {
+        private final CachedTimeThread cachedTimeThread;
+
+        private RelativeTimeInMillisSupplier(CachedTimeThread cachedTimeThread) {
+            this.cachedTimeThread = cachedTimeThread;
+        }
+
+        @Override
+        public long getAsLong() {
+            return cachedTimeThread.relativeTimeInMillis();
+        }
+
+        @Override
+        public String toString() {
+            return ThreadPool.class.getCanonicalName() + "::relativeTimeInMillis";
+        }
+    }
+
     static class TimeChangeChecker {
 
         private final long thresholdMillis;
diff --git a/server/src/main/java/org/elasticsearch/threadpool/internal/BuiltInExecutorBuilders.java b/server/src/main/java/org/elasticsearch/threadpool/internal/BuiltInExecutorBuilders.java
new file mode 100644
index 0000000000000..6709685af6a75
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/threadpool/internal/BuiltInExecutorBuilders.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.threadpool.internal;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ExecutorBuilder;
+
+import java.util.Map;
+
+public interface BuiltInExecutorBuilders {
+    @SuppressWarnings("rawtypes")
+    Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocatedProcessors);
+}
diff --git a/server/src/main/java/org/elasticsearch/threadpool/internal/package-info.java b/server/src/main/java/org/elasticsearch/threadpool/internal/package-info.java
new file mode 100644
index 0000000000000..821e27a4a854f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/threadpool/internal/package-info.java
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * This package can be selectively exported to control access to features
+ * that are not intended to be available to any arbitrary plugin.
+ */
+package org.elasticsearch.threadpool.internal;
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java
index 33ea35ecffd94..e5e8c52730a87 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -1396,6 +1396,11 @@ public void onConnectionClosed(Transport.Connection connection) {
                     protected void doRun() {
                         handler.handleException(exception);
                     }
+
+                    @Override
+                    public String toString() {
+                        return "onConnectionClosed/handleException[" + handler + "]";
+                    }
                 });
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/usage/UsageService.java b/server/src/main/java/org/elasticsearch/usage/UsageService.java
index e11b343c7055a..573332060f55d 100644
--- a/server/src/main/java/org/elasticsearch/usage/UsageService.java
+++ b/server/src/main/java/org/elasticsearch/usage/UsageService.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.usage;
 
 import org.elasticsearch.action.admin.cluster.node.usage.NodeUsage;
+import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry;
 import org.elasticsearch.rest.BaseRestHandler;
 
 import java.util.HashMap;
@@ -23,10 +24,12 @@ public class UsageService {
 
     private final Map<String, BaseRestHandler> handlers;
     private final SearchUsageHolder searchUsageHolder;
+    private final CCSUsageTelemetry ccsUsageHolder;
 
     public UsageService() {
         this.handlers = new HashMap<>();
         this.searchUsageHolder = new SearchUsageHolder();
+        this.ccsUsageHolder = new CCSUsageTelemetry();
     }
 
     /**
@@ -81,4 +84,8 @@ public Map<String, Long> getRestUsageStats() {
     public SearchUsageHolder getSearchUsageHolder() {
         return searchUsageHolder;
     }
+
+    public CCSUsageTelemetry getCcsUsageHolder() {
+        return ccsUsageHolder;
+    }
 }
diff --git a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
index 1bdc17b6cf6f8..90a1c29972ff3 100644
--- a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
+++ b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification
@@ -13,6 +13,7 @@ org.elasticsearch.cluster.service.TransportFeatures
 org.elasticsearch.cluster.metadata.MetadataFeatures
 org.elasticsearch.rest.RestFeatures
 org.elasticsearch.indices.IndicesFeatures
+org.elasticsearch.repositories.RepositoriesFeatures
 org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures
 org.elasticsearch.index.mapper.MapperFeatures
 org.elasticsearch.ingest.IngestGeoIpFeatures
@@ -20,3 +21,4 @@ org.elasticsearch.search.SearchFeatures
 org.elasticsearch.search.retriever.RetrieversFeatures
 org.elasticsearch.script.ScriptFeatures
 org.elasticsearch.reservedstate.service.FileSettingsFeatures
+org.elasticsearch.cluster.routing.RoutingFeatures
diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv
index 7d2697539fa13..8cd6fe9720039 100644
--- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv
+++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv
@@ -70,6 +70,7 @@
 7.17.20,7172099
 7.17.21,7172199
 7.17.22,7172299
+7.17.23,7172399
 8.0.0,8000099
 8.0.1,8000199
 8.1.0,8010099
@@ -125,3 +126,4 @@
 8.14.1,8636001
 8.14.2,8636001
 8.14.3,8636001
+8.15.0,8702002
diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json
index cd325ce9010ac..cc0bc5e2257c8 100644
--- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json
+++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json
@@ -2,8 +2,8 @@
   "INITIAL_MASTER_NODES": "important-settings.html#initial_master_nodes",
   "DISCOVERY_TROUBLESHOOTING": "discovery-troubleshooting.html",
   "UNSTABLE_CLUSTER_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html",
-  "LAGGING_NODE_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html#_diagnosing_lagging_nodes_2",
-  "SHARD_LOCK_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html#_diagnosing_shardlockobtainfailedexception_failures_2",
+  "LAGGING_NODE_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html#troubleshooting-unstable-cluster-lagging",
+  "SHARD_LOCK_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html#troubleshooting-unstable-cluster-shardlockobtainfailedexception",
   "CONCURRENT_REPOSITORY_WRITERS": "diagnosing-corrupted-repositories.html",
   "ARCHIVE_INDICES": "archive-indices.html",
   "HTTP_TRACER": "modules-network.html#http-rest-request-tracer",
@@ -38,5 +38,9 @@
   "NETWORK_BINDING_AND_PUBLISHING": "modules-network.html#modules-network-binding-publishing",
   "SNAPSHOT_REPOSITORY_ANALYSIS": "repo-analysis-api.html",
   "S3_COMPATIBLE_REPOSITORIES": "repository-s3.html#repository-s3-compatible-services",
-  "LUCENE_MAX_DOCS_LIMIT": "size-your-shards.html#troubleshooting-max-docs-limit"
+  "LUCENE_MAX_DOCS_LIMIT": "size-your-shards.html#troubleshooting-max-docs-limit",
+  "MAX_SHARDS_PER_NODE": "size-your-shards.html#troubleshooting-max-shards-open",
+  "FLOOD_STAGE_WATERMARK": "fix-watermark-errors.html",
+  "X_OPAQUE_ID": "api-conventions.html#x-opaque-id",
+  "FORMING_SINGLE_NODE_CLUSTERS": "modules-discovery-bootstrap-cluster.html#modules-discovery-bootstrap-cluster-joining"
 }
diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
index f177ab1468cb2..df0df2d05ba5b 100644
--- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
+++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
@@ -70,6 +70,7 @@
 7.17.20,7172099
 7.17.21,7172199
 7.17.22,7172299
+7.17.23,7172399
 8.0.0,8000099
 8.0.1,8000199
 8.1.0,8010099
@@ -125,3 +126,4 @@
 8.14.1,8505000
 8.14.2,8505000
 8.14.3,8505000
+8.15.0,8512000
diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 7ac4215670405..58f85474f38ed 100644
--- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -67,6 +67,7 @@
 import org.elasticsearch.indices.recovery.RecoverFilesRecoveryException;
 import org.elasticsearch.indices.recovery.RecoveryCommitTooNewException;
 import org.elasticsearch.ingest.GraphStructureException;
+import org.elasticsearch.ingest.IngestPipelineException;
 import org.elasticsearch.ingest.IngestProcessorException;
 import org.elasticsearch.persistent.NotPersistentTaskNodeException;
 import org.elasticsearch.persistent.PersistentTaskNodeNotAssignedException;
@@ -834,6 +835,7 @@ public void testIds() {
         ids.put(179, NotPersistentTaskNodeException.class);
         ids.put(180, PersistentTaskNodeNotAssignedException.class);
         ids.put(181, ResourceAlreadyUploadedException.class);
+        ids.put(182, IngestPipelineException.class);
 
         Map<Class<? extends ElasticsearchException>, Integer> reverse = new HashMap<>();
         for (Map.Entry<Integer, Class<? extends ElasticsearchException>> entry : ids.entrySet()) {
diff --git a/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java b/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java
index 744c0fafedb46..8f3d84b5e90da 100644
--- a/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java
@@ -221,4 +221,276 @@ public void testCauseCycle() {
         ExceptionsHelper.unwrap(e1, IOException.class);
         ExceptionsHelper.unwrapCorruption(e1);
     }
+
+    public void testLimitedStackTrace() {
+        // A normal exception is thrown several stack frames down
+        int maxTraces = between(0, 5);
+        RuntimeException exception = recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException);
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception, maxTraces);
+        int expectedLength = 1 + maxTraces + 1; // Exception message, traces, then the count of remaining traces
+        assertThat(limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    public void testLimitedStackTraceShortened() {
+        // An exception has a smaller trace than is requested
+        // Set max traces very, very high, since this test is sensitive to the number of method calls on the thread's stack.
+        int maxTraces = 5000;
+        RuntimeException exception = new RuntimeException("Regular Exception");
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception, maxTraces);
+        String fullTrace = ExceptionsHelper.stackTrace(exception);
+        int expectedLength = fullTrace.split("\n").length; // The resulting line count should not be reduced
+        assertThat(limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    public void testLimitedStackTraceWrappedExceptions() {
+        // An exception is thrown and is then wrapped several stack frames later
+        int maxTraces = between(0, 5);
+        RuntimeException exception = recurseAndCatchRuntime(
+            randomIntBetween(10, 15),
+            () -> throwExceptionCausedBy(recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException))
+        );
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception, maxTraces);
+        // (1) Exception message, (n) traces, (1) remaining traces, (1) caused by, (n) caused by traces, (1) remaining traces
+        int expectedLength = 4 + (2 * maxTraces);
+        assertThat(limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    public void testLimitedStackTraceSuppressingAnException() {
+        // A normal exception is thrown several stack frames down and then suppresses a new exception on the way back up
+        int maxTraces = between(0, 5);
+        RuntimeException exception = recurseAndCatchRuntime(randomIntBetween(10, 15), () -> {
+            RuntimeException original = recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException);
+            recurseUntil(randomIntBetween(10, 15), () -> suppressNewExceptionUnder(original));
+        });
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception, maxTraces);
+        // (1) Exception message, (n) traces, (1) remaining traces, then
+        // (1) suppressed, (n) suppressed by traces, (1) remaining lines
+        int expectedLength = 4 + (2 * maxTraces);
+        assertThat(limitedTrace, limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    public void testLimitedStackTraceSuppressedByAnException() {
+        // A normal exception is thrown several stack frames down and then gets suppressed on the way back up by a new exception
+        int maxTraces = between(0, 5);
+        RuntimeException exception = recurseAndCatchRuntime(randomIntBetween(10, 15), () -> {
+            RuntimeException original = recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException);
+            recurseUntil(randomIntBetween(10, 15), () -> throwNewExceptionThatSuppresses(original));
+        });
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception, maxTraces);
+        // (1) Exception message, (n) traces, (1) remaining traces, then
+        // (1) suppressed original exception, (n) suppressed traces, (1) remaining traces
+        int expectedLength = 4 + (2 * maxTraces);
+        assertThat(limitedTrace, limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    public void testLimitedStackTraceSuppressingAnExceptionWithACause() {
+        // A normal exception is thrown several stack frames down. On the way back up, a new exception with a nested cause is
+        // suppressed by it.
+        int maxTraces = between(0, 5);
+        RuntimeException exception = recurseAndCatchRuntime(randomIntBetween(10, 15), () -> {
+            RuntimeException original = recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException);
+            RuntimeException causedBy = recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException);
+            recurseUntil(randomIntBetween(10, 15), () -> suppressNewExceptionWithCauseUnder(original, causedBy));
+        });
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception, maxTraces);
+        // (1) Exception message, (n) traces, (1) remaining traces, then
+        // (1) suppressed exception, (n) suppressed traces, (1) remaining traces
+        // (1) suppressed caused by exception, (n) traces, (1) remaining traces
+        int expectedLength = 6 + (3 * maxTraces);
+        assertThat(limitedTrace, limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    public void testLimitedStackTraceSuppressedByAnExceptionWithACause() {
+        // A normal exception is thrown several stack frames down. On the way back up, a new exception with a nested cause
+        // suppresses it.
+        int maxTraces = between(0, 5);
+        RuntimeException exception = recurseAndCatchRuntime(randomIntBetween(10, 15), () -> {
+            RuntimeException original = recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException);
+            RuntimeException causedBy = recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException);
+            recurseUntil(randomIntBetween(10, 15), () -> throwNewExceptionWithCauseThatSuppresses(original, causedBy));
+        });
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception, maxTraces);
+        // (1) Exception message, (n) traces, (1) remaining traces, then
+        // (1) suppressed original exception, (n) traces, (1) remaining traces, then
+        // (1) caused by exception, (n) traces, (1) remaining traces
+        int expectedLength = 6 + (3 * maxTraces);
+        assertThat(limitedTrace, limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    public void testLimitedStackTraceWrappedAndSuppressingAWrappedException() {
+        // A normal exception is thrown several stack frames down. It gets wrapped on the way back up.
+        // Some "recovery" code runs and a new exception is thrown. It also gets wrapped on the way back up.
+        // The first chain of exceptions suppresses the second.
+        int maxTraces = between(0, 5);
+        RuntimeException exception = recurseAndCatchRuntime(randomIntBetween(10, 15), () -> {
+            RuntimeException original = recurseAndCatchRuntime(
+                randomIntBetween(10, 15),
+                () -> throwExceptionCausedBy(recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException))
+            );
+            RuntimeException causedBy = recurseAndCatchRuntime(
+                randomIntBetween(10, 15),
+                () -> throwExceptionCausedBy(recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException))
+            );
+            recurseUntil(randomIntBetween(10, 15), () -> suppressNewExceptionWithCauseUnder(original, causedBy));
+        });
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception, maxTraces);
+        // (1) wrapped exception message, (n) traces, (1) remaining traces, then
+        // (1) suppressed exception, (n) suppressed traces, (1) remaining traces, then
+        // (1) wrapped exception under suppressed exception, (n) traces, (1) remaining traces, then
+        // (1) root cause of suppressed exception chain, (n) traces, (1) remaining traces, then
+        // (1) root cause of the suppressing exception chain, (n) traces, (1) remaining traces
+        int expectedLength = 10 + (5 * maxTraces);
+        assertThat(limitedTrace, limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    public void testLimitedStackTraceWrappedAndSuppressedByAWrappedException() {
+        // A normal exception is thrown several stack frames down. It gets wrapped on the way back up.
+        // Some "recovery" code runs and a new exception is thrown. It also gets wrapped on the way back up.
+        // The first chain of exceptions suppresses the second.
+        int maxTraces = between(0, 5);
+        RuntimeException exception = recurseAndCatchRuntime(randomIntBetween(10, 15), () -> {
+            RuntimeException original = recurseAndCatchRuntime(
+                randomIntBetween(10, 15),
+                () -> throwExceptionCausedBy(recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException))
+            );
+            RuntimeException causedBy = recurseAndCatchRuntime(
+                randomIntBetween(10, 15),
+                () -> throwExceptionCausedBy(recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException))
+            );
+            throwNewExceptionWithCauseThatSuppresses(original, causedBy);
+        });
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception, maxTraces);
+        // (1) wrapped exception message, (n) traces, (1) remaining traces, then
+        // (1) suppressed wrapped exception, (n) traces, (1) remaining traces, then
+        // (1) root cause of suppressed exception chain, (n) traces, (1) remaining traces, then
+        // (1) wrapped exception, (n) traces, (1) remaining traces, then
+        // (1) root cause of exception chain, (n) traces, (1) remaining traces
+        int expectedLength = 10 + (5 * maxTraces);
+        assertThat(limitedTrace, limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    public void testLimitedStackTraceNestedSuppression() {
+        // The original exception is thrown and is then repeatedly suppressed by new exceptions
+        int maxTraces = between(0, 5);
+        RuntimeException exception = recurseAndCatchRuntime(
+            randomIntBetween(10, 15),
+            () -> throwNewExceptionWithCauseThatSuppresses(
+                recurseAndCatchRuntime(
+                    randomIntBetween(10, 15),
+                    () -> throwNewExceptionWithCauseThatSuppresses(
+                        recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException),
+                        recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException)
+                    )
+                ),
+                recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException)
+            )
+        );
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception, maxTraces);
+        // (1) first suppressing exception message, (n) traces, (1) remaining traces, then
+        // (1) second suppressing exception message, (n) traces, (1) remaining traces, then
+        // (1) suppressed original exception message, (n) traces, (1) remaining traces, then
+        // (1) cause of second suppressed exception, (n) traces, (1) remaining traces, then
+        // (1) cause of first suppressed exception, (n) traces, (1) remaining traces, then
+        int expectedLength = 10 + (5 * maxTraces);
+        assertThat(limitedTrace, limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    public void testLimitedStackTraceCircularCause() {
+        // An exception is thrown and then suppresses itself
+        int maxTraces = between(0, 5);
+        RuntimeException exception1 = recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException);
+        RuntimeException exception2 = recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException);
+        exception1.initCause(exception2);
+        exception2.initCause(exception1);
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception1, maxTraces);
+        // (1) first exception message, (n) traces, (1) remaining traces, then
+        // (1) caused by second exception, (n) traces, (1) remaining traces, then
+        // (1) caused by first exception message again, but no further traces
+        int expectedLength = 5 + (2 * maxTraces);
+        assertThat(limitedTrace, limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    public void testLimitedStackTraceCircularSuppression() {
+        // An exception is thrown and then suppresses itself
+        int maxTraces = between(0, 5);
+        RuntimeException exception1 = recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException);
+        RuntimeException exception2 = recurseAndCatchRuntime(randomIntBetween(10, 15), ExceptionsHelperTests::throwRegularException);
+        exception1.addSuppressed(exception2);
+        exception2.addSuppressed(exception1);
+        String limitedTrace = ExceptionsHelper.limitedStackTrace(exception1, maxTraces);
+        // (1) first exception message, (n) traces, (1) remaining traces, then
+        // (1) suppressed second exception, (n) traces, (1) remaining traces, then
+        // (1) suppressed first exception message again, but no further traces
+        int expectedLength = 5 + (2 * maxTraces);
+        assertThat(limitedTrace, limitedTrace.split("\n").length, equalTo(expectedLength));
+    }
+
+    private static void throwRegularException() {
+        throw new RuntimeException("Regular Exception");
+    }
+
+    private static void throwExceptionCausedBy(RuntimeException causedBy) {
+        throw new RuntimeException("Wrapping Exception", causedBy);
+    }
+
+    private static void suppressNewExceptionUnder(RuntimeException suppressor) {
+        suppressor.addSuppressed(new RuntimeException("Suppressed Exception"));
+        throw suppressor;
+    }
+
+    private static void throwNewExceptionThatSuppresses(RuntimeException suppressed) {
+        RuntimeException priority = new RuntimeException("Priority Exception");
+        priority.addSuppressed(suppressed);
+        throw priority;
+    }
+
+    private static void suppressNewExceptionWithCauseUnder(RuntimeException suppressor, RuntimeException suppressedCause) {
+        suppressor.addSuppressed(new RuntimeException("Suppressed Exception", suppressedCause));
+        throw suppressor;
+    }
+
+    private static void throwNewExceptionWithCauseThatSuppresses(RuntimeException suppressed, RuntimeException suppressorCause) {
+        RuntimeException priority = new RuntimeException("Priority Exception", suppressorCause);
+        priority.addSuppressed(suppressed);
+        throw priority;
+    };
+
+    private static RuntimeException recurseAndCatchRuntime(int depth, Runnable op) {
+        return expectThrows(RuntimeException.class, () -> doRecurse(depth, 0, op));
+    }
+
+    private static void recurseUntil(int depth, Runnable op) {
+        doRecurse(depth, 0, op);
+    }
+
+    private static void doRecurse(int depth, int current, Runnable op) {
+        if (depth == current) {
+            op.run();
+        } else {
+            doRecurse(depth, current + 1, op);
+        }
+    }
+
+    public void testCompressStackTraceElement() {
+        assertThat(compressPackages(""), equalTo(""));
+        assertThat(compressPackages("."), equalTo(""));
+        assertThat(compressPackages("ClassName"), equalTo("ClassName"));
+        assertThat(compressPackages("alfa.ClassName"), equalTo("a.ClassName"));
+        assertThat(compressPackages("alfa.bravo.ClassName"), equalTo("a.b.ClassName"));
+        assertThat(compressPackages(".ClassName"), equalTo("ClassName"));
+        assertThat(compressPackages(".alfa.ClassName"), equalTo("a.ClassName"));
+        assertThat(compressPackages(".alfa.bravo.ClassName"), equalTo("a.b.ClassName"));
+        assertThat(compressPackages("...alfa.....ClassName"), equalTo("a.ClassName"));
+        assertThat(compressPackages("...alfa....bravo.ClassName"), equalTo("a.b.ClassName"));
+        assertThat(compressPackages("...alfa....bravo.charliepackagenameisreallyreallylong.ClassName"), equalTo("a.b.c.ClassName"));
+        assertThat(compressPackages("alfa.bravo.charlie.OuterClassName.InnerClassName"), equalTo("a.b.c.O.InnerClassName"));
+
+        expectThrows(AssertionError.class, () -> compressPackages(null));
+    }
+
+    private static String compressPackages(String className) {
+        StringBuilder s = new StringBuilder();
+        ExceptionsHelper.compressPackages(s, className);
+        return s.toString();
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/ReleaseVersionsTests.java b/server/src/test/java/org/elasticsearch/ReleaseVersionsTests.java
index b80e953bd8aea..3b5f5eea57f66 100644
--- a/server/src/test/java/org/elasticsearch/ReleaseVersionsTests.java
+++ b/server/src/test/java/org/elasticsearch/ReleaseVersionsTests.java
@@ -17,19 +17,20 @@
 public class ReleaseVersionsTests extends ESTestCase {
 
     public void testReleaseVersions() {
-        IntFunction<String> versions = ReleaseVersions.generateVersionsLookup(ReleaseVersionsTests.class);
+        IntFunction<String> versions = ReleaseVersions.generateVersionsLookup(ReleaseVersionsTests.class, 23);
 
         assertThat(versions.apply(10), equalTo("8.0.0"));
         assertThat(versions.apply(14), equalTo("8.1.0-8.1.1"));
         assertThat(versions.apply(21), equalTo("8.2.0"));
         assertThat(versions.apply(22), equalTo("8.2.1"));
+        assertThat(versions.apply(23), equalTo(Version.CURRENT.toString()));
     }
 
     public void testReturnsRange() {
-        IntFunction<String> versions = ReleaseVersions.generateVersionsLookup(ReleaseVersionsTests.class);
+        IntFunction<String> versions = ReleaseVersions.generateVersionsLookup(ReleaseVersionsTests.class, 23);
 
         assertThat(versions.apply(17), equalTo("8.1.2-8.2.0"));
         assertThat(versions.apply(9), equalTo("0.0.0"));
-        assertThat(versions.apply(24), equalTo("8.2.2-snapshot[24]"));
+        assertThat(versions.apply(24), equalTo(new Version(Version.CURRENT.id + 100) + "-[24]"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/TransportVersionTests.java b/server/src/test/java/org/elasticsearch/TransportVersionTests.java
index 2de973622248b..a3728f20a23d4 100644
--- a/server/src/test/java/org/elasticsearch/TransportVersionTests.java
+++ b/server/src/test/java/org/elasticsearch/TransportVersionTests.java
@@ -186,6 +186,10 @@ public void testCURRENTIsLatest() {
         assertThat(Collections.max(TransportVersions.getAllVersions()), is(TransportVersion.current()));
     }
 
+    public void testToReleaseVersion() {
+        assertThat(TransportVersion.current().toReleaseVersion(), equalTo(Version.CURRENT.toString()));
+    }
+
     public void testToString() {
         assertEquals("5000099", TransportVersion.fromId(5_00_00_99).toString());
         assertEquals("2030099", TransportVersion.fromId(2_03_00_99).toString());
diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
index 7afa7adedc7bf..c015dc6177cad 100644
--- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.indices.TestIndexNameExpressionResolver;
@@ -88,7 +89,7 @@ public ActionRequestValidationException validate() {
         }
         class FakeTransportAction extends TransportAction<FakeRequest, ActionResponse> {
             protected FakeTransportAction(String actionName, ActionFilters actionFilters, TaskManager taskManager) {
-                super(actionName, actionFilters, taskManager);
+                super(actionName, actionFilters, taskManager, EsExecutors.DIRECT_EXECUTOR_SERVICE);
             }
 
             @Override
diff --git a/server/src/test/java/org/elasticsearch/action/ActionRunnableTests.java b/server/src/test/java/org/elasticsearch/action/ActionRunnableTests.java
index ebc9205f34d05..f2fd54e240e24 100644
--- a/server/src/test/java/org/elasticsearch/action/ActionRunnableTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ActionRunnableTests.java
@@ -47,11 +47,9 @@ public void testWrapReleasingNotRejected() throws Exception {
                 assertEquals("simulated", e.getMessage());
                 assertTrue(releaseListener.isDone());
                 l.onResponse(null);
-            }), () -> safeReleaseListener.onResponse(null), l -> executor.execute(() -> ActionListener.completeWith(l, () -> {
+            }), () -> safeReleaseListener.onResponse(null), l -> executor.execute(ActionRunnable.run(l, () -> {
                 if (randomBoolean()) {
                     throw new ElasticsearchException("simulated");
-                } else {
-                    return null;
                 }
             }))));
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
index 82f22a2572c1d..09b0a78849cd9 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -561,7 +561,12 @@ private static int expectedChunks(@Nullable NodeIndicesStats nodeIndicesStats, N
 
     private static CommonStats createIndexLevelCommonStats() {
         CommonStats stats = new CommonStats(new CommonStatsFlags().clear().set(CommonStatsFlags.Flag.Mappings, true));
-        stats.nodeMappings = new NodeMappingStats(randomNonNegativeLong(), randomNonNegativeLong());
+        stats.nodeMappings = new NodeMappingStats(
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong()
+        );
         return stats;
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index e541fef65a0f9..64b9b4f0b69d8 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -158,15 +158,11 @@ protected NodeResponse nodeOperation(CancellableNodeRequest request, Task task)
             if (shouldBlock) {
                 // Simulate a job that takes forever to finish
                 // Using periodic checks method to identify that the task was cancelled
-                try {
-                    waitUntil(() -> {
-                        ((CancellableTask) task).ensureNotCancelled();
-                        return false;
-                    });
-                    fail("It should have thrown an exception");
-                } catch (InterruptedException ex) {
-                    Thread.currentThread().interrupt();
-                }
+                waitUntil(() -> {
+                    ((CancellableTask) task).ensureNotCancelled();
+                    return false;
+                });
+                fail("It should have thrown an exception");
             }
             debugDelay("op4");
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index 6f2af8414187e..903ecfe2b2aa7 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -28,12 +28,12 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -283,16 +283,12 @@ protected void doExecute(Task task, NodesRequest request, ActionListener<NodesRe
         protected NodeResponse nodeOperation(NodeRequest request, Task task) {
             logger.info("Test task started on the node {}", clusterService.localNode());
             if (request.shouldBlock) {
-                try {
-                    waitUntil(() -> {
-                        if (((CancellableTask) task).isCancelled()) {
-                            throw new RuntimeException("Cancelled!");
-                        }
-                        return ((TestTask) task).isBlocked() == false;
-                    });
-                } catch (InterruptedException ex) {
-                    Thread.currentThread().interrupt();
-                }
+                waitUntil(() -> {
+                    if (((CancellableTask) task).isCancelled()) {
+                        throw new RuntimeException("Cancelled!");
+                    }
+                    return ((TestTask) task).isBlocked() == false;
+                });
             }
             logger.info("Test task finished on the node {}", clusterService.localNode());
             return new NodeResponse(clusterService.localNode());
@@ -301,9 +297,7 @@ protected NodeResponse nodeOperation(NodeRequest request, Task task) {
 
     public static class UnblockTestTaskResponse implements Writeable {
 
-        UnblockTestTaskResponse() {
-
-        }
+        UnblockTestTaskResponse() {}
 
         UnblockTestTaskResponse(StreamInput in) {}
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java
index 76abc4733d3ff..0d3c9dd203860 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java
@@ -19,7 +19,7 @@
 public class ClusterSearchShardsRequestTests extends ESTestCase {
 
     public void testSerialization() throws Exception {
-        ClusterSearchShardsRequest request = new ClusterSearchShardsRequest();
+        ClusterSearchShardsRequest request = new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT);
         if (randomBoolean()) {
             int numIndices = randomIntBetween(1, 5);
             String[] indices = new String[numIndices];
@@ -66,7 +66,7 @@ public void testSerialization() throws Exception {
     }
 
     public void testIndicesMustNotBeNull() {
-        ClusterSearchShardsRequest request = new ClusterSearchShardsRequest();
+        ClusterSearchShardsRequest request = new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT);
         assertNotNull(request.indices());
         expectThrows(NullPointerException.class, () -> request.indices((String[]) null));
         expectThrows(NullPointerException.class, () -> request.indices((String) null));
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
index 2886ca7be4821..999f22b4dc400 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
@@ -113,6 +113,7 @@ public void testToXContent() throws IOException {
             );
             processed.waitForCompletion(original.waitForCompletion());
             processed.masterNodeTimeout(original.masterNodeTimeout());
+            processed.uuid(original.uuid());
             processed.source(map);
 
             assertEquals(original, processed);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ApproximateMatcher.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ApproximateMatcher.java
new file mode 100644
index 0000000000000..3ceda1c7f4651
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ApproximateMatcher.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.hamcrest.Description;
+import org.hamcrest.TypeSafeMatcher;
+
+/**
+ * Matches a value that is within given range (currently 1%) of an expected value.
+ *
+ * We need this because histograms do not store exact values, but only value ranges.
+ * Since we have 2 significant digits, the value should be within 1% of the expected value.
+ */
+public class ApproximateMatcher extends TypeSafeMatcher<Long> {
+    public static double ACCURACY = 0.01;
+    private final long expectedValue;
+
+    public ApproximateMatcher(long expectedValue) {
+        this.expectedValue = expectedValue;
+    }
+
+    @Override
+    protected boolean matchesSafely(Long actualValue) {
+        double lowerBound = Math.floor(expectedValue * (1.00 - ACCURACY));
+        double upperBound = Math.ceil(expectedValue * (1.00 + ACCURACY));
+        return actualValue >= lowerBound && actualValue <= upperBound;
+    }
+
+    @Override
+    public void describeTo(Description description) {
+        description.appendText("a long value within 1% of ").appendValue(expectedValue);
+    }
+
+    /**
+     * Matches a value that is within given range (currently 1%) of an expected value.
+     */
+    public static ApproximateMatcher closeTo(long expectedValue) {
+        return new ApproximateMatcher(expectedValue);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java
new file mode 100644
index 0000000000000..9f08934503b69
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java
@@ -0,0 +1,324 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot.PerClusterCCSTelemetry;
+import org.elasticsearch.action.admin.cluster.stats.LongMetric.LongMetricValue;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.equalTo;
+
+public class CCSTelemetrySnapshotTests extends AbstractWireSerializingTestCase<CCSTelemetrySnapshot> {
+
+    private LongMetricValue randomLongMetricValue() {
+        LongMetric v = new LongMetric();
+        for (int i = 0; i < randomIntBetween(1, 10); i++) {
+            v.record(randomIntBetween(0, 1_000_000));
+        }
+        return v.getValue();
+    }
+
+    private PerClusterCCSTelemetry randomPerClusterCCSTelemetry() {
+        return new PerClusterCCSTelemetry(randomLongBetween(0, 1_000_000), randomLongBetween(0, 1_000_000), randomLongMetricValue());
+    }
+
+    @Override
+    protected CCSTelemetrySnapshot createTestInstance() {
+        if (randomBoolean()) {
+            return new CCSTelemetrySnapshot();
+        } else {
+            return randomCCSTelemetrySnapshot();
+        }
+    }
+
+    private CCSTelemetrySnapshot randomCCSTelemetrySnapshot() {
+        return new CCSTelemetrySnapshot(
+            randomLongBetween(0, 1_000_000),
+            randomLongBetween(0, 1_000_000),
+            Map.of(),
+            randomLongMetricValue(),
+            randomLongMetricValue(),
+            randomLongMetricValue(),
+            randomLongBetween(0, 1_000_000),
+            randomDoubleBetween(0.0, 100.0, false),
+            randomLongBetween(0, 1_000_000),
+            Map.of(),
+            Map.of(),
+            randomMap(1, 10, () -> new Tuple<>(randomAlphaOfLengthBetween(5, 10), randomPerClusterCCSTelemetry()))
+        );
+    }
+
+    @Override
+    protected Writeable.Reader<CCSTelemetrySnapshot> instanceReader() {
+        return CCSTelemetrySnapshot::new;
+    }
+
+    @Override
+    protected CCSTelemetrySnapshot mutateInstance(CCSTelemetrySnapshot instance) throws IOException {
+        // create a copy of CCSTelemetrySnapshot by extracting each field and mutating it
+        long totalCount = instance.getTotalCount();
+        long successCount = instance.getSuccessCount();
+        var failureReasons = instance.getFailureReasons();
+        LongMetricValue took = instance.getTook();
+        LongMetricValue tookMrtTrue = instance.getTookMrtTrue();
+        LongMetricValue tookMrtFalse = instance.getTookMrtFalse();
+        long skippedRemotes = instance.getSearchCountWithSkippedRemotes();
+        long remotesPerSearchMax = instance.getRemotesPerSearchMax();
+        double remotesPerSearchAvg = instance.getRemotesPerSearchAvg();
+        var featureCounts = instance.getFeatureCounts();
+        var clientCounts = instance.getClientCounts();
+        var perClusterCCSTelemetries = instance.getByRemoteCluster();
+
+        // Mutate values
+        int i = randomInt(11);
+        switch (i) {
+            case 0:
+                totalCount += randomNonNegativeLong();
+                break;
+            case 1:
+                successCount += randomNonNegativeLong();
+                break;
+            case 2:
+                failureReasons = new HashMap<>(failureReasons);
+                if (failureReasons.isEmpty() || randomBoolean()) {
+                    failureReasons.put(randomAlphaOfLengthBetween(5, 10), randomNonNegativeLong());
+                } else {
+                    // modify random element of the map
+                    String key = randomFrom(failureReasons.keySet());
+                    failureReasons.put(key, randomNonNegativeLong());
+                }
+                break;
+            case 3:
+                took = randomLongMetricValue();
+                break;
+            case 4:
+                tookMrtTrue = randomLongMetricValue();
+                break;
+            case 5:
+                tookMrtFalse = randomLongMetricValue();
+                break;
+            case 6:
+                skippedRemotes += randomNonNegativeLong();
+                break;
+            case 7:
+                remotesPerSearchMax += randomNonNegativeLong();
+                break;
+            case 8:
+                remotesPerSearchAvg = randomDoubleBetween(0.0, 100.0, false);
+                break;
+            case 9:
+                featureCounts = new HashMap<>(featureCounts);
+                if (featureCounts.isEmpty() || randomBoolean()) {
+                    featureCounts.put(randomAlphaOfLengthBetween(5, 10), randomNonNegativeLong());
+                } else {
+                    // modify random element of the map
+                    String key = randomFrom(featureCounts.keySet());
+                    featureCounts.put(key, randomNonNegativeLong());
+                }
+                break;
+            case 10:
+                clientCounts = new HashMap<>(clientCounts);
+                if (clientCounts.isEmpty() || randomBoolean()) {
+                    clientCounts.put(randomAlphaOfLengthBetween(5, 10), randomNonNegativeLong());
+                } else {
+                    // modify random element of the map
+                    String key = randomFrom(clientCounts.keySet());
+                    clientCounts.put(key, randomNonNegativeLong());
+                }
+                break;
+            case 11:
+                perClusterCCSTelemetries = new HashMap<>(perClusterCCSTelemetries);
+                if (perClusterCCSTelemetries.isEmpty() || randomBoolean()) {
+                    perClusterCCSTelemetries.put(randomAlphaOfLengthBetween(5, 10), randomPerClusterCCSTelemetry());
+                } else {
+                    // modify random element of the map
+                    String key = randomFrom(perClusterCCSTelemetries.keySet());
+                    perClusterCCSTelemetries.put(key, randomPerClusterCCSTelemetry());
+                }
+                break;
+        }
+        // Return new instance
+        return new CCSTelemetrySnapshot(
+            totalCount,
+            successCount,
+            failureReasons,
+            took,
+            tookMrtTrue,
+            tookMrtFalse,
+            remotesPerSearchMax,
+            remotesPerSearchAvg,
+            skippedRemotes,
+            featureCounts,
+            clientCounts,
+            perClusterCCSTelemetries
+        );
+    }
+
+    public void testAdd() {
+        CCSTelemetrySnapshot empty = new CCSTelemetrySnapshot();
+        CCSTelemetrySnapshot full = randomCCSTelemetrySnapshot();
+        empty.add(full);
+        assertThat(empty, equalTo(full));
+        // Add again
+        empty.add(full);
+        assertThat(empty.getTotalCount(), equalTo(full.getTotalCount() * 2));
+        assertThat(empty.getSuccessCount(), equalTo(full.getSuccessCount() * 2));
+        // check that each element of the map is doubled
+        empty.getFailureReasons().forEach((k, v) -> assertThat(v, equalTo(full.getFailureReasons().get(k) * 2)));
+        assertThat(empty.getTook().count(), equalTo(full.getTook().count() * 2));
+        assertThat(empty.getTookMrtTrue().count(), equalTo(full.getTookMrtTrue().count() * 2));
+        assertThat(empty.getTookMrtFalse().count(), equalTo(full.getTookMrtFalse().count() * 2));
+        assertThat(empty.getSearchCountWithSkippedRemotes(), equalTo(full.getSearchCountWithSkippedRemotes() * 2));
+        assertThat(empty.getRemotesPerSearchMax(), equalTo(full.getRemotesPerSearchMax()));
+        assertThat(empty.getRemotesPerSearchAvg(), closeTo(full.getRemotesPerSearchAvg(), 0.01));
+        empty.getFeatureCounts().forEach((k, v) -> assertThat(v, equalTo(full.getFeatureCounts().get(k) * 2)));
+        empty.getClientCounts().forEach((k, v) -> assertThat(v, equalTo(full.getClientCounts().get(k) * 2)));
+        empty.getByRemoteCluster().forEach((k, v) -> {
+            assertThat(v.getCount(), equalTo(full.getByRemoteCluster().get(k).getCount() * 2));
+            assertThat(v.getSkippedCount(), equalTo(full.getByRemoteCluster().get(k).getSkippedCount() * 2));
+            assertThat(v.getTook().count(), equalTo(full.getByRemoteCluster().get(k).getTook().count() * 2));
+        });
+    }
+
+    public void testAddTwo() {
+        CCSTelemetrySnapshot empty = new CCSTelemetrySnapshot();
+        CCSTelemetrySnapshot full = randomCCSTelemetrySnapshot();
+        CCSTelemetrySnapshot full2 = randomCCSTelemetrySnapshot();
+
+        empty.add(full);
+        empty.add(full2);
+        assertThat(empty.getTotalCount(), equalTo(full.getTotalCount() + full2.getTotalCount()));
+        assertThat(empty.getSuccessCount(), equalTo(full.getSuccessCount() + full2.getSuccessCount()));
+        empty.getFailureReasons()
+            .forEach(
+                (k, v) -> assertThat(
+                    v,
+                    equalTo(full.getFailureReasons().getOrDefault(k, 0L) + full2.getFailureReasons().getOrDefault(k, 0L))
+                )
+            );
+        assertThat(empty.getTook().count(), equalTo(full.getTook().count() + full2.getTook().count()));
+        assertThat(empty.getTookMrtTrue().count(), equalTo(full.getTookMrtTrue().count() + full2.getTookMrtTrue().count()));
+        assertThat(empty.getTookMrtFalse().count(), equalTo(full.getTookMrtFalse().count() + full2.getTookMrtFalse().count()));
+        assertThat(
+            empty.getSearchCountWithSkippedRemotes(),
+            equalTo(full.getSearchCountWithSkippedRemotes() + full2.getSearchCountWithSkippedRemotes())
+        );
+        assertThat(empty.getRemotesPerSearchMax(), equalTo(Math.max(full.getRemotesPerSearchMax(), full2.getRemotesPerSearchMax())));
+        double expectedAvg = (full.getRemotesPerSearchAvg() * full.getTotalCount() + full2.getRemotesPerSearchAvg() * full2.getTotalCount())
+            / empty.getTotalCount();
+        assertThat(empty.getRemotesPerSearchAvg(), closeTo(expectedAvg, 0.01));
+        empty.getFeatureCounts()
+            .forEach(
+                (k, v) -> assertThat(v, equalTo(full.getFeatureCounts().getOrDefault(k, 0L) + full2.getFeatureCounts().getOrDefault(k, 0L)))
+            );
+        empty.getClientCounts()
+            .forEach(
+                (k, v) -> assertThat(v, equalTo(full.getClientCounts().getOrDefault(k, 0L) + full2.getClientCounts().getOrDefault(k, 0L)))
+            );
+        PerClusterCCSTelemetry zeroDummy = new PerClusterCCSTelemetry();
+        empty.getByRemoteCluster().forEach((k, v) -> {
+            assertThat(
+                v.getCount(),
+                equalTo(
+                    full.getByRemoteCluster().getOrDefault(k, zeroDummy).getCount() + full2.getByRemoteCluster()
+                        .getOrDefault(k, zeroDummy)
+                        .getCount()
+                )
+            );
+            assertThat(
+                v.getSkippedCount(),
+                equalTo(
+                    full.getByRemoteCluster().getOrDefault(k, zeroDummy).getSkippedCount() + full2.getByRemoteCluster()
+                        .getOrDefault(k, zeroDummy)
+                        .getSkippedCount()
+                )
+            );
+            assertThat(
+                v.getTook().count(),
+                equalTo(
+                    full.getByRemoteCluster().getOrDefault(k, zeroDummy).getTook().count() + full2.getByRemoteCluster()
+                        .getOrDefault(k, zeroDummy)
+                        .getTook()
+                        .count()
+                )
+            );
+        });
+    }
+
+    private LongMetricValue manyValuesHistogram(long startingWith) {
+        LongMetric metric = new LongMetric();
+        // Produce 100 values from startingWith to 2 * startingWith with equal intervals
+        // We need to space values relative to initial value, otherwise the histogram would put them all in one bucket
+        for (long i = startingWith; i < 2 * startingWith; i += startingWith / 100) {
+            metric.record(i);
+        }
+        return metric.getValue();
+    }
+
+    public void testToXContent() throws IOException {
+        long totalCount = 10;
+        long successCount = 20;
+        // Using TreeMap's here to ensure consistent ordering in the JSON output
+        var failureReasons = new TreeMap<>(Map.of("reason1", 1L, "reason2", 2L, "unknown", 3L));
+        LongMetricValue took = manyValuesHistogram(1000);
+        LongMetricValue tookMrtTrue = manyValuesHistogram(5000);
+        LongMetricValue tookMrtFalse = manyValuesHistogram(10000);
+        long skippedRemotes = 5;
+        long remotesPerSearchMax = 6;
+        double remotesPerSearchAvg = 7.89;
+        var featureCounts = new TreeMap<>(Map.of("async", 10L, "mrt", 20L, "wildcard", 30L));
+        var clientCounts = new TreeMap<>(Map.of("kibana", 40L, "other", 500L));
+        var perClusterCCSTelemetries = new TreeMap<>(
+            Map.of(
+                "",
+                new PerClusterCCSTelemetry(12, 0, manyValuesHistogram(2000)),
+                "remote1",
+                new PerClusterCCSTelemetry(100, 22, manyValuesHistogram(2000)),
+                "remote2",
+                new PerClusterCCSTelemetry(300, 42, manyValuesHistogram(500000))
+            )
+        );
+
+        var snapshot = new CCSTelemetrySnapshot(
+            totalCount,
+            successCount,
+            failureReasons,
+            took,
+            tookMrtTrue,
+            tookMrtFalse,
+            remotesPerSearchMax,
+            remotesPerSearchAvg,
+            skippedRemotes,
+            featureCounts,
+            clientCounts,
+            perClusterCCSTelemetries
+        );
+        String expected = readJSONFromResource("telemetry_test.json");
+        assertEquals(expected, snapshot.toString());
+    }
+
+    private String readJSONFromResource(String fileName) throws IOException {
+        try (InputStream inputStream = getClass().getResourceAsStream("/org/elasticsearch/action/admin/cluster/stats/" + fileName)) {
+            if (inputStream == null) {
+                throw new IOException("Resource not found: " + fileName);
+            }
+            return new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java
new file mode 100644
index 0000000000000..bd36f89f38e4d
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetryTests.java
@@ -0,0 +1,342 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.stats;
+
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.action.admin.cluster.stats.ApproximateMatcher.closeTo;
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE;
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.KNOWN_CLIENTS;
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.MRT_FEATURE;
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.Result.CANCELED;
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.WILDCARD_FEATURE;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class CCSUsageTelemetryTests extends ESTestCase {
+
+    public void testSuccessfulSearchResults() {
+        CCSUsageTelemetry ccsUsageHolder = new CCSUsageTelemetry();
+
+        long expectedAsyncCount = 0L;
+        long expectedMinRTCount = 0L;
+        long expectedSearchesWithSkippedRemotes = 0L;
+        long took1 = 0L;
+        long took1Remote1 = 0L;
+
+        // first search
+        {
+            boolean minimizeRoundTrips = randomBoolean();
+            boolean async = randomBoolean();
+            took1 = randomLongBetween(5, 10000);
+            boolean skippedRemote = randomBoolean();
+            expectedSearchesWithSkippedRemotes = skippedRemote ? 1 : 0;
+            expectedAsyncCount = async ? 1 : 0;
+            expectedMinRTCount = minimizeRoundTrips ? 1 : 0;
+
+            // per cluster telemetry
+            long tookLocal = randomLongBetween(2, 8000);
+            took1Remote1 = randomLongBetween(2, 8000);
+
+            CCSUsage.Builder builder = new CCSUsage.Builder();
+            builder.took(took1).setRemotesCount(1);
+            if (async) {
+                builder.setFeature(ASYNC_FEATURE);
+            }
+            if (minimizeRoundTrips) {
+                builder.setFeature(MRT_FEATURE);
+            }
+            if (skippedRemote) {
+                builder.skippedRemote("remote1");
+            }
+            builder.perClusterUsage("(local)", new TimeValue(tookLocal));
+            builder.perClusterUsage("remote1", new TimeValue(took1Remote1));
+
+            CCSUsage ccsUsage = builder.build();
+            ccsUsageHolder.updateUsage(ccsUsage);
+
+            CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot();
+
+            assertThat(snapshot.getTotalCount(), equalTo(1L));
+            assertThat(snapshot.getSuccessCount(), equalTo(1L));
+            assertThat(snapshot.getFeatureCounts().getOrDefault(ASYNC_FEATURE, 0L), equalTo(expectedAsyncCount));
+            assertThat(snapshot.getFeatureCounts().getOrDefault(MRT_FEATURE, 0L), equalTo(expectedMinRTCount));
+            assertThat(snapshot.getSearchCountWithSkippedRemotes(), equalTo(expectedSearchesWithSkippedRemotes));
+            assertThat(snapshot.getTook().avg(), greaterThan(0L));
+            // Expect it to be within 1% of the actual value
+            assertThat(snapshot.getTook().avg(), closeTo(took1));
+            assertThat(snapshot.getTook().max(), closeTo(took1));
+            if (minimizeRoundTrips) {
+                assertThat(snapshot.getTookMrtTrue().count(), equalTo(1L));
+                assertThat(snapshot.getTookMrtTrue().avg(), greaterThan(0L));
+                assertThat(snapshot.getTookMrtTrue().avg(), closeTo(took1));
+                assertThat(snapshot.getTookMrtFalse().count(), equalTo(0L));
+                assertThat(snapshot.getTookMrtFalse().max(), equalTo(0L));
+            } else {
+                assertThat(snapshot.getTookMrtFalse().count(), equalTo(1L));
+                assertThat(snapshot.getTookMrtFalse().avg(), greaterThan(0L));
+                assertThat(snapshot.getTookMrtFalse().avg(), closeTo(took1));
+                assertThat(snapshot.getTookMrtTrue().count(), equalTo(0L));
+                assertThat(snapshot.getTookMrtTrue().max(), equalTo(0L));
+            }
+            // We currently don't count unknown clients
+            assertThat(snapshot.getClientCounts().size(), equalTo(0));
+
+            // per cluster telemetry asserts
+
+            var telemetryByCluster = snapshot.getByRemoteCluster();
+            assertThat(telemetryByCluster.size(), equalTo(2));
+            var localClusterTelemetry = telemetryByCluster.get("(local)");
+            assertNotNull(localClusterTelemetry);
+            assertThat(localClusterTelemetry.getCount(), equalTo(1L));
+            assertThat(localClusterTelemetry.getSkippedCount(), equalTo(0L));
+            assertThat(localClusterTelemetry.getTook().count(), equalTo(1L));
+            assertThat(localClusterTelemetry.getTook().avg(), greaterThan(0L));
+            assertThat(localClusterTelemetry.getTook().avg(), closeTo(tookLocal));
+            // assertThat(localClusterTelemetry.getTook().max(), greaterThanOrEqualTo(tookLocal));
+
+            var remote1ClusterTelemetry = telemetryByCluster.get("remote1");
+            assertNotNull(remote1ClusterTelemetry);
+            assertThat(remote1ClusterTelemetry.getCount(), equalTo(1L));
+            assertThat(remote1ClusterTelemetry.getSkippedCount(), equalTo(expectedSearchesWithSkippedRemotes));
+            assertThat(remote1ClusterTelemetry.getTook().avg(), greaterThan(0L));
+            assertThat(remote1ClusterTelemetry.getTook().count(), equalTo(1L));
+            assertThat(remote1ClusterTelemetry.getTook().avg(), greaterThan(0L));
+            assertThat(remote1ClusterTelemetry.getTook().avg(), closeTo(took1Remote1));
+            // assertThat(remote1ClusterTelemetry.getTook().max(), greaterThanOrEqualTo(took1Remote1));
+        }
+
+        // second search
+        {
+            boolean minimizeRoundTrips = randomBoolean();
+            boolean async = randomBoolean();
+            expectedAsyncCount += async ? 1 : 0;
+            expectedMinRTCount += minimizeRoundTrips ? 1 : 0;
+            long took2 = randomLongBetween(5, 10000);
+            boolean skippedRemote = randomBoolean();
+            expectedSearchesWithSkippedRemotes += skippedRemote ? 1 : 0;
+            long took2Remote1 = randomLongBetween(2, 8000);
+
+            CCSUsage.Builder builder = new CCSUsage.Builder();
+            builder.took(took2).setRemotesCount(1).setClient("kibana");
+            if (async) {
+                builder.setFeature(ASYNC_FEATURE);
+            }
+            if (minimizeRoundTrips) {
+                builder.setFeature(MRT_FEATURE);
+            }
+            if (skippedRemote) {
+                builder.skippedRemote("remote1");
+            }
+            builder.perClusterUsage("remote1", new TimeValue(took2Remote1));
+
+            CCSUsage ccsUsage = builder.build();
+            ccsUsageHolder.updateUsage(ccsUsage);
+
+            CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot();
+
+            assertThat(snapshot.getTotalCount(), equalTo(2L));
+            assertThat(snapshot.getSuccessCount(), equalTo(2L));
+            assertThat(snapshot.getFeatureCounts().getOrDefault(ASYNC_FEATURE, 0L), equalTo(expectedAsyncCount));
+            assertThat(snapshot.getFeatureCounts().getOrDefault(MRT_FEATURE, 0L), equalTo(expectedMinRTCount));
+            assertThat(snapshot.getSearchCountWithSkippedRemotes(), equalTo(expectedSearchesWithSkippedRemotes));
+            assertThat(snapshot.getTook().avg(), greaterThan(0L));
+            assertThat(snapshot.getTook().avg(), closeTo((took1 + took2) / 2));
+            // assertThat(snapshot.getTook().max(), greaterThanOrEqualTo(Math.max(took1, took2)));
+
+            // Counting only known clients
+            assertThat(snapshot.getClientCounts().get("kibana"), equalTo(1L));
+            assertThat(snapshot.getClientCounts().size(), equalTo(1));
+
+            // per cluster telemetry asserts
+
+            var telemetryByCluster = snapshot.getByRemoteCluster();
+            assertThat(telemetryByCluster.size(), equalTo(2));
+            var localClusterTelemetry = telemetryByCluster.get("(local)");
+            assertNotNull(localClusterTelemetry);
+            assertThat(localClusterTelemetry.getCount(), equalTo(1L));
+            assertThat(localClusterTelemetry.getSkippedCount(), equalTo(0L));
+            assertThat(localClusterTelemetry.getTook().count(), equalTo(1L));
+
+            var remote1ClusterTelemetry = telemetryByCluster.get("remote1");
+            assertNotNull(remote1ClusterTelemetry);
+            assertThat(remote1ClusterTelemetry.getCount(), equalTo(2L));
+            assertThat(remote1ClusterTelemetry.getSkippedCount(), equalTo(expectedSearchesWithSkippedRemotes));
+            assertThat(remote1ClusterTelemetry.getTook().avg(), greaterThan(0L));
+            assertThat(remote1ClusterTelemetry.getTook().count(), equalTo(2L));
+            assertThat(remote1ClusterTelemetry.getTook().avg(), greaterThan(0L));
+            assertThat(remote1ClusterTelemetry.getTook().avg(), closeTo((took1Remote1 + took2Remote1) / 2));
+            // assertThat(remote1ClusterTelemetry.getTook().max(), greaterThanOrEqualTo(Math.max(took1Remote1, took2Remote1)));
+        }
+    }
+
+    public void testClientsLimit() {
+        CCSUsageTelemetry ccsUsageHolder = new CCSUsageTelemetry();
+        // Add known clients
+        for (String knownClient : KNOWN_CLIENTS) {
+            CCSUsage.Builder builder = new CCSUsage.Builder();
+            builder.took(randomLongBetween(5, 10000)).setRemotesCount(1).setClient(knownClient);
+            CCSUsage ccsUsage = builder.build();
+            ccsUsageHolder.updateUsage(ccsUsage);
+        }
+        var counts = ccsUsageHolder.getCCSTelemetrySnapshot().getClientCounts();
+        for (String knownClient : KNOWN_CLIENTS) {
+            assertThat(counts.get(knownClient), equalTo(1L));
+        }
+        // Check that knowns are counted
+        for (String knownClient : KNOWN_CLIENTS) {
+            CCSUsage.Builder builder = new CCSUsage.Builder();
+            builder.took(randomLongBetween(5, 10000)).setRemotesCount(1).setClient(knownClient);
+            CCSUsage ccsUsage = builder.build();
+            ccsUsageHolder.updateUsage(ccsUsage);
+        }
+        counts = ccsUsageHolder.getCCSTelemetrySnapshot().getClientCounts();
+        for (String knownClient : KNOWN_CLIENTS) {
+            assertThat(counts.get(knownClient), equalTo(2L));
+        }
+        // Check that new clients are not counted
+        CCSUsage.Builder builder = new CCSUsage.Builder();
+        String randomClient = randomAlphaOfLength(10);
+        builder.took(randomLongBetween(5, 10000)).setRemotesCount(1).setClient(randomClient);
+        CCSUsage ccsUsage = builder.build();
+        ccsUsageHolder.updateUsage(ccsUsage);
+        counts = ccsUsageHolder.getCCSTelemetrySnapshot().getClientCounts();
+        assertThat(counts.get(randomClient), equalTo(null));
+    }
+
+    public void testFailures() {
+        CCSUsageTelemetry ccsUsageHolder = new CCSUsageTelemetry();
+
+        // first search
+        {
+            boolean skippedRemote = randomBoolean();
+            boolean minimizeRoundTrips = randomBoolean();
+            boolean async = randomBoolean();
+
+            CCSUsage.Builder builder = new CCSUsage.Builder();
+            builder.setRemotesCount(1).took(10L);
+            if (skippedRemote) {
+                builder.skippedRemote("remote1");
+            }
+            builder.perClusterUsage("(local)", new TimeValue(1));
+            builder.perClusterUsage("remote1", new TimeValue(2));
+            builder.setFailure(CANCELED);
+            if (async) {
+                builder.setFeature(ASYNC_FEATURE);
+            }
+            if (minimizeRoundTrips) {
+                builder.setFeature(MRT_FEATURE);
+            }
+
+            CCSUsage ccsUsage = builder.build();
+            ccsUsageHolder.updateUsage(ccsUsage);
+
+            CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot();
+
+            assertThat(snapshot.getTotalCount(), equalTo(1L));
+            assertThat(snapshot.getSuccessCount(), equalTo(0L));
+            assertThat(snapshot.getSearchCountWithSkippedRemotes(), equalTo(skippedRemote ? 1L : 0L));
+            assertThat(snapshot.getTook().count(), equalTo(0L));
+            assertThat(snapshot.getFailureReasons().size(), equalTo(1));
+            assertThat(snapshot.getFailureReasons().get(CANCELED.getName()), equalTo(1L));
+            // still counting features on failure
+            assertThat(snapshot.getFeatureCounts().getOrDefault(ASYNC_FEATURE, 0L), equalTo(async ? 1L : 0L));
+            assertThat(snapshot.getFeatureCounts().getOrDefault(MRT_FEATURE, 0L), equalTo(minimizeRoundTrips ? 1L : 0L));
+        }
+
+        // second search
+        {
+            CCSUsage.Builder builder = new CCSUsage.Builder();
+            boolean skippedRemote = randomBoolean();
+            builder.setRemotesCount(1).took(10L).setClient("kibana");
+            if (skippedRemote) {
+                builder.skippedRemote("remote1");
+            }
+            builder.setFailure(CANCELED);
+            CCSUsage ccsUsage = builder.build();
+
+            ccsUsageHolder.updateUsage(ccsUsage);
+
+            CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot();
+
+            assertThat(snapshot.getTotalCount(), equalTo(2L));
+            assertThat(snapshot.getSuccessCount(), equalTo(0L));
+            assertThat(snapshot.getTook().count(), equalTo(0L));
+            assertThat(snapshot.getFailureReasons().size(), equalTo(1));
+            assertThat(snapshot.getFailureReasons().get(CANCELED.getName()), equalTo(2L));
+            assertThat(snapshot.getClientCounts().get("kibana"), equalTo(1L));
+        }
+    }
+
+    public void testConcurrentUpdates() throws InterruptedException {
+        CCSUsageTelemetry ccsUsageHolder = new CCSUsageTelemetry();
+        CCSUsageTelemetry expectedHolder = new CCSUsageTelemetry();
+        int numSearches = randomIntBetween(1000, 5000);
+        int numThreads = randomIntBetween(10, 20);
+        Thread[] threads = new Thread[numThreads];
+        CCSUsage[] ccsUsages = new CCSUsage[numSearches];
+
+        // Make random usage objects
+        for (int i = 0; i < numSearches; i++) {
+            CCSUsage.Builder builder = new CCSUsage.Builder();
+            builder.took(randomLongBetween(5, 10000)).setRemotesCount(randomIntBetween(1, 10));
+            if (randomBoolean()) {
+                builder.setFeature(ASYNC_FEATURE);
+            }
+            if (randomBoolean()) {
+                builder.setFeature(WILDCARD_FEATURE);
+            }
+            if (randomBoolean()) {
+                builder.setFeature(MRT_FEATURE);
+            }
+            if (randomBoolean()) {
+                builder.setClient("kibana");
+            }
+            if (randomInt(20) == 7) {
+                // 5% of requests will fail
+                builder.setFailure(randomFrom(CCSUsageTelemetry.Result.values()));
+                ccsUsages[i] = builder.build();
+                continue;
+            }
+            builder.perClusterUsage("", new TimeValue(randomLongBetween(1, 10000)));
+            if (randomBoolean()) {
+                builder.skippedRemote("remote1");
+            } else {
+                builder.perClusterUsage("remote1", new TimeValue(randomLongBetween(1, 10000)));
+            }
+            builder.perClusterUsage(randomFrom("remote2", "remote3", "remote4"), new TimeValue(randomLongBetween(1, 10000)));
+            ccsUsages[i] = builder.build();
+        }
+
+        // Add each of the search objects to the telemetry holder in a different thread
+        for (int i = 0; i < numThreads; i++) {
+            final int threadNo = i;
+            threads[i] = new Thread(() -> {
+                for (int j = threadNo; j < numSearches; j += numThreads) {
+                    ccsUsageHolder.updateUsage(ccsUsages[j]);
+                }
+            });
+            threads[i].start();
+        }
+
+        for (int i = 0; i < numThreads; i++) {
+            threads[i].join();
+        }
+
+        // Add the same search objects to the expected holder in a single thread
+        for (int i = 0; i < numSearches; i++) {
+            expectedHolder.updateUsage(ccsUsages[i]);
+        }
+
+        CCSTelemetrySnapshot snapshot = ccsUsageHolder.getCCSTelemetrySnapshot();
+        CCSTelemetrySnapshot expectedSnapshot = ccsUsageHolder.getCCSTelemetrySnapshot();
+        assertThat(snapshot, equalTo(expectedSnapshot));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java
index 49528c204b042..20eae9833e4b0 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java
@@ -127,7 +127,8 @@ public void testCreation() {
             null,
             null,
             new ShardStats[] { shardStats },
-            null
+            new SearchUsageStats(),
+            RepositoryUsageStats.EMPTY
         );
 
         stats = VersionStats.of(metadata, Collections.singletonList(nodeResponse));
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java
index cf7bda6fc4c0b..f870588c61c7a 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java
@@ -19,7 +19,7 @@
 
 public class GetStoredScriptRequestTests extends ESTestCase {
     public void testGetIndexedScriptRequestSerialization() throws IOException {
-        GetStoredScriptRequest request = new GetStoredScriptRequest("id");
+        GetStoredScriptRequest request = new GetStoredScriptRequest(TEST_REQUEST_TIMEOUT, "id");
 
         BytesStreamOutput out = new BytesStreamOutput();
         out.setTransportVersion(randomVersion(random()));
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java
index aadd265c4a242..2a8bbed983d29 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java
@@ -25,6 +25,8 @@ public class PutStoredScriptRequestTests extends ESTestCase {
 
     public void testSerialization() throws IOException {
         PutStoredScriptRequest storedScriptRequest = new PutStoredScriptRequest(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
             "bar",
             "context",
             new BytesArray("{}"),
@@ -54,7 +56,7 @@ public void testToXContent() throws IOException {
 
         BytesReference expectedRequestBody = BytesReference.bytes(builder);
 
-        PutStoredScriptRequest request = new PutStoredScriptRequest();
+        PutStoredScriptRequest request = new PutStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.id("test1");
         request.content(expectedRequestBody, xContentType);
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
index d31c9fddf2712..4833d72484495 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComponentTemplateTests;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionTests;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.Strings;
@@ -45,8 +44,7 @@ protected Writeable.Reader<GetComponentTemplateAction.Response> instanceReader()
     protected GetComponentTemplateAction.Response createTestInstance() {
         return new GetComponentTemplateAction.Response(
             randomBoolean() ? Map.of() : randomTemplates(),
-            RolloverConfigurationTests.randomRolloverConditions(),
-            DataStreamGlobalRetentionTests.randomGlobalRetention()
+            RolloverConfigurationTests.randomRolloverConditions()
         );
     }
 
@@ -54,13 +52,11 @@ protected GetComponentTemplateAction.Response createTestInstance() {
     protected GetComponentTemplateAction.Response mutateInstance(GetComponentTemplateAction.Response instance) {
         var templates = instance.getComponentTemplates();
         var rolloverConditions = instance.getRolloverConfiguration();
-        var globalRetention = instance.getGlobalRetention();
-        switch (randomInt(2)) {
+        switch (randomInt(1)) {
             case 0 -> templates = templates == null ? randomTemplates() : null;
             case 1 -> rolloverConditions = randomValueOtherThan(rolloverConditions, RolloverConfigurationTests::randomRolloverConditions);
-            case 2 -> globalRetention = randomValueOtherThan(globalRetention, DataStreamGlobalRetentionTests::randomGlobalRetention);
         }
-        return new GetComponentTemplateAction.Response(templates, rolloverConditions, globalRetention);
+        return new GetComponentTemplateAction.Response(templates, rolloverConditions);
     }
 
     public void testXContentSerializationWithRolloverAndEffectiveRetention() throws IOException {
@@ -84,20 +80,15 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws
             null,
             false
         );
-        var globalRetention = DataStreamGlobalRetentionTests.randomGlobalRetention();
         var rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions();
-        var response = new GetComponentTemplateAction.Response(
-            Map.of(randomAlphaOfLength(10), template),
-            rolloverConfiguration,
-            globalRetention
-        );
+        var response = new GetComponentTemplateAction.Response(Map.of(randomAlphaOfLength(10), template), rolloverConfiguration);
 
         try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
             builder.humanReadable(true);
             response.toXContent(builder, EMPTY_PARAMS);
             String serialized = Strings.toString(builder);
             assertThat(serialized, containsString("rollover"));
-            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention))
+            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(null, randomBoolean()))
                 .getConditions()
                 .keySet()) {
                 assertThat(serialized, containsString(label));
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java
index aa9989257aa39..c3deabd849998 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java
@@ -8,11 +8,13 @@
 
 package org.elasticsearch.action.admin.indices.template.get;
 
+import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
+import org.elasticsearch.action.admin.indices.rollover.RolloverConfigurationTests;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionTests;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
 import java.util.HashMap;
@@ -26,19 +28,41 @@ protected Writeable.Reader<GetComposableIndexTemplateAction.Response> instanceRe
 
     @Override
     protected GetComposableIndexTemplateAction.Response createTestInstance() {
-        DataStreamGlobalRetention globalRetention = randomBoolean() ? null : DataStreamGlobalRetentionTests.randomGlobalRetention();
+        RolloverConfiguration rolloverConfiguration = randomBoolean() ? null : RolloverConfigurationTests.randomRolloverConditions();
         if (randomBoolean()) {
-            return new GetComposableIndexTemplateAction.Response(Map.of(), globalRetention);
+            return new GetComposableIndexTemplateAction.Response(Map.of(), rolloverConfiguration);
         }
         Map<String, ComposableIndexTemplate> templates = new HashMap<>();
         for (int i = 0; i < randomIntBetween(1, 4); i++) {
             templates.put(randomAlphaOfLength(4), ComposableIndexTemplateTests.randomInstance());
         }
-        return new GetComposableIndexTemplateAction.Response(templates, globalRetention);
+        return new GetComposableIndexTemplateAction.Response(templates, rolloverConfiguration);
     }
 
     @Override
     protected GetComposableIndexTemplateAction.Response mutateInstance(GetComposableIndexTemplateAction.Response instance) {
-        return randomValueOtherThan(instance, this::createTestInstance);
+        var rolloverConfiguration = instance.getRolloverConfiguration();
+        var templates = instance.indexTemplates();
+        switch (randomInt(1)) {
+            case 0 -> rolloverConfiguration = randomBoolean() || rolloverConfiguration == null
+                ? randomValueOtherThan(rolloverConfiguration, RolloverConfigurationTests::randomRolloverConditions)
+                : null;
+            case 1 -> {
+                var updatedTemplates = new HashMap<String, ComposableIndexTemplate>();
+                for (String name : templates.keySet()) {
+                    if (randomBoolean()) {
+                        updatedTemplates.put(name, templates.get(name));
+                    }
+                }
+                updatedTemplates.put(randomAlphaOfLength(4), ComposableIndexTemplateTests.randomInstance());
+                templates = updatedTemplates;
+            }
+        }
+        return new GetComposableIndexTemplateAction.Response(templates, rolloverConfiguration);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(IndicesModule.getNamedWriteables());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java
index 8cd0bb02b0e7c..32a74fef61209 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStreamFactoryRetention;
-import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionResolver;
+import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
@@ -26,6 +26,7 @@
 import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
 import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
@@ -75,7 +76,7 @@ public class ReservedComposableIndexTemplateActionTests extends ESTestCase {
     ClusterService clusterService;
     IndexScopedSettings indexScopedSettings;
     IndicesService indicesService;
-    private DataStreamGlobalRetentionResolver globalRetentionResolver;
+    private DataStreamGlobalRetentionSettings globalRetentionSettings;
 
     @Before
     public void setup() throws IOException {
@@ -92,7 +93,10 @@ public void setup() throws IOException {
         doReturn(mapperService).when(indexService).mapperService();
         doReturn(indexService).when(indicesService).createIndex(any(), any(), anyBoolean());
 
-        globalRetentionResolver = new DataStreamGlobalRetentionResolver(DataStreamFactoryRetention.emptyFactoryRetention());
+        globalRetentionSettings = DataStreamGlobalRetentionSettings.create(
+            ClusterSettings.createBuiltInClusterSettings(),
+            DataStreamFactoryRetention.emptyFactoryRetention()
+        );
         templateService = new MetadataIndexTemplateService(
             clusterService,
             mock(MetadataCreateIndexService.class),
@@ -101,7 +105,7 @@ public void setup() throws IOException {
             mock(NamedXContentRegistry.class),
             mock(SystemIndices.class),
             new IndexSettingProviders(Set.of()),
-            globalRetentionResolver
+            globalRetentionSettings
         );
     }
 
@@ -896,7 +900,7 @@ public void testTemplatesWithReservedPrefix() throws Exception {
             mock(NamedXContentRegistry.class),
             mock(SystemIndices.class),
             new IndexSettingProviders(Set.of()),
-            globalRetentionResolver
+            globalRetentionSettings
         );
 
         ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")).metadata(metadata).build();
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java
index 76bf8dc79b855..0c0e1de74a3e7 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.ClusterName;
@@ -60,9 +61,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -201,9 +200,6 @@ public void tearDownThreadpool() {
     public void testClusterBlockedFailsBulk() {
         NodeClient client = getNodeClient(assertNoClientInteraction());
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
         // Not retryable
         ClusterState state = ClusterState.builder(DEFAULT_STATE)
             .blocks(ClusterBlocks.builder().addGlobalBlock(Metadata.CLUSTER_READ_ONLY_BLOCK).build())
@@ -215,9 +211,10 @@ public void testClusterBlockedFailsBulk() {
         when(observer.isTimedOut()).thenReturn(false);
         doThrow(new AssertionError("Should not wait")).when(observer).waitForNextChange(any());
 
-        newBulkOperation(client, new BulkRequest(), state, observer, listener).run();
-
-        expectThrows(ExecutionException.class, ClusterBlockException.class, future::get);
+        assertThat(
+            safeAwaitFailure(BulkResponse.class, l -> newBulkOperation(client, new BulkRequest(), state, observer, l).run()),
+            instanceOf(ClusterBlockException.class)
+        );
     }
 
     /**
@@ -226,9 +223,6 @@ public void testClusterBlockedFailsBulk() {
     public void testTimeoutOnRetryableClusterBlockedFailsBulk() {
         NodeClient client = getNodeClient(assertNoClientInteraction());
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
         // Retryable
         final ClusterState state = ClusterState.builder(DEFAULT_STATE)
             .blocks(ClusterBlocks.builder().addGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_WRITES).build())
@@ -248,9 +242,11 @@ public void testTimeoutOnRetryableClusterBlockedFailsBulk() {
             return null;
         }).doThrow(new AssertionError("Should not wait")).when(observer).waitForNextChange(any());
 
-        newBulkOperation(client, new BulkRequest(), state, observer, listener).run();
+        assertThat(
+            safeAwaitFailure(BulkResponse.class, l -> newBulkOperation(client, new BulkRequest(), state, observer, l).run()),
+            instanceOf(ClusterBlockException.class)
+        );
 
-        expectThrows(ExecutionException.class, ClusterBlockException.class, future::get);
         verify(observer, times(2)).isTimedOut();
         verify(observer, times(1)).waitForNextChange(any());
     }
@@ -261,9 +257,6 @@ public void testTimeoutOnRetryableClusterBlockedFailsBulk() {
     public void testNodeClosedOnRetryableClusterBlockedFailsBulk() {
         NodeClient client = getNodeClient(assertNoClientInteraction());
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
         // Retryable
         final ClusterState state = ClusterState.builder(DEFAULT_STATE)
             .blocks(ClusterBlocks.builder().addGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_WRITES).build())
@@ -278,9 +271,10 @@ public void testNodeClosedOnRetryableClusterBlockedFailsBulk() {
             return null;
         }).doThrow(new AssertionError("Should not wait")).when(observer).waitForNextChange(any());
 
-        newBulkOperation(client, new BulkRequest(), state, observer, listener).run();
-
-        expectThrows(ExecutionException.class, NodeClosedException.class, future::get);
+        assertThat(
+            safeAwaitFailure(BulkResponse.class, l -> newBulkOperation(client, new BulkRequest(), state, observer, l).run()),
+            instanceOf(NodeClosedException.class)
+        );
         verify(observer, times(1)).isTimedOut();
         verify(observer, times(1)).waitForNextChange(any());
     }
@@ -296,12 +290,7 @@ public void testBulkToIndex() throws Exception {
 
         NodeClient client = getNodeClient(acceptAllShardWrites());
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, listener).run();
-
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, l).run());
         assertThat(bulkItemResponses.hasFailures(), is(false));
     }
 
@@ -318,12 +307,7 @@ public void testBulkToIndexFailingEntireShard() throws Exception {
             shardSpecificResponse(Map.of(new ShardId(indexMetadata.getIndex(), 0), failWithException(() -> new MapperException("test"))))
         );
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, listener).run();
-
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, l).run());
         assertThat(bulkItemResponses.hasFailures(), is(true));
         BulkItemResponse failedItem = Arrays.stream(bulkItemResponses.getItems())
             .filter(BulkItemResponse::isFailed)
@@ -344,12 +328,7 @@ public void testBulkToDataStream() throws Exception {
 
         NodeClient client = getNodeClient(acceptAllShardWrites());
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, listener).run();
-
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, l).run());
         assertThat(bulkItemResponses.hasFailures(), is(false));
     }
 
@@ -366,12 +345,7 @@ public void testBulkToDataStreamFailingEntireShard() throws Exception {
             shardSpecificResponse(Map.of(new ShardId(ds1BackingIndex2.getIndex(), 0), failWithException(() -> new MapperException("test"))))
         );
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, listener).run();
-
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, l).run());
         assertThat(bulkItemResponses.hasFailures(), is(true));
         BulkItemResponse failedItem = Arrays.stream(bulkItemResponses.getItems())
             .filter(BulkItemResponse::isFailed)
@@ -396,12 +370,7 @@ public void testFailingEntireShardRedirectsToFailureStore() throws Exception {
             shardSpecificResponse(Map.of(new ShardId(ds2BackingIndex1.getIndex(), 0), failWithException(() -> new MapperException("test"))))
         );
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, listener).run();
-
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, l).run());
         assertThat(bulkItemResponses.hasFailures(), is(false));
         BulkItemResponse failedItem = Arrays.stream(bulkItemResponses.getItems())
             .filter(item -> item.getIndex().equals(ds2FailureStore1.getIndex().getName()))
@@ -426,12 +395,7 @@ public void testFailingDocumentRedirectsToFailureStore() throws Exception {
             thatFailsDocuments(Map.of(new IndexAndId(ds2BackingIndex1.getIndex().getName(), "3"), () -> new MapperException("test")))
         );
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, listener).run();
-
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, l).run());
         assertThat(bulkItemResponses.hasFailures(), is(false));
         BulkItemResponse failedItem = Arrays.stream(bulkItemResponses.getItems())
             .filter(item -> item.getIndex().equals(ds2FailureStore1.getIndex().getName()))
@@ -465,12 +429,7 @@ public void testFailureStoreShardFailureRejectsDocument() throws Exception {
             )
         );
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, listener).run();
-
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, l).run());
         assertThat(bulkItemResponses.hasFailures(), is(true));
         BulkItemResponse failedItem = Arrays.stream(bulkItemResponses.getItems())
             .filter(BulkItemResponse::isFailed)
@@ -500,16 +459,12 @@ public void testFailedDocumentCanNotBeConvertedFails() throws Exception {
             thatFailsDocuments(Map.of(new IndexAndId(ds2BackingIndex1.getIndex().getName(), "3"), () -> new MapperException("root cause")))
         );
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
         // Mock a failure store document converter that always fails
         FailureStoreDocumentConverter mockConverter = mock(FailureStoreDocumentConverter.class);
         when(mockConverter.transformFailedRequest(any(), any(), any(), any())).thenThrow(new IOException("Could not serialize json"));
 
-        newBulkOperation(client, bulkRequest, mockConverter, listener).run();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, mockConverter, l).run());
 
-        BulkResponse bulkItemResponses = future.get();
         assertThat(bulkItemResponses.hasFailures(), is(true));
         BulkItemResponse failedItem = Arrays.stream(bulkItemResponses.getItems())
             .filter(BulkItemResponse::isFailed)
@@ -579,13 +534,10 @@ public void testRetryableBlockAcceptsFailureStoreDocument() throws Exception {
             return null;
         }).when(observer).waitForNextChange(any());
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.notifyOnce(
-            ActionListener.wrap(future::complete, future::completeExceptionally)
+        final SubscribableListener<BulkResponse> responseListener = SubscribableListener.newForked(
+            l -> newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, l).run()
         );
 
-        newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, listener).run();
-
         // The operation will attempt to write the documents in the request, receive a failure, wait for a stable cluster state, and then
         // redirect the failed documents to the failure store. Wait for that failure store write to start:
         if (readyToPerformFailureStoreWrite.await(30, TimeUnit.SECONDS) == false) {
@@ -595,7 +547,7 @@ public void testRetryableBlockAcceptsFailureStoreDocument() throws Exception {
         }
 
         // Check to make sure there is no response yet
-        if (future.isDone()) {
+        if (responseListener.isDone()) {
             // we're going to fail the test, but be a good citizen and unblock the other thread first
             beginFailureStoreWrite.countDown();
             fail("bulk operation completed prematurely");
@@ -605,7 +557,7 @@ public void testRetryableBlockAcceptsFailureStoreDocument() throws Exception {
         beginFailureStoreWrite.countDown();
 
         // Await final result and verify
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(responseListener);
         assertThat(bulkItemResponses.hasFailures(), is(false));
         BulkItemResponse failedItem = Arrays.stream(bulkItemResponses.getItems())
             .filter(item -> item.getIndex().equals(ds2FailureStore1.getIndex().getName()))
@@ -650,12 +602,7 @@ public void testBlockedClusterRejectsFailureStoreDocument() throws Exception {
         when(observer.isTimedOut()).thenReturn(false);
         doThrow(new AssertionError("Should not wait on non retryable block")).when(observer).waitForNextChange(any());
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, listener).run();
-
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, l).run());
         assertThat(bulkItemResponses.hasFailures(), is(true));
         BulkItemResponse failedItem = Arrays.stream(bulkItemResponses.getItems())
             .filter(BulkItemResponse::isFailed)
@@ -715,12 +662,7 @@ public void testOperationTimeoutRejectsFailureStoreDocument() throws Exception {
             return null;
         }).doThrow(new AssertionError("Should not wait any longer")).when(observer).waitForNextChange(any());
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, listener).run();
-
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, l).run());
         assertThat(bulkItemResponses.hasFailures(), is(true));
         BulkItemResponse failedItem = Arrays.stream(bulkItemResponses.getItems())
             .filter(BulkItemResponse::isFailed)
@@ -775,12 +717,10 @@ public void testNodeClosureRejectsFailureStoreDocument() {
             return null;
         }).doThrow(new AssertionError("Should not wait any longer")).when(observer).waitForNextChange(any());
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, listener).run();
-
-        expectThrows(ExecutionException.class, NodeClosedException.class, future::get);
+        assertThat(
+            safeAwaitFailure(BulkResponse.class, l -> newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, l).run()),
+            instanceOf(NodeClosedException.class)
+        );
 
         verify(observer, times(1)).isTimedOut();
         verify(observer, times(1)).waitForNextChange(any());
@@ -832,12 +772,7 @@ public void testLazilyRollingOverFailureStore() throws Exception {
         ClusterState rolledOverState = ClusterState.builder(DEFAULT_STATE).metadata(metadata).build();
         ClusterStateObserver observer = mockObserver(DEFAULT_STATE, DEFAULT_STATE, rolledOverState);
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, listener).run();
-
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, l).run());
         BulkItemResponse failedItem = Arrays.stream(bulkItemResponses.getItems())
             .filter(item -> item.getIndex().equals(ds3FailureStore2.getIndex().getName()))
             .findFirst()
@@ -880,12 +815,7 @@ public void testFailureWhileRollingOverFailureStore() throws Exception {
         ClusterState rolledOverState = ClusterState.builder(DEFAULT_STATE).metadata(metadata).build();
         ClusterStateObserver observer = mockObserver(DEFAULT_STATE, DEFAULT_STATE, rolledOverState);
 
-        CompletableFuture<BulkResponse> future = new CompletableFuture<>();
-        ActionListener<BulkResponse> listener = ActionListener.wrap(future::complete, future::completeExceptionally);
-
-        newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, listener).run();
-
-        BulkResponse bulkItemResponses = future.get();
+        BulkResponse bulkItemResponses = safeAwait(l -> newBulkOperation(client, bulkRequest, DEFAULT_STATE, observer, l).run());
         BulkItemResponse failedItem = Arrays.stream(bulkItemResponses.getItems())
             .filter(BulkItemResponse::isFailed)
             .findFirst()
@@ -1164,7 +1094,8 @@ private BulkOperation newBulkOperation(
             timeZero,
             listener,
             observer,
-            failureStoreDocumentConverter
+            failureStoreDocumentConverter,
+            FailureStoreMetrics.NOOP
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
index 5732e7f29f65d..61f5105301c30 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
@@ -290,13 +290,13 @@ public void testSmileIsSupported() throws IOException {
                 builder.endObject();
                 builder.endObject();
             }
-            out.write(xContentType.xContent().streamSeparator());
+            out.write(xContentType.xContent().bulkSeparator());
             try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) {
                 builder.startObject();
                 builder.field("field", "value");
                 builder.endObject();
             }
-            out.write(xContentType.xContent().streamSeparator());
+            out.write(xContentType.xContent().bulkSeparator());
             data = out.bytes();
         }
 
@@ -327,7 +327,7 @@ public void testToValidateUpsertRequestAndCASInBulkRequest() throws IOException
                 builder.endObject();
                 builder.endObject();
             }
-            out.write(xContentType.xContent().streamSeparator());
+            out.write(xContentType.xContent().bulkSeparator());
             try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) {
                 builder.startObject();
                 builder.startObject("doc").endObject();
@@ -338,7 +338,7 @@ public void testToValidateUpsertRequestAndCASInBulkRequest() throws IOException
                 builder.field("upsert", values);
                 builder.endObject();
             }
-            out.write(xContentType.xContent().streamSeparator());
+            out.write(xContentType.xContent().bulkSeparator());
             data = out.bytes();
         }
         BulkRequest bulkRequest = new BulkRequest();
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverterTests.java b/server/src/test/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverterTests.java
index 85cedff9145be..d33b2877d4280 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverterTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverterTests.java
@@ -83,18 +83,18 @@ public void testFailureStoreDocumentConversion() throws Exception {
 
         assertThat(ObjectPath.eval("document.id", convertedRequest.sourceAsMap()), is(equalTo("1")));
         assertThat(ObjectPath.eval("document.routing", convertedRequest.sourceAsMap()), is(equalTo("fake_routing")));
-        assertThat(ObjectPath.eval("document.index", convertedRequest.sourceAsMap()), is(equalTo(targetIndexName)));
+        assertThat(ObjectPath.eval("document.index", convertedRequest.sourceAsMap()), is(equalTo("original_index")));
         assertThat(ObjectPath.eval("document.source.key", convertedRequest.sourceAsMap()), is(equalTo("value")));
 
         assertThat(ObjectPath.eval("error.type", convertedRequest.sourceAsMap()), is(equalTo("exception")));
         assertThat(ObjectPath.eval("error.message", convertedRequest.sourceAsMap()), is(equalTo("Test exception please ignore")));
         assertThat(
             ObjectPath.eval("error.stack_trace", convertedRequest.sourceAsMap()),
-            startsWith("org.elasticsearch.ElasticsearchException: Test exception please ignore")
+            startsWith("o.e.ElasticsearchException: Test exception please ignore")
         );
         assertThat(
             ObjectPath.eval("error.stack_trace", convertedRequest.sourceAsMap()),
-            containsString("at org.elasticsearch.action.bulk.FailureStoreDocumentConverterTests.testFailureStoreDocumentConversion")
+            containsString("at o.e.a.b.FailureStoreDocumentConverterTests.testFailureStoreDocumentConversion")
         );
         assertThat(
             ObjectPath.eval("error.pipeline_trace", convertedRequest.sourceAsMap()),
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
index 4ca4e7158e454..1d3d514da13a3 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
@@ -130,7 +130,8 @@ public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state)
             mock(ActionFilters.class),
             indexNameExpressionResolver,
             new IndexingPressure(Settings.EMPTY),
-            EmptySystemIndices.INSTANCE
+            EmptySystemIndices.INSTANCE,
+            FailureStoreMetrics.NOOP
         ) {
             @Override
             void executeBulk(
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
index 3683c2c271739..609237f268807 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
@@ -69,7 +69,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
-import java.util.function.Predicate;
+import java.util.function.Function;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.sameInstance;
@@ -110,7 +110,7 @@ public class TransportBulkActionIngestTests extends ESTestCase {
 
     /** Arguments to callbacks we want to capture, but which require generics, so we must use @Captor */
     @Captor
-    ArgumentCaptor<Predicate<String>> redirectPredicate;
+    ArgumentCaptor<Function<String, Boolean>> redirectPredicate;
     @Captor
     ArgumentCaptor<TriConsumer<Integer, String, Exception>> redirectHandler;
     @Captor
@@ -155,7 +155,8 @@ class TestTransportBulkAction extends TransportBulkAction {
                 new ActionFilters(Collections.emptySet()),
                 TestIndexNameExpressionResolver.newInstance(),
                 new IndexingPressure(SETTINGS),
-                EmptySystemIndices.INSTANCE
+                EmptySystemIndices.INSTANCE,
+                FailureStoreMetrics.NOOP
             );
         }
 
@@ -410,9 +411,10 @@ public void testIngestLocal() throws Exception {
         Iterator<DocWriteRequest<?>> req = bulkDocsItr.getValue().iterator();
         failureHandler.getValue().accept(0, exception); // have an exception for our one index request
         indexRequest2.setPipeline(IngestService.NOOP_PIPELINE_NAME); // this is done by the real pipeline execution service when processing
-        assertTrue(redirectPredicate.getValue().test(WITH_FAILURE_STORE_ENABLED + "-1")); // ensure redirects on failure store data stream
-        assertFalse(redirectPredicate.getValue().test(WITH_DEFAULT_PIPELINE)); // no redirects for random existing indices
-        assertFalse(redirectPredicate.getValue().test("index")); // no redirects for non-existant indices with no templates
+        // ensure redirects on failure store data stream
+        assertTrue(redirectPredicate.getValue().apply(WITH_FAILURE_STORE_ENABLED + "-1"));
+        assertNull(redirectPredicate.getValue().apply(WITH_DEFAULT_PIPELINE)); // no redirects for random existing indices
+        assertNull(redirectPredicate.getValue().apply("index")); // no redirects for non-existent indices with no templates
         redirectHandler.getValue().apply(2, WITH_FAILURE_STORE_ENABLED + "-1", exception); // exception and redirect for request 3 (slot 2)
         completionHandler.getValue().accept(DUMMY_WRITE_THREAD, null); // all ingestion completed
         assertTrue(action.isExecuted);
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
index ca1d1ac49832e..ed7cc93f0ab43 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
@@ -33,6 +33,8 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.features.FeatureService;
@@ -54,6 +56,7 @@
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -68,6 +71,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 import static org.junit.Assume.assumeThat;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
@@ -100,7 +104,8 @@ class TestTransportBulkAction extends TransportBulkAction {
                 new ActionFilters(Collections.emptySet()),
                 new Resolver(),
                 new IndexingPressure(Settings.EMPTY),
-                EmptySystemIndices.INSTANCE
+                EmptySystemIndices.INSTANCE,
+                FailureStoreMetrics.NOOP
             );
         }
 
@@ -287,7 +292,7 @@ public void testProhibitCustomRoutingOnDataStream() throws Exception {
         prohibitCustomRoutingOnDataStream(writeRequestAgainstIndex, metadata.getIndicesLookup().get(writeRequestAgainstIndex.index()));
     }
 
-    public void testOnlySystem() {
+    public void testOnlySystem() throws IOException {
         SortedMap<String, IndexAbstraction> indicesLookup = new TreeMap<>();
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build();
         indicesLookup.put(
@@ -303,15 +308,23 @@ public void testOnlySystem() {
         );
         List<String> onlySystem = List.of(".foo", ".bar");
         assertTrue(TransportBulkAction.isOnlySystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices));
+        /* Test forwarded bulk requests (that are serialized then deserialized) */
+        assertTrue(TransportBulkAction.isOnlySystem(buildBulkStreamRequest(onlySystem), indicesLookup, systemIndices));
 
         onlySystem = List.of(".foo", ".bar", ".test");
         assertTrue(TransportBulkAction.isOnlySystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices));
+        /* Test forwarded bulk requests (that are serialized then deserialized) */
+        assertTrue(TransportBulkAction.isOnlySystem(buildBulkStreamRequest(onlySystem), indicesLookup, systemIndices));
 
         List<String> nonSystem = List.of("foo", "bar");
         assertFalse(TransportBulkAction.isOnlySystem(buildBulkRequest(nonSystem), indicesLookup, systemIndices));
+        /* Test forwarded bulk requests (that are serialized then deserialized) */
+        assertFalse(TransportBulkAction.isOnlySystem(buildBulkStreamRequest(nonSystem), indicesLookup, systemIndices));
 
         List<String> mixed = List.of(".foo", ".test", "other");
         assertFalse(TransportBulkAction.isOnlySystem(buildBulkRequest(mixed), indicesLookup, systemIndices));
+        /* Test forwarded bulk requests (that are serialized then deserialized) */
+        assertFalse(TransportBulkAction.isOnlySystem(buildBulkStreamRequest(mixed), indicesLookup, systemIndices));
     }
 
     private void blockWriteThreadPool(CountDownLatch blockingLatch) {
@@ -406,13 +419,16 @@ public void testResolveFailureStoreFromMetadata() throws Exception {
             .build();
 
         // Data stream with failure store should store failures
-        assertThat(TransportBulkAction.shouldStoreFailureInternal(dataStreamWithFailureStore, metadata, testTime), is(true));
+        assertThat(TransportBulkAction.resolveFailureInternal(dataStreamWithFailureStore, metadata, testTime), is(true));
         // Data stream without failure store should not
-        assertThat(TransportBulkAction.shouldStoreFailureInternal(dataStreamWithoutFailureStore, metadata, testTime), is(false));
+        assertThat(TransportBulkAction.resolveFailureInternal(dataStreamWithoutFailureStore, metadata, testTime), is(false));
         // An index should not be considered for failure storage
-        assertThat(TransportBulkAction.shouldStoreFailureInternal(backingIndex1.getIndex().getName(), metadata, testTime), is(false));
+        assertThat(TransportBulkAction.resolveFailureInternal(backingIndex1.getIndex().getName(), metadata, testTime), is(nullValue()));
         // even if that index is itself a failure store
-        assertThat(TransportBulkAction.shouldStoreFailureInternal(failureStoreIndex1.getIndex().getName(), metadata, testTime), is(false));
+        assertThat(
+            TransportBulkAction.resolveFailureInternal(failureStoreIndex1.getIndex().getName(), metadata, testTime),
+            is(nullValue())
+        );
     }
 
     public void testResolveFailureStoreFromTemplate() throws Exception {
@@ -443,11 +459,11 @@ public void testResolveFailureStoreFromTemplate() throws Exception {
             .build();
 
         // Data stream with failure store should store failures
-        assertThat(TransportBulkAction.shouldStoreFailureInternal(dsTemplateWithFailureStore + "-1", metadata, testTime), is(true));
+        assertThat(TransportBulkAction.resolveFailureInternal(dsTemplateWithFailureStore + "-1", metadata, testTime), is(true));
         // Data stream without failure store should not
-        assertThat(TransportBulkAction.shouldStoreFailureInternal(dsTemplateWithoutFailureStore + "-1", metadata, testTime), is(false));
+        assertThat(TransportBulkAction.resolveFailureInternal(dsTemplateWithoutFailureStore + "-1", metadata, testTime), is(false));
         // An index template should not be considered for failure storage
-        assertThat(TransportBulkAction.shouldStoreFailureInternal(indexTemplate + "-1", metadata, testTime), is(false));
+        assertThat(TransportBulkAction.resolveFailureInternal(indexTemplate + "-1", metadata, testTime), is(nullValue()));
     }
 
     private BulkRequest buildBulkRequest(List<String> indices) {
@@ -463,4 +479,12 @@ private BulkRequest buildBulkRequest(List<String> indices) {
         }
         return request;
     }
+
+    private BulkRequest buildBulkStreamRequest(List<String> indices) throws IOException {
+        BulkRequest request = buildBulkRequest(indices);
+        BytesStreamOutput out = new BytesStreamOutput();
+        request.writeTo(out);
+        StreamInput streamInput = out.bytes().streamInput();
+        return (new BulkRequest(streamInput));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index 09513351652b8..626f07fe61216 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -254,7 +254,8 @@ static class TestTransportBulkAction extends TransportBulkAction {
                 indexNameExpressionResolver,
                 new IndexingPressure(Settings.EMPTY),
                 EmptySystemIndices.INSTANCE,
-                relativeTimeProvider
+                relativeTimeProvider,
+                FailureStoreMetrics.NOOP
             );
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
index 1f54d8dd1edd5..b83b26321a3e7 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
@@ -599,7 +599,7 @@ public void testUpdateRequestWithConflictFailure() throws Exception {
 
         IndexRequest updateResponse = new IndexRequest("index").id("id")
             .source(Requests.INDEX_CONTENT_TYPE, "field", "value")
-            .noParsedBytesToReport();// let's pretend this was modified by a script
+            .setNormalisedBytesParsed(0);// let's pretend this was modified by a script
         DocumentParsingProvider documentParsingProvider = mock(DocumentParsingProvider.class);
 
         Exception err = new VersionConflictEngineException(shardId, "id", "I'm conflicted <(;_;)>");
@@ -655,7 +655,7 @@ public void testUpdateRequestWithConflictFailure() throws Exception {
 
         // we have set 0 value on normalisedBytesParsed on the IndexRequest, like it happens with updates by script.
         ArgumentCaptor<IndexRequest> argument = ArgumentCaptor.forClass(IndexRequest.class);
-        verify(documentParsingProvider, times(retries + 1)).newDocumentSizeObserver(argument.capture());
+        verify(documentParsingProvider, times(retries + 1)).newMeteringParserDecorator(argument.capture());
         IndexRequest value = argument.getValue();
         assertThat(value.getNormalisedBytesParsed(), equalTo(0L));
     }
@@ -722,7 +722,7 @@ public void testUpdateRequestWithSuccess() throws Exception {
         assertThat(response.getSeqNo(), equalTo(13L));
 
         ArgumentCaptor<IndexRequest> argument = ArgumentCaptor.forClass(IndexRequest.class);
-        verify(documentParsingProvider, times(1)).newDocumentSizeObserver(argument.capture());
+        verify(documentParsingProvider, times(1)).newMeteringParserDecorator(argument.capture());
         IndexRequest value = argument.getValue();
         assertThat(value.getNormalisedBytesParsed(), equalTo(100L));
     }
diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/ModifyDataStreamsRequestTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/ModifyDataStreamsRequestTests.java
index c9bf4b38cd433..c0fd159a859f0 100644
--- a/server/src/test/java/org/elasticsearch/action/datastreams/ModifyDataStreamsRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/datastreams/ModifyDataStreamsRequestTests.java
@@ -31,7 +31,7 @@ protected Request createTestInstance() {
         for (int k = 1; k <= numActions; k++) {
             actions.add(DataStreamActionTests.createTestInstance());
         }
-        return new Request(actions);
+        return new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, actions);
     }
 
     @Override
@@ -41,6 +41,6 @@ protected Request mutateInstance(Request request) {
         for (int k = 1; k <= moreActions; k++) {
             actions.add(DataStreamActionTests.createTestInstance());
         }
-        return new Request(actions);
+        return new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, actions);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java
index c769e504ef15b..920e844055f6c 100644
--- a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java
@@ -34,10 +34,10 @@ public void testDataStreamLifecycleToXContent() throws Exception {
         TimeValue globalMaxRetention = TimeValue.timeValueDays(50);
         DataStreamLifecycle lifecycle = new DataStreamLifecycle(new DataStreamLifecycle.Retention(configuredRetention), null, null);
         {
-            boolean isSystemDataStream = true;
+            boolean isInternalDataStream = true;
             GetDataStreamLifecycleAction.Response.DataStreamLifecycle explainIndexDataStreamLifecycle = createDataStreamLifecycle(
                 lifecycle,
-                isSystemDataStream
+                isInternalDataStream
             );
             Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention);
             Map<String, Object> lifecycleResult = (Map<String, Object>) resultMap.get("lifecycle");
@@ -46,10 +46,10 @@ public void testDataStreamLifecycleToXContent() throws Exception {
             assertThat(lifecycleResult.get("retention_determined_by"), equalTo("data_stream_configuration"));
         }
         {
-            boolean isSystemDataStream = false;
+            boolean isInternalDataStream = false;
             GetDataStreamLifecycleAction.Response.DataStreamLifecycle explainIndexDataStreamLifecycle = createDataStreamLifecycle(
                 lifecycle,
-                isSystemDataStream
+                isInternalDataStream
             );
             Map<String, Object> resultMap = getXContentMap(explainIndexDataStreamLifecycle, globalDefaultRetention, globalMaxRetention);
             Map<String, Object> lifecycleResult = (Map<String, Object>) resultMap.get("lifecycle");
@@ -61,9 +61,9 @@ public void testDataStreamLifecycleToXContent() throws Exception {
 
     private GetDataStreamLifecycleAction.Response.DataStreamLifecycle createDataStreamLifecycle(
         DataStreamLifecycle lifecycle,
-        boolean isSystemDataStream
+        boolean isInternalDataStream
     ) {
-        return new GetDataStreamLifecycleAction.Response.DataStreamLifecycle(randomAlphaOfLength(50), lifecycle, isSystemDataStream);
+        return new GetDataStreamLifecycleAction.Response.DataStreamLifecycle(randomAlphaOfLength(50), lifecycle, isInternalDataStream);
     }
 
     /*
diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponseTests.java
index 0f5ba959c5ed8..d487f584a20bb 100644
--- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponseTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.test.ESTestCase;
 
@@ -60,9 +61,10 @@ static List<FieldCapabilitiesIndexResponse> randomIndexResponsesWithMappingHash(
         final List<FieldCapabilitiesIndexResponse> responses = new ArrayList<>();
         for (Map.Entry<String, List<String>> e : mappingHashToIndices.entrySet()) {
             Map<String, IndexFieldCapabilities> fieldCaps = randomFieldCaps();
+            var indexMode = randomFrom(IndexMode.values());
             String mappingHash = e.getKey();
             for (String index : e.getValue()) {
-                responses.add(new FieldCapabilitiesIndexResponse(index, mappingHash, fieldCaps, true));
+                responses.add(new FieldCapabilitiesIndexResponse(index, mappingHash, fieldCaps, true, indexMode));
             }
         }
         return responses;
@@ -73,7 +75,8 @@ static List<FieldCapabilitiesIndexResponse> randomIndexResponsesWithoutMappingHa
         int numIndices = between(0, 10);
         for (int i = 0; i < numIndices; i++) {
             String index = "index_without_mapping_hash_" + i;
-            responses.add(new FieldCapabilitiesIndexResponse(index, null, randomFieldCaps(), randomBoolean()));
+            var indexMode = randomFrom(IndexMode.values());
+            responses.add(new FieldCapabilitiesIndexResponse(index, null, randomFieldCaps(), randomBoolean(), indexMode));
         }
         return responses;
     }
diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeResponseTests.java
index 0802e498c43a7..c39cc6ebfd665 100644
--- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeResponseTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.test.TransportVersionUtils;
@@ -49,7 +50,9 @@ protected FieldCapabilitiesNodeResponse createTestInstance() {
         List<FieldCapabilitiesIndexResponse> responses = new ArrayList<>();
         int numResponse = randomIntBetween(0, 10);
         for (int i = 0; i < numResponse; i++) {
-            responses.add(new FieldCapabilitiesIndexResponse("index_" + i, null, randomFieldCaps(), randomBoolean()));
+            responses.add(
+                new FieldCapabilitiesIndexResponse("index_" + i, null, randomFieldCaps(), randomBoolean(), randomFrom(IndexMode.values()))
+            );
         }
         int numUnmatched = randomIntBetween(0, 3);
         Set<ShardId> shardIds = new HashSet<>();
@@ -69,21 +72,38 @@ protected FieldCapabilitiesNodeResponse mutateInstance(FieldCapabilitiesNodeResp
         List<FieldCapabilitiesIndexResponse> newResponses = new ArrayList<>(response.getIndexResponses());
         int mutation = response.getIndexResponses().isEmpty() ? 0 : randomIntBetween(0, 3);
         switch (mutation) {
-            case 0 -> newResponses.add(new FieldCapabilitiesIndexResponse("extra_index", null, randomFieldCaps(), randomBoolean()));
+            case 0 -> newResponses.add(
+                new FieldCapabilitiesIndexResponse("extra_index", null, randomFieldCaps(), randomBoolean(), randomFrom(IndexMode.values()))
+            );
             case 1 -> {
                 int toRemove = randomInt(newResponses.size() - 1);
                 newResponses.remove(toRemove);
             }
             case 2 -> {
                 int toReplace = randomInt(newResponses.size() - 1);
-                newResponses.set(toReplace, new FieldCapabilitiesIndexResponse("new_index", null, randomFieldCaps(), randomBoolean()));
+                newResponses.set(
+                    toReplace,
+                    new FieldCapabilitiesIndexResponse(
+                        "new_index",
+                        null,
+                        randomFieldCaps(),
+                        randomBoolean(),
+                        randomFrom(IndexMode.values())
+                    )
+                );
             }
             case 3 -> {
                 int toReplace = randomInt(newResponses.size() - 1);
                 FieldCapabilitiesIndexResponse resp = newResponses.get(toReplace);
                 newResponses.set(
                     toReplace,
-                    new FieldCapabilitiesIndexResponse(resp.getIndexName(), UUIDs.randomBase64UUID(), resp.get(), true)
+                    new FieldCapabilitiesIndexResponse(
+                        resp.getIndexName(),
+                        UUIDs.randomBase64UUID(),
+                        resp.get(),
+                        true,
+                        randomFrom(IndexMode.values())
+                    )
                 );
             }
         }
@@ -194,9 +214,10 @@ public void testReadNodeResponseFromPre82() throws Exception {
                         "blue_field",
                         new IndexFieldCapabilities("blue_field", "long", false, true, true, false, null, Map.of())
                     ),
-                    true
+                    true,
+                    IndexMode.STANDARD
                 ),
-                new FieldCapabilitiesIndexResponse("index_02", null, Map.of(), false),
+                new FieldCapabilitiesIndexResponse("index_02", null, Map.of(), false, IndexMode.STANDARD),
                 new FieldCapabilitiesIndexResponse(
                     "index_03",
                     null,
@@ -206,7 +227,8 @@ public void testReadNodeResponseFromPre82() throws Exception {
                         "_seq_no",
                         new IndexFieldCapabilities("_seq_no", "long", true, true, true, false, null, Map.of())
                     ),
-                    true
+                    true,
+                    IndexMode.STANDARD
                 )
             )
         );
diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java
index 461000fc22b02..cc4d4de1e0f39 100644
--- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xcontent.ToXContent;
@@ -54,7 +55,8 @@ protected FieldCapabilitiesResponse createTestInstance() {
         int numResponse = randomIntBetween(0, 10);
         for (int i = 0; i < numResponse; i++) {
             Map<String, IndexFieldCapabilities> fieldCaps = FieldCapabilitiesIndexResponseTests.randomFieldCaps();
-            responses.add(new FieldCapabilitiesIndexResponse("index_" + i, null, fieldCaps, randomBoolean()));
+            var indexMode = randomFrom(IndexMode.values());
+            responses.add(new FieldCapabilitiesIndexResponse("index_" + i, null, fieldCaps, randomBoolean(), indexMode));
         }
         randomResponse = new FieldCapabilitiesResponse(responses, Collections.emptyList());
         return randomResponse;
@@ -267,9 +269,10 @@ public void testReadCCSResponseFromPre82() throws Exception {
                         "blue_field",
                         new IndexFieldCapabilities("blue_field", "long", false, true, true, false, null, Map.of())
                     ),
-                    true
+                    true,
+                    IndexMode.STANDARD
                 ),
-                new FieldCapabilitiesIndexResponse("index_02", null, Map.of(), false),
+                new FieldCapabilitiesIndexResponse("index_02", null, Map.of(), false, IndexMode.STANDARD),
                 new FieldCapabilitiesIndexResponse(
                     "index_03",
                     null,
@@ -279,7 +282,8 @@ public void testReadCCSResponseFromPre82() throws Exception {
                         "_seq_no",
                         new IndexFieldCapabilities("_seq_no", "long", true, true, true, false, null, Map.of())
                     ),
-                    true
+                    true,
+                    IndexMode.STANDARD
                 )
             )
         );
diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/RequestDispatcherTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/RequestDispatcherTests.java
index f5f35c52044d7..96e615afb498f 100644
--- a/server/src/test/java/org/elasticsearch/action/fieldcaps/RequestDispatcherTests.java
+++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/RequestDispatcherTests.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.RangeQueryBuilder;
@@ -890,7 +891,13 @@ static FieldCapabilitiesNodeResponse randomNodeResponse(
                 indicesWithMappingHash.computeIfAbsent(index, k -> new ArrayList<>()).add(index);
             } else {
                 indexResponses.add(
-                    new FieldCapabilitiesIndexResponse(index, null, FieldCapabilitiesIndexResponseTests.randomFieldCaps(), true)
+                    new FieldCapabilitiesIndexResponse(
+                        index,
+                        null,
+                        FieldCapabilitiesIndexResponseTests.randomFieldCaps(),
+                        true,
+                        randomFrom(IndexMode.values())
+                    )
                 );
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java b/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java
index aea3359e18bf6..b620495472e28 100644
--- a/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.action.bulk.FailureStoreMetrics;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -90,7 +91,8 @@ public void setup() {
             Collections.singletonList(DUMMY_PLUGIN),
             client,
             null,
-            DocumentParsingProvider.EMPTY_INSTANCE
+            DocumentParsingProvider.EMPTY_INSTANCE,
+            FailureStoreMetrics.NOOP
         );
         Map<String, Processor.Factory> factories = ingestService.getProcessorFactories();
         assertTrue(factories.containsKey("set"));
diff --git a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
index 70c4d73f578b3..c450fd8a9c39c 100644
--- a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.action.search;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.CanMatchNodeResponse.ResponseOrFailure;
@@ -26,8 +27,6 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexMode;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -38,6 +37,7 @@
 import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardLongFieldRange;
+import org.elasticsearch.indices.DateFieldRangeInfo;
 import org.elasticsearch.search.CanMatchShardResponse;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.SignificantTermsAggregationBuilder;
@@ -72,6 +72,7 @@
 import static org.elasticsearch.action.search.SearchAsyncActionTests.getShardsIter;
 import static org.elasticsearch.core.Types.forciblyCast;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.mockito.Mockito.mock;
 
@@ -464,7 +465,17 @@ public void sendCanMatch(
         }
     }
 
-    public void testCanMatchFilteringOnCoordinatorThatCanBeSkipped() throws Exception {
+    // test using @timestamp
+    public void testCanMatchFilteringOnCoordinatorThatCanBeSkippedUsingTimestamp() throws Exception {
+        doCanMatchFilteringOnCoordinatorThatCanBeSkipped(DataStream.TIMESTAMP_FIELD_NAME);
+    }
+
+    // test using event.ingested
+    public void testCanMatchFilteringOnCoordinatorThatCanBeSkippedUsingEventIngested() throws Exception {
+        doCanMatchFilteringOnCoordinatorThatCanBeSkipped(IndexMetadata.EVENT_INGESTED_FIELD_NAME);
+    }
+
+    public void doCanMatchFilteringOnCoordinatorThatCanBeSkipped(String timestampField) throws Exception {
         Index dataStreamIndex1 = new Index(".ds-mydata0001", UUIDs.base64UUID());
         Index dataStreamIndex2 = new Index(".ds-mydata0002", UUIDs.base64UUID());
         DataStream dataStream = DataStreamTestHelper.newInstance("mydata", List.of(dataStreamIndex1, dataStreamIndex2));
@@ -475,15 +486,10 @@ public void testCanMatchFilteringOnCoordinatorThatCanBeSkipped() throws Exceptio
         long indexMaxTimestamp = randomLongBetween(indexMinTimestamp, 5000 * 2);
         StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder();
         for (Index dataStreamIndex : dataStream.getIndices()) {
-            contextProviderBuilder.addIndexMinMaxTimestamps(
-                dataStreamIndex,
-                DataStream.TIMESTAMP_FIELD_NAME,
-                indexMinTimestamp,
-                indexMaxTimestamp
-            );
+            contextProviderBuilder.addIndexMinMaxTimestamps(dataStreamIndex, timestampField, indexMinTimestamp, indexMaxTimestamp);
         }
 
-        RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME);
+        RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(timestampField);
         // We query a range outside of the timestamp range covered by both datastream indices
         rangeQueryBuilder.from(indexMaxTimestamp + 1).to(indexMaxTimestamp + 2);
 
@@ -516,12 +522,12 @@ public void testCanMatchFilteringOnCoordinatorThatCanBeSkipped() throws Exceptio
                 // When all the shards can be skipped we should query at least 1
                 // in order to get a valid search response.
                 if (regularIndexShardCount == 0) {
-                    assertThat(nonSkippedShards.size(), equalTo(1));
+                    assertThat(nonSkippedShards.size(), equalTo(1)); // FIXME - fails here with expected 1 but was 11 OR
                 } else {
                     boolean allNonSkippedShardsAreFromRegularIndices = nonSkippedShards.stream()
                         .allMatch(shardIterator -> regularIndices.contains(shardIterator.shardId().getIndex()));
 
-                    assertThat(allNonSkippedShardsAreFromRegularIndices, equalTo(true));
+                    assertThat(allNonSkippedShardsAreFromRegularIndices, equalTo(true)); // FIXME - OR fails here with "false"
                 }
 
                 boolean allSkippedShardAreFromDataStream = skippedShards.stream()
@@ -535,26 +541,107 @@ public void testCanMatchFilteringOnCoordinatorThatCanBeSkipped() throws Exceptio
         );
     }
 
-    public void testCanMatchFilteringOnCoordinatorParsingFails() throws Exception {
-        Index dataStreamIndex1 = new Index(".ds-mydata0001", UUIDs.base64UUID());
-        Index dataStreamIndex2 = new Index(".ds-mydata0002", UUIDs.base64UUID());
+    public void testCoordinatorCanMatchFilteringThatCanBeSkippedUsingBothTimestamps() throws Exception {
+        Index dataStreamIndex1 = new Index(".ds-twoTimestamps0001", UUIDs.base64UUID());
+        Index dataStreamIndex2 = new Index(".ds-twoTimestamps0002", UUIDs.base64UUID());
         DataStream dataStream = DataStreamTestHelper.newInstance("mydata", List.of(dataStreamIndex1, dataStreamIndex2));
 
-        List<Index> regularIndices = randomList(0, 2, () -> new Index(randomAlphaOfLength(10), UUIDs.base64UUID()));
+        List<Index> regularIndices = randomList(1, 2, () -> new Index(randomAlphaOfLength(10), UUIDs.base64UUID()));
 
         long indexMinTimestamp = randomLongBetween(0, 5000);
         long indexMaxTimestamp = randomLongBetween(indexMinTimestamp, 5000 * 2);
         StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder();
         for (Index dataStreamIndex : dataStream.getIndices()) {
-            contextProviderBuilder.addIndexMinMaxTimestamps(
+            // use same range for both @timestamp and event.ingested
+            contextProviderBuilder.addIndexMinMaxForTimestampAndEventIngested(
                 dataStreamIndex,
-                DataStream.TIMESTAMP_FIELD_NAME,
+                indexMinTimestamp,
+                indexMaxTimestamp,
                 indexMinTimestamp,
                 indexMaxTimestamp
             );
         }
 
-        RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME);
+        /**
+         * Expected behavior: if either @timestamp or 'event.ingested' filters in the query are "out of range" (do not
+         * overlap the range in cluster state), then all shards in the datastream should be skipped.
+         * Only if both @timestamp or 'event.ingested' filters are "in range" should the data stream shards be searched
+         */
+        boolean timestampQueryOutOfRange = randomBoolean();
+        boolean eventIngestedQueryOutOfRange = randomBoolean();
+        int timestampOffset = timestampQueryOutOfRange ? 1 : -500;
+        int eventIngestedOffset = eventIngestedQueryOutOfRange ? 1 : -500;
+
+        RangeQueryBuilder tsRangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME);
+        tsRangeQueryBuilder.from(indexMaxTimestamp + timestampOffset).to(indexMaxTimestamp + 2);
+
+        RangeQueryBuilder eventIngestedRangeQueryBuilder = new RangeQueryBuilder(IndexMetadata.EVENT_INGESTED_FIELD_NAME);
+        eventIngestedRangeQueryBuilder.from(indexMaxTimestamp + eventIngestedOffset).to(indexMaxTimestamp + 2);
+
+        BoolQueryBuilder queryBuilder = new BoolQueryBuilder().filter(tsRangeQueryBuilder).filter(eventIngestedRangeQueryBuilder);
+
+        if (randomBoolean()) {
+            // Add an additional filter that cannot be evaluated in the coordinator but shouldn't
+            // affect the end result as we're filtering
+            queryBuilder.filter(new TermQueryBuilder("fake", "value"));
+        }
+
+        assignShardsAndExecuteCanMatchPhase(
+            List.of(dataStream),
+            regularIndices,
+            contextProviderBuilder.build(),
+            queryBuilder,
+            List.of(),
+            null,
+            (updatedSearchShardIterators, requests) -> {
+                List<SearchShardIterator> skippedShards = updatedSearchShardIterators.stream().filter(SearchShardIterator::skip).toList();
+                List<SearchShardIterator> nonSkippedShards = updatedSearchShardIterators.stream()
+                    .filter(searchShardIterator -> searchShardIterator.skip() == false)
+                    .toList();
+
+                if (timestampQueryOutOfRange || eventIngestedQueryOutOfRange) {
+                    // data stream shards should have been skipped
+                    assertThat(skippedShards.size(), greaterThan(0));
+                    boolean allSkippedShardAreFromDataStream = skippedShards.stream()
+                        .allMatch(shardIterator -> dataStream.getIndices().contains(shardIterator.shardId().getIndex()));
+                    assertThat(allSkippedShardAreFromDataStream, equalTo(true));
+
+                    boolean allNonSkippedShardsAreFromRegularIndices = nonSkippedShards.stream()
+                        .allMatch(shardIterator -> regularIndices.contains(shardIterator.shardId().getIndex()));
+                    assertThat(allNonSkippedShardsAreFromRegularIndices, equalTo(true));
+
+                    boolean allRequestsWereTriggeredAgainstRegularIndices = requests.stream()
+                        .allMatch(request -> regularIndices.contains(request.shardId().getIndex()));
+                    assertThat(allRequestsWereTriggeredAgainstRegularIndices, equalTo(true));
+
+                } else {
+                    assertThat(skippedShards.size(), equalTo(0));
+                    long countSkippedShardsFromDatastream = nonSkippedShards.stream()
+                        .filter(iter -> dataStream.getIndices().contains(iter.shardId().getIndex()))
+                        .count();
+                    assertThat(countSkippedShardsFromDatastream, greaterThan(0L));
+                }
+            }
+        );
+    }
+
+    public void testCanMatchFilteringOnCoordinatorParsingFails() throws Exception {
+        Index dataStreamIndex1 = new Index(".ds-mydata0001", UUIDs.base64UUID());
+        Index dataStreamIndex2 = new Index(".ds-mydata0002", UUIDs.base64UUID());
+        DataStream dataStream = DataStreamTestHelper.newInstance("mydata", List.of(dataStreamIndex1, dataStreamIndex2));
+
+        List<Index> regularIndices = randomList(0, 2, () -> new Index(randomAlphaOfLength(10), UUIDs.base64UUID()));
+
+        String timeField = randomFrom(DataStream.TIMESTAMP_FIELD_NAME, IndexMetadata.EVENT_INGESTED_FIELD_NAME);
+
+        long indexMinTimestamp = randomLongBetween(0, 5000);
+        long indexMaxTimestamp = randomLongBetween(indexMinTimestamp, 5000 * 2);
+        StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder();
+        for (Index dataStreamIndex : dataStream.getIndices()) {
+            contextProviderBuilder.addIndexMinMaxTimestamps(dataStreamIndex, timeField, indexMinTimestamp, indexMaxTimestamp);
+        }
+
+        RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(timeField);
         // Query with a non default date format
         rangeQueryBuilder.from("2020-1-01").to("2021-1-01");
 
@@ -585,23 +672,20 @@ public void testCanMatchFilteringOnCoordinatorThatCanNotBeSkipped() throws Excep
 
         List<Index> regularIndices = randomList(0, 2, () -> new Index(randomAlphaOfLength(10), UUIDs.base64UUID()));
 
+        String timeField = randomFrom(DataStream.TIMESTAMP_FIELD_NAME, IndexMetadata.EVENT_INGESTED_FIELD_NAME);
+
         long indexMinTimestamp = 10;
         long indexMaxTimestamp = 20;
         StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder();
         for (Index dataStreamIndex : dataStream.getIndices()) {
-            contextProviderBuilder.addIndexMinMaxTimestamps(
-                dataStreamIndex,
-                DataStream.TIMESTAMP_FIELD_NAME,
-                indexMinTimestamp,
-                indexMaxTimestamp
-            );
+            contextProviderBuilder.addIndexMinMaxTimestamps(dataStreamIndex, timeField, indexMinTimestamp, indexMaxTimestamp);
         }
 
         BoolQueryBuilder queryBuilder = new BoolQueryBuilder();
         // Query inside of the data stream index range
         if (randomBoolean()) {
             // Query generation
-            RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME);
+            RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(timeField);
             // We query a range within the timestamp range covered by both datastream indices
             rangeQueryBuilder.from(indexMinTimestamp).to(indexMaxTimestamp);
 
@@ -614,8 +698,7 @@ public void testCanMatchFilteringOnCoordinatorThatCanNotBeSkipped() throws Excep
             }
         } else {
             // We query a range outside of the timestamp range covered by both datastream indices
-            RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME).from(indexMaxTimestamp + 1)
-                .to(indexMaxTimestamp + 2);
+            RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(timeField).from(indexMaxTimestamp + 1).to(indexMaxTimestamp + 2);
 
             TermQueryBuilder termQueryBuilder = new TermQueryBuilder("fake", "value");
 
@@ -635,17 +718,86 @@ public void testCanMatchFilteringOnCoordinatorThatCanNotBeSkipped() throws Excep
         );
     }
 
+    public void testCanMatchFilteringOnCoordinatorWithTimestampAndEventIngestedThatCanNotBeSkipped() throws Exception {
+        // Generate indices
+        Index dataStreamIndex1 = new Index(".ds-mydata0001", UUIDs.base64UUID());
+        Index dataStreamIndex2 = new Index(".ds-mydata0002", UUIDs.base64UUID());
+        DataStream dataStream = DataStreamTestHelper.newInstance("mydata", List.of(dataStreamIndex1, dataStreamIndex2));
+
+        List<Index> regularIndices = randomList(0, 2, () -> new Index(randomAlphaOfLength(10), UUIDs.base64UUID()));
+
+        long indexMinTimestampForTs = 10;
+        long indexMaxTimestampForTs = 20;
+        long indexMinTimestampForEventIngested = 10;
+        long indexMaxTimestampForEventIngested = 20;
+        StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder();
+        for (Index dataStreamIndex : dataStream.getIndices()) {
+            contextProviderBuilder.addIndexMinMaxForTimestampAndEventIngested(
+                dataStreamIndex,
+                indexMinTimestampForTs,
+                indexMaxTimestampForTs,
+                indexMinTimestampForEventIngested,
+                indexMaxTimestampForEventIngested
+            );
+        }
+
+        BoolQueryBuilder queryBuilder = new BoolQueryBuilder();
+        // Query inside of the data stream index range
+        if (randomBoolean()) {
+            // Query generation
+            // We query a range within both timestamp ranges covered by both datastream indices
+            RangeQueryBuilder tsRangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME);
+            tsRangeQueryBuilder.from(indexMinTimestampForTs).to(indexMaxTimestampForTs);
+
+            RangeQueryBuilder eventIngestedRangeQueryBuilder = new RangeQueryBuilder(IndexMetadata.EVENT_INGESTED_FIELD_NAME);
+            eventIngestedRangeQueryBuilder.from(indexMinTimestampForEventIngested).to(indexMaxTimestampForEventIngested);
+
+            queryBuilder.filter(tsRangeQueryBuilder).filter(eventIngestedRangeQueryBuilder);
+
+            if (randomBoolean()) {
+                // Add an additional filter that cannot be evaluated in the coordinator but shouldn't
+                // affect the end result as we're filtering
+                queryBuilder.filter(new TermQueryBuilder("fake", "value"));
+            }
+        } else {
+            // We query a range outside of the both ranges covered by both datastream indices
+            RangeQueryBuilder tsRangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME).from(indexMaxTimestampForTs + 1)
+                .to(indexMaxTimestampForTs + 2);
+            RangeQueryBuilder eventIngestedRangeQueryBuilder = new RangeQueryBuilder(IndexMetadata.EVENT_INGESTED_FIELD_NAME).from(
+                indexMaxTimestampForEventIngested + 1
+            ).to(indexMaxTimestampForEventIngested + 2);
+
+            TermQueryBuilder termQueryBuilder = new TermQueryBuilder("fake", "value");
+
+            // This is always evaluated as true in the coordinator as we cannot determine there if
+            // the term query clause is false.
+            queryBuilder.should(tsRangeQueryBuilder).should(eventIngestedRangeQueryBuilder).should(termQueryBuilder);
+        }
+
+        assignShardsAndExecuteCanMatchPhase(
+            List.of(dataStream),
+            regularIndices,
+            contextProviderBuilder.build(),
+            queryBuilder,
+            List.of(),
+            null,
+            this::assertAllShardsAreQueried
+        );
+    }
+
     public void testCanMatchFilteringOnCoordinator_withSignificantTermsAggregation_withDefaultBackgroundFilter() throws Exception {
         Index index1 = new Index("index1", UUIDs.base64UUID());
         Index index2 = new Index("index2", UUIDs.base64UUID());
         Index index3 = new Index("index3", UUIDs.base64UUID());
 
+        String timeField = randomFrom(DataStream.TIMESTAMP_FIELD_NAME, IndexMetadata.EVENT_INGESTED_FIELD_NAME);
+
         StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder();
-        contextProviderBuilder.addIndexMinMaxTimestamps(index1, DataStream.TIMESTAMP_FIELD_NAME, 0, 999);
-        contextProviderBuilder.addIndexMinMaxTimestamps(index2, DataStream.TIMESTAMP_FIELD_NAME, 1000, 1999);
-        contextProviderBuilder.addIndexMinMaxTimestamps(index3, DataStream.TIMESTAMP_FIELD_NAME, 2000, 2999);
+        contextProviderBuilder.addIndexMinMaxTimestamps(index1, timeField, 0, 999);
+        contextProviderBuilder.addIndexMinMaxTimestamps(index2, timeField, 1000, 1999);
+        contextProviderBuilder.addIndexMinMaxTimestamps(index3, timeField, 2000, 2999);
 
-        QueryBuilder query = new BoolQueryBuilder().filter(new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME).from(2100).to(2200));
+        QueryBuilder query = new BoolQueryBuilder().filter(new RangeQueryBuilder(timeField).from(2100).to(2200));
         AggregationBuilder aggregation = new SignificantTermsAggregationBuilder("significant_terms");
 
         assignShardsAndExecuteCanMatchPhase(
@@ -661,20 +813,22 @@ public void testCanMatchFilteringOnCoordinator_withSignificantTermsAggregation_w
     }
 
     public void testCanMatchFilteringOnCoordinator_withSignificantTermsAggregation_withBackgroundFilter() throws Exception {
+        String timestampField = randomFrom(IndexMetadata.EVENT_INGESTED_FIELD_NAME, DataStream.TIMESTAMP_FIELD_NAME);
+
         Index index1 = new Index("index1", UUIDs.base64UUID());
         Index index2 = new Index("index2", UUIDs.base64UUID());
         Index index3 = new Index("index3", UUIDs.base64UUID());
         Index index4 = new Index("index4", UUIDs.base64UUID());
 
         StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder();
-        contextProviderBuilder.addIndexMinMaxTimestamps(index1, DataStream.TIMESTAMP_FIELD_NAME, 0, 999);
-        contextProviderBuilder.addIndexMinMaxTimestamps(index2, DataStream.TIMESTAMP_FIELD_NAME, 1000, 1999);
-        contextProviderBuilder.addIndexMinMaxTimestamps(index3, DataStream.TIMESTAMP_FIELD_NAME, 2000, 2999);
-        contextProviderBuilder.addIndexMinMaxTimestamps(index4, DataStream.TIMESTAMP_FIELD_NAME, 3000, 3999);
+        contextProviderBuilder.addIndexMinMaxTimestamps(index1, timestampField, 0, 999);
+        contextProviderBuilder.addIndexMinMaxTimestamps(index2, timestampField, 1000, 1999);
+        contextProviderBuilder.addIndexMinMaxTimestamps(index3, timestampField, 2000, 2999);
+        contextProviderBuilder.addIndexMinMaxTimestamps(index4, timestampField, 3000, 3999);
 
-        QueryBuilder query = new BoolQueryBuilder().filter(new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME).from(3100).to(3200));
+        QueryBuilder query = new BoolQueryBuilder().filter(new RangeQueryBuilder(timestampField).from(3100).to(3200));
         AggregationBuilder aggregation = new SignificantTermsAggregationBuilder("significant_terms").backgroundFilter(
-            new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME).from(0).to(1999)
+            new RangeQueryBuilder(timestampField).from(0).to(1999)
         );
 
         assignShardsAndExecuteCanMatchPhase(
@@ -703,14 +857,53 @@ public void testCanMatchFilteringOnCoordinator_withSignificantTermsAggregation_w
         Index index2 = new Index("index2", UUIDs.base64UUID());
         Index index3 = new Index("index3", UUIDs.base64UUID());
 
+        String timestampField = randomFrom(IndexMetadata.EVENT_INGESTED_FIELD_NAME, DataStream.TIMESTAMP_FIELD_NAME);
+
+        StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder();
+        contextProviderBuilder.addIndexMinMaxTimestamps(index1, timestampField, 0, 999);
+        contextProviderBuilder.addIndexMinMaxTimestamps(index2, timestampField, 1000, 1999);
+        contextProviderBuilder.addIndexMinMaxTimestamps(index3, timestampField, 2000, 2999);
+
+        QueryBuilder query = new BoolQueryBuilder().filter(new RangeQueryBuilder(timestampField).from(2100).to(2200));
+        AggregationBuilder aggregation = new SignificantTermsAggregationBuilder("significant_terms").backgroundFilter(
+            new RangeQueryBuilder(timestampField).from(2000).to(2300)
+        );
+        SuggestBuilder suggest = new SuggestBuilder().setGlobalText("test");
+
+        assignShardsAndExecuteCanMatchPhase(
+            List.of(),
+            List.of(index1, index2, index3),
+            contextProviderBuilder.build(),
+            query,
+            List.of(aggregation),
+            suggest,
+            // The query and aggregation and match only index3, but suggest should match everything.
+            this::assertAllShardsAreQueried
+        );
+    }
+
+    public void testCanMatchFilteringOnCoordinator_withSignificantTermsAggregation_withSuggest_withTwoTimestamps() throws Exception {
+        Index index1 = new Index("index1", UUIDs.base64UUID());
+        Index index2 = new Index("index2", UUIDs.base64UUID());
+        Index index3 = new Index("index3", UUIDs.base64UUID());
+
         StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder();
-        contextProviderBuilder.addIndexMinMaxTimestamps(index1, DataStream.TIMESTAMP_FIELD_NAME, 0, 999);
-        contextProviderBuilder.addIndexMinMaxTimestamps(index2, DataStream.TIMESTAMP_FIELD_NAME, 1000, 1999);
-        contextProviderBuilder.addIndexMinMaxTimestamps(index3, DataStream.TIMESTAMP_FIELD_NAME, 2000, 2999);
+        contextProviderBuilder.addIndexMinMaxForTimestampAndEventIngested(index1, 0, 999, 0, 999);
+        contextProviderBuilder.addIndexMinMaxForTimestampAndEventIngested(index2, 1000, 1999, 1000, 1999);
+        contextProviderBuilder.addIndexMinMaxForTimestampAndEventIngested(index3, 2000, 2999, 2000, 2999);
+
+        String fieldInRange = IndexMetadata.EVENT_INGESTED_FIELD_NAME;
+        String fieldOutOfRange = DataStream.TIMESTAMP_FIELD_NAME;
 
-        QueryBuilder query = new BoolQueryBuilder().filter(new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME).from(2100).to(2200));
+        if (randomBoolean()) {
+            fieldInRange = DataStream.TIMESTAMP_FIELD_NAME;
+            fieldOutOfRange = IndexMetadata.EVENT_INGESTED_FIELD_NAME;
+        }
+
+        QueryBuilder query = new BoolQueryBuilder().filter(new RangeQueryBuilder(fieldInRange).from(2100).to(2200))
+            .filter(new RangeQueryBuilder(fieldOutOfRange).from(8888).to(9999));
         AggregationBuilder aggregation = new SignificantTermsAggregationBuilder("significant_terms").backgroundFilter(
-            new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME).from(2000).to(2300)
+            new RangeQueryBuilder(fieldInRange).from(2000).to(2300)
         );
         SuggestBuilder suggest = new SuggestBuilder().setGlobalText("test");
 
@@ -744,13 +937,13 @@ public void testCanMatchFilteringOnCoordinatorThatCanBeSkippedTsdb() throws Exce
         long indexMaxTimestamp = randomLongBetween(indexMinTimestamp, 5000 * 2);
         StaticCoordinatorRewriteContextProviderBuilder contextProviderBuilder = new StaticCoordinatorRewriteContextProviderBuilder();
         for (Index index : dataStream1.getIndices()) {
-            contextProviderBuilder.addIndexMinMaxTimestamps(index, indexMinTimestamp, indexMaxTimestamp);
+            contextProviderBuilder.addIndexMinMaxTimestamps(index, DataStream.TIMESTAMP_FIELD_NAME, indexMinTimestamp, indexMaxTimestamp);
         }
         for (Index index : dataStream2.getIndices()) {
             contextProviderBuilder.addIndex(index);
         }
 
-        RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder("@timestamp");
+        RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder(DataStream.TIMESTAMP_FIELD_NAME);
         // We query a range outside of the timestamp range covered by both datastream indices
         rangeQueryBuilder.from(indexMaxTimestamp + 1).to(indexMaxTimestamp + 2);
 
@@ -954,9 +1147,9 @@ public void sendCanMatch(
         canMatchResultsConsumer.accept(updatedSearchShardIterators, requests);
     }
 
-    private static class StaticCoordinatorRewriteContextProviderBuilder {
+    static class StaticCoordinatorRewriteContextProviderBuilder {
         private ClusterState clusterState = ClusterState.EMPTY_STATE;
-        private final Map<Index, DateFieldMapper.DateFieldType> fields = new HashMap<>();
+        private final Map<Index, DateFieldRangeInfo> fields = new HashMap<>();
 
         private void addIndexMinMaxTimestamps(Index index, String fieldName, long minTimeStamp, long maxTimestamp) {
             if (clusterState.metadata().index(index) != null) {
@@ -974,35 +1167,64 @@ private void addIndexMinMaxTimestamps(Index index, String fieldName, long minTim
             IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(index.getName())
                 .settings(indexSettings)
                 .numberOfShards(1)
-                .numberOfReplicas(0)
-                .timestampRange(timestampRange);
+                .numberOfReplicas(0);
+            if (fieldName.equals(DataStream.TIMESTAMP_FIELD_NAME)) {
+                indexMetadataBuilder.timestampRange(timestampRange);
+                fields.put(index, new DateFieldRangeInfo(new DateFieldMapper.DateFieldType(fieldName), null, null, null));
+            } else if (fieldName.equals(IndexMetadata.EVENT_INGESTED_FIELD_NAME)) {
+                indexMetadataBuilder.eventIngestedRange(timestampRange, TransportVersion.current());
+                fields.put(index, new DateFieldRangeInfo(null, null, new DateFieldMapper.DateFieldType(fieldName), null));
+            }
 
             Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()).put(indexMetadataBuilder);
-
             clusterState = ClusterState.builder(clusterState).metadata(metadataBuilder).build();
-
-            fields.put(index, new DateFieldMapper.DateFieldType(fieldName));
         }
 
-        private void addIndexMinMaxTimestamps(Index index, long minTimestamp, long maxTimestamp) {
+        /**
+         * Add min/max timestamps to IndexMetadata for the specified index for both @timestamp and 'event.ingested'
+         */
+        private void addIndexMinMaxForTimestampAndEventIngested(
+            Index index,
+            long minTimestampForTs,
+            long maxTimestampForTs,
+            long minTimestampForEventIngested,
+            long maxTimestampForEventIngested
+        ) {
             if (clusterState.metadata().index(index) != null) {
                 throw new IllegalArgumentException("Min/Max timestamps for " + index + " were already defined");
             }
 
-            Settings.Builder indexSettings = settings(IndexVersion.current()).put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID())
-                .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
-                .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "a_field")
-                .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(minTimestamp))
-                .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(maxTimestamp));
+            IndexLongFieldRange tsTimestampRange = IndexLongFieldRange.NO_SHARDS.extendWithShardRange(
+                0,
+                1,
+                ShardLongFieldRange.of(minTimestampForTs, maxTimestampForTs)
+            );
+            IndexLongFieldRange eventIngestedTimestampRange = IndexLongFieldRange.NO_SHARDS.extendWithShardRange(
+                0,
+                1,
+                ShardLongFieldRange.of(minTimestampForEventIngested, maxTimestampForEventIngested)
+            );
+
+            Settings.Builder indexSettings = settings(IndexVersion.current()).put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID());
 
             IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(index.getName())
                 .settings(indexSettings)
                 .numberOfShards(1)
-                .numberOfReplicas(0);
+                .numberOfReplicas(0)
+                .timestampRange(tsTimestampRange)
+                .eventIngestedRange(eventIngestedTimestampRange, TransportVersion.current());
 
             Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()).put(indexMetadataBuilder);
             clusterState = ClusterState.builder(clusterState).metadata(metadataBuilder).build();
-            fields.put(index, new DateFieldMapper.DateFieldType("@timestamp"));
+            fields.put(
+                index,
+                new DateFieldRangeInfo(
+                    new DateFieldMapper.DateFieldType(DataStream.TIMESTAMP_FIELD_NAME),
+                    null,
+                    new DateFieldMapper.DateFieldType(IndexMetadata.EVENT_INGESTED_FIELD_NAME),
+                    null
+                )
+            );
         }
 
         private void addIndex(Index index) {
@@ -1018,7 +1240,7 @@ private void addIndex(Index index) {
 
             Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()).put(indexMetadataBuilder);
             clusterState = ClusterState.builder(clusterState).metadata(metadataBuilder).build();
-            fields.put(index, new DateFieldMapper.DateFieldType("@timestamp"));
+            fields.put(index, new DateFieldRangeInfo(new DateFieldMapper.DateFieldType(DataStream.TIMESTAMP_FIELD_NAME), null, null, null));
         }
 
         public CoordinatorRewriteContextProvider build() {
diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
index 47dbe8f126556..3c698f1b790e5 100644
--- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -353,12 +353,14 @@ public void testRewriteShardSearchRequestWithRank() {
         KnnScoreDocQueryBuilder ksdqb0 = new KnnScoreDocQueryBuilder(
             new ScoreDoc[] { new ScoreDoc(1, 3.0f, 1), new ScoreDoc(4, 1.5f, 1) },
             "vector",
-            VectorData.fromFloats(new float[] { 0.0f })
+            VectorData.fromFloats(new float[] { 0.0f }),
+            null
         );
         KnnScoreDocQueryBuilder ksdqb1 = new KnnScoreDocQueryBuilder(
             new ScoreDoc[] { new ScoreDoc(1, 2.0f, 1) },
             "vector2",
-            VectorData.fromFloats(new float[] { 0.0f })
+            VectorData.fromFloats(new float[] { 0.0f }),
+            null
         );
         assertEquals(
             List.of(bm25, ksdqb0, ksdqb1),
diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java
index fde0f190737e1..54c60dd11e149 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java
@@ -165,7 +165,6 @@ public String getLocalNodeId() {
 
         if (controlledClock) {
             return new TransportMultiSearchAction(
-                threadPool,
                 actionFilters,
                 transportService,
                 clusterService,
@@ -187,7 +186,6 @@ void executeSearch(
             };
         } else {
             return new TransportMultiSearchAction(
-                threadPool,
                 actionFilters,
                 transportService,
                 clusterService,
diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index a45730a82dbc2..67c8599f47029 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -45,6 +45,7 @@
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest;
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -582,11 +583,12 @@ public void testFailOnExtraCharacters() throws IOException {
                 """, null);
             fail("should have caught second line; extra closing brackets");
         } catch (XContentParseException e) {
-            assertEquals(
-                "[1:31] Unexpected close marker '}': expected ']' (for root starting at "
-                    + "[Source: (byte[])\"{ \"query\": {\"match_all\": {}}}}}}different error message\"; line: 1, column: 0])\n "
-                    + "at [Source: (byte[])\"{ \"query\": {\"match_all\": {}}}}}}different error message\"; line: 1, column: 31]",
-                e.getMessage()
+            assertThat(
+                e.getMessage(),
+                containsString(
+                    "Unexpected close marker '}': expected ']' (for root starting at "
+                        + "[Source: (byte[])\"{ \"query\": {\"match_all\": {}}}}}}different error message\""
+                )
             );
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/action/search/RestOpenPointInTimeActionTests.java b/server/src/test/java/org/elasticsearch/action/search/RestOpenPointInTimeActionTests.java
index dda977565af45..e7b5e898684f6 100644
--- a/server/src/test/java/org/elasticsearch/action/search/RestOpenPointInTimeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/RestOpenPointInTimeActionTests.java
@@ -31,7 +31,7 @@ public void testMaxConcurrentSearchRequests() {
         verifyingClient.setExecuteVerifier(((actionType, transportRequest) -> {
             assertThat(transportRequest, instanceOf(OpenPointInTimeRequest.class));
             transportRequests.add((OpenPointInTimeRequest) transportRequest);
-            return new OpenPointInTimeResponse(new BytesArray("n/a"));
+            return new OpenPointInTimeResponse(new BytesArray("n/a"), 1, 1, 0, 0);
         }));
         {
             RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST)
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java
index 32157e09e628f..af7068152648f 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchContextIdTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
@@ -18,6 +19,7 @@
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.test.ESTestCase;
 
@@ -52,40 +54,82 @@ public void testEncode() {
         final AtomicArray<SearchPhaseResult> queryResults = TransportSearchHelperTests.generateQueryResults();
         final TransportVersion version = TransportVersion.current();
         final Map<String, AliasFilter> aliasFilters = new HashMap<>();
+        Map<SearchShardTarget, ShardSearchFailure> shardSearchFailures = new HashMap<>();
+        int idx = 0;
         for (SearchPhaseResult result : queryResults.asList()) {
-            final AliasFilter aliasFilter;
             if (randomBoolean()) {
-                aliasFilter = AliasFilter.of(randomQueryBuilder());
-            } else if (randomBoolean()) {
-                aliasFilter = AliasFilter.of(randomQueryBuilder(), "alias-" + between(1, 10));
+                shardSearchFailures.put(
+                    result.getSearchShardTarget(),
+                    new ShardSearchFailure(
+                        new NoShardAvailableActionException(result.getSearchShardTarget().getShardId()),
+                        result.getSearchShardTarget()
+                    )
+                );
+                queryResults.set(idx, null);
             } else {
-                aliasFilter = AliasFilter.EMPTY;
-            }
-            if (randomBoolean()) {
-                aliasFilters.put(result.getSearchShardTarget().getShardId().getIndex().getUUID(), aliasFilter);
+                final AliasFilter aliasFilter;
+                if (randomBoolean()) {
+                    aliasFilter = AliasFilter.of(randomQueryBuilder());
+                } else if (randomBoolean()) {
+                    aliasFilter = AliasFilter.of(randomQueryBuilder(), "alias-" + between(1, 10));
+                } else {
+                    aliasFilter = AliasFilter.EMPTY;
+                }
+                if (randomBoolean()) {
+                    aliasFilters.put(result.getSearchShardTarget().getShardId().getIndex().getUUID(), aliasFilter);
+                }
             }
+            idx += 1;
         }
-        final BytesReference id = SearchContextId.encode(queryResults.asList(), aliasFilters, version);
+        final BytesReference id = SearchContextId.encode(
+            queryResults.asList(),
+            aliasFilters,
+            version,
+            shardSearchFailures.values().toArray(ShardSearchFailure[]::new)
+        );
         final SearchContextId context = SearchContextId.decode(namedWriteableRegistry, id);
         assertThat(context.shards().keySet(), hasSize(3));
+        // TODO assertThat(context.failedShards().keySet(), hasSize(shardsFailed));
         assertThat(context.aliasFilter(), equalTo(aliasFilters));
-        SearchContextIdForNode node1 = context.shards().get(new ShardId("idx", "uuid1", 2));
+
+        ShardId shardIdForNode1 = new ShardId("idx", "uuid1", 2);
+        SearchShardTarget shardTargetForNode1 = new SearchShardTarget("node_1", shardIdForNode1, "cluster_x");
+        SearchContextIdForNode node1 = context.shards().get(shardIdForNode1);
         assertThat(node1.getClusterAlias(), equalTo("cluster_x"));
-        assertThat(node1.getNode(), equalTo("node_1"));
-        assertThat(node1.getSearchContextId().getId(), equalTo(1L));
-        assertThat(node1.getSearchContextId().getSessionId(), equalTo("a"));
+        if (shardSearchFailures.containsKey(shardTargetForNode1)) {
+            assertNull(node1.getNode());
+            assertNull(node1.getSearchContextId());
+        } else {
+            assertThat(node1.getNode(), equalTo("node_1"));
+            assertThat(node1.getSearchContextId().getId(), equalTo(1L));
+            assertThat(node1.getSearchContextId().getSessionId(), equalTo("a"));
+        }
 
-        SearchContextIdForNode node2 = context.shards().get(new ShardId("idy", "uuid2", 42));
+        ShardId shardIdForNode2 = new ShardId("idy", "uuid2", 42);
+        SearchShardTarget shardTargetForNode2 = new SearchShardTarget("node_2", shardIdForNode2, "cluster_y");
+        SearchContextIdForNode node2 = context.shards().get(shardIdForNode2);
         assertThat(node2.getClusterAlias(), equalTo("cluster_y"));
-        assertThat(node2.getNode(), equalTo("node_2"));
-        assertThat(node2.getSearchContextId().getId(), equalTo(12L));
-        assertThat(node2.getSearchContextId().getSessionId(), equalTo("b"));
+        if (shardSearchFailures.containsKey(shardTargetForNode2)) {
+            assertNull(node2.getNode());
+            assertNull(node2.getSearchContextId());
+        } else {
+            assertThat(node2.getNode(), equalTo("node_2"));
+            assertThat(node2.getSearchContextId().getId(), equalTo(12L));
+            assertThat(node2.getSearchContextId().getSessionId(), equalTo("b"));
+        }
 
-        SearchContextIdForNode node3 = context.shards().get(new ShardId("idy", "uuid2", 43));
+        ShardId shardIdForNode3 = new ShardId("idy", "uuid2", 43);
+        SearchShardTarget shardTargetForNode3 = new SearchShardTarget("node_3", shardIdForNode3, null);
+        SearchContextIdForNode node3 = context.shards().get(shardIdForNode3);
         assertThat(node3.getClusterAlias(), nullValue());
-        assertThat(node3.getNode(), equalTo("node_3"));
-        assertThat(node3.getSearchContextId().getId(), equalTo(42L));
-        assertThat(node3.getSearchContextId().getSessionId(), equalTo("c"));
+        if (shardSearchFailures.containsKey(shardTargetForNode3)) {
+            assertNull(node3.getNode());
+            assertNull(node3.getSearchContextId());
+        } else {
+            assertThat(node3.getNode(), equalTo("node_3"));
+            assertThat(node3.getSearchContextId().getId(), equalTo(42L));
+            assertThat(node3.getSearchContextId().getSessionId(), equalTo("c"));
+        }
 
         final String[] indices = SearchContextId.decodeIndices(id);
         assertThat(indices.length, equalTo(3));
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index ed02328d388b6..585e7c775da35 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -58,7 +58,6 @@
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.search.rank.RankShardResult;
-import org.elasticsearch.search.rank.TestRankDoc;
 import org.elasticsearch.search.rank.TestRankShardResult;
 import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
 import org.elasticsearch.search.suggest.SortBy;
@@ -463,10 +462,10 @@ private static AtomicArray<SearchPhaseResult> generateQueryResults(
                 topDocs = Lucene.EMPTY_TOP_DOCS;
             } else if (rank) {
                 int nDocs = randomIntBetween(0, searchHitsSize);
-                TestRankDoc[] rankDocs = new TestRankDoc[nDocs];
+                RankDoc[] rankDocs = new RankDoc[nDocs];
                 for (int i = 0; i < nDocs; i++) {
                     float score = useConstantScore ? 1.0F : Math.abs(randomFloat());
-                    rankDocs[i] = new TestRankDoc(i, score, shardIndex);
+                    rankDocs[i] = new RankDoc(i, score, shardIndex);
                     maxScore = Math.max(score, maxScore);
                 }
                 querySearchResult.setRankShardResult(new TestRankShardResult(rankDocs));
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
index 475f44238f36e..2b1b95d8595f0 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.ArrayUtils;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.search.AbstractSearchTestCase;
@@ -29,6 +30,7 @@
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.rank.TestRankBuilder;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
+import org.elasticsearch.search.retriever.RetrieverBuilder;
 import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
@@ -37,6 +39,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -257,6 +260,128 @@ public void testValidate() throws IOException {
             assertEquals(1, validationErrors.validationErrors().size());
             assertEquals("cannot use `collapse` in a scroll context", validationErrors.validationErrors().get(0));
         }
+        {
+            // allow_partial_results and compound retriever
+            SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder().retriever(new RetrieverBuilder() {
+                @Override
+                public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
+                    // no-op
+                }
+
+                @Override
+                public String getName() {
+                    return "compound_retriever";
+                }
+
+                @Override
+                protected void doToXContent(XContentBuilder builder, Params params) throws IOException {}
+
+                @Override
+                protected boolean doEquals(Object o) {
+                    return false;
+                }
+
+                @Override
+                protected int doHashCode() {
+                    return 0;
+                }
+
+                @Override
+                public boolean isCompound() {
+                    return true;
+                }
+
+                @Override
+                public QueryBuilder topDocsQuery() {
+                    return null;
+                }
+            }));
+            searchRequest.allowPartialSearchResults(true);
+            searchRequest.scroll((Scroll) null);
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNotNull(validationErrors);
+            assertEquals(1, validationErrors.validationErrors().size());
+            assertEquals(
+                "cannot specify a compound retriever and [allow_partial_search_results]",
+                validationErrors.validationErrors().get(0)
+            );
+        }
+        {
+            // allow_partial_results and non-compound retriever
+            SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder().retriever(new RetrieverBuilder() {
+                @Override
+                public QueryBuilder topDocsQuery() {
+                    return null;
+                }
+
+                @Override
+                public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
+                    // no-op
+                }
+
+                @Override
+                public String getName() {
+                    return "not_a_compound_retriever";
+                }
+
+                @Override
+                protected void doToXContent(XContentBuilder builder, Params params) throws IOException {}
+
+                @Override
+                protected boolean doEquals(Object o) {
+                    return false;
+                }
+
+                @Override
+                protected int doHashCode() {
+                    return 0;
+                }
+            }));
+            searchRequest.allowPartialSearchResults(true);
+            searchRequest.scroll((Scroll) null);
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNull(validationErrors);
+        }
+        {
+            // allow_partial_results not defined and compound retriever
+            SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder().retriever(new RetrieverBuilder() {
+                @Override
+                public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
+                    // no-op
+                }
+
+                @Override
+                public String getName() {
+                    return "compound_retriever";
+                }
+
+                @Override
+                protected void doToXContent(XContentBuilder builder, Params params) throws IOException {}
+
+                @Override
+                protected boolean doEquals(Object o) {
+                    return false;
+                }
+
+                @Override
+                protected int doHashCode() {
+                    return 0;
+                }
+
+                @Override
+                public boolean isCompound() {
+                    return true;
+                }
+
+                @Override
+                public QueryBuilder topDocsQuery() {
+                    return null;
+                }
+            }));
+            searchRequest.scroll((Scroll) null);
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNull(validationErrors);
+        }
         {
             // search_after and `from` isn't valid
             SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder());
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
index 561bca2695337..7694553f4de21 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
@@ -54,7 +55,7 @@ public void testParentTaskId() throws Exception {
         Settings settings = Settings.builder().put("node.name", TransportMultiSearchActionTests.class.getSimpleName()).build();
         ActionFilters actionFilters = mock(ActionFilters.class);
         when(actionFilters.filters()).thenReturn(new ActionFilter[0]);
-        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
         try {
             TransportService transportService = new TransportService(
                 Settings.EMPTY,
@@ -99,7 +100,6 @@ public String getLocalNodeId() {
                 }
             };
             TransportMultiSearchAction action = new TransportMultiSearchAction(
-                threadPool,
                 actionFilters,
                 transportService,
                 clusterService,
@@ -122,7 +122,7 @@ public void testBatchExecute() throws ExecutionException, InterruptedException {
         Settings settings = Settings.builder().put("node.name", TransportMultiSearchActionTests.class.getSimpleName()).build();
         ActionFilters actionFilters = mock(ActionFilters.class);
         when(actionFilters.filters()).thenReturn(new ActionFilter[0]);
-        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
         TransportService transportService = new TransportService(
             Settings.EMPTY,
             mock(Transport.class),
@@ -192,7 +192,6 @@ public String getLocalNodeId() {
         };
 
         TransportMultiSearchAction action = new TransportMultiSearchAction(
-            threadPool,
             actionFilters,
             transportService,
             clusterService,
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
index 98de321d792e0..f68e5f06bcf08 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
@@ -85,6 +85,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.NodeDisconnectedException;
@@ -98,6 +99,7 @@
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.usage.UsageService;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -1646,7 +1648,7 @@ public void testLocalShardIteratorFromPointInTime() {
         }
         TimeValue keepAlive = randomBoolean() ? null : TimeValue.timeValueSeconds(between(30, 3600));
 
-        final List<SearchShardIterator> shardIterators = TransportSearchAction.getLocalLocalShardsIteratorFromPointInTime(
+        final List<SearchShardIterator> shardIterators = TransportSearchAction.getLocalShardsIteratorFromPointInTime(
             clusterState,
             null,
             null,
@@ -1691,7 +1693,7 @@ public void testLocalShardIteratorFromPointInTime() {
             )
         );
         IndexNotFoundException error = expectThrows(IndexNotFoundException.class, () -> {
-            TransportSearchAction.getLocalLocalShardsIteratorFromPointInTime(
+            TransportSearchAction.getLocalShardsIteratorFromPointInTime(
                 clusterState,
                 null,
                 null,
@@ -1702,7 +1704,7 @@ public void testLocalShardIteratorFromPointInTime() {
         });
         assertThat(error.getIndex().getName(), equalTo("another-index"));
         // Ok when some indices don't exist and `allowPartialSearchResults` is true.
-        Optional<SearchShardIterator> anotherShardIterator = TransportSearchAction.getLocalLocalShardsIteratorFromPointInTime(
+        Optional<SearchShardIterator> anotherShardIterator = TransportSearchAction.getLocalShardsIteratorFromPointInTime(
             clusterState,
             null,
             null,
@@ -1722,7 +1724,7 @@ public void testCCSCompatibilityCheck() throws Exception {
         ActionFilters actionFilters = mock(ActionFilters.class);
         when(actionFilters.filters()).thenReturn(new ActionFilter[0]);
         TransportVersion transportVersion = TransportVersionUtils.getNextVersion(TransportVersions.MINIMUM_CCS_VERSION, true);
-        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
         try {
             TransportService transportService = MockTransportService.createNewService(
                 Settings.EMPTY,
@@ -1741,7 +1743,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
             NodeClient client = new NodeClient(settings, threadPool);
 
             SearchService searchService = mock(SearchService.class);
-            when(searchService.getRewriteContext(any(), any())).thenReturn(new QueryRewriteContext(null, null, null));
+            when(searchService.getRewriteContext(any(), any(), any())).thenReturn(new QueryRewriteContext(null, null, null, null, null));
             ClusterService clusterService = new ClusterService(
                 settings,
                 new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
@@ -1764,7 +1766,8 @@ protected void doWriteTo(StreamOutput out) throws IOException {
                 null,
                 new SearchTransportAPMMetrics(TelemetryProvider.NOOP.getMeterRegistry()),
                 new SearchResponseMetrics(TelemetryProvider.NOOP.getMeterRegistry()),
-                client
+                client,
+                new UsageService()
             );
 
             CountDownLatch latch = new CountDownLatch(1);
diff --git a/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java
index 808d539646550..08357e7e5805a 100644
--- a/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java
@@ -526,6 +526,45 @@ private static <T> void assertComplete(SubscribableListener<T> listener, @Nullab
         }
     }
 
+    public void testAndThenDropResultSuccess() {
+        final var initialListener = new SubscribableListener<>();
+        final var forked = new AtomicReference<ActionListener<Object>>();
+
+        final var chainedListener = initialListener.andThen(forked::set);
+        assertNull(forked.get());
+
+        final var o1 = new Object();
+        initialListener.onResponse(o1);
+        assertSame(chainedListener, forked.get());
+        assertFalse(chainedListener.isDone());
+    }
+
+    public void testAndThenDropResultThrowException() {
+        final var initialListener = new SubscribableListener<>();
+        final var forked = new AtomicReference<ActionListener<Object>>();
+
+        final var chainedListener = initialListener.andThen(l -> {
+            forked.set(l);
+            throw new ElasticsearchException("simulated");
+        });
+        assertNull(forked.get());
+
+        final var o1 = new Object();
+        initialListener.onResponse(o1);
+        assertSame(chainedListener, forked.get());
+        assertComplete(chainedListener, "simulated");
+    }
+
+    public void testAndThenDropResultFailure() {
+        final var initialListener = new SubscribableListener<>();
+
+        final var chainedListener = initialListener.andThen(l -> fail("should not be called"));
+        assertFalse(chainedListener.isDone());
+
+        initialListener.onFailure(new ElasticsearchException("simulated"));
+        assertComplete(chainedListener, "simulated");
+    }
+
     public void testAndThenApplySuccess() throws Exception {
         final var initialListener = new SubscribableListener<>();
         final var result = new AtomicReference<>();
diff --git a/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainRefCountingTests.java b/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainRefCountingTests.java
index a199003fc59c4..bd46a4ad07d26 100644
--- a/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainRefCountingTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainRefCountingTests.java
@@ -15,12 +15,12 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.ActionType;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.RefCounted;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.Task;
@@ -152,7 +152,7 @@ public static class TestAction extends TransportAction<Request, Response> {
 
         @Inject
         public TestAction(TransportService transportService, ActionFilters actionFilters) {
-            super(TYPE.name(), actionFilters, transportService.getTaskManager());
+            super(TYPE.name(), actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
             threadPool = transportService.getThreadPool();
         }
 
diff --git a/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java b/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
index 82c204b1d0b88..29963ac98d957 100644
--- a/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
@@ -16,12 +16,14 @@
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -53,7 +55,8 @@ public void init() throws Exception {
         counter = new AtomicInteger();
         threadPool = new ThreadPool(
             Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "TransportActionFilterChainTests").build(),
-            MeterRegistry.NOOP
+            MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders()
         );
     }
 
@@ -79,7 +82,8 @@ public void testActionFiltersRequest() throws InterruptedException {
         TransportAction<TestRequest, TestResponse> transportAction = new TransportAction<TestRequest, TestResponse>(
             actionName,
             actionFilters,
-            new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet())
+            new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()),
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
         ) {
             @Override
             protected void doExecute(Task task, TestRequest request, ActionListener<TestResponse> listener) {
@@ -165,7 +169,8 @@ public <Request extends ActionRequest, Response extends ActionResponse> void exe
         TransportAction<TestRequest, TestResponse> transportAction = new TransportAction<TestRequest, TestResponse>(
             actionName,
             actionFilters,
-            new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet())
+            new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()),
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
         ) {
             @Override
             protected void doExecute(Task task, TestRequest request, ActionListener<TestResponse> listener) {
diff --git a/server/src/test/java/org/elasticsearch/action/support/TransportActionTests.java b/server/src/test/java/org/elasticsearch/action/support/TransportActionTests.java
new file mode 100644
index 0000000000000..2fa91937a7dcc
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/support/TransportActionTests.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.Collections;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+
+public class TransportActionTests extends ESTestCase {
+
+    private ThreadPool threadPool;
+
+    @Before
+    public void init() throws Exception {
+        threadPool = new ThreadPool(
+            Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "TransportActionTests").build(),
+            MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders()
+        );
+    }
+
+    @After
+    public void shutdown() throws Exception {
+        terminate(threadPool);
+    }
+
+    public void testDirectExecuteRunsOnCallingThread() throws ExecutionException, InterruptedException {
+
+        String actionName = randomAlphaOfLength(randomInt(30));
+        var testExecutor = new Executor() {
+            @Override
+            public void execute(Runnable command) {
+                fail("executeDirect should not run a TransportAction on a different executor");
+            }
+        };
+
+        var transportAction = getTestTransportAction(actionName, testExecutor);
+
+        PlainActionFuture<TestResponse> future = new PlainActionFuture<>();
+
+        transportAction.executeDirect(null, new TestRequest(), future);
+
+        var response = future.get();
+        assertThat(response, notNullValue());
+        assertThat(response.executingThreadName, equalTo(Thread.currentThread().getName()));
+        assertThat(response.executingThreadId, equalTo(Thread.currentThread().getId()));
+    }
+
+    public void testExecuteRunsOnExecutor() throws ExecutionException, InterruptedException {
+
+        String actionName = randomAlphaOfLength(randomInt(30));
+
+        boolean[] executedOnExecutor = new boolean[1];
+        var testExecutor = new Executor() {
+            @Override
+            public void execute(Runnable command) {
+                command.run();
+                executedOnExecutor[0] = true;
+            }
+        };
+
+        var transportAction = getTestTransportAction(actionName, testExecutor);
+
+        PlainActionFuture<TestResponse> future = new PlainActionFuture<>();
+
+        ActionTestUtils.execute(transportAction, null, new TestRequest(), future);
+
+        var response = future.get();
+        assertThat(response, notNullValue());
+        assertTrue(executedOnExecutor[0]);
+    }
+
+    public void testExecuteWithGenericExecutorRunsOnDifferentThread() throws ExecutionException, InterruptedException {
+
+        String actionName = randomAlphaOfLength(randomInt(30));
+        var transportAction = getTestTransportAction(actionName, threadPool.executor(ThreadPool.Names.GENERIC));
+
+        PlainActionFuture<TestResponse> future = new PlainActionFuture<>();
+
+        ActionTestUtils.execute(transportAction, null, new TestRequest(), future);
+
+        var response = future.get();
+        assertThat(response, notNullValue());
+        assertThat(response.executingThreadName, not(equalTo(Thread.currentThread().getName())));
+        assertThat(response.executingThreadName, containsString("[generic]"));
+        assertThat(response.executingThreadId, not(equalTo(Thread.currentThread().getId())));
+    }
+
+    public void testExecuteWithDirectExecutorRunsOnCallingThread() throws ExecutionException, InterruptedException {
+
+        String actionName = randomAlphaOfLength(randomInt(30));
+        var transportAction = getTestTransportAction(actionName, EsExecutors.DIRECT_EXECUTOR_SERVICE);
+
+        PlainActionFuture<TestResponse> future = new PlainActionFuture<>();
+
+        ActionTestUtils.execute(transportAction, null, new TestRequest(), future);
+
+        var response = future.get();
+        assertThat(response, notNullValue());
+        assertThat(response, notNullValue());
+        assertThat(response.executingThreadName, equalTo(Thread.currentThread().getName()));
+        assertThat(response.executingThreadId, equalTo(Thread.currentThread().getId()));
+    }
+
+    private TransportAction<TestRequest, TestResponse> getTestTransportAction(String actionName, Executor executor) {
+        ActionFilters actionFilters = new ActionFilters(Collections.emptySet());
+        TransportAction<TestRequest, TestResponse> transportAction = new TransportAction<>(
+            actionName,
+            actionFilters,
+            new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()),
+            executor
+        ) {
+            @Override
+            protected void doExecute(Task task, TestRequest request, ActionListener<TestResponse> listener) {
+                listener.onResponse(new TestResponse(Thread.currentThread().getName(), Thread.currentThread().getId()));
+            }
+        };
+        return transportAction;
+    }
+
+    private static class TestRequest extends ActionRequest {
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+    }
+
+    private static class TestResponse extends ActionResponse {
+
+        private final String executingThreadName;
+        private final long executingThreadId;
+
+        TestResponse(String executingThreadName, long executingThreadId) {
+            this.executingThreadName = executingThreadName;
+            this.executingThreadId = executingThreadId;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) {}
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/support/UnsafePlainActionFutureTests.java b/server/src/test/java/org/elasticsearch/action/support/UnsafePlainActionFutureTests.java
new file mode 100644
index 0000000000000..68d934eb43bf3
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/support/UnsafePlainActionFutureTests.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class UnsafePlainActionFutureTests extends ESTestCase {
+
+    public void testOneArg() {
+        String unsafeExecutorName = "unsafe-executor";
+        String otherExecutorName = "other-executor";
+        UnsafePlainActionFuture<?> future = new UnsafePlainActionFuture<Void>(unsafeExecutorName);
+        Thread other1 = getThread(otherExecutorName);
+        Thread other2 = getThread(otherExecutorName);
+        assertFalse(future.allowedExecutors(other1, other2));
+        Thread unsafe1 = getThread(unsafeExecutorName);
+        Thread unsafe2 = getThread(unsafeExecutorName);
+        assertTrue(future.allowedExecutors(unsafe1, unsafe2));
+
+        assertTrue(future.allowedExecutors(unsafe1, other1));
+    }
+
+    public void testTwoArg() {
+        String unsafeExecutorName1 = "unsafe-executor-1";
+        String unsafeExecutorName2 = "unsafe-executor-2";
+        String otherExecutorName = "other-executor";
+        UnsafePlainActionFuture<?> future = new UnsafePlainActionFuture<Void>(unsafeExecutorName1, unsafeExecutorName2);
+        Thread other1 = getThread(otherExecutorName);
+        Thread other2 = getThread(otherExecutorName);
+        assertFalse(future.allowedExecutors(other1, other2));
+        Thread unsafe1Thread1 = getThread(unsafeExecutorName1);
+        Thread unsafe2Thread1 = getThread(unsafeExecutorName2);
+        Thread unsafe1Thread2 = getThread(unsafeExecutorName1);
+        Thread unsafe2Thread2 = getThread(unsafeExecutorName2);
+        assertTrue(future.allowedExecutors(unsafe1Thread1, unsafe1Thread2));
+        assertTrue(future.allowedExecutors(unsafe2Thread1, unsafe2Thread2));
+
+        assertTrue(future.allowedExecutors(unsafe1Thread1, unsafe2Thread2));
+        assertTrue(future.allowedExecutors(unsafe2Thread1, other1));
+        assertTrue(future.allowedExecutors(other1, unsafe2Thread2));
+    }
+
+    private static Thread getThread(String executorName) {
+        Thread t = new Thread("[" + executorName + "][]");
+        assertThat(EsExecutors.executorName(t), equalTo(executorName));
+        return t;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index 04ad7d410e9b0..1e891aa80b696 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -1519,7 +1519,8 @@ private class TestAction extends TransportReplicationAction<Request, Request, Te
                 Request::new,
                 EsExecutors.DIRECT_EXECUTOR_SERVICE,
                 SyncGlobalCheckpointAfterOperation.DoNotSync,
-                primaryActionExecution
+                primaryActionExecution,
+                ReplicaActionExecution.SubjectToCircuitBreaker
             );
         }
 
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java
index c5642fd9681ac..946e0475c95f9 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java
@@ -458,7 +458,8 @@ private abstract class TestAction extends TransportReplicationAction<Request, Re
                 Request::new,
                 EsExecutors.DIRECT_EXECUTOR_SERVICE,
                 SyncGlobalCheckpointAfterOperation.DoNotSync,
-                PrimaryActionExecution.RejectOnOverload
+                PrimaryActionExecution.RejectOnOverload,
+                ReplicaActionExecution.SubjectToCircuitBreaker
             );
             this.shardId = Objects.requireNonNull(shardId);
             this.primary = Objects.requireNonNull(primary);
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
index 5cc0e55942818..a89415b8b7887 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
@@ -428,7 +428,8 @@ protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentF
                 (service, ignore) -> EsExecutors.DIRECT_EXECUTOR_SERVICE,
                 PrimaryActionExecution.RejectOnOverload,
                 new IndexingPressure(Settings.EMPTY),
-                EmptySystemIndices.INSTANCE
+                EmptySystemIndices.INSTANCE,
+                ReplicaActionExecution.SubjectToCircuitBreaker
             );
             this.withDocumentFailureOnPrimary = withDocumentFailureOnPrimary;
             this.withDocumentFailureOnReplica = withDocumentFailureOnReplica;
@@ -456,7 +457,8 @@ protected TestAction(
                 (service, ignore) -> EsExecutors.DIRECT_EXECUTOR_SERVICE,
                 PrimaryActionExecution.RejectOnOverload,
                 new IndexingPressure(settings),
-                EmptySystemIndices.INSTANCE
+                EmptySystemIndices.INSTANCE,
+                ReplicaActionExecution.SubjectToCircuitBreaker
             );
             this.withDocumentFailureOnPrimary = false;
             this.withDocumentFailureOnReplica = false;
diff --git a/server/src/test/java/org/elasticsearch/client/internal/AbstractClientHeadersTestCase.java b/server/src/test/java/org/elasticsearch/client/internal/AbstractClientHeadersTestCase.java
index 39e424adecfce..453ca8ab873b6 100644
--- a/server/src/test/java/org/elasticsearch/client/internal/AbstractClientHeadersTestCase.java
+++ b/server/src/test/java/org/elasticsearch/client/internal/AbstractClientHeadersTestCase.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction;
 import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction;
 import org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction;
 import org.elasticsearch.action.admin.indices.cache.clear.TransportClearIndicesCacheAction;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
@@ -29,6 +30,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentType;
 
@@ -78,7 +80,7 @@ public void setUp() throws Exception {
             .put("node.name", "test-" + getTestName())
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
-        threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
         client = buildClient(settings, ACTIONS);
     }
 
@@ -101,10 +103,11 @@ public void testActions() {
         client.prepareGet("idx", "id").execute(new AssertingActionListener<>(TransportGetAction.TYPE.name(), client.threadPool()));
         client.prepareSearch().execute(new AssertingActionListener<>(TransportSearchAction.TYPE.name(), client.threadPool()));
         client.prepareDelete("idx", "id").execute(new AssertingActionListener<>(TransportDeleteAction.NAME, client.threadPool()));
-        client.admin()
-            .cluster()
-            .prepareDeleteStoredScript("id")
-            .execute(new AssertingActionListener<>(TransportDeleteStoredScriptAction.TYPE.name(), client.threadPool()));
+        client.execute(
+            TransportDeleteStoredScriptAction.TYPE,
+            new DeleteStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "id"),
+            new AssertingActionListener<>(TransportDeleteStoredScriptAction.TYPE.name(), client.threadPool())
+        );
         client.prepareIndex("idx")
             .setId("id")
             .setSource("source", XContentType.JSON)
diff --git a/server/src/test/java/org/elasticsearch/client/internal/node/NodeClientHeadersTests.java b/server/src/test/java/org/elasticsearch/client/internal/node/NodeClientHeadersTests.java
index 9aea310180410..84537359e0399 100644
--- a/server/src/test/java/org/elasticsearch/client/internal/node/NodeClientHeadersTests.java
+++ b/server/src/test/java/org/elasticsearch/client/internal/node/NodeClientHeadersTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.client.internal.AbstractClientHeadersTestCase;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.transport.Transport;
@@ -54,7 +55,7 @@ private Actions(ActionType<?>[] actions, TaskManager taskManager) {
     private static class InternalTransportAction extends TransportAction<ActionRequest, ActionResponse> {
 
         private InternalTransportAction(String actionName, TaskManager taskManager) {
-            super(actionName, EMPTY_FILTERS, taskManager);
+            super(actionName, EMPTY_FILTERS, taskManager, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
index ff0f166eb8339..544151d2adcd4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
@@ -34,7 +34,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -43,6 +42,7 @@
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.indices.EmptySystemIndices;
+import org.elasticsearch.injection.guice.ModuleTestCase;
 import org.elasticsearch.plugins.ClusterPlugin;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.telemetry.TelemetryProvider;
diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index 4f1c5b7fa5dc5..8cb2f7c0a7ce1 100644
--- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -607,12 +608,12 @@ private void runConnectionBlock(CheckedRunnable<Exception> connectionBlock) thro
         public void openConnection(DiscoveryNode node, ConnectionProfile profile, ActionListener<Connection> listener) {
             final CheckedRunnable<Exception> connectionBlock = nodeConnectionBlocks.get(node);
             if (profile == null && randomConnectionExceptions && randomBoolean()) {
-                threadPool.generic().execute(() -> ActionListener.completeWith(listener, () -> {
+                threadPool.generic().execute(ActionRunnable.run(listener, () -> {
                     runConnectionBlock(connectionBlock);
                     throw new ConnectTransportException(node, "simulated");
                 }));
             } else {
-                threadPool.generic().execute(() -> ActionListener.completeWith(listener, () -> {
+                threadPool.generic().execute(ActionRunnable.supply(listener, () -> {
                     runConnectionBlock(connectionBlock);
                     return new Connection() {
                         private final SubscribableListener<Void> closeListener = new SubscribableListener<>();
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceTests.java
index cbfd43d646d0d..897b4d39c8504 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationDiagnosticsServiceTests.java
@@ -1378,7 +1378,7 @@ private static String randomNodeId() {
      */
     private static MasterHistoryService createMasterHistoryService(ClusterService clusterService) throws Exception {
         ThreadPool threadPool = mock(ThreadPool.class);
-        when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis());
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(System::currentTimeMillis);
         MasterHistory localMasterHistory = new MasterHistory(threadPool, clusterService);
         MasterHistoryService masterHistoryService = mock(MasterHistoryService.class);
         when(masterHistoryService.getLocalMasterHistory()).thenReturn(localMasterHistory);
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java
index b19cce96a2208..bf64b29d364e0 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterStateUpdateStats;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -79,6 +80,8 @@
 import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING;
 import static org.elasticsearch.monitor.StatusInfo.Status.HEALTHY;
 import static org.elasticsearch.monitor.StatusInfo.Status.UNHEALTHY;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -1762,7 +1765,13 @@ public void testLogsWarningPeriodicallyIfSingleNodeClusterHasSeedHosts() {
                         @Override
                         public void match(LogEvent event) {
                             final String message = event.getMessage().getFormattedMessage();
-                            assertThat(message, startsWith("This node is a fully-formed single-node cluster with cluster UUID"));
+                            assertThat(
+                                message,
+                                allOf(
+                                    startsWith("This node is a fully-formed single-node cluster with cluster UUID"),
+                                    containsString(ReferenceDocs.FORMING_SINGLE_NODE_CLUSTERS.toString())
+                                )
+                            );
                             loggedClusterUuid = (String) event.getMessage().getParameters()[0];
                         }
 
@@ -2047,7 +2056,15 @@ public void testElectionSchedulingAfterDiscoveryOutage() {
                     + 4 * DEFAULT_DELAY_VARIABILITY
                     // Then a commit of the new leader's first cluster state
                     + DEFAULT_CLUSTER_STATE_UPDATE_DELAY
-                    // Then the remaining node may join
+                    // Then the remaining node may experience a disconnect (see comment in PeerFinder#closePeers) for which it is
+                    // removed from the cluster, and its fault detection must also detect its removal
+                    + Math.max(
+                        DEFAULT_CLUSTER_STATE_UPDATE_DELAY,
+                        defaultMillis(LEADER_CHECK_TIMEOUT_SETTING) * LEADER_CHECK_RETRY_COUNT_SETTING.get(Settings.EMPTY)
+                    )
+                    // then it does another round of discovery
+                    + defaultMillis(DISCOVERY_FIND_PEERS_INTERVAL_SETTING)
+                    // and finally it joins the master
                     + DEFAULT_CLUSTER_STATE_UPDATE_DELAY
             );
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/MasterHistoryServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/MasterHistoryServiceTests.java
index 737c20cc5f41e..e339272914f87 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/MasterHistoryServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/MasterHistoryServiceTests.java
@@ -72,11 +72,11 @@ public void testGetRemoteHistory() throws Exception {
         assertNull(remoteHistory);
     }
 
-    private static MasterHistoryService createMasterHistoryService() throws Exception {
+    private static MasterHistoryService createMasterHistoryService() {
         var clusterService = mock(ClusterService.class);
         when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
         ThreadPool threadPool = mock(ThreadPool.class);
-        when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis());
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(System::currentTimeMillis);
         TransportService transportService = mock(TransportService.class);
         return new MasterHistoryService(transportService, threadPool, clusterService);
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/MasterHistoryTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/MasterHistoryTests.java
index c746f9fde755c..6f792917fbea4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/MasterHistoryTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/MasterHistoryTests.java
@@ -26,6 +26,7 @@
 
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -59,7 +60,7 @@ public void testGetBasicUse() {
         var clusterService = mock(ClusterService.class);
         when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
         ThreadPool threadPool = mock(ThreadPool.class);
-        when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis());
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(System::currentTimeMillis);
         MasterHistory masterHistory = new MasterHistory(threadPool, clusterService);
         assertNull(masterHistory.getMostRecentMaster());
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, nullMasterClusterState));
@@ -82,10 +83,11 @@ public void testHasMasterGoneNull() {
         when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
         AtomicReference<ClusterState> clusterStateReference = new AtomicReference<>(nullMasterClusterState);
         when(clusterService.state()).thenAnswer((Answer<ClusterState>) invocation -> clusterStateReference.get());
+        AtomicLong currentTimeMillis = new AtomicLong(System.currentTimeMillis());
         ThreadPool threadPool = mock(ThreadPool.class);
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(currentTimeMillis::get);
         MasterHistory masterHistory = new MasterHistory(threadPool, clusterService);
-        long oneHourAgo = System.currentTimeMillis() - (60 * 60 * 1000);
-        when(threadPool.relativeTimeInMillis()).thenReturn(oneHourAgo);
+        currentTimeMillis.set(System.currentTimeMillis() - (60 * 60 * 1000));
         assertFalse(masterHistory.hasMasterGoneNullAtLeastNTimes(3));
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, nullMasterClusterState));
         assertFalse(masterHistory.hasMasterGoneNullAtLeastNTimes(3));
@@ -110,23 +112,24 @@ public void testHasMasterGoneNull() {
 
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState));
         assertTrue(masterHistory.hasMasterGoneNullAtLeastNTimes(3));
-        when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis());
+        currentTimeMillis.set(System.currentTimeMillis());
         assertFalse(masterHistory.hasMasterGoneNullAtLeastNTimes(3));
     }
 
     public void testTime() {
         var clusterService = mock(ClusterService.class);
         when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
+        AtomicLong currentTimeMillis = new AtomicLong(System.currentTimeMillis());
         ThreadPool threadPool = mock(ThreadPool.class);
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(currentTimeMillis::get);
         MasterHistory masterHistory = new MasterHistory(threadPool, clusterService);
-        long oneHourAgo = System.currentTimeMillis() - (60 * 60 * 1000);
-        when(threadPool.relativeTimeInMillis()).thenReturn(oneHourAgo);
+        currentTimeMillis.set(System.currentTimeMillis() - (60 * 60 * 1000));
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, nullMasterClusterState));
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState));
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node2MasterClusterState, node1MasterClusterState));
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node3MasterClusterState, node2MasterClusterState));
         assertThat(masterHistory.getMostRecentMaster(), equalTo(node3MasterClusterState.nodes().getMasterNode()));
-        when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis());
+        currentTimeMillis.set(System.currentTimeMillis());
         assertThat(masterHistory.getMostRecentMaster(), equalTo(node3MasterClusterState.nodes().getMasterNode()));
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node3MasterClusterState));
         assertTrue(masterHistory.hasSeenMasterInLastNSeconds(5));
@@ -136,7 +139,9 @@ public void testTime() {
     public void testHasSeenMasterInLastNSeconds() {
         var clusterService = mock(ClusterService.class);
         when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
+        AtomicLong currentTimeMillis = new AtomicLong(System.currentTimeMillis());
         ThreadPool threadPool = mock(ThreadPool.class);
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(currentTimeMillis::get);
         MasterHistory masterHistory = new MasterHistory(threadPool, clusterService);
 
         /*
@@ -144,8 +149,7 @@ public void testHasSeenMasterInLastNSeconds() {
          * null -> node1 -> node2 -> node3
          * Except for when only null had been master, there has been a non-null master node in the last 5 seconds all along
          */
-        long sixtyMinutesAgo = System.currentTimeMillis() - new TimeValue(60, TimeUnit.MINUTES).getMillis();
-        when(threadPool.relativeTimeInMillis()).thenReturn(sixtyMinutesAgo);
+        currentTimeMillis.set(System.currentTimeMillis() - new TimeValue(60, TimeUnit.MINUTES).getMillis());
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, nullMasterClusterState));
         assertFalse(masterHistory.hasSeenMasterInLastNSeconds(5));
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState));
@@ -158,8 +162,7 @@ public void testHasSeenMasterInLastNSeconds() {
          * null -> node1 -> null -> null -> node1
          * There has been a non-null master for the last 5 seconds every step at this time
          */
-        long fourtyMinutesAgo = System.currentTimeMillis() - new TimeValue(40, TimeUnit.MINUTES).getMillis();
-        when(threadPool.relativeTimeInMillis()).thenReturn(fourtyMinutesAgo);
+        currentTimeMillis.set(System.currentTimeMillis() - new TimeValue(40, TimeUnit.MINUTES).getMillis());
         assertTrue(masterHistory.hasSeenMasterInLastNSeconds(5));
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node3MasterClusterState));
         assertTrue(masterHistory.hasSeenMasterInLastNSeconds(5));
@@ -178,8 +181,7 @@ public void testHasSeenMasterInLastNSeconds() {
          * than 5 seconds ago (and more than the age of history we keep, 30 minutes), the transition from it to null was just now, so we
          * still say that there has been a master recently.
          */
-        long sixSecondsAgo = System.currentTimeMillis() - new TimeValue(6, TimeUnit.SECONDS).getMillis();
-        when(threadPool.relativeTimeInMillis()).thenReturn(sixSecondsAgo);
+        currentTimeMillis.set(System.currentTimeMillis() - new TimeValue(6, TimeUnit.SECONDS).getMillis());
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, node1MasterClusterState));
         assertTrue(masterHistory.hasSeenMasterInLastNSeconds(5));
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, nullMasterClusterState));
@@ -192,8 +194,7 @@ public void testHasSeenMasterInLastNSeconds() {
          * transitioned from a non-null master 6 seconds ago). After the transition to node1, we again have seen a non-null master in the
          *  last 5 seconds.
          */
-        long now = System.currentTimeMillis();
-        when(threadPool.relativeTimeInMillis()).thenReturn(now);
+        currentTimeMillis.set(System.currentTimeMillis());
         assertFalse(masterHistory.hasSeenMasterInLastNSeconds(5));
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, nullMasterClusterState, nullMasterClusterState));
         assertFalse(masterHistory.hasSeenMasterInLastNSeconds(5));
@@ -207,6 +208,7 @@ public void testGetNumberOfMasterChanges() {
         AtomicReference<ClusterState> clusterStateReference = new AtomicReference<>(nullMasterClusterState);
         when(clusterService.state()).thenAnswer((Answer<ClusterState>) invocation -> clusterStateReference.get());
         ThreadPool threadPool = mock(ThreadPool.class);
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(() -> 0L);
         MasterHistory masterHistory = new MasterHistory(threadPool, clusterService);
         assertThat(MasterHistory.getNumberOfMasterIdentityChanges(masterHistory.getNodes()), equalTo(0));
         masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState));
@@ -238,6 +240,7 @@ public void testMaxSize() {
         when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
         when(clusterService.state()).thenReturn(nullMasterClusterState);
         ThreadPool threadPool = mock(ThreadPool.class);
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(() -> 0L);
         MasterHistory masterHistory = new MasterHistory(threadPool, clusterService);
         for (int i = 0; i < MasterHistory.MAX_HISTORY_SIZE; i++) {
             masterHistory.clusterChanged(new ClusterChangedEvent(TEST_SOURCE, node1MasterClusterState, nullMasterClusterState));
@@ -252,6 +255,7 @@ public void testGetNodesAndGetRawNodes() {
         AtomicReference<ClusterState> clusterStateReference = new AtomicReference<>(createClusterState((DiscoveryNode) null));
         when(clusterService.state()).thenAnswer((Answer<ClusterState>) invocation -> clusterStateReference.get());
         ThreadPool threadPool = mock(ThreadPool.class);
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(() -> 0L);
         MasterHistory masterHistory = new MasterHistory(threadPool, clusterService);
 
         assertThat(masterHistory.getRawNodes().size(), equalTo(0));
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorServiceTests.java
index 77c59fe9e8209..91a4d1bdb5636 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorServiceTests.java
@@ -285,7 +285,7 @@ private MasterHistoryService createMasterHistoryService() throws Exception {
         when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
         when(clusterService.state()).thenReturn(nullMasterClusterState);
         ThreadPool threadPool = mock(ThreadPool.class);
-        when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis());
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(System::currentTimeMillis);
         MasterHistory localMasterHistory = new MasterHistory(threadPool, clusterService);
         MasterHistoryService masterHistoryService = mock(MasterHistoryService.class);
         when(masterHistoryService.getLocalMasterHistory()).thenReturn(localMasterHistory);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
index b93ccb0f978af..58a9e839e08f2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
@@ -294,9 +295,9 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws
             template.toXContent(builder, withEffectiveRetention, rolloverConfiguration);
             String serialized = Strings.toString(builder);
             assertThat(serialized, containsString("rollover"));
-            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention))
-                .getConditions()
-                .keySet()) {
+            for (String label : rolloverConfiguration.resolveRolloverConditions(
+                lifecycle.getEffectiveDataRetention(globalRetention, randomBoolean())
+            ).getConditions().keySet()) {
                 assertThat(serialized, containsString(label));
             }
             /*
@@ -307,4 +308,22 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws
             assertThat(serialized, not(containsString("effective_retention")));
         }
     }
+
+    public void testHangingParsing() throws IOException {
+        String cutDown = """
+            {
+              "template": {
+                "aliases": {
+                  "foo": "bar"
+                },
+                "food": "eggplant"
+              },
+              "potato": true
+            }
+            """;
+
+        try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, cutDown)) {
+            expectThrows(Exception.class, () -> ComponentTemplate.parse(parser));
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java
index 22c54ff2ad057..8d205a198feeb 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java
@@ -243,9 +243,9 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws
             template.toXContent(builder, withEffectiveRetention, rolloverConfiguration);
             String serialized = Strings.toString(builder);
             assertThat(serialized, containsString("rollover"));
-            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention))
-                .getConditions()
-                .keySet()) {
+            for (String label : rolloverConfiguration.resolveRolloverConditions(
+                lifecycle.getEffectiveDataRetention(globalRetention, randomBoolean())
+            ).getConditions().keySet()) {
                 assertThat(serialized, containsString(label));
             }
             /*
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionResolverTests.java
deleted file mode 100644
index 5d5317b61a8f0..0000000000000
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionResolverTests.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.cluster.metadata;
-
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.Map;
-
-import static org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionTests.randomGlobalRetention;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class DataStreamGlobalRetentionResolverTests extends ESTestCase {
-
-    public void testOnlyGlobalRetentionMetadata() {
-        DataStreamFactoryRetention factoryRetention = randomBoolean()
-            ? DataStreamFactoryRetention.emptyFactoryRetention()
-            : randomNonEmptyFactoryRetention();
-        DataStreamGlobalRetentionResolver resolver = new DataStreamGlobalRetentionResolver(factoryRetention);
-        DataStreamGlobalRetention expectedGlobalRetention = randomGlobalRetention();
-        DataStreamGlobalRetention globalRetention = resolver.resolve(
-            ClusterState.builder(ClusterName.DEFAULT).customs(Map.of(DataStreamGlobalRetention.TYPE, expectedGlobalRetention)).build()
-        );
-        assertThat(globalRetention, notNullValue());
-        assertThat(globalRetention.getDefaultRetention(), equalTo(expectedGlobalRetention.getDefaultRetention()));
-        assertThat(globalRetention.getMaxRetention(), equalTo(expectedGlobalRetention.getMaxRetention()));
-    }
-
-    public void testOnlyFactoryRetentionFallback() {
-        DataStreamFactoryRetention factoryRetention = randomNonEmptyFactoryRetention();
-        DataStreamGlobalRetentionResolver resolver = new DataStreamGlobalRetentionResolver(factoryRetention);
-        DataStreamGlobalRetention globalRetention = resolver.resolve(ClusterState.EMPTY_STATE);
-        assertThat(globalRetention, notNullValue());
-        assertThat(globalRetention.getDefaultRetention(), equalTo(factoryRetention.getDefaultRetention()));
-        assertThat(globalRetention.getMaxRetention(), equalTo(factoryRetention.getMaxRetention()));
-    }
-
-    private static DataStreamFactoryRetention randomNonEmptyFactoryRetention() {
-        boolean withDefault = randomBoolean();
-        TimeValue defaultRetention = withDefault ? TimeValue.timeValueDays(randomIntBetween(10, 20)) : null;
-        TimeValue maxRetention = withDefault && randomBoolean() ? null : TimeValue.timeValueDays(randomIntBetween(50, 200));
-        return new DataStreamFactoryRetention() {
-            @Override
-            public TimeValue getMaxRetention() {
-                return maxRetention;
-            }
-
-            @Override
-            public TimeValue getDefaultRetention() {
-                return defaultRetention;
-            }
-
-            @Override
-            public void init(ClusterSettings clusterSettings) {
-
-            }
-        };
-    }
-
-    public void testNoRetentionConfiguration() {
-        DataStreamGlobalRetentionResolver resolver = new DataStreamGlobalRetentionResolver(
-            DataStreamFactoryRetention.emptyFactoryRetention()
-        );
-        assertThat(resolver.resolve(ClusterState.EMPTY_STATE), nullValue());
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSettingsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSettingsTests.java
new file mode 100644
index 0000000000000..78184fd7568e5
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionSettingsTests.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class DataStreamGlobalRetentionSettingsTests extends ESTestCase {
+
+    public void testDefaults() {
+        DataStreamGlobalRetentionSettings globalRetentionSettings = DataStreamGlobalRetentionSettings.create(
+            ClusterSettings.createBuiltInClusterSettings(),
+            DataStreamFactoryRetention.emptyFactoryRetention()
+        );
+
+        assertThat(globalRetentionSettings.getDefaultRetention(), nullValue());
+        assertThat(globalRetentionSettings.getMaxRetention(), nullValue());
+
+        // Fallback to factory settings
+        TimeValue maxFactoryValue = randomPositiveTimeValue();
+        TimeValue defaultFactoryValue = randomPositiveTimeValue();
+        DataStreamGlobalRetentionSettings withFactorySettings = DataStreamGlobalRetentionSettings.create(
+            ClusterSettings.createBuiltInClusterSettings(),
+            new DataStreamFactoryRetention() {
+                @Override
+                public TimeValue getMaxRetention() {
+                    return maxFactoryValue;
+                }
+
+                @Override
+                public TimeValue getDefaultRetention() {
+                    return defaultFactoryValue;
+                }
+
+                @Override
+                public void init(ClusterSettings clusterSettings) {
+
+                }
+            }
+        );
+
+        assertThat(withFactorySettings.getDefaultRetention(), equalTo(defaultFactoryValue));
+        assertThat(withFactorySettings.getMaxRetention(), equalTo(maxFactoryValue));
+    }
+
+    public void testMonitorsDefaultRetention() {
+        ClusterSettings clusterSettings = ClusterSettings.createBuiltInClusterSettings();
+        DataStreamGlobalRetentionSettings globalRetentionSettings = DataStreamGlobalRetentionSettings.create(
+            clusterSettings,
+            DataStreamFactoryRetention.emptyFactoryRetention()
+        );
+
+        // Test valid update
+        TimeValue newDefaultRetention = TimeValue.timeValueDays(randomIntBetween(1, 10));
+        Settings newSettings = Settings.builder()
+            .put(
+                DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING.getKey(),
+                newDefaultRetention.toHumanReadableString(0)
+            )
+            .build();
+        clusterSettings.applySettings(newSettings);
+
+        assertThat(newDefaultRetention, equalTo(globalRetentionSettings.getDefaultRetention()));
+
+        // Test invalid update
+        Settings newInvalidSettings = Settings.builder()
+            .put(DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING.getKey(), TimeValue.ZERO)
+            .build();
+        IllegalArgumentException exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> clusterSettings.applySettings(newInvalidSettings)
+        );
+        assertThat(
+            exception.getCause().getMessage(),
+            containsString("Setting 'data_streams.lifecycle.retention.default' should be greater than")
+        );
+    }
+
+    public void testMonitorsMaxRetention() {
+        ClusterSettings clusterSettings = ClusterSettings.createBuiltInClusterSettings();
+        DataStreamGlobalRetentionSettings globalRetentionSettings = DataStreamGlobalRetentionSettings.create(
+            clusterSettings,
+            DataStreamFactoryRetention.emptyFactoryRetention()
+        );
+
+        // Test valid update
+        TimeValue newMaxRetention = TimeValue.timeValueDays(randomIntBetween(10, 30));
+        Settings newSettings = Settings.builder()
+            .put(DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING.getKey(), newMaxRetention.toHumanReadableString(0))
+            .build();
+        clusterSettings.applySettings(newSettings);
+
+        assertThat(newMaxRetention, equalTo(globalRetentionSettings.getMaxRetention()));
+
+        // Test invalid update
+        Settings newInvalidSettings = Settings.builder()
+            .put(DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING.getKey(), TimeValue.ZERO)
+            .build();
+        IllegalArgumentException exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> clusterSettings.applySettings(newInvalidSettings)
+        );
+        assertThat(
+            exception.getCause().getMessage(),
+            containsString("Setting 'data_streams.lifecycle.retention.max' should be greater than")
+        );
+    }
+
+    public void testCombinationValidation() {
+        ClusterSettings clusterSettings = ClusterSettings.createBuiltInClusterSettings();
+        DataStreamGlobalRetentionSettings.create(clusterSettings, DataStreamFactoryRetention.emptyFactoryRetention());
+
+        // Test invalid update
+        Settings newInvalidSettings = Settings.builder()
+            .put(DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING.getKey(), TimeValue.timeValueDays(90))
+            .put(DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING.getKey(), TimeValue.timeValueDays(30))
+            .build();
+        IllegalArgumentException exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> clusterSettings.applySettings(newInvalidSettings)
+        );
+        assertThat(
+            exception.getCause().getMessage(),
+            containsString(
+                "Setting [data_streams.lifecycle.retention.default=90d] cannot be greater than [data_streams.lifecycle.retention.max=30d]"
+            )
+        );
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionTests.java
index 193b18f1b6908..54b365689db8b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetentionTests.java
@@ -8,55 +8,26 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.Diff;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
-import org.elasticsearch.test.SimpleDiffableWireSerializationTestCase;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.util.List;
-
-public class DataStreamGlobalRetentionTests extends SimpleDiffableWireSerializationTestCase<ClusterState.Custom> {
-
-    @Override
-    protected ClusterState.Custom makeTestChanges(ClusterState.Custom testInstance) {
-        if (randomBoolean()) {
-            return testInstance;
-        }
-        return mutateInstance(testInstance);
-    }
-
-    @Override
-    protected Writeable.Reader<Diff<ClusterState.Custom>> diffReader() {
-        return DataStreamGlobalRetention::readDiffFrom;
-    }
+public class DataStreamGlobalRetentionTests extends AbstractWireSerializingTestCase<DataStreamGlobalRetention> {
 
     @Override
-    protected Writeable.Reader<ClusterState.Custom> instanceReader() {
+    protected Writeable.Reader<DataStreamGlobalRetention> instanceReader() {
         return DataStreamGlobalRetention::read;
     }
 
     @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(
-            List.of(
-                new NamedWriteableRegistry.Entry(ClusterState.Custom.class, DataStreamGlobalRetention.TYPE, DataStreamGlobalRetention::read)
-            )
-        );
-    }
-
-    @Override
-    protected ClusterState.Custom createTestInstance() {
+    protected DataStreamGlobalRetention createTestInstance() {
         return randomGlobalRetention();
     }
 
     @Override
-    protected ClusterState.Custom mutateInstance(ClusterState.Custom instance) {
-        DataStreamGlobalRetention metadata = (DataStreamGlobalRetention) instance;
-        var defaultRetention = metadata.getDefaultRetention();
-        var maxRetention = metadata.getMaxRetention();
+    protected DataStreamGlobalRetention mutateInstance(DataStreamGlobalRetention instance) {
+        var defaultRetention = instance.defaultRetention();
+        var maxRetention = instance.maxRetention();
         switch (randomInt(1)) {
             case 0 -> {
                 defaultRetention = randomValueOtherThan(
@@ -83,10 +54,6 @@ public static DataStreamGlobalRetention randomGlobalRetention() {
         );
     }
 
-    public void testChunking() {
-        AbstractChunkedSerializingTestCase.assertChunkCount(createTestInstance(), ignored -> 1);
-    }
-
     public void testValidation() {
         expectThrows(
             IllegalArgumentException.class,
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java
index 01d77d6eba2be..6a46b9907b5fe 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java
@@ -39,7 +39,6 @@
 import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.DATA_STREAM_CONFIGURATION;
 import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.DEFAULT_GLOBAL_RETENTION;
 import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.MAX_GLOBAL_RETENTION;
-import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
@@ -117,10 +116,10 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws
             RolloverConfiguration rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions();
             DataStreamGlobalRetention globalRetention = DataStreamGlobalRetentionTests.randomGlobalRetention();
             ToXContent.Params withEffectiveRetention = new ToXContent.MapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS);
-            lifecycle.toXContent(builder, withEffectiveRetention, rolloverConfiguration, globalRetention);
+            lifecycle.toXContent(builder, withEffectiveRetention, rolloverConfiguration, globalRetention, false);
             String serialized = Strings.toString(builder);
             assertThat(serialized, containsString("rollover"));
-            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention))
+            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention, false))
                 .getConditions()
                 .keySet()) {
                 assertThat(serialized, containsString(label));
@@ -135,7 +134,7 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws
             } else {
                 assertThat(serialized, containsString("data_retention"));
             }
-            boolean globalRetentionIsNotNull = globalRetention.getDefaultRetention() != null || globalRetention.getMaxRetention() != null;
+            boolean globalRetentionIsNotNull = globalRetention.defaultRetention() != null || globalRetention.maxRetention() != null;
             boolean configuredLifeCycleIsNotNull = lifecycle.getDataRetention() != null && lifecycle.getDataRetention().value() != null;
             if (lifecycle.isEnabled() && (globalRetentionIsNotNull || configuredLifeCycleIsNotNull)) {
                 assertThat(serialized, containsString("effective_retention"));
@@ -283,24 +282,27 @@ public void testEffectiveRetention() {
             TimeValue maxRetention = TimeValue.timeValueDays(randomIntBetween(50, 100));
             TimeValue defaultRetention = TimeValue.timeValueDays(randomIntBetween(1, 50));
             Tuple<TimeValue, DataStreamLifecycle.RetentionSource> effectiveDataRetentionWithSource = noRetentionLifecycle
-                .getEffectiveDataRetentionWithSource(null);
+                .getEffectiveDataRetentionWithSource(null, randomBoolean());
             assertThat(effectiveDataRetentionWithSource.v1(), nullValue());
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION));
 
             effectiveDataRetentionWithSource = noRetentionLifecycle.getEffectiveDataRetentionWithSource(
-                new DataStreamGlobalRetention(null, maxRetention)
+                new DataStreamGlobalRetention(null, maxRetention),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(maxRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(MAX_GLOBAL_RETENTION));
 
             effectiveDataRetentionWithSource = noRetentionLifecycle.getEffectiveDataRetentionWithSource(
-                new DataStreamGlobalRetention(defaultRetention, null)
+                new DataStreamGlobalRetention(defaultRetention, null),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(defaultRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DEFAULT_GLOBAL_RETENTION));
 
             effectiveDataRetentionWithSource = noRetentionLifecycle.getEffectiveDataRetentionWithSource(
-                new DataStreamGlobalRetention(defaultRetention, maxRetention)
+                new DataStreamGlobalRetention(defaultRetention, maxRetention),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(defaultRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DEFAULT_GLOBAL_RETENTION));
@@ -316,19 +318,21 @@ public void testEffectiveRetention() {
             TimeValue defaultRetention = TimeValue.timeValueDays(randomIntBetween(1, (int) dataStreamRetention.getDays() - 1));
 
             Tuple<TimeValue, DataStreamLifecycle.RetentionSource> effectiveDataRetentionWithSource = lifecycleRetention
-                .getEffectiveDataRetentionWithSource(null);
+                .getEffectiveDataRetentionWithSource(null, randomBoolean());
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(dataStreamRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION));
 
             effectiveDataRetentionWithSource = lifecycleRetention.getEffectiveDataRetentionWithSource(
-                new DataStreamGlobalRetention(defaultRetention, null)
+                new DataStreamGlobalRetention(defaultRetention, null),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(dataStreamRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION));
 
             TimeValue maxGlobalRetention = randomBoolean() ? dataStreamRetention : TimeValue.timeValueDays(dataStreamRetention.days() + 1);
             effectiveDataRetentionWithSource = lifecycleRetention.getEffectiveDataRetentionWithSource(
-                new DataStreamGlobalRetention(defaultRetention, maxGlobalRetention)
+                new DataStreamGlobalRetention(defaultRetention, maxGlobalRetention),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(dataStreamRetention));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION));
@@ -340,29 +344,52 @@ public void testEffectiveRetention() {
                         ? null
                         : TimeValue.timeValueDays(randomIntBetween(1, (int) (maxRetentionLessThanDataStream.days() - 1))),
                     maxRetentionLessThanDataStream
-                )
+                ),
+                false
             );
             assertThat(effectiveDataRetentionWithSource.v1(), equalTo(maxRetentionLessThanDataStream));
             assertThat(effectiveDataRetentionWithSource.v2(), equalTo(MAX_GLOBAL_RETENTION));
         }
-    }
 
-    public void testEffectiveRetentionParams() {
-        {
-            ToXContent.Params params = DataStreamLifecycle.maybeAddEffectiveRetentionParams(new ToXContent.MapParams(Map.of()));
-            assertThat(params.paramAsBoolean(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false), equalTo(false));
-        }
+        // Global retention does not apply to internal data streams
         {
-            ToXContent.Params params = DataStreamLifecycle.maybeAddEffectiveRetentionParams(
-                new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "not-serverless"))
+            // Pick the values in such a way that global retention should have kicked in
+            boolean dataStreamWithRetention = randomBoolean();
+            TimeValue dataStreamRetention = dataStreamWithRetention ? TimeValue.timeValueDays(365) : null;
+            DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(
+                TimeValue.timeValueDays(7),
+                TimeValue.timeValueDays(90)
             );
-            assertThat(params.paramAsBoolean(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false), equalTo(false));
+            DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(dataStreamRetention).build();
+
+            // Verify that global retention should have kicked in
+            var effectiveDataRetentionWithSource = lifecycle.getEffectiveDataRetentionWithSource(globalRetention, false);
+            if (dataStreamWithRetention) {
+                assertThat(effectiveDataRetentionWithSource.v1(), equalTo(globalRetention.maxRetention()));
+                assertThat(effectiveDataRetentionWithSource.v2(), equalTo(MAX_GLOBAL_RETENTION));
+            } else {
+                assertThat(effectiveDataRetentionWithSource.v1(), equalTo(globalRetention.defaultRetention()));
+                assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DEFAULT_GLOBAL_RETENTION));
+            }
+
+            // Now verify that internal data streams do not use global retention
+            // Verify that global retention should have kicked in
+            effectiveDataRetentionWithSource = lifecycle.getEffectiveDataRetentionWithSource(globalRetention, true);
+            if (dataStreamWithRetention) {
+                assertThat(effectiveDataRetentionWithSource.v1(), equalTo(dataStreamRetention));
+            } else {
+                assertThat(effectiveDataRetentionWithSource.v1(), nullValue());
+            }
+            assertThat(effectiveDataRetentionWithSource.v2(), equalTo(DATA_STREAM_CONFIGURATION));
         }
-        {
-            ToXContent.Params params = DataStreamLifecycle.maybeAddEffectiveRetentionParams(
-                new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "serverless"))
-            );
-            assertThat(params.paramAsBoolean(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false), equalTo(true));
+    }
+
+    public void testEffectiveRetentionParams() {
+        Map<String, String> initialParams = randomMap(0, 10, () -> Tuple.tuple(randomAlphaOfLength(10), randomAlphaOfLength(10)));
+        ToXContent.Params params = DataStreamLifecycle.addEffectiveRetentionParams(new ToXContent.MapParams(initialParams));
+        assertThat(params.paramAsBoolean(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false), equalTo(true));
+        for (String key : initialParams.keySet()) {
+            assertThat(initialParams.get(key), equalTo(params.param(key)));
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java
index 922a1405bddff..06aa349bbd8c7 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java
@@ -8,7 +8,6 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.HeaderWarning;
@@ -59,7 +58,7 @@ public void testNoHeaderWarning() {
         HeaderWarning.setThreadContext(threadContext);
 
         DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.newBuilder().downsampling(randomDownsampling()).build();
-        noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(null);
+        noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(null, randomBoolean());
         Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
         assertThat(responseHeaders.isEmpty(), is(true));
 
@@ -72,7 +71,7 @@ public void testNoHeaderWarning() {
             TimeValue.timeValueDays(2),
             TimeValue.timeValueDays(dataStreamRetention.days() + randomIntBetween(1, 5))
         );
-        lifecycleWithRetention.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
+        lifecycleWithRetention.addWarningHeaderIfDataRetentionNotEffective(globalRetention, false);
         responseHeaders = threadContext.getResponseHeaders();
         assertThat(responseHeaders.isEmpty(), is(true));
     }
@@ -86,14 +85,14 @@ public void testDefaultRetentionHeaderWarning() {
             randomTimeValue(2, 10, TimeUnit.DAYS),
             randomBoolean() ? null : TimeValue.timeValueDays(20)
         );
-        noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
+        noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention, false);
         Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
         assertThat(responseHeaders.size(), is(1));
         assertThat(
             responseHeaders.get("Warning").get(0),
             containsString(
                 "Not providing a retention is not allowed for this project. The default retention of ["
-                    + globalRetention.getDefaultRetention().getStringRep()
+                    + globalRetention.defaultRetention().getStringRep()
                     + "] will be applied."
             )
         );
@@ -108,7 +107,7 @@ public void testMaxRetentionHeaderWarning() {
             .downsampling(randomDownsampling())
             .build();
         DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(null, maxRetention);
-        lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention);
+        lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention, false);
         Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
         assertThat(responseHeaders.size(), is(1));
         String userRetentionPart = lifecycle.getDataStreamRetention() == null
@@ -129,24 +128,22 @@ public void testUpdatingLifecycleOnADataStream() {
         HeaderWarning.setThreadContext(threadContext);
         String dataStream = randomAlphaOfLength(5);
         TimeValue defaultRetention = randomTimeValue(2, 100, TimeUnit.DAYS);
-
-        DataStreamFactoryRetention factoryRetention;
         ClusterState before = ClusterState.builder(
             DataStreamTestHelper.getClusterStateWithDataStreams(List.of(new Tuple<>(dataStream, 2)), List.of())
         ).build();
-        if (randomBoolean()) {
-            factoryRetention = DataStreamFactoryRetention.emptyFactoryRetention();
-            before = ClusterState.builder(before)
-                .putCustom(DataStreamGlobalRetention.TYPE, new DataStreamGlobalRetention(defaultRetention, null))
-                .build();
-        } else {
-            factoryRetention = getDefaultFactoryRetention(defaultRetention);
-        }
+
+        Settings settingsWithDefaultRetention = builder().put(
+            DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING.getKey(),
+            defaultRetention
+        ).build();
 
         MetadataDataStreamsService metadataDataStreamsService = new MetadataDataStreamsService(
             mock(ClusterService.class),
             mock(IndicesService.class),
-            new DataStreamGlobalRetentionResolver(factoryRetention)
+            DataStreamGlobalRetentionSettings.create(
+                ClusterSettings.createBuiltInClusterSettings(settingsWithDefaultRetention),
+                DataStreamFactoryRetention.emptyFactoryRetention()
+            )
         );
 
         ClusterState after = metadataDataStreamsService.updateDataLifecycle(before, List.of(dataStream), DataStreamLifecycle.DEFAULT);
@@ -191,6 +188,24 @@ public void testValidateLifecycleIndexTemplateWithWarning() {
         );
     }
 
+    public void testValidateInternalDataStreamRetentionWithoutWarning() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        HeaderWarning.setThreadContext(threadContext);
+        TimeValue defaultRetention = randomTimeValue(2, 100, TimeUnit.DAYS);
+        MetadataIndexTemplateService.validateLifecycle(
+            Metadata.builder().build(),
+            randomAlphaOfLength(10),
+            ComposableIndexTemplate.builder()
+                .template(new Template(null, null, null, DataStreamLifecycle.DEFAULT))
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                .indexPatterns(List.of("." + randomAlphaOfLength(10)))
+                .build(),
+            new DataStreamGlobalRetention(defaultRetention, null)
+        );
+        Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+        assertThat(responseHeaders.size(), is(0));
+    }
+
     /**
      * Make sure we still take into account component templates during validation (and not just the index template).
      */
@@ -254,17 +269,10 @@ public void testValidateLifecycleInComponentTemplate() throws Exception {
             new IndexSettingProviders(Set.of())
         );
         TimeValue defaultRetention = randomTimeValue(2, 100, TimeUnit.DAYS);
-        DataStreamFactoryRetention factoryRetention;
-        ClusterState state;
-        if (randomBoolean()) {
-            factoryRetention = DataStreamFactoryRetention.emptyFactoryRetention();
-            state = ClusterState.builder(ClusterName.DEFAULT)
-                .putCustom(DataStreamGlobalRetention.TYPE, new DataStreamGlobalRetention(defaultRetention, null))
-                .build();
-        } else {
-            state = ClusterState.EMPTY_STATE;
-            factoryRetention = getDefaultFactoryRetention(defaultRetention);
-        }
+        Settings settingsWithDefaultRetention = Settings.builder()
+            .put(DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING.getKey(), defaultRetention)
+            .build();
+        ClusterState state = ClusterState.EMPTY_STATE;
         MetadataIndexTemplateService metadataIndexTemplateService = new MetadataIndexTemplateService(
             clusterService,
             createIndexService,
@@ -273,7 +281,10 @@ public void testValidateLifecycleInComponentTemplate() throws Exception {
             xContentRegistry(),
             EmptySystemIndices.INSTANCE,
             new IndexSettingProviders(Set.of()),
-            new DataStreamGlobalRetentionResolver(factoryRetention)
+            DataStreamGlobalRetentionSettings.create(
+                ClusterSettings.createBuiltInClusterSettings(settingsWithDefaultRetention),
+                DataStreamFactoryRetention.emptyFactoryRetention()
+            )
         );
 
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
@@ -301,23 +312,4 @@ public void testValidateLifecycleInComponentTemplate() throws Exception {
             )
         );
     }
-
-    private DataStreamFactoryRetention getDefaultFactoryRetention(TimeValue defaultRetention) {
-        return new DataStreamFactoryRetention() {
-            @Override
-            public TimeValue getMaxRetention() {
-                return null;
-            }
-
-            @Override
-            public TimeValue getDefaultRetention() {
-                return defaultRetention;
-            }
-
-            @Override
-            public void init(ClusterSettings clusterSettings) {
-
-            }
-        };
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
index 07481a68c5176..9b4ec1fa89ee2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
@@ -1039,11 +1039,7 @@ public void testValidate() {
                 dataStream.validate(
                     (index) -> IndexMetadata.builder(index)
                         .settings(
-                            Settings.builder()
-                                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-                                .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current())
-                                .build()
+                            indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()).build()
                         )
                         .build()
                 );
@@ -1058,10 +1054,7 @@ public void testValidate() {
                     () -> dataStream.validate(
                         (index) -> IndexMetadata.builder(index)
                             .settings(
-                                Settings.builder()
-                                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-                                    .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current())
+                                indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current())
                                     .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
                                     .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), start3.toEpochMilli())
                                     .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), end3.toEpochMilli())
@@ -1900,15 +1893,15 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws
             dataStream.toXContent(builder, withEffectiveRetention, rolloverConfiguration, globalRetention);
             String serialized = Strings.toString(builder);
             assertThat(serialized, containsString("rollover"));
-            for (String label : rolloverConfiguration.resolveRolloverConditions(lifecycle.getEffectiveDataRetention(globalRetention))
-                .getConditions()
-                .keySet()) {
+            for (String label : rolloverConfiguration.resolveRolloverConditions(
+                lifecycle.getEffectiveDataRetention(globalRetention, dataStream.isInternal())
+            ).getConditions().keySet()) {
                 assertThat(serialized, containsString(label));
             }
             // We check that even if there was no retention provided by the user, the global retention applies
             assertThat(serialized, not(containsString("data_retention")));
-            if (dataStream.isSystem() == false
-                && (globalRetention.getDefaultRetention() != null || globalRetention.getMaxRetention() != null)) {
+            if (dataStream.isInternal() == false
+                && (globalRetention.defaultRetention() != null || globalRetention.maxRetention() != null)) {
                 assertThat(serialized, containsString("effective_retention"));
             } else {
                 assertThat(serialized, not(containsString("effective_retention")));
@@ -2212,6 +2205,17 @@ public void testIsFailureIndex() {
         assertThat(failureStoreDataStreamWithEmptyFailureIndices.isFailureStoreIndex(randomAlphaOfLength(10)), is(false));
     }
 
+    public void testInternalDataStream() {
+        var nonInternalDataStream = createTestInstance().copy().setSystem(false).setName(randomAlphaOfLength(10)).build();
+        assertThat(nonInternalDataStream.isInternal(), is(false));
+
+        var systemDataStream = nonInternalDataStream.copy().setSystem(true).setHidden(true).setName(randomAlphaOfLength(10)).build();
+        assertThat(systemDataStream.isInternal(), is(true));
+
+        var dotPrefixedDataStream = nonInternalDataStream.copy().setSystem(false).setName("." + randomAlphaOfLength(10)).build();
+        assertThat(dotPrefixedDataStream.isInternal(), is(true));
+    }
+
     private record DataStreamMetadata(Long creationTimeInMillis, Long rolloverTimeInMillis, Long originationTimeInMillis) {
         public static DataStreamMetadata dataStreamMetadata(Long creationTimeInMillis, Long rolloverTimeInMillis) {
             return new DataStreamMetadata(creationTimeInMillis, rolloverTimeInMillis, null);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
index 8a487e5653627..f7d343b43b29c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -967,13 +967,7 @@ public void testClusterStateCreateIndexThrowsWriteIndexValidationException() thr
         assertThat(
             expectThrows(
                 IllegalStateException.class,
-                () -> clusterStateCreateIndex(
-                    currentClusterState,
-                    Set.of(),
-                    newIndex,
-                    null,
-                    TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
-                )
+                () -> clusterStateCreateIndex(currentClusterState, newIndex, null, TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             ).getMessage(),
             startsWith("alias [alias1] has more than one write index [")
         );
@@ -991,7 +985,6 @@ public void testClusterStateCreateIndex() {
 
         ClusterState updatedClusterState = clusterStateCreateIndex(
             currentClusterState,
-            Set.of(INDEX_READ_ONLY_BLOCK),
             newIndexMetadata,
             null,
             TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
@@ -1037,7 +1030,6 @@ public void testClusterStateCreateIndexWithMetadataTransaction() {
 
         ClusterState updatedClusterState = clusterStateCreateIndex(
             currentClusterState,
-            Set.of(INDEX_READ_ONLY_BLOCK),
             newIndexMetadata,
             metadataTransformer,
             TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
index d4639c3d3118e..e0f4936300c0e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
@@ -400,7 +401,10 @@ public void testUpdateLifecycle() {
         MetadataDataStreamsService service = new MetadataDataStreamsService(
             mock(ClusterService.class),
             mock(IndicesService.class),
-            new DataStreamGlobalRetentionResolver(DataStreamFactoryRetention.emptyFactoryRetention())
+            DataStreamGlobalRetentionSettings.create(
+                ClusterSettings.createBuiltInClusterSettings(),
+                DataStreamFactoryRetention.emptyFactoryRetention()
+            )
         );
         {
             // Remove lifecycle
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
index d9c01953fbaab..e66dd32b718b7 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.PutRequest;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
@@ -2158,6 +2159,23 @@ public void testDataStreamsUsingTemplates() throws Exception {
         MetadataIndexTemplateService.innerRemoveIndexTemplateV2(stateWithTwoTemplates, "logs");
     }
 
+    public void testDataStreamsUsingMatchAllTemplate() throws Exception {
+        ClusterState state = ClusterState.EMPTY_STATE;
+        final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
+
+        ComposableIndexTemplate template = ComposableIndexTemplate.builder()
+            .indexPatterns(Collections.singletonList("*"))
+            .priority(100L)
+            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+            .build();
+        final String templateName = "all-data-streams-template";
+        state = service.addIndexTemplateV2(state, false, templateName, template);
+        // When creating a data stream, we'll look for templates. The data stream is not hidden
+        assertThat(MetadataIndexTemplateService.findV2Template(state.metadata(), "some-data-stream", false), equalTo(templateName));
+        // The write index for a data stream will be a hidden index. We need to make sure it matches the same template:
+        assertThat(MetadataIndexTemplateService.findV2Template(state.metadata(), "some-data-stream", true), equalTo(templateName));
+    }
+
     public void testRemovingHigherOrderTemplateOfDataStreamWithMultipleTemplates() throws Exception {
         ClusterState state = ClusterState.EMPTY_STATE;
         final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
@@ -2484,7 +2502,10 @@ private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistr
             xContentRegistry,
             EmptySystemIndices.INSTANCE,
             new IndexSettingProviders(Set.of()),
-            new DataStreamGlobalRetentionResolver(DataStreamFactoryRetention.emptyFactoryRetention())
+            DataStreamGlobalRetentionSettings.create(
+                ClusterSettings.createBuiltInClusterSettings(),
+                DataStreamFactoryRetention.emptyFactoryRetention()
+            )
         );
 
         final List<Throwable> throwables = new ArrayList<>();
@@ -2526,9 +2547,6 @@ public void onFailure(Exception e) {
     private MetadataIndexTemplateService getMetadataIndexTemplateService() {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         ClusterService clusterService = getInstanceFromNode(ClusterService.class);
-        DataStreamGlobalRetentionResolver dataStreamGlobalRetentionResolver = new DataStreamGlobalRetentionResolver(
-            DataStreamFactoryRetention.emptyFactoryRetention()
-        );
         MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(
             Settings.EMPTY,
             clusterService,
@@ -2551,7 +2569,10 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() {
             xContentRegistry(),
             EmptySystemIndices.INSTANCE,
             new IndexSettingProviders(Set.of()),
-            dataStreamGlobalRetentionResolver
+            DataStreamGlobalRetentionSettings.create(
+                ClusterSettings.createBuiltInClusterSettings(),
+                DataStreamFactoryRetention.emptyFactoryRetention()
+            )
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java
index 062cb20f2e1b3..881e84f07af9a 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java
@@ -34,6 +34,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -121,6 +122,39 @@ public void testSigtermIsRemoveInOlderVersions() throws IOException {
         assertThat(new SingleNodeShutdownMetadata(out.bytes().streamInput()).getType(), equalTo(SingleNodeShutdownMetadata.Type.SIGTERM));
     }
 
+    public void testIsNodeMarkedForRemoval() {
+        SingleNodeShutdownMetadata.Type type;
+        SingleNodeShutdownMetadata.Builder builder = SingleNodeShutdownMetadata.builder()
+            .setNodeId("thenode")
+            .setReason("myReason")
+            .setStartedAtMillis(0L);
+        switch (type = randomFrom(SingleNodeShutdownMetadata.Type.values())) {
+            case SIGTERM -> {
+                var metadata = new NodesShutdownMetadata(
+                    Map.of("thenode", builder.setType(type).setGracePeriod(TimeValue.ONE_MINUTE).build())
+                );
+                assertThat(metadata.isNodeMarkedForRemoval("thenode"), is(true));
+                assertThat(metadata.isNodeMarkedForRemoval("anotherNode"), is(false));
+            }
+            case REMOVE -> {
+                var metadata = new NodesShutdownMetadata(Map.of("thenode", builder.setType(type).build()));
+                assertThat(metadata.isNodeMarkedForRemoval("thenode"), is(true));
+                assertThat(metadata.isNodeMarkedForRemoval("anotherNode"), is(false));
+            }
+            case REPLACE -> {
+                var metadata = new NodesShutdownMetadata(
+                    Map.of("thenode", builder.setType(type).setTargetNodeName("newnodecoming").build())
+                );
+                assertThat(metadata.isNodeMarkedForRemoval("thenode"), is(true));
+                assertThat(metadata.isNodeMarkedForRemoval("anotherNode"), is(false));
+            }
+            case RESTART -> {
+                var metadata = new NodesShutdownMetadata(Map.of("thenode", builder.setType(type).build()));
+                assertThat(metadata.isNodeMarkedForRemoval("thenode"), is(false));
+            }
+        }
+    }
+
     @Override
     protected Writeable.Reader<Diff<Metadata.Custom>> diffReader() {
         return NodesShutdownMetadata.NodeShutdownMetadataDiff::new;
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java
index d76e874c7061b..ad39a7017b465 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java
@@ -589,6 +589,14 @@ public void testRoutingPathNumbersInSource() throws IOException {
         assertIndexShard(routing, Map.of("foo", 123), Math.floorMod(hash(List.of("foo", "123")), shards));
     }
 
+    public void testRoutingPathBooleansInSource() throws IOException {
+        int shards = between(2, 1000);
+        IndexRouting routing = indexRoutingForPath(shards, "foo");
+        assertIndexShard(routing, Map.of("foo", true), Math.floorMod(hash(List.of("foo", "true")), shards));
+        assertIndexShard(routing, Map.of("foo", false), Math.floorMod(hash(List.of("foo", "false")), shards));
+
+    }
+
     public void testRoutingPathBwc() throws IOException {
         IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
         IndexRouting routing = indexRoutingForPath(version, 8, "dim.*,other.*,top");
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java
index b731fd79c82fe..994e892e3ac3c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityActionGuideTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.cluster.routing.allocation.shards.ShardsAvailabilityHealthIndicatorService;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.test.ESTestCase;
 
@@ -33,14 +34,17 @@
 import static org.elasticsearch.cluster.routing.allocation.shards.ShardsAvailabilityHealthIndicatorService.TIER_CAPACITY_ACTION_GUIDE;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class ShardsAvailabilityActionGuideTests extends ESTestCase {
 
-    private final ShardsAvailabilityHealthIndicatorService service = new ShardsAvailabilityHealthIndicatorService(
-        mock(ClusterService.class),
-        mock(AllocationService.class),
-        mock(SystemIndices.class)
-    );
+    private final ShardsAvailabilityHealthIndicatorService service;
+
+    public ShardsAvailabilityActionGuideTests() {
+        ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.getClusterSettings()).thenReturn(ClusterSettings.createBuiltInClusterSettings());
+        service = new ShardsAvailabilityHealthIndicatorService(clusterService, mock(AllocationService.class), mock(SystemIndices.class));
+    }
 
     public void testRestoreFromSnapshotAction() {
         assertThat(ACTION_RESTORE_FROM_SNAPSHOT.helpURL(), is(RESTORE_FROM_SNAPSHOT_ACTION_GUIDE));
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
index 0de27aea5b08f..615405645d6b2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
@@ -54,6 +54,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.shard.ShardId;
@@ -79,6 +80,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiPredicate;
 import java.util.function.Consumer;
@@ -1208,7 +1210,11 @@ public void testRebalanceDoesNotCauseHotSpots() {
             new ConcurrentRebalanceAllocationDecider(clusterSettings),
             new ThrottlingAllocationDecider(clusterSettings) };
 
-        var reconciler = new DesiredBalanceReconciler(clusterSettings, mock(ThreadPool.class), mock(MeterRegistry.class));
+        var reconciler = new DesiredBalanceReconciler(
+            clusterSettings,
+            new DeterministicTaskQueue().getThreadPool(),
+            mock(MeterRegistry.class)
+        );
 
         var totalOutgoingMoves = new HashMap<String, AtomicInteger>();
         for (int i = 0; i < numberOfNodes; i++) {
@@ -1275,7 +1281,7 @@ public void testShouldLogOnTooManyUndesiredAllocations() {
             .build();
 
         var threadPool = mock(ThreadPool.class);
-        when(threadPool.relativeTimeInMillis()).thenReturn(1L).thenReturn(2L).thenReturn(3L);
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(new AtomicLong()::incrementAndGet);
 
         var reconciler = new DesiredBalanceReconciler(createBuiltInClusterSettings(), threadPool, mock(MeterRegistry.class));
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java
index 916683c4a536a..ad30c79a01334 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java
@@ -42,6 +42,8 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.health.Diagnosis;
 import org.elasticsearch.health.HealthIndicatorDetails;
 import org.elasticsearch.health.HealthIndicatorImpact;
@@ -61,9 +63,9 @@
 import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.mockito.Mockito;
 import org.mockito.stubbing.Answer;
 
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -104,6 +106,7 @@
 import static org.elasticsearch.cluster.routing.allocation.shards.ShardsAvailabilityHealthIndicatorServiceTests.ShardState.UNAVAILABLE;
 import static org.elasticsearch.common.util.CollectionUtils.concatLists;
 import static org.elasticsearch.core.TimeValue.timeValueSeconds;
+import static org.elasticsearch.core.Tuple.tuple;
 import static org.elasticsearch.health.Diagnosis.Resource.Type.FEATURE_STATE;
 import static org.elasticsearch.health.Diagnosis.Resource.Type.INDEX;
 import static org.elasticsearch.health.HealthStatus.GREEN;
@@ -337,7 +340,8 @@ public void testAllReplicasUnassigned() {
                 status,
                 clusterState,
                 NodesShutdownMetadata.EMPTY,
-                randomBoolean()
+                randomBoolean(),
+                timeValueSeconds(0)
             );
             assertFalse(status.replicas.doAnyIndicesHaveAllUnavailable());
         }
@@ -359,7 +363,8 @@ public void testAllReplicasUnassigned() {
                 status,
                 clusterState,
                 NodesShutdownMetadata.EMPTY,
-                randomBoolean()
+                randomBoolean(),
+                timeValueSeconds(0)
             );
             assertFalse(status.replicas.doAnyIndicesHaveAllUnavailable());
         }
@@ -381,7 +386,8 @@ public void testAllReplicasUnassigned() {
                 status,
                 clusterState,
                 NodesShutdownMetadata.EMPTY,
-                randomBoolean()
+                randomBoolean(),
+                timeValueSeconds(0)
             );
             assertTrue(status.replicas.doAnyIndicesHaveAllUnavailable());
         }
@@ -398,13 +404,15 @@ public void testAllReplicasUnassigned() {
                 ),
                 List.of()
             );
+
             var service = createShardsAvailabilityIndicatorService(clusterState);
             ShardAllocationStatus status = service.createNewStatus(clusterState.metadata());
             ShardsAvailabilityHealthIndicatorService.updateShardAllocationStatus(
                 status,
                 clusterState,
                 NodesShutdownMetadata.EMPTY,
-                randomBoolean()
+                randomBoolean(),
+                timeValueSeconds(0)
             );
             assertTrue(status.replicas.doAnyIndicesHaveAllUnavailable());
         }
@@ -440,7 +448,8 @@ public void testAllReplicasUnassigned() {
                 status,
                 clusterState,
                 NodesShutdownMetadata.EMPTY,
-                randomBoolean()
+                randomBoolean(),
+                timeValueSeconds(0)
             );
             // Here because the replica is unassigned due to the primary being created, it's treated as though the replica can be ignored.
             assertFalse(
@@ -469,7 +478,8 @@ public void testAllReplicasUnassigned() {
                 status,
                 clusterState,
                 NodesShutdownMetadata.EMPTY,
-                randomBoolean()
+                randomBoolean(),
+                timeValueSeconds(0)
             );
             var shardRouting = routingTable.shardsWithState(ShardRoutingState.UNASSIGNED).get(0);
             assertTrue(service.areAllShardsOfThisTypeUnavailable(shardRouting, clusterState));
@@ -492,7 +502,8 @@ public void testAllReplicasUnassigned() {
                 status,
                 clusterState,
                 NodesShutdownMetadata.EMPTY,
-                randomBoolean()
+                randomBoolean(),
+                timeValueSeconds(0)
             );
             var shardRouting = clusterState.routingTable().index("myindex").shardsWithState(ShardRoutingState.UNASSIGNED).get(0);
             assertFalse(service.areAllShardsOfThisTypeUnavailable(shardRouting, clusterState));
@@ -922,7 +933,7 @@ public void testRestoreFromSnapshotReportsFeatureStates() {
         );
         HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO);
 
-        assertThat(result.status(), is(HealthStatus.RED));
+        assertThat(result.status(), is(RED));
         assertThat(result.diagnosisList().size(), is(1));
         Diagnosis diagnosis = result.diagnosisList().get(0);
         List<Diagnosis.Resource> affectedResources = diagnosis.affectedResources();
@@ -1925,7 +1936,7 @@ private SystemIndices getSystemIndices(
     // We expose the indicator name and the diagnoses in the x-pack usage API. In order to index them properly in a telemetry index
     // they need to be declared in the health-api-indexer.edn in the telemetry repository.
     public void testMappedFieldsForTelemetry() {
-        assertThat(ShardsAvailabilityHealthIndicatorService.NAME, equalTo("shards_availability"));
+        assertThat(NAME, equalTo("shards_availability"));
         assertThat(
             ACTION_RESTORE_FROM_SNAPSHOT.getUniqueId(),
             equalTo("elasticsearch:health:shards_availability:diagnosis:restore_from_snapshot")
@@ -1970,8 +1981,10 @@ public void testMappedFieldsForTelemetry() {
             DIAGNOSIS_WAIT_FOR_INITIALIZATION.getUniqueId(),
             equalTo("elasticsearch:health:shards_availability:diagnosis:initializing_shards")
         );
+        ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.getClusterSettings()).thenReturn(ClusterSettings.createBuiltInClusterSettings());
         var service = new ShardsAvailabilityHealthIndicatorService(
-            mock(ClusterService.class),
+            clusterService,
             mock(AllocationService.class),
             mock(SystemIndices.class)
         );
@@ -2004,6 +2017,7 @@ public void testMappedFieldsForTelemetry() {
     }
 
     public void testIsNewlyCreatedAndInitializingReplica() {
+
         ShardId id = new ShardId("index", "uuid", 0);
         IndexMetadata idxMeta = IndexMetadata.builder("index")
             .numberOfShards(1)
@@ -2017,56 +2031,156 @@ public void testIsNewlyCreatedAndInitializingReplica() {
                     .build()
             )
             .build();
-        ShardRouting primary = createShardRouting(id, true, new ShardAllocation("node", AVAILABLE));
-        var state = createClusterStateWith(List.of(index("index", new ShardAllocation("node", AVAILABLE))), List.of());
-        assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(primary, state));
-
-        ShardRouting replica = createShardRouting(id, false, new ShardAllocation("node", AVAILABLE));
-        state = createClusterStateWith(List.of(index("index", new ShardAllocation("node", AVAILABLE))), List.of());
-        assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(replica, state));
-
-        ShardRouting unassignedReplica = createShardRouting(id, false, new ShardAllocation("node", UNAVAILABLE));
-        state = createClusterStateWith(
-            List.of(idxMeta),
-            List.of(index("index", "uuid", new ShardAllocation("node", UNAVAILABLE))),
-            List.of(),
-            List.of()
-        );
-        assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(unassignedReplica, state));
 
-        UnassignedInfo.Reason reason = randomFrom(UnassignedInfo.Reason.NODE_LEFT, UnassignedInfo.Reason.NODE_RESTARTING);
-        ShardAllocation allocation = new ShardAllocation(
-            "node",
-            UNAVAILABLE,
-            new UnassignedInfo(
-                reason,
-                "message",
-                null,
-                0,
-                0,
-                0,
-                randomBoolean(),
-                randomFrom(UnassignedInfo.AllocationStatus.values()),
-                Set.of(),
-                reason == UnassignedInfo.Reason.NODE_LEFT ? null : randomAlphaOfLength(20)
-            )
-        );
-        ShardRouting unallocatedReplica = createShardRouting(id, false, allocation);
-        state = createClusterStateWith(
-            List.of(idxMeta),
-            List.of(index(idxMeta, new ShardAllocation("node", UNAVAILABLE), allocation)),
-            List.of(),
-            List.of()
-        );
-        assertFalse(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(unallocatedReplica, state));
+        ClusterState state;
 
-        state = createClusterStateWith(
-            List.of(idxMeta),
-            List.of(index(idxMeta, new ShardAllocation("node", CREATING), allocation)),
-            List.of(),
-            List.of()
-        );
-        assertTrue(ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(unallocatedReplica, state));
+        // --------- Test conditions that don't depend on threshold ---------
+
+        TimeValue replicaUnassignedThreshold = randomFrom(timeValueSeconds(3), timeValueSeconds(0));
+        {
+            // active, whether primary or replica
+            boolean primary = randomBoolean();
+            ShardAllocation primaryAllocation = new ShardAllocation("node", AVAILABLE);
+            ShardRouting shard = createShardRouting(id, primary, primaryAllocation);
+            state = createClusterStateWith(List.of(index("index", primaryAllocation)), List.of());
+            assertFalse(
+                ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(
+                    shard,
+                    state,
+                    Instant.now().toEpochMilli() - replicaUnassignedThreshold.millis()
+                )
+            );
+        }
+
+        {   // primary, but not active
+            var primaryAllocation = new ShardAllocation("node", INITIALIZING);
+            ShardRouting primary = createShardRouting(id, true, primaryAllocation);
+            state = createClusterStateWith(List.of(index("index", primaryAllocation)), List.of());
+            assertFalse(
+                ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(
+                    primary,
+                    state,
+                    Instant.now().toEpochMilli() - replicaUnassignedThreshold.millis()
+                )
+            );
+        }
+
+        // --------- Test conditions that depend on threshold, but with threshold of 0 ---------
+        replicaUnassignedThreshold = timeValueSeconds(0);
+        long now = Instant.now().toEpochMilli();
+        TimeValue afterCutoffTime = TimeValue.timeValueMillis(now);
+        {
+            var unassignedInfo = randomFrom(decidersNo(afterCutoffTime), unassignedInfoNoFailures(afterCutoffTime));
+            var replicaAllocation = new ShardAllocation("node", UNAVAILABLE, unassignedInfo);
+            var primaryAllocation = new ShardAllocation("node", randomFrom(INITIALIZING, UNAVAILABLE, AVAILABLE, RESTARTING));
+
+            ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation);
+            state = createClusterStateWith(
+                List.of(idxMeta),
+                List.of(index(idxMeta, primaryAllocation, replicaAllocation)),
+                List.of(),
+                List.of()
+            );
+            assertFalse(
+                ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(
+                    unallocatedReplica,
+                    state,
+                    now - replicaUnassignedThreshold.millis()
+                )
+            );
+        }
+
+        {
+            var unassignedInfo = randomFrom(decidersNo(afterCutoffTime), unassignedInfoNoFailures(afterCutoffTime));
+            var replicaAllocation = new ShardAllocation("node", UNAVAILABLE, unassignedInfo);
+            var primaryAllocation = new ShardAllocation("node", CREATING);
+
+            ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation);
+            state = createClusterStateWith(
+                List.of(idxMeta),
+                List.of(index(idxMeta, primaryAllocation, replicaAllocation)),
+                List.of(),
+                List.of()
+            );
+            assertTrue(
+                ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(
+                    unallocatedReplica,
+                    state,
+                    now - replicaUnassignedThreshold.millis()
+                )
+            );
+        }
+
+        // --------- Test conditions that do depend on threshold, but with non-zero threshold ---------
+
+        replicaUnassignedThreshold = timeValueSeconds(3);
+        afterCutoffTime = TimeValue.timeValueMillis(now - 3000);
+        TimeValue beforeCutoffTime = TimeValue.timeValueMillis(now - 2999);
+        {
+            List<Tuple<ShardState, UnassignedInfo>> configs = new ArrayList<>();
+
+            // return false if primary is not creating and if unassigned info has failed allocations or is after cutoff
+            var uis = List.of(decidersNo(afterCutoffTime), decidersNo(beforeCutoffTime), unassignedInfoNoFailures(afterCutoffTime));
+            var shardStates = List.of(UNAVAILABLE, INITIALIZING, RESTARTING, AVAILABLE);
+            for (var shardState : shardStates) {
+                for (var ui : uis) {
+                    configs.add(tuple(shardState, ui));
+                }
+            }
+            // return false if primary is not creating or available and unassigned time is before cutoff
+            for (var shardState : List.of(UNAVAILABLE, INITIALIZING, RESTARTING)) {
+                configs.add(tuple(shardState, unassignedInfoNoFailures(beforeCutoffTime)));
+            }
+
+            for (var config : configs) {
+                var replicaAllocation = new ShardAllocation("node", UNAVAILABLE, config.v2());
+                var primaryAllocation = new ShardAllocation("node", config.v1());
+                ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation);
+                state = createClusterStateWith(
+                    List.of(idxMeta),
+                    List.of(index(idxMeta, primaryAllocation, replicaAllocation)),
+                    List.of(),
+                    List.of()
+                );
+                assertFalse(
+                    ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(
+                        unallocatedReplica,
+                        state,
+                        now - replicaUnassignedThreshold.millis()
+                    )
+                );
+            }
+        }
+
+        {
+            var configs = List.of(
+                // return true because primary is still creating
+                tuple(CREATING, decidersNo(afterCutoffTime)),
+                tuple(CREATING, decidersNo(beforeCutoffTime)),
+                tuple(CREATING, unassignedInfoNoFailures(afterCutoffTime)),
+                tuple(CREATING, unassignedInfoNoFailures(beforeCutoffTime)),
+
+                // returns true because unassigned time is before cutoff, and no failedAllocations
+                tuple(AVAILABLE, unassignedInfoNoFailures(beforeCutoffTime))
+            );
+
+            for (var config : configs) {
+                var replicaAllocation = new ShardAllocation("node", UNAVAILABLE, config.v2());
+                var primaryAllocation = new ShardAllocation("node", config.v1());
+
+                ShardRouting unallocatedReplica = createShardRouting(id, false, replicaAllocation);
+                IndexRoutingTable index = index(idxMeta, primaryAllocation, replicaAllocation);
+
+                state = createClusterStateWith(List.of(idxMeta), List.of(index), List.of(), List.of());
+                assertTrue(
+                    ShardsAvailabilityHealthIndicatorService.isNewlyCreatedAndInitializingReplica(
+                        unallocatedReplica,
+                        state,
+                        now - replicaUnassignedThreshold.millis()
+                    )
+                );
+            }
+        }
     }
 
     private HealthIndicatorResult createExpectedResult(
@@ -2109,10 +2223,7 @@ private static List<IndexMetadata> createIndexMetadataForIndexNameToPriorityMap(
             for (Map.Entry<String, Integer> indexNameToPriority : indexNameToPriorityMap.entrySet()) {
                 String indexName = indexNameToPriority.getKey();
                 IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder(indexName);
-                Settings settings = Settings.builder()
-                    .put(IndexMetadata.SETTING_PRIORITY, indexNameToPriority.getValue())
-                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                Settings settings = indexSettings(1, 1).put(IndexMetadata.SETTING_PRIORITY, indexNameToPriority.getValue())
                     .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current())
                     .build();
                 indexMetadataBuilder.settings(settings);
@@ -2376,14 +2487,34 @@ private static UnassignedInfo nodeLeft() {
         );
     }
 
+    private static UnassignedInfo unassignedInfoNoFailures(TimeValue unassignedTime) {
+        UnassignedInfo.Reason reason = randomFrom(UnassignedInfo.Reason.NODE_LEFT, UnassignedInfo.Reason.NODE_RESTARTING);
+        return new UnassignedInfo(
+            reason,
+            "message",
+            null,
+            0,
+            unassignedTime.nanos(),
+            unassignedTime.millis(),
+            randomBoolean(),
+            randomValueOtherThan(UnassignedInfo.AllocationStatus.DECIDERS_NO, () -> randomFrom(UnassignedInfo.AllocationStatus.values())),
+            Set.of(),
+            reason == UnassignedInfo.Reason.NODE_LEFT ? null : randomAlphaOfLength(20)
+        );
+    }
+
     private static UnassignedInfo decidersNo() {
+        return decidersNo(TimeValue.timeValueMillis(0));
+    }
+
+    private static UnassignedInfo decidersNo(TimeValue unassignedTime) {
         return new UnassignedInfo(
             UnassignedInfo.Reason.ALLOCATION_FAILED,
             null,
             null,
             1,
-            0,
-            0,
+            unassignedTime.nanos(),
+            unassignedTime.millis(),
             false,
             UnassignedInfo.AllocationStatus.DECIDERS_NO,
             Collections.emptySet(),
@@ -2426,7 +2557,7 @@ private static ShardsAvailabilityHealthIndicatorService createAllocationHealthIn
         when(clusterService.state()).thenReturn(clusterState);
         var clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
-        var allocationService = Mockito.mock(AllocationService.class);
+        var allocationService = mock(AllocationService.class);
         when(allocationService.explainShardAllocation(any(), any())).thenAnswer((Answer<ShardAllocationDecision>) invocation -> {
             ShardRouting shardRouting = invocation.getArgument(0);
             var key = new ShardRoutingKey(shardRouting.getIndexName(), shardRouting.getId(), shardRouting.primary());
diff --git a/server/src/test/java/org/elasticsearch/common/UUIDTests.java b/server/src/test/java/org/elasticsearch/common/UUIDTests.java
index 5af036ce0648f..3229049b67b4c 100644
--- a/server/src/test/java/org/elasticsearch/common/UUIDTests.java
+++ b/server/src/test/java/org/elasticsearch/common/UUIDTests.java
@@ -176,4 +176,20 @@ protected byte[] macAddress() {
         );
         return bytesPerDoc;
     }
+
+    public void testStringLength() {
+        assertEquals(UUIDs.RANDOM_BASED_UUID_STRING_LENGTH, getUnpaddedBase64StringLength(RandomBasedUUIDGenerator.SIZE_IN_BYTES));
+        assertEquals(UUIDs.RANDOM_BASED_UUID_STRING_LENGTH, UUIDs.randomBase64UUID().length());
+        assertEquals(UUIDs.RANDOM_BASED_UUID_STRING_LENGTH, UUIDs.randomBase64UUID(random()).length());
+        try (var secureString = UUIDs.randomBase64UUIDSecureString()) {
+            assertEquals(UUIDs.RANDOM_BASED_UUID_STRING_LENGTH, secureString.toString().length());
+        }
+
+        assertEquals(UUIDs.TIME_BASED_UUID_STRING_LENGTH, getUnpaddedBase64StringLength(TimeBasedUUIDGenerator.SIZE_IN_BYTES));
+        assertEquals(UUIDs.TIME_BASED_UUID_STRING_LENGTH, UUIDs.base64UUID().length());
+    }
+
+    private static int getUnpaddedBase64StringLength(int sizeInBytes) {
+        return (int) Math.ceil(sizeInBytes * 4.0 / 3.0);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java
index 1a6f52fabbd1b..ae686afcbb296 100644
--- a/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.common.io.stream;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -25,6 +26,7 @@
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TransportVersionUtils;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -48,6 +50,10 @@
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
+import static java.time.Instant.ofEpochSecond;
+import static java.time.ZonedDateTime.ofInstant;
+import static org.elasticsearch.TransportVersions.ZDT_NANOS_SUPPORT;
+import static org.elasticsearch.TransportVersions.ZDT_NANOS_SUPPORT_BROKEN;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
@@ -717,24 +723,77 @@ public void testReadAfterReachingEndOfStream() throws IOException {
             input.readBytes(new byte[len], 0, len);
 
             assertEquals(-1, input.read());
+            assertEquals(-1, input.read(new byte[2], 0, 2));
+        }
+    }
+
+    public void testZonedDateTimeSerialization() throws IOException {
+        checkZonedDateTimeSerialization(ZDT_NANOS_SUPPORT);
+    }
+
+    public void testZonedDateTimeMillisBwcSerializationV1() throws IOException {
+        checkZonedDateTimeSerialization(TransportVersionUtils.getPreviousVersion(ZDT_NANOS_SUPPORT_BROKEN));
+    }
+
+    public void testZonedDateTimeMillisBwcSerialization() throws IOException {
+        checkZonedDateTimeSerialization(TransportVersionUtils.getPreviousVersion(ZDT_NANOS_SUPPORT));
+    }
+
+    public void checkZonedDateTimeSerialization(TransportVersion tv) throws IOException {
+        assertGenericRoundtrip(ofInstant(Instant.EPOCH, randomZone()), tv);
+        assertGenericRoundtrip(ofInstant(ofEpochSecond(1), randomZone()), tv);
+        // just want to test a large number that will use 5+ bytes
+        long maxEpochSecond = Integer.MAX_VALUE;
+        long minEpochSecond = tv.between(ZDT_NANOS_SUPPORT_BROKEN, ZDT_NANOS_SUPPORT) ? 0 : Integer.MIN_VALUE;
+        assertGenericRoundtrip(ofInstant(ofEpochSecond(maxEpochSecond), randomZone()), tv);
+        assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond)), randomZone()), tv);
+        assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond), 1_000_000), randomZone()), tv);
+        assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond), 999_000_000), randomZone()), tv);
+        if (tv.onOrAfter(ZDT_NANOS_SUPPORT)) {
+            assertGenericRoundtrip(
+                ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond), 999_999_999), randomZone()),
+                tv
+            );
+            assertGenericRoundtrip(
+                ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond), randomIntBetween(0, 999_999_999)), randomZone()),
+                tv
+            );
+        }
+    }
+
+    public void testOptional() throws IOException {
+        try (var output = new BytesStreamOutput()) {
+            output.writeOptional(StreamOutput::writeString, "not-null");
+            output.writeOptional(StreamOutput::writeString, null);
+
+            final var input = getStreamInput(output.bytes());
+            assertEquals("not-null", input.readOptional(StreamInput::readString));
+            assertNull(input.readOptional(StreamInput::readString));
         }
     }
 
     private void assertSerialization(
         CheckedConsumer<StreamOutput, IOException> outputAssertions,
-        CheckedConsumer<StreamInput, IOException> inputAssertions
+        CheckedConsumer<StreamInput, IOException> inputAssertions,
+        TransportVersion transportVersion
     ) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.setTransportVersion(transportVersion);
             outputAssertions.accept(output);
             final StreamInput input = getStreamInput(output.bytes());
+            input.setTransportVersion(transportVersion);
             inputAssertions.accept(input);
         }
     }
 
     private void assertGenericRoundtrip(Object original) throws IOException {
+        assertGenericRoundtrip(original, TransportVersion.current());
+    }
+
+    private void assertGenericRoundtrip(Object original, TransportVersion transportVersion) throws IOException {
         assertSerialization(output -> { output.writeGenericValue(original); }, input -> {
             Object read = input.readGenericValue();
             assertThat(read, equalTo(original));
-        });
+        }, transportVersion);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/StreamInputTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/StreamInputTests.java
index 645461778f637..cda1f9b0e29de 100644
--- a/server/src/test/java/org/elasticsearch/common/io/stream/StreamInputTests.java
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/StreamInputTests.java
@@ -24,6 +24,15 @@
 public class StreamInputTests extends ESTestCase {
 
     private StreamInput in = Mockito.spy(StreamInput.class);
+
+    {
+        try {
+            Mockito.when(in.skip(anyLong())).thenAnswer(a -> a.getArguments()[0]);
+        } catch (IOException e) {
+            throw new AssertionError(e);
+        }
+    }
+
     byte[] bytes = "0123456789".getBytes(UTF_8);
 
     public void testCalculateByteLengthOfAscii() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/common/logging/ConsoleThrowablePatternConverterTests.java b/server/src/test/java/org/elasticsearch/common/logging/ConsoleThrowablePatternConverterTests.java
index 897aa31763ab8..87ab1352b7abb 100644
--- a/server/src/test/java/org/elasticsearch/common/logging/ConsoleThrowablePatternConverterTests.java
+++ b/server/src/test/java/org/elasticsearch/common/logging/ConsoleThrowablePatternConverterTests.java
@@ -11,8 +11,8 @@
 import org.apache.logging.log4j.core.LogEvent;
 import org.apache.logging.log4j.core.impl.Log4jLogEvent;
 import org.elasticsearch.bootstrap.StartupException;
-import org.elasticsearch.common.inject.CreationException;
-import org.elasticsearch.common.inject.spi.Message;
+import org.elasticsearch.injection.guice.CreationException;
+import org.elasticsearch.injection.guice.spi.Message;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
diff --git a/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java b/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java
index f010b233f40c2..bebc245172c1a 100644
--- a/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java
+++ b/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java
@@ -14,6 +14,7 @@
 import java.io.IOException;
 import java.util.Locale;
 import java.util.Random;
+import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
 import static org.elasticsearch.test.LambdaMatchers.falseWith;
@@ -236,4 +237,16 @@ public void testSimpleMatcher() {
         assertThat(Regex.simpleMatcher("a*c", "x*z"), falseWith("abd"));
         assertThat(Regex.simpleMatcher("a*c", "x*z"), falseWith("xyy"));
     }
+
+    public void testThousandsAndLongPattern() throws IOException {
+        String[] patterns = new String[10000];
+        for (int i = 0; i < patterns.length / 2; i++) {
+            patterns[i * 2] = randomAlphaOfLength(10);
+            patterns[i * 2 + 1] = patterns[i * 2] + ".*";
+        }
+        Predicate<String> predicate = Regex.simpleMatcher(patterns);
+        for (int i = 0; i < patterns.length / 2; i++) {
+            assertTrue(predicate.test(patterns[i]));
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/scheduler/SchedulerEngineTests.java b/server/src/test/java/org/elasticsearch/common/scheduler/SchedulerEngineTests.java
index e10898da978be..8672189220a9f 100644
--- a/server/src/test/java/org/elasticsearch/common/scheduler/SchedulerEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/common/scheduler/SchedulerEngineTests.java
@@ -166,7 +166,7 @@ public void testCancellingDuringRunPreventsRescheduling() throws Exception {
         final String jobId = randomAlphaOfLength(4);
         try {
             engine.register(event -> {
-                assertThat(event.getJobName(), is(jobId));
+                assertThat(event.jobName(), is(jobId));
                 calledCount.incrementAndGet();
                 jobRunningLatch.countDown();
                 try {
diff --git a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
index 94c0b849edf8d..8d25b41619fcd 100644
--- a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
@@ -924,6 +924,26 @@ public void testDiff() throws IOException {
         assertThat(diff.getAsInt("foo.bar", null), equalTo(1));
     }
 
+    public void testDiffWithFallbackDefaultSetting() {
+        final String fallbackSettingName = "fallback";
+        final Setting<Integer> fallbackSetting = Setting.intSetting(fallbackSettingName, 1, Property.Dynamic, Property.NodeScope);
+
+        final String settingName = "setting.with.fallback";
+        final Setting<Integer> dependentSetting = new Setting<>(
+            settingName,
+            fallbackSetting,
+            (s) -> Setting.parseInt(s, 1, settingName),
+            value -> {},
+            Property.Dynamic,
+            Property.NodeScope
+        );
+
+        ClusterSettings settings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(fallbackSetting, dependentSetting)));
+
+        final Settings diff = settings.diff(Settings.EMPTY, Settings.builder().put(fallbackSettingName, 2).build());
+        assertThat(diff.getAsInt(settingName, null), equalTo(2));
+    }
+
     public void testDiffWithDependentSettings() {
         final String dependedSettingName = "this.setting.is.depended.on";
         Setting<Integer> dependedSetting = Setting.intSetting(dependedSettingName, 1, Property.Dynamic, Property.NodeScope);
diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
index a8395145d2e40..3d94b51aeeec3 100644
--- a/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
@@ -8,8 +8,8 @@
 
 package org.elasticsearch.common.settings;
 
-import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.injection.guice.ModuleTestCase;
 import org.hamcrest.Matchers;
 
 import java.util.Arrays;
diff --git a/server/src/test/java/org/elasticsearch/common/util/CancellableSingleObjectCacheTests.java b/server/src/test/java/org/elasticsearch/common/util/CancellableSingleObjectCacheTests.java
index b038b6effd08f..c744b01fdc82c 100644
--- a/server/src/test/java/org/elasticsearch/common/util/CancellableSingleObjectCacheTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/CancellableSingleObjectCacheTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@@ -202,7 +203,7 @@ protected void refresh(
                     BooleanSupplier supersedeIfStale,
                     ActionListener<Integer> listener
                 ) {
-                    threadPool.generic().execute(() -> ActionListener.completeWith(listener, () -> {
+                    threadPool.generic().execute(ActionRunnable.supply(listener, () -> {
                         ensureNotCancelled.run();
                         if (s.equals("FAIL")) {
                             throw new ElasticsearchException("simulated");
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index 11a871e9fbbce..568fa3e36c769 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -7,12 +7,16 @@
  */
 package org.elasticsearch.common.util.concurrent;
 
+import org.apache.logging.log4j.Level;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.http.HttpTransportSettings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.MockLog;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -28,6 +32,7 @@
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiLettersOfLengthBetween;
 import static org.elasticsearch.tasks.Task.HEADERS_TO_COPY;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
@@ -633,6 +638,53 @@ public void testResponseHeaders() {
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/112256")
+    public void testDropWarningsExceedingMaxSettings() {
+        Settings settings = Settings.builder()
+            .put(HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT.getKey(), 1)
+            .put(HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey(), "50b")
+            .build();
+
+        try (var mockLog = MockLog.capture(ThreadContext.class)) {
+            mockLog.addExpectation(
+                new MockLog.SeenEventExpectation(
+                    "too many warnings",
+                    ThreadContext.class.getCanonicalName(),
+                    Level.WARN,
+                    "Dropping a warning header,* count reached the maximum allowed of [1] set in [http.max_warning_header_count]!"
+                        + ("* X-Opaque-Id header*, see " + ReferenceDocs.X_OPAQUE_ID + "*")
+                )
+            );
+            mockLog.addExpectation(
+                new MockLog.SeenEventExpectation(
+                    "warnings too long",
+                    ThreadContext.class.getCanonicalName(),
+                    Level.WARN,
+                    "Dropping a warning header for request [{X-Opaque-Id=abc, X-elastic-product-origin=product}], "
+                        + "* size reached the maximum allowed of [50] bytes set in [http.max_warning_header_size]!"
+                )
+            );
+
+            final ThreadContext threadContext = new ThreadContext(settings);
+
+            threadContext.addResponseHeader("Warning", "warning");
+            threadContext.addResponseHeader("Warning", "dropped, too many");
+
+            threadContext.putHeader(Task.X_OPAQUE_ID_HTTP_HEADER, "abc");
+            threadContext.putHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER, "product");
+            threadContext.putHeader("other", "filtered out");
+
+            threadContext.addResponseHeader("Warning", "dropped, size exceeded " + randomAlphaOfLength(50));
+
+            final Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+            final List<String> warnings = responseHeaders.get("Warning");
+
+            assertThat(warnings, contains("warning"));
+            mockLog.assertAllExpectationsMatched();
+        }
+
+    }
+
     public void testCopyHeaders() {
         Settings build = Settings.builder().put("request.headers.default", "1").build();
         ThreadContext threadContext = new ThreadContext(build);
diff --git a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
index d2ad92320cada..ee5b4972c13ad 100644
--- a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
@@ -726,10 +726,7 @@ public void testFailsIfIndexMetadataIsDuplicated() throws IOException {
                                         .version(1L)
                                         .putMapping(randomMappingMetadataOrNull())
                                         .settings(
-                                            Settings.builder()
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                                                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                            indexSettings(1, 0).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                                 .put(IndexMetadata.SETTING_INDEX_UUID, indexUUID)
                                         )
                                 )
@@ -797,10 +794,7 @@ public void testPersistsAndReloadsIndexMetadataIffVersionOrTermChanges() throws
                                         .version(indexMetadataVersion - 1) // -1 because it's incremented in .put()
                                         .putMapping(randomMappingMetadataOrNull())
                                         .settings(
-                                            Settings.builder()
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                                                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                            indexSettings(1, 0).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                                 .put(IndexMetadata.SETTING_INDEX_UUID, indexUUID)
                                         )
                                 )
@@ -931,10 +925,7 @@ public void testPersistsAndReloadsIndexMetadataForMultipleIndices() throws IOExc
                                         .putMapping(randomMappingMetadataOrNull())
                                         .version(randomLongBetween(0L, Long.MAX_VALUE - 1) - 1) // -1 because it's incremented in .put()
                                         .settings(
-                                            Settings.builder()
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                            indexSettings(1, 1).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                                 .put(IndexMetadata.SETTING_INDEX_UUID, updatedIndexUuid)
                                         )
                                 )
@@ -943,10 +934,7 @@ public void testPersistsAndReloadsIndexMetadataForMultipleIndices() throws IOExc
                                         .putMapping(randomMappingMetadataOrNull())
                                         .version(randomLongBetween(0L, Long.MAX_VALUE - 1) - 1) // -1 because it's incremented in .put()
                                         .settings(
-                                            Settings.builder()
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                            indexSettings(1, 1).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                                 .put(IndexMetadata.SETTING_INDEX_UUID, deletedIndexUuid)
                                         )
                                 )
@@ -990,10 +978,7 @@ public void testPersistsAndReloadsIndexMetadataForMultipleIndices() throws IOExc
                                         .version(randomLongBetween(0L, Long.MAX_VALUE - 1) - 1) // -1 because it's incremented in .put()
                                         .putMapping(randomMappingMetadataOrNull())
                                         .settings(
-                                            Settings.builder()
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                            indexSettings(1, 1).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                                 .put(IndexMetadata.SETTING_INDEX_UUID, addedIndexUuid)
                                         )
                                 )
@@ -1040,10 +1025,7 @@ public void testReloadsMetadataAcrossMultipleSegments() throws IOException {
                                         IndexMetadata.builder(index.getName())
                                             .putMapping(randomMappingMetadataOrNull())
                                             .settings(
-                                                Settings.builder()
-                                                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                                                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                                indexSettings(1, 0).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                                     .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID())
                                             )
                                     )
@@ -1074,10 +1056,7 @@ public void testHandlesShuffledDocuments() throws IOException {
                     IndexMetadata.builder("test-" + i)
                         .putMapping(randomMappingMetadataOrNull())
                         .settings(
-                            Settings.builder()
-                                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                            indexSettings(1, 0).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                 .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))
                         )
                 );
@@ -1254,10 +1233,7 @@ public void testSlowLogging() throws IOException, IllegalAccessException {
                                 IndexMetadata.builder("test")
                                     .putMapping(randomMappingMetadata())
                                     .settings(
-                                        Settings.builder()
-                                            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                                            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                        indexSettings(1, 0).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                             .put(IndexMetadata.SETTING_INDEX_UUID, "test-uuid")
                                     )
                             )
@@ -1369,10 +1345,7 @@ public void testLimitsFileCount() throws IOException {
                                     IndexMetadata.builder("index-" + i)
                                         .putMapping(randomMappingMetadataOrNull())
                                         .settings(
-                                            Settings.builder()
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                                                .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                            indexSettings(1, 0).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                                 .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))
                                         )
                                 )
@@ -1592,10 +1565,7 @@ public void testFailsIfMappingIsDuplicated() throws IOException {
                             IndexMetadata.builder("test-1")
                                 .putMapping(randomMappingMetadata())
                                 .settings(
-                                    Settings.builder()
-                                        .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                        .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                                        .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                    indexSettings(1, 0).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                         .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))
                                 )
                         )
@@ -1647,10 +1617,7 @@ public void testFailsIfMappingIsMissing() throws IOException {
                             IndexMetadata.builder("test-1")
                                 .putMapping(randomMappingMetadata())
                                 .settings(
-                                    Settings.builder()
-                                        .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                        .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                                        .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                    indexSettings(1, 0).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                         .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))
                                 )
                         )
@@ -1713,10 +1680,7 @@ public void testDeduplicatedMappings() throws IOException {
                         IndexMetadata.builder("test-" + i)
                             .putMapping(mapping1)
                             .settings(
-                                Settings.builder()
-                                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                indexSettings(1, 0).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                     .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))
                             )
                     );
@@ -1740,10 +1704,7 @@ public void testDeduplicatedMappings() throws IOException {
                         IndexMetadata.builder("test-" + 99)
                             .putMapping(mapping2)
                             .settings(
-                                Settings.builder()
-                                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+                                indexSettings(1, 0).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
                                     .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))
                             )
                     );
diff --git a/server/src/test/java/org/elasticsearch/http/HttpClientStatsTrackerTests.java b/server/src/test/java/org/elasticsearch/http/HttpClientStatsTrackerTests.java
index 2dfaaf34bb1f1..0ae494428939b 100644
--- a/server/src/test/java/org/elasticsearch/http/HttpClientStatsTrackerTests.java
+++ b/server/src/test/java/org/elasticsearch/http/HttpClientStatsTrackerTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.net.InetSocketAddress;
@@ -438,7 +439,11 @@ private static class FakeTimeThreadPool extends ThreadPool {
         private final long absoluteTimeOffset = randomLong();
 
         FakeTimeThreadPool() {
-            super(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), MeterRegistry.NOOP);
+            super(
+                Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(),
+                MeterRegistry.NOOP,
+                new DefaultBuiltInExecutorBuilders()
+            );
             stopCachedTimeThread();
             setRandomTime();
         }
diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
index c743a83208a24..75b63da0227ed 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
@@ -27,10 +27,10 @@
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineTestCase;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.ParsedDocument.DocumentSize;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentParseException;
 import org.elasticsearch.xcontent.XContentType;
@@ -211,7 +211,7 @@ public void testSlowLogMessageHasJsonFields() throws IOException {
             source,
             XContentType.JSON,
             null,
-            DocumentSizeObserver.EMPTY_INSTANCE
+            DocumentSize.UNKNOWN
         );
         Index index = new Index("foo", "123");
         // Turning off document logging doesn't log source[]
@@ -240,7 +240,7 @@ public void testSlowLogMessageHasAdditionalFields() throws IOException {
             source,
             XContentType.JSON,
             null,
-            DocumentSizeObserver.EMPTY_INSTANCE
+            DocumentSize.UNKNOWN
         );
         Index index = new Index("foo", "123");
         // Turning off document logging doesn't log source[]
@@ -270,7 +270,7 @@ public void testEmptyRoutingField() throws IOException {
             source,
             XContentType.JSON,
             null,
-            DocumentSizeObserver.EMPTY_INSTANCE
+            DocumentSize.UNKNOWN
         );
         Index index = new Index("foo", "123");
 
@@ -289,7 +289,7 @@ public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException {
             source,
             XContentType.JSON,
             null,
-            DocumentSizeObserver.EMPTY_INSTANCE
+            DocumentSize.UNKNOWN
         );
         Index index = new Index("foo", "123");
         // Turning off document logging doesn't log source[]
@@ -321,7 +321,7 @@ public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException {
             source,
             XContentType.JSON,
             null,
-            DocumentSizeObserver.EMPTY_INSTANCE
+            DocumentSize.UNKNOWN
         );
 
         final XContentParseException e = expectThrows(
diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecIntegrationTests.java
new file mode 100644
index 0000000000000..05b9cf42e6236
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/codec/CodecIntegrationTests.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CodecIntegrationTests extends ESSingleNodeTestCase {
+
+    public void testCanConfigureLegacySettings() {
+        assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled());
+
+        createIndex("index1", Settings.builder().put("index.codec", "legacy_default").build());
+        var codec = client().admin().indices().prepareGetSettings("index1").execute().actionGet().getSetting("index1", "index.codec");
+        assertThat(codec, equalTo("legacy_default"));
+
+        createIndex("index2", Settings.builder().put("index.codec", "legacy_best_compression").build());
+        codec = client().admin().indices().prepareGetSettings("index2").execute().actionGet().getSetting("index2", "index.codec");
+        assertThat(codec, equalTo("legacy_best_compression"));
+    }
+
+    public void testDefaultCodecLogsdb() {
+        assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled());
+
+        var indexService = createIndex("index1", Settings.builder().put("index.mode", "logsdb").build());
+        var storedFieldsFormat = (Zstd814StoredFieldsFormat) indexService.getShard(0)
+            .getEngineOrNull()
+            .config()
+            .getCodec()
+            .storedFieldsFormat();
+        assertThat(storedFieldsFormat.getMode(), equalTo(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION));
+    }
+
+    public void testDefaultCodec() {
+        assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled());
+
+        var indexService = createIndex("index1");
+        var storedFieldsFormat = (Zstd814StoredFieldsFormat) indexService.getShard(0)
+            .getEngineOrNull()
+            .config()
+            .getCodec()
+            .storedFieldsFormat();
+        assertThat(storedFieldsFormat.getMode(), equalTo(Zstd814StoredFieldsFormat.Mode.BEST_SPEED));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
index 2c2af49e7d062..002e42b3198e6 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
@@ -38,7 +38,9 @@
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -96,6 +98,29 @@ public void testLegacyBestCompression() throws Exception {
         }
     }
 
+    public void testCodecRetrievalForUnknownCodec() throws Exception {
+        CodecService codecService = createCodecService();
+        IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> codecService.codec("unknown_codec"));
+        assertEquals("failed to find codec [unknown_codec]", exception.getMessage());
+    }
+
+    public void testAvailableCodecsContainsExpectedCodecs() throws Exception {
+        CodecService codecService = createCodecService();
+        String[] availableCodecs = codecService.availableCodecs();
+        List<String> codecList = Arrays.asList(availableCodecs);
+        int expectedCodecCount = Codec.availableCodecs().size() + 5;
+
+        assertTrue(codecList.contains(CodecService.DEFAULT_CODEC));
+        assertTrue(codecList.contains(CodecService.LEGACY_DEFAULT_CODEC));
+        assertTrue(codecList.contains(CodecService.BEST_COMPRESSION_CODEC));
+        assertTrue(codecList.contains(CodecService.LEGACY_BEST_COMPRESSION_CODEC));
+        assertTrue(codecList.contains(CodecService.LUCENE_DEFAULT_CODEC));
+
+        assertFalse(codecList.contains("unknown_codec"));
+
+        assertEquals(expectedCodecCount, availableCodecs.length);
+    }
+
     private CodecService createCodecService() throws IOException {
         Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
         IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesCodecDuelTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesCodecDuelTests.java
index e8949dda78f7f..20ae59e113c33 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesCodecDuelTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesCodecDuelTests.java
@@ -141,6 +141,9 @@ private void assertSortedDocValues(LeafReader baselineReader, LeafReader contend
             for (int i = 0; i < docIdsToAdvanceTo.length; i++) {
                 int docId = docIdsToAdvanceTo[i];
                 int baselineTarget = assertAdvance(docId, baselineReader, contenderReader, baseline, contender);
+                if (baselineTarget == NO_MORE_DOCS) {
+                    break;
+                }
                 assertEquals(baseline.ordValue(), contender.ordValue());
                 assertEquals(baseline.lookupOrd(baseline.ordValue()), contender.lookupOrd(contender.ordValue()));
                 i = shouldSkipDocIds(i, docId, baselineTarget, docIdsToAdvanceTo);
@@ -232,6 +235,9 @@ private void assertSortedSetDocValues(LeafReader baselineReader, LeafReader cont
             for (int i = 0; i < docIdsToAdvanceTo.length; i++) {
                 int docId = docIdsToAdvanceTo[i];
                 int baselineTarget = assertAdvance(docId, baselineReader, contenderReader, baseline, contender);
+                if (baselineTarget == NO_MORE_DOCS) {
+                    break;
+                }
                 assertEquals(baseline.docValueCount(), contender.docValueCount());
                 for (int j = 0; j < baseline.docValueCount(); j++) {
                     long baselineOrd = baseline.nextOrd();
@@ -255,12 +261,14 @@ private void assertSortedSetDocValues(LeafReader baselineReader, LeafReader cont
                 boolean contenderFound = contender.advanceExact(docId);
                 assertEquals(baselineFound, contenderFound);
                 assertEquals(baseline.docID(), contender.docID());
-                assertEquals(baseline.docValueCount(), contender.docValueCount());
-                for (int i = 0; i < baseline.docValueCount(); i++) {
-                    long baselineOrd = baseline.nextOrd();
-                    long contenderOrd = contender.nextOrd();
-                    assertEquals(baselineOrd, contenderOrd);
-                    assertEquals(baseline.lookupOrd(baselineOrd), contender.lookupOrd(contenderOrd));
+                if (baselineFound) {
+                    assertEquals(baseline.docValueCount(), contender.docValueCount());
+                    for (int i = 0; i < baseline.docValueCount(); i++) {
+                        long baselineOrd = baseline.nextOrd();
+                        long contenderOrd = contender.nextOrd();
+                        assertEquals(baselineOrd, contenderOrd);
+                        assertEquals(baseline.lookupOrd(baselineOrd), contender.lookupOrd(contenderOrd));
+                    }
                 }
             }
         }
@@ -328,6 +336,9 @@ private void assertSortedNumericDocValues(LeafReader baselineReader, LeafReader
             for (int i = 0; i < docIdsToAdvanceTo.length; i++) {
                 int docId = docIdsToAdvanceTo[i];
                 int baselineTarget = assertAdvance(docId, baselineReader, contenderReader, baseline, contender);
+                if (baselineTarget == NO_MORE_DOCS) {
+                    break;
+                }
                 assertEquals(baseline.docValueCount(), contender.docValueCount());
                 for (int j = 0; j < baseline.docValueCount(); j++) {
                     long baselineValue = baseline.nextValue();
@@ -349,11 +360,13 @@ private void assertSortedNumericDocValues(LeafReader baselineReader, LeafReader
                 boolean contenderResult = contender.advanceExact(docId);
                 assertEquals(baselineResult, contenderResult);
                 assertEquals(baseline.docID(), contender.docID());
-                assertEquals(baseline.docValueCount(), contender.docValueCount());
-                for (int i = 0; i < baseline.docValueCount(); i++) {
-                    long baselineValue = baseline.nextValue();
-                    long contenderValue = contender.nextValue();
-                    assertEquals(baselineValue, contenderValue);
+                if (baselineResult) {
+                    assertEquals(baseline.docValueCount(), contender.docValueCount());
+                    for (int i = 0; i < baseline.docValueCount(); i++) {
+                        long baselineValue = baseline.nextValue();
+                        long contenderValue = contender.nextValue();
+                        assertEquals(baselineValue, contenderValue);
+                    }
                 }
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java
new file mode 100644
index 0000000000000..93e9911746d18
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.zstd;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.tests.index.ForceMergePolicy;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.codec.LegacyPerFieldMapperCodec;
+import org.elasticsearch.index.codec.PerFieldMapperCodec;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class StoredFieldCodecDuelTests extends ESTestCase {
+
+    private static final String STRING_FIELD = "string_field_1";
+    private static final String BINARY_FIELD = "binary_field_2";
+    private static final String INT_FIELD = "int_field_3";
+    private static final String LONG_FIELD = "long_field_4";
+    private static final String FLOAT_FIELD = "float_field_5";
+    private static final String DOUBLE_FIELD = "double_field_5";
+
+    public void testDuelBestSpeed() throws IOException {
+        var baseline = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, null, BigArrays.NON_RECYCLING_INSTANCE);
+        var contender = new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, null, BigArrays.NON_RECYCLING_INSTANCE);
+        doTestDuel(baseline, contender);
+    }
+
+    public void testDuelBestCompression() throws IOException {
+        var baseline = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_COMPRESSION, null, BigArrays.NON_RECYCLING_INSTANCE);
+        var contender = new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, null, BigArrays.NON_RECYCLING_INSTANCE);
+        doTestDuel(baseline, contender);
+    }
+
+    static void doTestDuel(Codec baslineCodec, Codec contenderCodec) throws IOException {
+        try (var baselineDirectory = newDirectory(); var contenderDirectory = newDirectory()) {
+            int numDocs = randomIntBetween(256, 8096);
+
+            var mergePolicy = new ForceMergePolicy(newLogMergePolicy());
+            var baselineConfig = newIndexWriterConfig();
+            baselineConfig.setMergePolicy(mergePolicy);
+            baselineConfig.setCodec(baslineCodec);
+            var contenderConf = newIndexWriterConfig();
+            contenderConf.setCodec(contenderCodec);
+            contenderConf.setMergePolicy(mergePolicy);
+
+            try (
+                var baselineIw = new RandomIndexWriter(random(), baselineDirectory, baselineConfig);
+                var contenderIw = new RandomIndexWriter(random(), contenderDirectory, contenderConf)
+            ) {
+                for (int i = 0; i < numDocs; i++) {
+                    Document doc = new Document();
+                    doc.add(new StoredField(STRING_FIELD, randomAlphaOfLength(randomIntBetween(1, 4096))));
+                    doc.add(new StoredField(BINARY_FIELD, randomByteArrayOfLength(randomIntBetween(1, 4096))));
+                    doc.add(new StoredField(INT_FIELD, randomInt()));
+                    doc.add(new StoredField(LONG_FIELD, randomLong()));
+                    doc.add(new StoredField(FLOAT_FIELD, randomFloat()));
+                    doc.add(new StoredField(DOUBLE_FIELD, randomDouble()));
+                    baselineIw.addDocument(doc);
+                    contenderIw.addDocument(doc);
+                }
+                baselineIw.forceMerge(1);
+                contenderIw.forceMerge(1);
+            }
+            try (var baselineIr = DirectoryReader.open(baselineDirectory); var contenderIr = DirectoryReader.open(contenderDirectory)) {
+                assertEquals(1, baselineIr.leaves().size());
+                assertEquals(1, contenderIr.leaves().size());
+
+                var baseLeafReader = baselineIr.leaves().get(0).reader();
+                var contenderLeafReader = contenderIr.leaves().get(0).reader();
+                assertEquals(baseLeafReader.maxDoc(), contenderLeafReader.maxDoc());
+
+                for (int docId = 0; docId < contenderLeafReader.maxDoc(); docId++) {
+                    Document baselineDoc = baseLeafReader.storedFields().document(docId);
+                    Document contenderDoc = contenderLeafReader.storedFields().document(docId);
+                    assertThat(contenderDoc.getFields().size(), equalTo(baselineDoc.getFields().size()));
+                    for (int i = 0; i < baselineDoc.getFields().size(); i++) {
+                        var baselineField = baselineDoc.getFields().get(i);
+                        var contenderField = contenderDoc.getFields().get(i);
+                        assertThat(contenderField.name(), equalTo(baselineField.name()));
+                        switch (baselineField.name()) {
+                            case STRING_FIELD -> assertThat(contenderField.stringValue(), equalTo(baselineField.stringValue()));
+                            case BINARY_FIELD -> assertThat(contenderField.binaryValue(), equalTo(baselineField.binaryValue()));
+                            case INT_FIELD, LONG_FIELD, FLOAT_FIELD, DOUBLE_FIELD -> assertThat(
+                                contenderField.numericValue(),
+                                equalTo(baselineField.numericValue())
+                            );
+                            default -> fail("unexpected field [" + baselineField.name() + "]");
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index cc636fb1bc995..77f5fa301c78d 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -108,6 +108,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.ParsedDocument.DocumentSize;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.Uid;
@@ -129,7 +130,6 @@
 import org.elasticsearch.index.translog.TranslogDeletionPolicy;
 import org.elasticsearch.index.translog.TranslogOperationsUtils;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -5510,7 +5510,7 @@ public void testSeqNoGenerator() throws IOException {
                 source,
                 XContentType.JSON,
                 null,
-                DocumentSizeObserver.EMPTY_INSTANCE
+                DocumentSize.UNKNOWN
             );
 
             final Engine.Index index = new Engine.Index(
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
index 1289095ae8d2c..28e65a6803ae3 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
@@ -358,7 +358,9 @@ public void testRequireDocValuesOnDoubles() {
 
     public void testRequireDocValuesOnBools() {
         doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field"));
-        doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap()));
+        doTestRequireDocValues(
+            new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap(), false)
+        );
     }
 
     public void testFieldDataCacheExpire() {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
index 70e2fee7a003a..e08a443bd74cb 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
@@ -13,8 +13,12 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.script.BooleanFieldScript;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -24,6 +28,7 @@
 import java.util.List;
 import java.util.function.Function;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class BooleanFieldMapperTests extends MapperTestCase {
@@ -44,6 +49,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
         checker.registerConflictCheck("index", b -> b.field("index", false));
         checker.registerConflictCheck("store", b -> b.field("store", true));
         checker.registerConflictCheck("null_value", b -> b.field("null_value", true));
+
+        registerDimensionChecks(checker);
     }
 
     public void testExistsQueryDocValuesDisabled() throws IOException {
@@ -207,6 +214,74 @@ public void testScriptAndPrecludedParameters() {
         assertThat(e.getMessage(), equalTo("Failed to parse mapping: Field [null_value] cannot be set in conjunction with field [script]"));
     }
 
+    public void testDimension() throws IOException {
+        // Test default setting
+        MapperService mapperService = createMapperService(fieldMapping(b -> minimalMapping(b)));
+        BooleanFieldMapper.BooleanFieldType ft = (BooleanFieldMapper.BooleanFieldType) mapperService.fieldType("field");
+        assertFalse(ft.isDimension());
+
+        assertDimension(true, BooleanFieldMapper.BooleanFieldType::isDimension);
+        assertDimension(false, BooleanFieldMapper.BooleanFieldType::isDimension);
+    }
+
+    public void testDimensionIndexedAndDocvalues() {
+        {
+            Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
+                minimalMapping(b);
+                b.field("time_series_dimension", true).field("index", false).field("doc_values", false);
+            })));
+            assertThat(
+                e.getCause().getMessage(),
+                containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true")
+            );
+        }
+        {
+            Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
+                minimalMapping(b);
+                b.field("time_series_dimension", true).field("index", true).field("doc_values", false);
+            })));
+            assertThat(
+                e.getCause().getMessage(),
+                containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true")
+            );
+        }
+        {
+            Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
+                minimalMapping(b);
+                b.field("time_series_dimension", true).field("index", false).field("doc_values", true);
+            })));
+            assertThat(
+                e.getCause().getMessage(),
+                containsString("Field [time_series_dimension] requires that [index] and [doc_values] are true")
+            );
+        }
+    }
+
+    public void testDimensionMultiValuedField() throws IOException {
+        XContentBuilder mapping = fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("time_series_dimension", true);
+        });
+        DocumentMapper mapper = randomBoolean() ? createDocumentMapper(mapping) : createTimeSeriesModeDocumentMapper(mapping);
+
+        Exception e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> b.array("field", true, false))));
+        assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field"));
+    }
+
+    public void testDimensionInRoutingPath() throws IOException {
+        MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "keyword").field("time_series_dimension", true)));
+        IndexSettings settings = createIndexSettings(
+            IndexVersion.current(),
+            Settings.builder()
+                .put(IndexSettings.MODE.getKey(), "time_series")
+                .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field")
+                .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
+                .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
+                .build()
+        );
+        mapper.documentMapper().validate(settings, false);  // Doesn't throw
+    }
+
     @Override
     protected List<ExampleMalformedValue> exampleMalformedValues() {
         return List.of(exampleMalformedValue("a").errorMatches("Failed to parse value [a] as only [true] or [false] are allowed."));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java
index 55f076e985c4d..0466525288d7b 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java
@@ -79,7 +79,8 @@ public void testFetchSourceValue() throws IOException {
             true,
             true,
             null,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            false
         );
         assertEquals(List.of(true), fetchSourceValue(nullFieldType, null));
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java
index 0cdc9568f1fac..89d5e300112b4 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java
@@ -142,6 +142,11 @@ public boolean needs_score() {
                                 return false;
                             }
 
+                            @Override
+                            public boolean needs_termStats() {
+                                return false;
+                            }
+
                             @Override
                             public ScoreScript newInstance(DocReader docReader) {
                                 return new ScoreScript(Map.of(), searchContext.lookup(), docReader) {
@@ -165,6 +170,11 @@ public boolean needs_score() {
                                 return false;
                             }
 
+                            @Override
+                            public boolean needs_termStats() {
+                                return false;
+                            }
+
                             @Override
                             public ScoreScript newInstance(DocReader docReader) {
                                 return new ScoreScript(Map.of(), searchContext.lookup(), docReader) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
index 2f8af6c5521f9..d90517e4be274 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
@@ -152,6 +152,9 @@ public void testPostingsFormat() throws IOException {
             assertThat(codec, instanceOf(PerFieldMapperCodec.class));
             assertThat(((PerFieldMapperCodec) codec).getPostingsFormatForField("field"), instanceOf(Completion99PostingsFormat.class));
         } else {
+            if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
+                codec = deduplicateFieldInfosCodec.delegate();
+            }
             assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
             assertThat(
                 ((LegacyPerFieldMapperCodec) codec).getPostingsFormatForField("field"),
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoaderTests.java
new file mode 100644
index 0000000000000..8cbd1e52d3f44
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoaderTests.java
@@ -0,0 +1,226 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.LeafReader;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class CompositeSyntheticFieldLoaderTests extends ESTestCase {
+    public void testComposingMultipleStoredFields() throws IOException {
+        var sut = new CompositeSyntheticFieldLoader(
+            "foo",
+            "bar.baz.foo",
+            List.of(new CompositeSyntheticFieldLoader.StoredFieldLayer("foo.one") {
+                @Override
+                protected void writeValue(Object value, XContentBuilder b) throws IOException {
+                    b.value((long) value);
+                }
+            }, new CompositeSyntheticFieldLoader.StoredFieldLayer("foo.two") {
+                @Override
+                protected void writeValue(Object value, XContentBuilder b) throws IOException {
+                    b.value((long) value);
+                }
+            })
+        );
+
+        var storedFieldLoaders = sut.storedFieldLoaders().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        storedFieldLoaders.get("foo.one").advanceToDoc(0);
+        storedFieldLoaders.get("foo.one").load(List.of(45L, 46L));
+        storedFieldLoaders.get("foo.two").advanceToDoc(0);
+        storedFieldLoaders.get("foo.two").load(List.of(1L));
+
+        var result = XContentBuilder.builder(XContentType.JSON.xContent());
+        result.startObject();
+        sut.write(result);
+        result.endObject();
+
+        assertEquals("""
+            {"foo":[45,46,1]}""", Strings.toString(result));
+    }
+
+    public void testLoadStoredFieldAndAdvance() throws IOException {
+        var sut = new CompositeSyntheticFieldLoader(
+            "foo",
+            "bar.baz.foo",
+            List.of(new CompositeSyntheticFieldLoader.StoredFieldLayer("foo.one") {
+                @Override
+                protected void writeValue(Object value, XContentBuilder b) throws IOException {
+                    b.value((long) value);
+                }
+            })
+        );
+
+        var storedFieldLoaders = sut.storedFieldLoaders().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        storedFieldLoaders.get("foo.one").advanceToDoc(0);
+        storedFieldLoaders.get("foo.one").load(List.of(45L));
+
+        var result = XContentBuilder.builder(XContentType.JSON.xContent());
+        result.startObject();
+        sut.write(result);
+        result.endObject();
+
+        assertEquals("""
+            {"foo":45}""", Strings.toString(result));
+
+        storedFieldLoaders.get("foo.one").advanceToDoc(1);
+
+        var empty = XContentBuilder.builder(XContentType.JSON.xContent());
+        empty.startObject();
+        sut.write(result);
+        empty.endObject();
+
+        assertEquals("{}", Strings.toString(empty));
+    }
+
+    public void testComposingMultipleDocValuesFields() throws IOException {
+        var sut = new CompositeSyntheticFieldLoader("foo", "bar.baz.foo", List.of(new CompositeSyntheticFieldLoader.Layer() {
+            @Override
+            public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
+                return Stream.empty();
+            }
+
+            @Override
+            public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
+                return (docId -> true);
+            }
+
+            @Override
+            public boolean hasValue() {
+                return true;
+            }
+
+            @Override
+            public void write(XContentBuilder b) throws IOException {
+                b.value(45L);
+                b.value(46L);
+            }
+
+            @Override
+            public String fieldName() {
+                return "";
+            }
+
+            @Override
+            public long valueCount() {
+                return 2;
+            }
+        }, new CompositeSyntheticFieldLoader.Layer() {
+            @Override
+            public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
+                return Stream.empty();
+            }
+
+            @Override
+            public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
+                return (docId -> true);
+            }
+
+            @Override
+            public boolean hasValue() {
+                return true;
+            }
+
+            @Override
+            public void write(XContentBuilder b) throws IOException {
+                b.value(1L);
+            }
+
+            @Override
+            public String fieldName() {
+                return "";
+            }
+
+            @Override
+            public long valueCount() {
+                return 1;
+            }
+        }));
+
+        sut.docValuesLoader(null, new int[0]).advanceToDoc(0);
+
+        var result = XContentBuilder.builder(XContentType.JSON.xContent());
+        result.startObject();
+        sut.write(result);
+        result.endObject();
+
+        assertEquals("""
+            {"foo":[45,46,1]}""", Strings.toString(result));
+    }
+
+    public void testComposingStoredFieldsWithDocValues() throws IOException {
+        var sut = new CompositeSyntheticFieldLoader(
+            "foo",
+            "bar.baz.foo",
+            List.of(new CompositeSyntheticFieldLoader.StoredFieldLayer("foo.one") {
+                @Override
+                protected void writeValue(Object value, XContentBuilder b) throws IOException {
+                    b.value((long) value);
+                }
+            }, new CompositeSyntheticFieldLoader.Layer() {
+                @Override
+                public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
+                    return Stream.empty();
+                }
+
+                @Override
+                public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
+                    return (docId -> true);
+                }
+
+                @Override
+                public boolean hasValue() {
+                    return true;
+                }
+
+                @Override
+                public void write(XContentBuilder b) throws IOException {
+                    b.value(1L);
+                }
+
+                @Override
+                public String fieldName() {
+                    return "";
+                }
+
+                @Override
+                public long valueCount() {
+                    return 1;
+                }
+            })
+        );
+
+        var storedFieldLoaders = sut.storedFieldLoaders().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        storedFieldLoaders.get("foo.one").advanceToDoc(0);
+        storedFieldLoaders.get("foo.one").load(List.of(45L, 46L));
+
+        sut.docValuesLoader(null, new int[0]).advanceToDoc(0);
+
+        var result = XContentBuilder.builder(XContentType.JSON.xContent());
+        result.startObject();
+        sut.write(result);
+        result.endObject();
+
+        assertEquals("""
+            {"foo":[45,46,1]}""", Strings.toString(result));
+    }
+
+    public void testFieldName() {
+        var sut = new CompositeSyntheticFieldLoader("foo", "bar.baz.foo");
+        assertEquals("bar.baz.foo", sut.fieldName());
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index d9894df9104a1..9cfdb2c46a291 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -731,7 +731,18 @@ public void execute() {
 
     @Override
     protected Function<Object, Object> loadBlockExpected() {
-        return v -> ((Number) v).longValue();
+        return v -> asJacksonNumberOutput(((Number) v).longValue());
+    }
+
+    protected static Object asJacksonNumberOutput(long l) {
+        // If a long value fits in int, Jackson will write it as int in NumberOutput.outputLong()
+        // and we hit this during serialization of expected values.
+        // Code below mimics that behaviour in order for matching to work.
+        if (l < 0 && l >= Integer.MIN_VALUE || l >= 0 && l <= Integer.MAX_VALUE) {
+            return (int) l;
+        } else {
+            return l;
+        }
     }
 
     public void testLegacyField() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java
index 3728943cae418..0507a37636370 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java
@@ -233,6 +233,11 @@ public boolean needs_score() {
                         return false;
                     }
 
+                    @Override
+                    public boolean needs_termStats() {
+                        return false;
+                    }
+
                     @Override
                     public ScoreScript newInstance(DocReader docReader) throws IOException {
                         return new ScoreScript(Map.of(), searchContext.lookup(), docReader) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 7fa08acd53882..07a1f732d818f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -2642,7 +2642,7 @@ same name need to be part  of the same mappings (hence the same document). If th
     }
 
     public void testDeeplyNestedDocument() throws Exception {
-        int depth = 10000;
+        int depth = 20;
 
         DocumentMapper docMapper = createMapperService(Settings.builder().put(getIndexSettings()).build(), mapping(b -> {}))
             .documentMapper();
@@ -3245,7 +3245,7 @@ protected String contentType() {
 
             @Override
             public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
-                return new StringStoredFieldFieldLoader(fullPath(), leafName(), null) {
+                return new StringStoredFieldFieldLoader(fullPath(), leafName()) {
                     @Override
                     protected void write(XContentBuilder b, Object value) throws IOException {
                         BytesRef ref = (BytesRef) value;
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java
index 9547b4f9cb9a3..6e12778f87cf9 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java
@@ -141,6 +141,11 @@ public boolean needs_score() {
                         return false;
                     }
 
+                    @Override
+                    public boolean needs_termStats() {
+                        return false;
+                    }
+
                     @Override
                     public ScoreScript newInstance(DocReader docReader) {
                         return new ScoreScript(Map.of(), searchContext.lookup(), docReader) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java
index a138f0910e6ec..878bdc91bba06 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java
@@ -8,7 +8,6 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
@@ -19,6 +18,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
@@ -69,7 +69,7 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti
         SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON);
 
         SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false).setSynthetic().build();
-        RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
+        RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
             new PassThroughObjectMapper.Builder("labels").setPriority(0).setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE)
         ).build(MapperBuilderContext.root(false, false));
         Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of());
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
index cd8a59a1a6ad5..a5a5d9726f233 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
+import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator;
 import org.elasticsearch.test.XContentTestUtils;
 import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -739,7 +739,7 @@ public void testTemplateWithoutMatchPredicates() throws Exception {
                     XContentType.JSON,
                     null,
                     Map.of("foo", "geo_point"),
-                    DocumentSizeObserver.EMPTY_INSTANCE
+                    XContentMeteringParserDecorator.NOOP
                 )
             );
         assertThat(doc.rootDoc().getFields("foo"), hasSize(2));
@@ -1132,6 +1132,14 @@ public void testDynamicRuntimeWithDynamicTemplate() throws IOException {
     }
 
     private MapperService createDynamicTemplateNoSubobjects() throws IOException {
+        return createDynamicTemplateWithSubobjects("false");
+    }
+
+    private MapperService createDynamicTemplateAutoSubobjects() throws IOException {
+        return createDynamicTemplateWithSubobjects("auto");
+    }
+
+    private MapperService createDynamicTemplateWithSubobjects(String subobjects) throws IOException {
         return createMapperService(topMapping(b -> {
             b.startArray("dynamic_templates");
             {
@@ -1141,7 +1149,7 @@ private MapperService createDynamicTemplateNoSubobjects() throws IOException {
                     {
                         b.field("match_mapping_type", "object");
                         b.field("match", "metric");
-                        b.startObject("mapping").field("type", "object").field("subobjects", false).endObject();
+                        b.startObject("mapping").field("type", "object").field("subobjects", subobjects).endObject();
                     }
                     b.endObject();
                 }
@@ -1388,7 +1396,7 @@ public void testDynamicSubobjectsFalseDynamicFalse() throws Exception {
         assertEquals(ObjectMapper.Dynamic.FALSE, metrics.dynamic());
         assertEquals(1, metrics.mappers.size());
         ObjectMapper service = (ObjectMapper) metrics.getMapper("service");
-        assertFalse(service.subobjects());
+        assertEquals(ObjectMapper.Subobjects.DISABLED, service.subobjects());
         assertEquals(1, service.mappers.size());
         assertNotNull(service.getMapper("time"));
     }
@@ -1434,6 +1442,255 @@ public void testSubobjectsFalseWithInnerNestedFromDynamicTemplate() {
         );
     }
 
+    public void testSubobjectsAutoFlatPaths() throws IOException {
+        MapperService mapperService = createDynamicTemplateAutoSubobjects();
+        ParsedDocument doc = mapperService.documentMapper().parse(source(b -> {
+            b.field("foo.metric.count", 10);
+            b.field("foo.bar.baz", 10);
+            b.field("foo.metric.count.min", 4);
+            b.field("foo.metric.count.max", 15);
+        }));
+        merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate()));
+        assertNoSubobjects(mapperService);
+    }
+
+    public void testSubobjectsAutoStructuredPaths() throws IOException {
+        MapperService mapperService = createDynamicTemplateAutoSubobjects();
+        ParsedDocument doc = mapperService.documentMapper().parse(source(b -> {
+            b.startObject("foo");
+            {
+                b.startObject("metric");
+                {
+                    b.field("count", 10);
+                    b.field("count.min", 4);
+                    b.field("count.max", 15);
+                }
+                b.endObject();
+                b.startObject("bar");
+                b.field("baz", 10);
+                b.endObject();
+            }
+            b.endObject();
+        }));
+        merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate()));
+        assertNoSubobjects(mapperService);
+    }
+
+    public void testSubobjectsAutoArrayOfObjects() throws IOException {
+        MapperService mapperService = createDynamicTemplateAutoSubobjects();
+        ParsedDocument doc = mapperService.documentMapper().parse(source(b -> {
+            b.startObject("foo");
+            {
+                b.startArray("metric");
+                {
+                    b.startObject();
+                    {
+                        b.field("count", 10);
+                        b.field("count.min", 4);
+                        b.field("count.max", 15);
+                    }
+                    b.endObject();
+                    b.startObject();
+                    {
+                        b.field("count", 5);
+                        b.field("count.min", 3);
+                        b.field("count.max", 50);
+                    }
+                    b.endObject();
+                }
+                b.endArray();
+                b.startObject("bar");
+                b.field("baz", 10);
+                b.endObject();
+            }
+            b.endObject();
+        }));
+        merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate()));
+        assertNoSubobjects(mapperService);
+    }
+
+    public void testSubobjectAutoDynamicNested() throws IOException {
+        DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                b.startObject("nested");
+                {
+                    b.field("match", "object");
+                    b.startObject("mapping");
+                    {
+                        b.field("type", "nested");
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+                b.endObject();
+            }
+            b.endArray();
+            b.startObject("properties");
+            b.startObject("metrics").field("type", "object").field("subobjects", "auto").endObject();
+            b.endObject();
+        }));
+
+        ParsedDocument doc = mapper.parse(source("""
+            {
+              "metrics.object" : {
+                "foo" : "bar"
+              }
+            }
+            """));
+
+        assertNotNull(doc.docs().get(0).get("metrics.object.foo"));
+        assertThat(
+            ((ObjectMapper) doc.dynamicMappingsUpdate().getRoot().getMapper("metrics")).getMapper("object"),
+            instanceOf(NestedObjectMapper.class)
+        );
+    }
+
+    public void testRootSubobjectAutoDynamicNested() throws IOException {
+        DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                b.startObject("nested");
+                {
+                    b.field("match", "object");
+                    b.startObject("mapping");
+                    {
+                        b.field("type", "nested");
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+                b.endObject();
+            }
+            b.endArray();
+            b.field("subobjects", "auto");
+        }));
+
+        ParsedDocument doc = mapper.parse(source("""
+            {
+              "object" : {
+                "foo" : "bar"
+              }
+            }
+            """));
+
+        assertNotNull(doc.docs().get(0).get("object.foo"));
+        assertThat(doc.dynamicMappingsUpdate().getRoot().getMapper("object"), instanceOf(NestedObjectMapper.class));
+    }
+
+    public void testDynamicSubobjectsAutoDynamicFalse() throws Exception {
+        // verify that we read the dynamic value properly from the parent mapper. DocumentParser#dynamicOrDefault splits the field
+        // name where dots are found, but it does that only for the parent prefix e.g. metrics.service and not for the leaf suffix time.max
+        DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                b.startObject("metrics");
+                {
+                    b.field("match", "metrics");
+                    b.startObject("mapping");
+                    {
+                        b.field("type", "object");
+                        b.field("dynamic", "false");
+                        b.startObject("properties");
+                        {
+                            b.startObject("service");
+                            {
+                                b.field("type", "object");
+                                b.field("subobjects", "auto");
+                                b.startObject("properties");
+                                {
+                                    b.startObject("time");
+                                    b.field("type", "keyword");
+                                    b.endObject();
+                                }
+                                b.endObject();
+                            }
+                            b.endObject();
+                        }
+                        b.endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+                b.endObject();
+            }
+            b.endArray();
+        }));
+
+        ParsedDocument doc = mapper.parse(source("""
+            {
+              "metrics": {
+                "service": {
+                  "time" : 10,
+                  "time.max" : 500
+                }
+              }
+            }
+            """));
+
+        assertNotNull(doc.rootDoc().getField("metrics.service.time"));
+        assertNull(doc.rootDoc().getField("metrics.service.time.max"));
+        assertNotNull(doc.dynamicMappingsUpdate());
+        ObjectMapper metrics = (ObjectMapper) doc.dynamicMappingsUpdate().getRoot().getMapper("metrics");
+        assertEquals(ObjectMapper.Dynamic.FALSE, metrics.dynamic());
+        assertEquals(1, metrics.mappers.size());
+        ObjectMapper service = (ObjectMapper) metrics.getMapper("service");
+        assertEquals(ObjectMapper.Subobjects.AUTO, service.subobjects());
+        assertEquals(1, service.mappers.size());
+        assertNotNull(service.getMapper("time"));
+    }
+
+    public void testSubobjectsAutoWithInnerNestedFromDynamicTemplate() throws IOException {
+        DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
+            b.startArray("dynamic_templates");
+            {
+                b.startObject();
+                {
+                    b.startObject("test");
+                    {
+                        b.field("match", "metrics");
+                        b.startObject("mapping");
+                        {
+                            b.field("type", "object").field("subobjects", "auto");
+                            b.startObject("properties");
+                            {
+                                b.startObject("time");
+                                b.field("type", "nested");
+                                b.endObject();
+                            }
+                            b.endObject();
+                        }
+                        b.endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endArray();
+        }));
+
+        ParsedDocument doc = mapper.parse(source("""
+            {
+              "metrics": {
+                "time" : {
+                  "foo" : "bar"
+                },
+                "time.max" : 500
+              }
+            }
+            """));
+
+        assertNotNull(doc.rootDoc().get("metrics.time.max"));
+        assertNotNull(doc.docs().get(0).get("metrics.time.foo"));
+        assertThat(
+            ((ObjectMapper) doc.dynamicMappingsUpdate().getRoot().getMapper("metrics")).getMapper("time"),
+            instanceOf(NestedObjectMapper.class)
+        );
+    }
+
     public void testDynamicSubobject() throws IOException {
         MapperService mapperService = createMapperService(topMapping(b -> {
             b.startArray("dynamic_templates");
@@ -1803,7 +2060,7 @@ public void testSubobjectsFalseDocWithEmptyObject() throws IOException {
         Mapping mapping = doc.dynamicMappingsUpdate();
         ObjectMapper artifacts = (ObjectMapper) mapping.getRoot().getMapper("artifacts");
         ObjectMapper leaf = (ObjectMapper) artifacts.getMapper("leaf");
-        assertFalse(leaf.subobjects());
+        assertEquals(ObjectMapper.Subobjects.DISABLED, leaf.subobjects());
     }
 
     public void testSubobjectsFalseFlattened() throws IOException {
@@ -1853,6 +2110,53 @@ public void testSubobjectsFalseFlattened() throws IOException {
         assertEquals("flattened", fooStructuredMapper.typeName());
     }
 
+    public void testSubobjectsAutoFlattened() throws IOException {
+        String mapping = """
+            {
+              "_doc": {
+                "properties": {
+                  "attributes": {
+                    "type": "object",
+                    "subobjects": "auto"
+                  }
+                },
+                "dynamic_templates": [
+                  {
+                    "test": {
+                      "path_match": "attributes.resource.*",
+                      "match_mapping_type": "object",
+                      "mapping": {
+                        "type": "flattened"
+                      }
+                    }
+                  }
+                ]
+              }
+            }
+            """;
+        String docJson = """
+            {
+              "attributes.resource": {
+                "complex.attribute": {
+                  "a": "b"
+                },
+                "foo.bar": "baz"
+              }
+            }
+            """;
+
+        MapperService mapperService = createMapperService(mapping);
+        ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(docJson));
+        merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate()));
+
+        Mapper fooBarMapper = mapperService.documentMapper().mappers().getMapper("attributes.resource.foo.bar");
+        assertNotNull(fooBarMapper);
+        assertEquals("text", fooBarMapper.typeName());
+        Mapper fooStructuredMapper = mapperService.documentMapper().mappers().getMapper("attributes.resource.complex.attribute");
+        assertNotNull(fooStructuredMapper);
+        assertEquals("flattened", fooStructuredMapper.typeName());
+    }
+
     public void testMatchWithArrayOfFieldNames() throws IOException {
         String mapping = """
             {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java
index d913b86aed2d5..a8669a0befd0d 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java
@@ -16,6 +16,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyList;
@@ -176,7 +177,7 @@ private static ObjectMapper createObjectMapper(String name) {
             name,
             name,
             Explicit.IMPLICIT_TRUE,
-            Explicit.IMPLICIT_TRUE,
+            Optional.empty(),
             Explicit.IMPLICIT_FALSE,
             ObjectMapper.Dynamic.FALSE,
             emptyMap()
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java
index 3289e46941a45..863f6a0819554 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java
@@ -150,6 +150,11 @@ public boolean needs_score() {
                         return false;
                     }
 
+                    @Override
+                    public boolean needs_termStats() {
+                        return false;
+                    }
+
                     @Override
                     public ScoreScript newInstance(DocReader docReader) {
                         return new ScoreScript(Map.of(), searchContext.lookup(), docReader) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
index fc30b9b6677f1..dcb5cd1711c8c 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java
@@ -8,8 +8,10 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.DirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.test.FieldMaskingReader;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.hamcrest.Matchers;
 
@@ -19,6 +21,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
 public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
 
@@ -633,6 +636,132 @@ public void testArrayWithinArray() throws IOException {
             {"path":[{"to":[{"name":"A"},{"name":"B"}]},{"to":[{"name":"C"},{"name":"D"}]}]}""", booleanValue), syntheticSource);
     }
 
+    public void testDisabledObjectWithinHigherLevelArray() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path");
+            {
+                b.field("type", "object");
+                b.startObject("properties");
+                {
+                    b.startObject("to").field("type", "object").field("enabled", false);
+                    {
+                        b.startObject("properties");
+                        {
+                            b.startObject("name").field("type", "keyword").endObject();
+                        }
+                        b.endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        })).documentMapper();
+
+        boolean booleanValue = randomBoolean();
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startArray("path");
+            {
+                b.startObject();
+                {
+                    b.startObject("to").field("name", "A").endObject();
+                }
+                b.endObject();
+                b.startObject();
+                {
+                    b.startObject("to").field("name", "B").endObject();
+                }
+                b.endObject();
+            }
+            b.endArray();
+        });
+        assertEquals(String.format(Locale.ROOT, """
+            {"path":{"to":[{"name":"A"},{"name":"B"}]}}""", booleanValue), syntheticSource);
+    }
+
+    public void testStoredArrayWithinHigherLevelArray() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path");
+            {
+                b.field("type", "object");
+                b.startObject("properties");
+                {
+                    b.startObject("to").field("type", "object").field("store_array_source", true);
+                    {
+                        b.startObject("properties");
+                        {
+                            b.startObject("name").field("type", "keyword").endObject();
+                        }
+                        b.endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        })).documentMapper();
+
+        boolean booleanValue = randomBoolean();
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startArray("path");
+            {
+                b.startObject();
+                {
+                    b.startArray("to");
+                    {
+                        b.startObject().field("name", "A").endObject();
+                        b.startObject().field("name", "B").endObject();
+                    }
+                    b.endArray();
+                }
+                b.endObject();
+                b.startObject();
+                {
+                    b.startArray("to");
+                    {
+                        b.startObject().field("name", "C").endObject();
+                        b.startObject().field("name", "D").endObject();
+                    }
+                    b.endArray();
+                }
+                b.endObject();
+            }
+            b.endArray();
+        });
+        assertEquals(String.format(Locale.ROOT, """
+            {"path":{"to":[{"name":"A"},{"name":"B"},{"name":"C"},{"name":"D"}]}}""", booleanValue), syntheticSource);
+    }
+
+    public void testFallbackFieldWithinHigherLevelArray() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path");
+            {
+                b.field("type", "object");
+                b.startObject("properties");
+                {
+                    b.startObject("name").field("type", "keyword").field("doc_values", false).endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        })).documentMapper();
+
+        boolean booleanValue = randomBoolean();
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startArray("path");
+            {
+
+                b.startObject().field("name", "A").endObject();
+                b.startObject().field("name", "B").endObject();
+                b.startObject().field("name", "C").endObject();
+                b.startObject().field("name", "D").endObject();
+            }
+            b.endArray();
+        });
+        assertEquals(String.format(Locale.ROOT, """
+            {"path":{"name":["A","B","C","D"]}}""", booleanValue), syntheticSource);
+    }
+
     public void testFieldOrdering() throws IOException {
         DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
             b.startObject("A").field("type", "integer").endObject();
@@ -1055,4 +1184,58 @@ public void testRuntimeDynamicObjectNestedArray() throws IOException {
         assertEquals("""
             {"path":[{"to":{"foo":"A","bar":"B"}},{"to":{"foo":"C","bar":"D"}}]}""", syntheticSource);
     }
+
+    public void testDisabledSubObjectWithNameOverlappingParentName() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path");
+            b.startObject("properties");
+            {
+                b.startObject("at").field("type", "object").field("enabled", "false").endObject();
+            }
+            b.endObject();
+            b.endObject();
+        })).documentMapper();
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startObject("path");
+            {
+                b.startObject("at").field("foo", "A").endObject();
+            }
+            b.endObject();
+        });
+        assertEquals("""
+            {"path":{"at":{"foo":"A"}}}""", syntheticSource);
+    }
+
+    public void testStoredNestedSubObjectWithNameOverlappingParentName() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path");
+            b.startObject("properties");
+            {
+                b.startObject("at").field("type", "nested").field("store_array_source", "true").endObject();
+            }
+            b.endObject();
+            b.endObject();
+        })).documentMapper();
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startObject("path");
+            {
+                b.startObject("at").field("foo", "A").endObject();
+            }
+            b.endObject();
+        });
+        assertEquals("""
+            {"path":{"at":{"foo":"A"}}}""", syntheticSource);
+    }
+
+    protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader)
+        throws IOException {
+        // We exclude ignored source field since in some cases it contains an exact copy of a part of document source.
+        // Sometime synthetic source is different in this case (structurally but not logically)
+        // and since the copy is exact, contents of ignored source are different.
+        assertReaderEquals(
+            "round trip " + syntheticSource,
+            new FieldMaskingReader(Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME), reader),
+            new FieldMaskingReader(Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME), roundTripReader)
+        );
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java
index 4726424ada5f2..4593b149e13db 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java
@@ -155,6 +155,11 @@ public boolean needs_score() {
                         return false;
                     }
 
+                    @Override
+                    public boolean needs_termStats() {
+                        return false;
+                    }
+
                     @Override
                     public ScoreScript newInstance(DocReader docReader) {
                         return new ScoreScript(Map.of(), searchContext.lookup(), docReader) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java
index 6912194625bb7..b5270b358ec40 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java
@@ -136,6 +136,11 @@ public boolean needs_score() {
                         return false;
                     }
 
+                    @Override
+                    public boolean needs_termStats() {
+                        return false;
+                    }
+
                     @Override
                     public ScoreScript newInstance(DocReader docReader) {
                         return new ScoreScript(Map.of(), searchContext.lookup(), docReader) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java
index 83b3dbe858471..c9ac1516b6f8e 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java
@@ -177,6 +177,11 @@ public boolean needs_score() {
                         return false;
                     }
 
+                    @Override
+                    public boolean needs_termStats() {
+                        return false;
+                    }
+
                     @Override
                     public ScoreScript newInstance(DocReader docReader) {
                         return new ScoreScript(Map.of(), searchContext.lookup(), docReader) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java
index 251b0ae62f3c5..6a790f7e91118 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java
@@ -26,6 +26,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyList;
@@ -81,7 +82,7 @@ public void testSubfieldOverride() {
             "object",
             "object",
             Explicit.EXPLICIT_TRUE,
-            Explicit.IMPLICIT_TRUE,
+            Optional.empty(),
             Explicit.IMPLICIT_FALSE,
             ObjectMapper.Dynamic.TRUE,
             Collections.singletonMap("object.subfield", fieldMapper)
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
index 306887099849b..13bd5955d67a5 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
@@ -1575,11 +1575,11 @@ public void testStoreArraySourceinSyntheticSourceMode() throws IOException {
         assertNotNull(mapper.mapping().getRoot().getMapper("o"));
     }
 
-    public void testStoreArraySourceThrowsInNonSyntheticSourceMode() {
-        var exception = expectThrows(MapperParsingException.class, () -> createDocumentMapper(mapping(b -> {
+    public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOException {
+        DocumentMapper mapper = createDocumentMapper(mapping(b -> {
             b.startObject("o").field("type", "nested").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject();
-        })));
-        assertEquals("Parameter [store_array_source] can only be set in synthetic source mode.", exception.getMessage());
+        }));
+        assertNotNull(mapper.mapping().getRoot().getMapper("o"));
     }
 
     public void testSyntheticNestedWithObject() throws IOException {
@@ -1737,6 +1737,97 @@ public void testSyntheticNestedWithIncludeInRoot() throws IOException {
             {"path":{"bar":"B","foo":"A"}}""", syntheticSource);
     }
 
+    public void testSyntheticNestedWithEmptyObject() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path").field("type", "nested");
+            {
+                b.startObject("properties");
+                {
+                    b.startObject("foo").field("type", "keyword").endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        })).documentMapper();
+        var syntheticSource = syntheticSource(documentMapper, b -> { b.startObject("path").nullField("foo").endObject(); });
+        assertEquals("""
+            {"path":{}}""", syntheticSource);
+    }
+
+    public void testSyntheticNestedWithEmptySubObject() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path").field("type", "nested");
+            {
+                b.startObject("properties");
+                {
+                    b.startObject("to").startObject("properties");
+                    {
+                        b.startObject("foo").field("type", "keyword").endObject();
+                    }
+                    b.endObject().endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        })).documentMapper();
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startObject("path");
+            {
+                b.startObject("to").nullField("foo").endObject();
+            }
+            b.endObject();
+        });
+        assertEquals("""
+            {"path":{}}""", syntheticSource);
+    }
+
+    public void testSyntheticNestedWithArrayContainingEmptyObject() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path").field("type", "nested");
+            {
+                b.startObject("properties");
+                {
+                    b.startObject("foo").field("type", "keyword").endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        })).documentMapper();
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startArray("path");
+            {
+                b.startObject().field("foo", "A").endObject();
+                b.startObject().nullField("foo").endObject();
+            }
+            b.endArray();
+        });
+        assertEquals("""
+            {"path":[{"foo":"A"},{}]}""", syntheticSource);
+    }
+
+    public void testSyntheticNestedWithArrayContainingOnlyEmptyObject() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path").field("type", "nested");
+            {
+                b.startObject("properties");
+                {
+                    b.startObject("foo").field("type", "keyword").endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        })).documentMapper();
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startArray("path");
+            {
+                b.startObject().nullField("foo").endObject();
+            }
+            b.endArray();
+        });
+        assertEquals("""
+            {"path":{}}""", syntheticSource);
+    }
+
     private NestedObjectMapper createNestedObjectMapperWithAllParametersSet(CheckedConsumer<XContentBuilder, IOException> propertiesBuilder)
         throws IOException {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NodeMappingStatsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NodeMappingStatsTests.java
index 166e2c33fe263..563a860b084ea 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NodeMappingStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NodeMappingStatsTests.java
@@ -7,52 +7,62 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-public class NodeMappingStatsTests extends ESTestCase {
-
-    public void testSerialize() throws IOException {
-        NodeMappingStats stats = randomNodeMappingStats();
-        BytesStreamOutput out = new BytesStreamOutput();
-        stats.writeTo(out);
-        StreamInput input = out.bytes().streamInput();
-        NodeMappingStats read = new NodeMappingStats(input);
-        assertEquals(-1, input.read());
-        assertEquals(stats.getTotalCount(), read.getTotalCount());
-        assertEquals(stats.getTotalEstimatedOverhead(), read.getTotalEstimatedOverhead());
-    }
+public class NodeMappingStatsTests extends AbstractWireSerializingTestCase<NodeMappingStats> {
 
-    public void testEqualityAndHashCode() {
-        NodeMappingStats stats = randomNodeMappingStats();
-        assertEquals(stats, stats);
-        assertEquals(stats.hashCode(), stats.hashCode());
+    @Override
+    protected Writeable.Reader<NodeMappingStats> instanceReader() {
+        return NodeMappingStats::new;
+    }
 
-        NodeMappingStats stats1 = new NodeMappingStats(1L, 2L);
-        NodeMappingStats stats2 = new NodeMappingStats(3L, 5L);
-        NodeMappingStats stats3 = new NodeMappingStats(3L, 5L);
+    @Override
+    protected NodeMappingStats createTestInstance() {
+        return new NodeMappingStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
+    }
 
-        assertNotEquals(stats1, stats2);
-        assertNotEquals(stats1, stats3);
-        assertEquals(stats2, stats3);
+    @Override
+    protected NodeMappingStats mutateInstance(NodeMappingStats in) throws IOException {
+        return switch (between(0, 3)) {
+            case 0 -> new NodeMappingStats(
+                randomValueOtherThan(in.getTotalCount(), ESTestCase::randomNonNegativeLong),
+                in.getTotalEstimatedOverhead().getBytes(),
+                in.getTotalSegments(),
+                in.getTotalSegmentFields()
+            );
+            case 1 -> new NodeMappingStats(
+                in.getTotalCount(),
+                randomValueOtherThan(in.getTotalCount(), ESTestCase::randomNonNegativeLong),
+                in.getTotalSegments(),
+                in.getTotalSegmentFields()
+            );
+            case 2 -> new NodeMappingStats(
+                in.getTotalCount(),
+                in.getTotalEstimatedOverhead().getBytes(),
+                randomValueOtherThan(in.getTotalSegments(), ESTestCase::randomNonNegativeLong),
+                in.getTotalSegmentFields()
+            );
+            case 3 -> new NodeMappingStats(
+                in.getTotalCount(),
+                in.getTotalEstimatedOverhead().getBytes(),
+                in.getTotalSegments(),
+                randomValueOtherThan(in.getTotalSegmentFields(), ESTestCase::randomNonNegativeLong)
+            );
+            default -> throw new AssertionError("invalid option");
+        };
     }
 
     public void testAdd() {
-        NodeMappingStats stats1 = new NodeMappingStats(1L, 2L);
-        NodeMappingStats stats2 = new NodeMappingStats(2L, 3L);
-        NodeMappingStats stats3 = new NodeMappingStats(3L, 5L);
+        NodeMappingStats stats1 = new NodeMappingStats(1L, 2L, 4L, 6L);
+        NodeMappingStats stats2 = new NodeMappingStats(2L, 3L, 10L, 20L);
+        NodeMappingStats stats3 = new NodeMappingStats(3L, 5L, 14L, 26L);
 
         stats1.add(stats2);
         assertEquals(stats1, stats3);
         assertEquals(stats1.hashCode(), stats3.hashCode());
     }
-
-    private static NodeMappingStats randomNodeMappingStats() {
-        long totalCount = randomIntBetween(1, 100);
-        long estimatedOverhead = totalCount * 1024;
-        return new NodeMappingStats(totalCount, estimatedOverhead);
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java
index b3bb8cbe697a5..ea6ddf0257d6f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java
@@ -7,11 +7,11 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.common.Explicit;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;
+import java.util.Optional;
 
 import static org.elasticsearch.index.mapper.MapperService.MergeReason.INDEX_TEMPLATE;
 import static org.elasticsearch.index.mapper.MapperService.MergeReason.MAPPING_UPDATE;
@@ -26,9 +26,9 @@ private RootObjectMapper createMapping(
         boolean includeBarField,
         boolean includeBazField
     ) {
-        RootObjectMapper.Builder rootBuilder = new RootObjectMapper.Builder("type1", Explicit.IMPLICIT_TRUE);
-        rootBuilder.add(new ObjectMapper.Builder("disabled", Explicit.IMPLICIT_TRUE).enabled(disabledFieldEnabled));
-        ObjectMapper.Builder fooBuilder = new ObjectMapper.Builder("foo", Explicit.IMPLICIT_TRUE).enabled(fooFieldEnabled);
+        RootObjectMapper.Builder rootBuilder = new RootObjectMapper.Builder("type1", Optional.empty());
+        rootBuilder.add(new ObjectMapper.Builder("disabled", Optional.empty()).enabled(disabledFieldEnabled));
+        ObjectMapper.Builder fooBuilder = new ObjectMapper.Builder("foo", Optional.empty()).enabled(fooFieldEnabled);
         if (includeBarField) {
             fooBuilder.add(new TextFieldMapper.Builder("bar", createDefaultIndexAnalyzers(), false));
         }
@@ -77,8 +77,8 @@ public void testMergeWhenDisablingField() {
     public void testMergeDisabledField() {
         // GIVEN a mapping with "foo" field disabled
         // the field is disabled, and we are not trying to re-enable it, hence merge should work
-        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
-            new ObjectMapper.Builder("disabled", Explicit.IMPLICIT_TRUE)
+        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
+            new ObjectMapper.Builder("disabled", Optional.empty())
         ).build(MapperBuilderContext.root(false, false));
 
         RootObjectMapper merged = rootObjectMapper.merge(mergeWith, MapperMergeContext.root(false, false, MAPPING_UPDATE, Long.MAX_VALUE));
@@ -100,10 +100,8 @@ public void testMergeEnabled() {
 
     public void testMergeEnabledForRootMapper() {
         String type = MapperService.SINGLE_MAPPING_NAME;
-        ObjectMapper firstMapper = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).build(
-            MapperBuilderContext.root(false, false)
-        );
-        ObjectMapper secondMapper = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).enabled(false)
+        ObjectMapper firstMapper = new RootObjectMapper.Builder("_doc", Optional.empty()).build(MapperBuilderContext.root(false, false));
+        ObjectMapper secondMapper = new RootObjectMapper.Builder("_doc", Optional.empty()).enabled(false)
             .build(MapperBuilderContext.root(false, false));
 
         MapperException e = expectThrows(
@@ -144,12 +142,10 @@ public void testMergedFieldNamesFieldWithDotsSubobjectsFalseAtRoot() {
     }
 
     public void testMergedFieldNamesFieldWithDotsSubobjectsFalse() {
-        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
-            createObjectSubobjectsFalseLeafWithDots()
-        ).build(MapperBuilderContext.root(false, false));
-        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
-            createObjectSubobjectsFalseLeafWithDots()
-        ).build(MapperBuilderContext.root(false, false));
+        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Optional.empty()).add(createObjectSubobjectsFalseLeafWithDots())
+            .build(MapperBuilderContext.root(false, false));
+        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add(createObjectSubobjectsFalseLeafWithDots())
+            .build(MapperBuilderContext.root(false, false));
 
         final ObjectMapper merged = mergeInto.merge(mergeWith, MapperMergeContext.root(false, false, MAPPING_UPDATE, Long.MAX_VALUE));
 
@@ -161,9 +157,9 @@ public void testMergedFieldNamesFieldWithDotsSubobjectsFalse() {
     }
 
     public void testMergedFieldNamesMultiFields() {
-        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(createTextKeywordMultiField("text"))
+        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Optional.empty()).add(createTextKeywordMultiField("text"))
             .build(MapperBuilderContext.root(false, false));
-        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(createTextKeywordMultiField("text"))
+        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add(createTextKeywordMultiField("text"))
             .build(MapperBuilderContext.root(false, false));
 
         final ObjectMapper merged = mergeInto.merge(mergeWith, MapperMergeContext.root(false, false, MAPPING_UPDATE, Long.MAX_VALUE));
@@ -177,10 +173,10 @@ public void testMergedFieldNamesMultiFields() {
     }
 
     public void testMergedFieldNamesMultiFieldsWithinSubobjectsFalse() {
-        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
+        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
             createObjectSubobjectsFalseLeafWithMultiField()
         ).build(MapperBuilderContext.root(false, false));
-        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
+        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
             createObjectSubobjectsFalseLeafWithMultiField()
         ).build(MapperBuilderContext.root(false, false));
 
@@ -212,9 +208,9 @@ public void testMergeWithLimit() {
     }
 
     public void testMergeWithLimitTruncatedObjectField() {
-        RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).build(MapperBuilderContext.root(false, false));
-        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
-            new ObjectMapper.Builder("parent", Explicit.IMPLICIT_FALSE).add(
+        RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).build(MapperBuilderContext.root(false, false));
+        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
+            new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.DISABLED)).add(
                 new KeywordFieldMapper.Builder("child1", IndexVersion.current())
             ).add(new KeywordFieldMapper.Builder("child2", IndexVersion.current()))
         ).build(MapperBuilderContext.root(false, false));
@@ -243,11 +239,11 @@ public void testMergeWithLimitTruncatedObjectField() {
     }
 
     public void testMergeSameObjectDifferentFields() {
-        RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
-            new ObjectMapper.Builder("parent", Explicit.IMPLICIT_TRUE).add(new KeywordFieldMapper.Builder("child1", IndexVersion.current()))
+        RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
+            new ObjectMapper.Builder("parent", Optional.empty()).add(new KeywordFieldMapper.Builder("child1", IndexVersion.current()))
         ).build(MapperBuilderContext.root(false, false));
-        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
-            new ObjectMapper.Builder("parent", Explicit.IMPLICIT_TRUE).add(
+        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
+            new ObjectMapper.Builder("parent", Optional.empty()).add(
                 new KeywordFieldMapper.Builder("child1", IndexVersion.current()).ignoreAbove(42)
             ).add(new KeywordFieldMapper.Builder("child2", IndexVersion.current()))
         ).build(MapperBuilderContext.root(false, false));
@@ -270,10 +266,10 @@ public void testMergeSameObjectDifferentFields() {
     }
 
     public void testMergeWithLimitMultiField() {
-        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
+        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
             createTextKeywordMultiField("text", "keyword1")
         ).build(MapperBuilderContext.root(false, false));
-        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
+        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
             createTextKeywordMultiField("text", "keyword2")
         ).build(MapperBuilderContext.root(false, false));
 
@@ -287,10 +283,10 @@ public void testMergeWithLimitMultiField() {
     }
 
     public void testMergeWithLimitRuntimeField() {
-        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).addRuntimeField(
+        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Optional.empty()).addRuntimeField(
             new TestRuntimeField("existing_runtime_field", "keyword")
         ).add(createTextKeywordMultiField("text", "keyword1")).build(MapperBuilderContext.root(false, false));
-        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).addRuntimeField(
+        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).addRuntimeField(
             new TestRuntimeField("existing_runtime_field", "keyword")
         ).addRuntimeField(new TestRuntimeField("new_runtime_field", "keyword")).build(MapperBuilderContext.root(false, false));
 
@@ -304,12 +300,12 @@ public void testMergeWithLimitRuntimeField() {
     }
 
     public void testMergeSubobjectsFalseWithObject() {
-        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
-            new ObjectMapper.Builder("parent", Explicit.IMPLICIT_FALSE)
+        RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
+            new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.DISABLED))
         ).build(MapperBuilderContext.root(false, false));
-        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
-            new ObjectMapper.Builder("parent", Explicit.IMPLICIT_TRUE).add(
-                new ObjectMapper.Builder("child", Explicit.IMPLICIT_TRUE).add(
+        RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
+            new ObjectMapper.Builder("parent", Optional.empty()).add(
+                new ObjectMapper.Builder("child", Optional.empty()).add(
                     new KeywordFieldMapper.Builder("grandchild", IndexVersion.current())
                 )
             )
@@ -326,7 +322,7 @@ private static RootObjectMapper createRootSubobjectFalseLeafWithDots() {
         FieldMapper fieldMapper = fieldBuilder.build(MapperBuilderContext.root(false, false));
         assertEquals("host.name", fieldMapper.leafName());
         assertEquals("host.name", fieldMapper.fullPath());
-        return new RootObjectMapper.Builder("_doc", Explicit.EXPLICIT_FALSE).add(fieldBuilder)
+        return new RootObjectMapper.Builder("_doc", Optional.of(ObjectMapper.Subobjects.DISABLED)).add(fieldBuilder)
             .build(MapperBuilderContext.root(false, false));
     }
 
@@ -346,7 +342,7 @@ private static ObjectMapper.Builder createObjectSubobjectsFalseLeafWithDots() {
         assertEquals("host.name", fieldMapper.leafName());
         assertEquals("foo.metrics.host.name", fieldMapper.fullPath());
         return new ObjectMapper.Builder("foo", ObjectMapper.Defaults.SUBOBJECTS).add(
-            new ObjectMapper.Builder("metrics", Explicit.EXPLICIT_FALSE).add(fieldBuilder)
+            new ObjectMapper.Builder("metrics", Optional.of(ObjectMapper.Subobjects.DISABLED)).add(fieldBuilder)
         );
     }
 
@@ -369,7 +365,7 @@ private ObjectMapper.Builder createObjectSubobjectsFalseLeafWithMultiField() {
         assertEquals("keyword", fieldMapper.leafName());
         assertEquals("foo.metrics.host.name.keyword", fieldMapper.fullPath());
         return new ObjectMapper.Builder("foo", ObjectMapper.Defaults.SUBOBJECTS).add(
-            new ObjectMapper.Builder("metrics", Explicit.EXPLICIT_FALSE).add(fieldBuilder)
+            new ObjectMapper.Builder("metrics", Optional.of(ObjectMapper.Subobjects.DISABLED)).add(fieldBuilder)
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java
index 6687a28883716..49d8ba9c2ca29 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java
@@ -8,7 +8,6 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -21,9 +20,11 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
+import org.hamcrest.core.IsInstanceOf;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Optional;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
@@ -164,7 +165,7 @@ public void testMergeEnabledForIndexTemplates() throws IOException {
         ObjectMapper objectMapper = mapper.mappers().objectMappers().get("object");
         assertNotNull(objectMapper);
         assertFalse(objectMapper.isEnabled());
-        assertTrue(objectMapper.subobjects());
+        assertEquals(ObjectMapper.Subobjects.ENABLED, objectMapper.subobjects());
         assertFalse(objectMapper.storeArraySource());
 
         // Setting 'enabled' to true is allowed, and updates the mapping.
@@ -175,7 +176,7 @@ public void testMergeEnabledForIndexTemplates() throws IOException {
                 .startObject("object")
                 .field("type", "object")
                 .field("enabled", true)
-                .field("subobjects", false)
+                .field("subobjects", "auto")
                 .field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true)
                 .endObject()
                 .endObject()
@@ -186,7 +187,7 @@ public void testMergeEnabledForIndexTemplates() throws IOException {
         objectMapper = mapper.mappers().objectMappers().get("object");
         assertNotNull(objectMapper);
         assertTrue(objectMapper.isEnabled());
-        assertFalse(objectMapper.subobjects());
+        assertEquals(ObjectMapper.Subobjects.AUTO, objectMapper.subobjects());
         assertTrue(objectMapper.storeArraySource());
     }
 
@@ -500,6 +501,141 @@ public void testSubobjectsCannotBeUpdatedOnRoot() throws IOException {
         assertEquals("the [subobjects] parameter can't be updated for the object mapping [_doc]", exception.getMessage());
     }
 
+    public void testSubobjectsAuto() throws Exception {
+        MapperService mapperService = createMapperService(mapping(b -> {
+            b.startObject("metrics.service");
+            {
+                b.field("subobjects", "auto");
+                b.startObject("properties");
+                {
+                    b.startObject("time");
+                    b.field("type", "long");
+                    b.endObject();
+                    b.startObject("time.max");
+                    b.field("type", "long");
+                    b.endObject();
+                    b.startObject("attributes");
+                    {
+                        b.field("type", "object");
+                        b.field("enabled", "false");
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }));
+        assertNotNull(mapperService.fieldType("metrics.service.time"));
+        assertNotNull(mapperService.fieldType("metrics.service.time.max"));
+        assertNotNull(mapperService.documentMapper().mappers().objectMappers().get("metrics.service.attributes"));
+    }
+
+    public void testSubobjectsAutoWithInnerObject() throws IOException {
+        MapperService mapperService = createMapperService(mapping(b -> {
+            b.startObject("metrics.service");
+            {
+                b.field("subobjects", "auto");
+                b.startObject("properties");
+                {
+                    b.startObject("time");
+                    {
+                        b.startObject("properties");
+                        {
+                            b.startObject("max");
+                            b.field("type", "long");
+                            b.endObject();
+                        }
+                        b.endObject();
+                    }
+                    b.endObject();
+                    b.startObject("foo");
+                    b.field("type", "keyword");
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }));
+        assertNull(mapperService.fieldType("metrics.service.time"));
+        assertNotNull(mapperService.fieldType("metrics.service.time.max"));
+        assertNotNull(mapperService.fieldType("metrics.service.foo"));
+        assertNotNull(mapperService.documentMapper().mappers().objectMappers().get("metrics.service.time"));
+        assertNotNull(mapperService.documentMapper().mappers().getMapper("metrics.service.foo"));
+    }
+
+    public void testSubobjectsAutoWithInnerNested() throws IOException {
+        MapperService mapperService = createMapperService(mapping(b -> {
+            b.startObject("metrics.service");
+            {
+                b.field("subobjects", "auto");
+                b.startObject("properties");
+                {
+                    b.startObject("time");
+                    b.field("type", "nested");
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }));
+        assertThat(
+            mapperService.documentMapper().mappers().objectMappers().get("metrics.service.time"),
+            IsInstanceOf.instanceOf(NestedObjectMapper.class)
+        );
+    }
+
+    public void testSubobjectsAutoRoot() throws Exception {
+        MapperService mapperService = createMapperService(mappingWithSubobjects(b -> {
+            b.startObject("metrics.service.time");
+            b.field("type", "long");
+            b.endObject();
+            b.startObject("metrics.service.time.max");
+            b.field("type", "long");
+            b.endObject();
+            b.startObject("metrics.attributes");
+            {
+                b.field("type", "object");
+                b.field("enabled", "false");
+            }
+            b.endObject();
+        }, "auto"));
+        assertNotNull(mapperService.fieldType("metrics.service.time"));
+        assertNotNull(mapperService.fieldType("metrics.service.time.max"));
+        assertNotNull(mapperService.documentMapper().mappers().objectMappers().get("metrics.attributes"));
+    }
+
+    public void testSubobjectsAutoRootWithInnerObject() throws IOException {
+        MapperService mapperService = createMapperService(mappingWithSubobjects(b -> {
+            b.startObject("metrics.service.time");
+            {
+                b.startObject("properties");
+                {
+                    b.startObject("max");
+                    b.field("type", "long");
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }, "auto"));
+        assertNull(mapperService.fieldType("metrics.service.time"));
+        assertNotNull(mapperService.fieldType("metrics.service.time.max"));
+        assertNotNull(mapperService.documentMapper().mappers().objectMappers().get("metrics.service.time"));
+        assertNotNull(mapperService.documentMapper().mappers().getMapper("metrics.service.time.max"));
+    }
+
+    public void testSubobjectsAutoRootWithInnerNested() throws IOException {
+        MapperService mapperService = createMapperService(mappingWithSubobjects(b -> {
+            b.startObject("metrics.service");
+            b.field("type", "nested");
+            b.endObject();
+        }, "auto"));
+        assertThat(
+            mapperService.documentMapper().mappers().objectMappers().get("metrics.service"),
+            IsInstanceOf.instanceOf(NestedObjectMapper.class)
+        );
+    }
+
     /**
      * Makes sure that an empty object mapper returns {@code null} from
      * {@link SourceLoader.SyntheticFieldLoader#docValuesLoader}. This
@@ -546,16 +682,16 @@ public void testStoreArraySourceinSyntheticSourceMode() throws IOException {
         assertNotNull(mapper.mapping().getRoot().getMapper("o"));
     }
 
-    public void testStoreArraySourceThrowsInNonSyntheticSourceMode() {
-        var exception = expectThrows(MapperParsingException.class, () -> createDocumentMapper(mapping(b -> {
+    public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOException {
+        DocumentMapper mapper = createDocumentMapper(mapping(b -> {
             b.startObject("o").field("type", "object").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject();
-        })));
-        assertEquals("Parameter [store_array_source] can only be set in synthetic source mode.", exception.getMessage());
+        }));
+        assertNotNull(mapper.mapping().getRoot().getMapper("o"));
     }
 
     public void testNestedObjectWithMultiFieldsgetTotalFieldsCount() {
-        ObjectMapper.Builder mapperBuilder = new ObjectMapper.Builder("parent_size_1", Explicit.IMPLICIT_TRUE).add(
-            new ObjectMapper.Builder("child_size_2", Explicit.IMPLICIT_TRUE).add(
+        ObjectMapper.Builder mapperBuilder = new ObjectMapper.Builder("parent_size_1", Optional.empty()).add(
+            new ObjectMapper.Builder("child_size_2", Optional.empty()).add(
                 new TextFieldMapper.Builder("grand_child_size_3", createDefaultIndexAnalyzers(), false).addMultiField(
                     new KeywordFieldMapper.Builder("multi_field_size_4", IndexVersion.current())
                 )
@@ -602,10 +738,26 @@ private ObjectMapper createObjectMapperWithAllParametersSet(CheckedConsumer<XCon
 
     public void testFlatten() {
         MapperBuilderContext rootContext = MapperBuilderContext.root(false, false);
-        ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Explicit.IMPLICIT_TRUE).add(
-            new ObjectMapper.Builder("child", Explicit.IMPLICIT_TRUE).add(
-                new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())
-            )
+        ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.empty()).add(
+            new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current()))
+        ).add(new KeywordFieldMapper.Builder("keyword1", IndexVersion.current())).build(rootContext);
+        List<String> fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList();
+        assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2"));
+    }
+
+    public void testFlattenSubobjectsAuto() {
+        MapperBuilderContext rootContext = MapperBuilderContext.root(false, false);
+        ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.AUTO)).add(
+            new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current()))
+        ).add(new KeywordFieldMapper.Builder("keyword1", IndexVersion.current())).build(rootContext);
+        List<String> fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList();
+        assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2"));
+    }
+
+    public void testFlattenSubobjectsFalse() {
+        MapperBuilderContext rootContext = MapperBuilderContext.root(false, false);
+        ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.DISABLED)).add(
+            new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current()))
         ).add(new KeywordFieldMapper.Builder("keyword1", IndexVersion.current())).build(rootContext);
         List<String> fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList();
         assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2"));
@@ -613,8 +765,8 @@ public void testFlatten() {
 
     public void testFlattenDynamicIncompatible() {
         MapperBuilderContext rootContext = MapperBuilderContext.root(false, false);
-        ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Explicit.IMPLICIT_TRUE).add(
-            new ObjectMapper.Builder("child", Explicit.IMPLICIT_TRUE).dynamic(Dynamic.FALSE)
+        ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.empty()).add(
+            new ObjectMapper.Builder("child", Optional.empty()).dynamic(Dynamic.FALSE)
         ).build(rootContext);
 
         IllegalArgumentException exception = expectThrows(
@@ -631,7 +783,7 @@ public void testFlattenDynamicIncompatible() {
 
     public void testFlattenEnabledFalse() {
         MapperBuilderContext rootContext = MapperBuilderContext.root(false, false);
-        ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Explicit.IMPLICIT_TRUE).enabled(false).build(rootContext);
+        ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.empty()).enabled(false).build(rootContext);
 
         IllegalArgumentException exception = expectThrows(
             IllegalArgumentException.class,
@@ -646,7 +798,7 @@ public void testFlattenEnabledFalse() {
 
     public void testFlattenExplicitSubobjectsTrue() {
         MapperBuilderContext rootContext = MapperBuilderContext.root(false, false);
-        ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Explicit.EXPLICIT_TRUE).build(rootContext);
+        ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.ENABLED)).build(rootContext);
 
         IllegalArgumentException exception = expectThrows(
             IllegalArgumentException.class,
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java
index 0c176a0302620..0c2e7b5e06f3f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java
@@ -12,13 +12,13 @@
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.name.Named;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.ByteUtils;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.injection.guice.name.Named;
 import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java
index cd5b43d0af771..a4532bca67778 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java
@@ -8,9 +8,11 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -21,8 +23,12 @@
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.util.Arrays;
 import java.util.Base64;
+import java.util.Collection;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -168,4 +174,94 @@ public void testCloneSubContextWithParser() throws IOException {
         assertEquals(data, dataInParser(tuple.v2()));
         assertTrue(tuple.v1().getClonedSource());
     }
+
+    public void testWriteMergedWithSingleValue() throws IOException {
+        testWriteMergedWithSingleValue(randomLong());
+        testWriteMergedWithSingleValue(randomDouble());
+        testWriteMergedWithSingleValue(randomBoolean());
+        testWriteMergedWithSingleValue(randomAlphaOfLength(5));
+        testWriteMergedWithSingleValue(null);
+        testWriteMergedWithSingleValue(Map.of("object_field", randomAlphaOfLength(5)));
+        testWriteMergedWithSingleValue(Map.of("object_field", Map.of("nested_object_field", randomAlphaOfLength(5))));
+    }
+
+    private void testWriteMergedWithSingleValue(Object value) throws IOException {
+        var map = executeWriteMergeOnRepeated(value);
+        assertEquals(Arrays.asList(value, value), map.get("foo"));
+    }
+
+    public void testWriteMergedWithMultipleValues() throws IOException {
+        testWriteMergedWithMultipleValues(List.of(randomLong(), randomLong()));
+        testWriteMergedWithMultipleValues(List.of(randomDouble(), randomDouble()));
+        testWriteMergedWithMultipleValues(List.of(randomBoolean(), randomBoolean()));
+        testWriteMergedWithMultipleValues(List.of(randomAlphaOfLength(5), randomAlphaOfLength(5)));
+        testWriteMergedWithMultipleValues(Arrays.asList(null, null));
+        testWriteMergedWithMultipleValues(
+            List.of(Map.of("object_field", randomAlphaOfLength(5)), Map.of("object_field", randomAlphaOfLength(5)))
+        );
+        testWriteMergedWithMultipleValues(
+            List.of(
+                Map.of("object_field", Map.of("nested_object_field", randomAlphaOfLength(5))),
+                Map.of("object_field", Map.of("nested_object_field", randomAlphaOfLength(5)))
+            )
+        );
+    }
+
+    private void testWriteMergedWithMultipleValues(List<Object> value) throws IOException {
+        var map = executeWriteMergeOnRepeated(value);
+        var expected = Stream.of(value, value).flatMap(Collection::stream).toList();
+        assertEquals(expected, map.get("foo"));
+    }
+
+    public void testWriteMergedWithMixedValues() throws IOException {
+        testWriteMergedWithMixedValues(randomLong(), List.of(randomLong(), randomLong()));
+        testWriteMergedWithMixedValues(randomDouble(), List.of(randomDouble(), randomDouble()));
+        testWriteMergedWithMixedValues(randomBoolean(), List.of(randomBoolean(), randomBoolean()));
+        testWriteMergedWithMixedValues(randomAlphaOfLength(5), List.of(randomAlphaOfLength(5), randomAlphaOfLength(5)));
+        testWriteMergedWithMixedValues(null, Arrays.asList(null, null));
+        testWriteMergedWithMixedValues(
+            Map.of("object_field", randomAlphaOfLength(5)),
+            List.of(Map.of("object_field", randomAlphaOfLength(5)), Map.of("object_field", randomAlphaOfLength(5)))
+        );
+        testWriteMergedWithMixedValues(
+            Map.of("object_field", Map.of("nested_object_field", randomAlphaOfLength(5))),
+            List.of(
+                Map.of("object_field", Map.of("nested_object_field", randomAlphaOfLength(5))),
+                Map.of("object_field", Map.of("nested_object_field", randomAlphaOfLength(5)))
+            )
+        );
+    }
+
+    private void testWriteMergedWithMixedValues(Object value, List<Object> multipleValues) throws IOException {
+        var map = executeWriteMergeOnTwoEncodedValues(value, multipleValues);
+        var expected = Stream.concat(Stream.of(value), multipleValues.stream()).toList();
+        assertEquals(expected, map.get("foo"));
+    }
+
+    private Map<String, Object> executeWriteMergeOnRepeated(Object value) throws IOException {
+        return executeWriteMergeOnTwoEncodedValues(value, value);
+    }
+
+    private Map<String, Object> executeWriteMergeOnTwoEncodedValues(Object first, Object second) throws IOException {
+        var xContentType = randomFrom(XContentType.values());
+
+        var firstEncoded = encodeSingleValue(first, xContentType);
+        var secondEncoded = encodeSingleValue(second, xContentType);
+
+        var destination = XContentFactory.contentBuilder(xContentType);
+        destination.startObject();
+        XContentDataHelper.writeMerged(destination, "foo", List.of(firstEncoded, secondEncoded));
+        destination.endObject();
+
+        return XContentHelper.convertToMap(BytesReference.bytes(destination), false, xContentType).v2();
+    }
+
+    private BytesRef encodeSingleValue(Object value, XContentType xContentType) throws IOException {
+        var builder = XContentFactory.contentBuilder(xContentType);
+        builder.value(value);
+
+        XContentParser parser = createParser(builder);
+        parser.nextToken();
+        return XContentDataHelper.encodeToken(parser);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
index 4c6d7a6fec2fd..83b8a8fa991c2 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
@@ -48,6 +48,7 @@
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.search.vectors.VectorData;
+import org.elasticsearch.simdvec.VectorScorerFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -1842,6 +1843,9 @@ public void testKnnVectorsFormat() throws IOException {
             assertThat(codec, instanceOf(PerFieldMapperCodec.class));
             knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
         } else {
+            if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
+                codec = deduplicateFieldInfosCodec.delegate();
+            }
             assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
             knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
         }
@@ -1877,19 +1881,27 @@ public void testKnnQuantizedFlatVectorsFormat() throws IOException {
                 assertThat(codec, instanceOf(PerFieldMapperCodec.class));
                 knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
             } else {
+                if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
+                    codec = deduplicateFieldInfosCodec.delegate();
+                }
                 assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
                 knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
             }
+            VectorScorerFactory factory = VectorScorerFactory.instance().orElse(null);
             String expectedString = "ES813Int8FlatVectorFormat(name=ES813Int8FlatVectorFormat, innerFormat="
-                + "Lucene99ScalarQuantizedVectorsFormat(name=Lucene99ScalarQuantizedVectorsFormat,"
+                + "ES814ScalarQuantizedVectorsFormat(name=ES814ScalarQuantizedVectorsFormat,"
                 + " confidenceInterval="
                 + (setConfidenceInterval ? Float.toString(confidenceInterval) : (quantizedFlatFormat.equals("int4_flat") ? "0.0" : null))
                 + ", bits="
                 + (quantizedFlatFormat.equals("int4_flat") ? 4 : 7)
-                + ", compress="
+                + ", compressed="
                 + quantizedFlatFormat.equals("int4_flat")
-                + ", flatVectorScorer=ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer()),"
-                + " rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())))";
+                + ", flatVectorScorer=ESFlatVectorsScorer("
+                + "delegate=ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())"
+                + ", factory="
+                + (factory != null ? factory : "null")
+                + "), "
+                + "rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())))";
             assertEquals(expectedString, knnVectorsFormat.toString());
         }
     }
@@ -1920,9 +1932,13 @@ public void testKnnQuantizedHNSWVectorsFormat() throws IOException {
             assertThat(codec, instanceOf(PerFieldMapperCodec.class));
             knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
         } else {
+            if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
+                codec = deduplicateFieldInfosCodec.delegate();
+            }
             assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
             knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
         }
+        VectorScorerFactory factory = VectorScorerFactory.instance().orElse(null);
         String expectedString = "ES814HnswScalarQuantizedVectorsFormat(name=ES814HnswScalarQuantizedVectorsFormat, maxConn="
             + m
             + ", beamWidth="
@@ -1930,7 +1946,12 @@ public void testKnnQuantizedHNSWVectorsFormat() throws IOException {
             + ", flatVectorFormat=ES814ScalarQuantizedVectorsFormat("
             + "name=ES814ScalarQuantizedVectorsFormat, confidenceInterval="
             + (setConfidenceInterval ? confidenceInterval : null)
-            + ", bits=7, compressed=false, rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())"
+            + ", bits=7, compressed=false, "
+            + "flatVectorScorer=ESFlatVectorsScorer(delegate=ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer()), "
+            + "factory="
+            + (factory != null ? factory : "null")
+            + "), "
+            + "rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())"
             + "))";
         assertEquals(expectedString, knnVectorsFormat.toString());
     }
@@ -1961,9 +1982,13 @@ public void testKnnHalfByteQuantizedHNSWVectorsFormat() throws IOException {
             assertThat(codec, instanceOf(PerFieldMapperCodec.class));
             knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
         } else {
+            if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
+                codec = deduplicateFieldInfosCodec.delegate();
+            }
             assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
             knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
         }
+        VectorScorerFactory factory = VectorScorerFactory.instance().orElse(null);
         String expectedString = "ES814HnswScalarQuantizedVectorsFormat(name=ES814HnswScalarQuantizedVectorsFormat, maxConn="
             + m
             + ", beamWidth="
@@ -1971,7 +1996,12 @@ public void testKnnHalfByteQuantizedHNSWVectorsFormat() throws IOException {
             + ", flatVectorFormat=ES814ScalarQuantizedVectorsFormat("
             + "name=ES814ScalarQuantizedVectorsFormat, confidenceInterval="
             + (setConfidenceInterval ? confidenceInterval : 0.0f)
-            + ", bits=4, compressed=true, rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())"
+            + ", bits=4, compressed=true, "
+            + "flatVectorScorer=ESFlatVectorsScorer(delegate=ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer()), "
+            + "factory="
+            + (factory != null ? factory : "null")
+            + "), "
+            + "rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())"
             + "))";
         assertEquals(expectedString, knnVectorsFormat.toString());
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
index 9ee895f6de003..9ef2d0df90cce 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
@@ -215,7 +215,7 @@ public void testExactKnnQuery() {
             for (int i = 0; i < dims; i++) {
                 queryVector[i] = randomFloat();
             }
-            Query query = field.createExactKnnQuery(VectorData.fromFloats(queryVector));
+            Query query = field.createExactKnnQuery(VectorData.fromFloats(queryVector), null);
             assertTrue(query instanceof DenseVectorQuery.Floats);
         }
         {
@@ -233,7 +233,7 @@ public void testExactKnnQuery() {
             for (int i = 0; i < dims; i++) {
                 queryVector[i] = randomByte();
             }
-            Query query = field.createExactKnnQuery(VectorData.fromBytes(queryVector));
+            Query query = field.createExactKnnQuery(VectorData.fromBytes(queryVector), null);
             assertTrue(query instanceof DenseVectorQuery.Bytes);
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
index c29957f04c515..a4cc96a2063dc 100644
--- a/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
@@ -32,7 +32,9 @@
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.Matchers.not;
 
 public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilder> {
     @Override
@@ -463,4 +465,39 @@ public void testMustRewrite() throws IOException {
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> boolQuery.toQuery(context));
         assertEquals("Rewrite first", e.getMessage());
     }
+
+    public void testShallowCopy() {
+        BoolQueryBuilder orig = createTestQueryBuilder();
+        BoolQueryBuilder shallowCopy = orig.shallowCopy();
+        assertThat(shallowCopy.adjustPureNegative(), equalTo(orig.adjustPureNegative()));
+        assertThat(shallowCopy.minimumShouldMatch(), equalTo(orig.minimumShouldMatch()));
+        assertThat(shallowCopy.must(), equalTo(orig.must()));
+        assertThat(shallowCopy.mustNot(), equalTo(orig.mustNot()));
+        assertThat(shallowCopy.should(), equalTo(orig.should()));
+        assertThat(shallowCopy.filter(), equalTo(orig.filter()));
+
+        QueryBuilder b = new MatchQueryBuilder("foo", "bar");
+        switch (between(0, 3)) {
+            case 0 -> {
+                shallowCopy.must(b);
+                assertThat(shallowCopy.must(), hasItem(b));
+                assertThat(orig.must(), not(hasItem(b)));
+            }
+            case 1 -> {
+                shallowCopy.mustNot(b);
+                assertThat(shallowCopy.mustNot(), hasItem(b));
+                assertThat(orig.mustNot(), not(hasItem(b)));
+            }
+            case 2 -> {
+                shallowCopy.should(b);
+                assertThat(shallowCopy.should(), hasItem(b));
+                assertThat(orig.should(), not(hasItem(b)));
+            }
+            case 3 -> {
+                shallowCopy.filter(b);
+                assertThat(shallowCopy.filter(), hasItem(b));
+                assertThat(orig.filter(), not(hasItem(b)));
+            }
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
index 9cd1df700a618..ffca4352f0ae6 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
@@ -27,7 +27,6 @@
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
@@ -90,6 +89,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.BiFunction;
 import java.util.function.Function;
@@ -384,7 +384,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() {
     public void testSyntheticSourceSearchLookup() throws IOException {
         // Build a mapping using synthetic source
         SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false).setSynthetic().build();
-        RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(
+        RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
             new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100)
         ).build(MapperBuilderContext.root(true, false));
         Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of());
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 142c03cdfa053..28497dc4a8a6b 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -1784,6 +1784,67 @@ public void testExternalRefreshMetric() throws IOException {
         closeShards(shard);
     }
 
+    public void testShardFieldStats() throws IOException {
+        Settings settings = Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE).build();
+        IndexShard shard = newShard(true, settings);
+        assertNull(shard.getShardFieldStats());
+        recoverShardFromStore(shard);
+        ShardFieldStats stats = shard.getShardFieldStats();
+        assertNotNull(stats);
+        assertThat(stats.numSegments(), equalTo(0));
+        assertThat(stats.totalFields(), equalTo(0));
+        // index some documents
+        int numDocs = between(1, 10);
+        for (int i = 0; i < numDocs; i++) {
+            indexDoc(shard, "_doc", "first_" + i, """
+                {
+                    "f1": "foo",
+                    "f2": "bar"
+                }
+                """);
+        }
+        assertThat(shard.getShardFieldStats(), sameInstance(stats));
+        shard.refresh("test");
+        stats = shard.getShardFieldStats();
+        assertThat(stats.numSegments(), equalTo(1));
+        // _id, _source, _version, _primary_term, _seq_no, f1, f1.keyword, f2, f2.keyword,
+        assertThat(stats.totalFields(), equalTo(9));
+        // don't re-compute on refresh without change
+        if (randomBoolean()) {
+            shard.refresh("test");
+        } else {
+            // trigger internal refresh
+            shard.newChangesSnapshot("test", 0, Long.MAX_VALUE, false, randomBoolean(), randomBoolean()).close();
+        }
+        assertThat(shard.getShardFieldStats(), sameInstance(stats));
+        // index more docs
+        numDocs = between(1, 10);
+        for (int i = 0; i < numDocs; i++) {
+            indexDoc(shard, "_doc", "first_" + i, """
+                {
+                    "f1": "foo",
+                    "f2": "bar",
+                    "f3": "foobar"
+                }
+                """);
+        }
+        if (randomBoolean()) {
+            shard.refresh("test");
+        } else {
+            // trigger internal refresh
+            shard.newChangesSnapshot("test", 0, Long.MAX_VALUE, false, randomBoolean(), randomBoolean()).close();
+        }
+        stats = shard.getShardFieldStats();
+        assertThat(stats.numSegments(), equalTo(2));
+        // 9 + _id, _source, _version, _primary_term, _seq_no, f1, f1.keyword, f2, f2.keyword, f3, f3.keyword
+        assertThat(stats.totalFields(), equalTo(21));
+        shard.forceMerge(new ForceMergeRequest().maxNumSegments(1).flush(true));
+        stats = shard.getShardFieldStats();
+        assertThat(stats.numSegments(), equalTo(1));
+        assertThat(stats.totalFields(), equalTo(12));
+        closeShards(shard);
+    }
+
     public void testIndexingOperationsListeners() throws IOException {
         IndexShard shard = newStartedShard(true);
         indexDoc(shard, "_doc", "0", "{\"foo\" : \"bar\"}");
diff --git a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index 2b333277e2d4a..55bfc2480d4d6 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.index.mapper.LuceneDocument;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.ParsedDocument.DocumentSize;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.seqno.RetentionLeases;
@@ -52,7 +53,6 @@
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -567,7 +567,7 @@ private Engine.IndexResult index(String id, String testFieldValue) throws IOExce
             source,
             XContentType.JSON,
             null,
-            DocumentSizeObserver.EMPTY_INSTANCE
+            DocumentSize.UNKNOWN
         );
         Engine.Index index = new Engine.Index(uid, engine.config().getPrimaryTermSupplier().getAsLong(), doc);
         return engine.index(index);
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index 8a277e400ad6c..c0de7947dbb60 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -55,6 +55,7 @@
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.LuceneDocument;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.ParsedDocument.DocumentSize;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.seqno.LocalCheckpointTracker;
@@ -62,7 +63,6 @@
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog.Location;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.TransportVersionUtils;
@@ -3394,7 +3394,7 @@ public void testTranslogOpSerialization() throws Exception {
             B_1,
             XContentType.JSON,
             null,
-            DocumentSizeObserver.EMPTY_INSTANCE
+            DocumentSize.UNKNOWN
         );
 
         Engine.Index eIndex = new Engine.Index(
diff --git a/server/src/test/java/org/elasticsearch/indices/ShardLimitValidatorTests.java b/server/src/test/java/org/elasticsearch/indices/ShardLimitValidatorTests.java
index e1b3bb4fe9c49..0eea536ddbff1 100644
--- a/server/src/test/java/org/elasticsearch/indices/ShardLimitValidatorTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/ShardLimitValidatorTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.shards.ShardCounts;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -79,7 +80,8 @@ private void testOverShardLimit(CheckShardLimitMethod targetMethod, String group
                 + maxShards
                 + "] maximum "
                 + group
-                + " shards open",
+                + " shards open; for more information, see "
+                + ReferenceDocs.MAX_SHARDS_PER_NODE,
             ShardLimitValidator.errorMessageFrom(shardLimitsResult)
         );
         assertEquals(shardLimitsResult.maxShardsInCluster(), maxShards);
@@ -151,7 +153,9 @@ public void testValidateShardLimitOpenIndices() {
                 + maxShards
                 + "] maximum "
                 + group
-                + " shards open;",
+                + " shards open; for more information, see "
+                + ReferenceDocs.MAX_SHARDS_PER_NODE
+                + ";",
             exception.getMessage()
         );
     }
@@ -179,7 +183,9 @@ public void testValidateShardLimitUpdateReplicas() {
                 + shardsPerNode * nodesInCluster
                 + "] maximum "
                 + group
-                + " shards open;",
+                + " shards open; for more information, see "
+                + ReferenceDocs.MAX_SHARDS_PER_NODE
+                + ";",
             exception.getMessage()
         );
     }
diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java
index 108ce6b63cec8..d805f8528897e 100644
--- a/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java
@@ -208,8 +208,18 @@ public void testManagerSkipsIndicesWithUpToDateMappings() {
         );
     }
 
-    // TODO[wrb]: add test where we have the old mappings version but not the new one
-    // Is this where we "placeholder" a "distant future" version string?
+    /**
+     * Check that the manager will try to upgrade indices when we have the old mappings version but not the new one
+     */
+    public void testManagerProcessesIndicesWithOldMappingsVersion() {
+        assertThat(
+            SystemIndexMappingUpdateService.getUpgradeStatus(
+                markShardsAvailable(createClusterState(Strings.toString(getMappings("1.0.0", null)))),
+                DESCRIPTOR
+            ),
+            equalTo(UpgradeStatus.NEEDS_MAPPINGS_UPDATE)
+        );
+    }
 
     /**
      * Check that the manager will try to upgrade indices where their mappings are out-of-date.
diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerTelemetryTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerTelemetryTests.java
deleted file mode 100644
index 2cbe1202520df..0000000000000
--- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerTelemetryTests.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.indices.breaker;
-
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.breaker.CircuitBreakingException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.PluginsService;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.telemetry.Measurement;
-import org.elasticsearch.telemetry.RecordingInstruments;
-import org.elasticsearch.telemetry.RecordingMeterRegistry;
-import org.elasticsearch.telemetry.TestTelemetryPlugin;
-import org.elasticsearch.telemetry.metric.LongCounter;
-import org.elasticsearch.telemetry.metric.MeterRegistry;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.hamcrest.Matchers;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.stream.Stream;
-
-import static org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.CIRCUIT_BREAKER_TYPE_ATTRIBUTE;
-import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING;
-import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING;
-import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING;
-import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING;
-import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING;
-import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING;
-import static org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING;
-
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, supportsDedicatedMasters = true)
-public class HierarchyCircuitBreakerTelemetryTests extends ESIntegTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return List.of(TestCircuitBreakerTelemetryPlugin.class);
-    }
-
-    public static class TestCircuitBreakerTelemetryPlugin extends TestTelemetryPlugin {
-        protected final MeterRegistry meter = new RecordingMeterRegistry() {
-            private final LongCounter tripCount = new RecordingInstruments.RecordingLongCounter(
-                CircuitBreakerMetrics.ES_BREAKER_TRIP_COUNT_TOTAL,
-                recorder
-            ) {
-                @Override
-                public void incrementBy(long inc) {
-                    throw new UnsupportedOperationException();
-                }
-
-                @Override
-                public void incrementBy(long inc, Map<String, Object> attributes) {
-                    throw new UnsupportedOperationException();
-                }
-            };
-
-            @Override
-            protected LongCounter buildLongCounter(String name, String description, String unit) {
-                if (name.equals(tripCount.getName())) {
-                    return tripCount;
-                }
-                throw new IllegalArgumentException("Unknown counter metric name [" + name + "]");
-            }
-
-            @Override
-            public LongCounter registerLongCounter(String name, String description, String unit) {
-                assertCircuitBreakerName(name);
-                return super.registerLongCounter(name, description, unit);
-            }
-
-            @Override
-            public LongCounter getLongCounter(String name) {
-                assertCircuitBreakerName(name);
-                return super.getLongCounter(name);
-            }
-
-            private void assertCircuitBreakerName(final String name) {
-                assertThat(name, Matchers.oneOf(CircuitBreakerMetrics.ES_BREAKER_TRIP_COUNT_TOTAL));
-            }
-        };
-    }
-
-    public void testCircuitBreakerTripCountMetric() {
-        final Settings circuitBreakerSettings = Settings.builder()
-            .put(FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), 100, ByteSizeUnit.BYTES)
-            .put(FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0)
-            .put(REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), 100, ByteSizeUnit.BYTES)
-            .put(REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0)
-            .put(IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), 100, ByteSizeUnit.BYTES)
-            .put(IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0)
-            .put(TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), 150, ByteSizeUnit.BYTES)
-            .put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), false)
-            .build();
-        String dataNodeName = null;
-        String masterNodeName = null;
-        try {
-            // NOTE: we start with empty circuitBreakerSettings to allow cluster formation
-            masterNodeName = internalCluster().startMasterOnlyNode(Settings.EMPTY);
-            dataNodeName = internalCluster().startDataOnlyNode(Settings.EMPTY);
-            assertTrue(clusterAdmin().prepareUpdateSettings().setPersistentSettings(circuitBreakerSettings).get().isAcknowledged());
-            assertTrue(
-                client().admin()
-                    .indices()
-                    .prepareCreate("test")
-                    .setSettings(
-                        Settings.builder()
-                            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                            .build()
-                    )
-                    .get()
-                    .isAcknowledged()
-            );
-            assertEquals(
-                RestStatus.OK.getStatus(),
-                client().prepareIndex("test").setWaitForActiveShards(1).setSource("field", "value").get().status().getStatus()
-            );
-        } catch (CircuitBreakingException cbex) {
-            final List<Measurement> dataNodeMeasurements = getMeasurements(dataNodeName);
-            final List<Measurement> masterNodeMeasurements = getMeasurements(masterNodeName);
-            final List<Measurement> allMeasurements = Stream.concat(dataNodeMeasurements.stream(), masterNodeMeasurements.stream())
-                .toList();
-            assertThat(allMeasurements, Matchers.not(Matchers.empty()));
-            final Measurement measurement = allMeasurements.get(0);
-            assertThat(1L, Matchers.equalTo(measurement.getLong()));
-            assertThat(1L, Matchers.equalTo(measurement.value()));
-            assertThat(Map.of(CIRCUIT_BREAKER_TYPE_ATTRIBUTE, "inflight_requests"), Matchers.equalTo(measurement.attributes()));
-            assertThat(true, Matchers.equalTo(measurement.isLong()));
-            return;
-        }
-        fail("Expected exception not thrown");
-    }
-
-    private List<Measurement> getMeasurements(String dataNodeName) {
-        final TestTelemetryPlugin dataNodeTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName)
-            .filterPlugins(TestCircuitBreakerTelemetryPlugin.class)
-            .toList()
-            .get(0);
-        return Measurement.combine(
-            Stream.of(dataNodeTelemetryPlugin.getLongCounterMeasurement(CircuitBreakerMetrics.ES_BREAKER_TRIP_COUNT_TOTAL).stream())
-                .flatMap(Function.identity())
-                .toList()
-        );
-    }
-
-    // Make sure circuit breaker telemetry on trip count reports the same values as circuit breaker stats
-    private void assertCircuitBreakerTripCount(
-        final HierarchyCircuitBreakerService circuitBreakerService,
-        final String circuitBreakerName,
-        int firstBytesEstimate,
-        int secondBytesEstimate,
-        long expectedTripCountValue
-    ) {
-        try {
-            circuitBreakerService.getBreaker(circuitBreakerName).addEstimateBytesAndMaybeBreak(firstBytesEstimate, randomAlphaOfLength(5));
-            circuitBreakerService.getBreaker(circuitBreakerName).addEstimateBytesAndMaybeBreak(secondBytesEstimate, randomAlphaOfLength(5));
-        } catch (final CircuitBreakingException cbex) {
-            final CircuitBreakerStats circuitBreakerStats = Arrays.stream(circuitBreakerService.stats().getAllStats())
-                .filter(stats -> circuitBreakerName.equals(stats.getName()))
-                .findAny()
-                .get();
-            assertThat(circuitBreakerService.getBreaker(circuitBreakerName).getTrippedCount(), Matchers.equalTo(expectedTripCountValue));
-            assertThat(circuitBreakerStats.getTrippedCount(), Matchers.equalTo(expectedTripCountValue));
-        }
-    }
-
-}
diff --git a/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java b/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java
index 06a80eb55128b..fd244464922b6 100644
--- a/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java
+++ b/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java
@@ -23,12 +23,12 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.Task;
diff --git a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java
index 3a6de10b5901d..546b252615b28 100644
--- a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.IngestConditionalScript;
 import org.elasticsearch.script.MockScriptEngine;
@@ -25,7 +26,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
@@ -242,14 +242,14 @@ public boolean execute(Map<String, Object> ctx) {
 
     private static void assertMutatingCtxThrows(Consumer<Map<String, Object>> mutation) throws Exception {
         String scriptName = "conditionalScript";
-        CompletableFuture<Exception> expectedException = new CompletableFuture<>();
+        PlainActionFuture<Exception> expectedException = new PlainActionFuture<>();
         ScriptService scriptService = new ScriptService(
             Settings.builder().build(),
             Map.of(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Map.of(scriptName, ctx -> {
                 try {
                     mutation.accept(ctx);
                 } catch (Exception e) {
-                    expectedException.complete(e);
+                    expectedException.onResponse(e);
                 }
                 return false;
             }), Map.of())),
@@ -267,7 +267,7 @@ private static void assertMutatingCtxThrows(Consumer<Map<String, Object>> mutati
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         ingestDocument.setFieldValue("listField", new ArrayList<>());
         execProcessor(processor, ingestDocument, (result, e) -> {});
-        Exception e = expectedException.get();
+        Exception e = safeGet(expectedException);
         assertThat(e, instanceOf(UnsupportedOperationException.class));
         assertEquals("Mutating ingest documents in conditionals is not supported", e.getMessage());
         assertStats(processor, 0, 0, 0);
diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
index 5621ed468f557..5c07c2344cf13 100644
--- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.FailureStoreMetrics;
 import org.elasticsearch.action.bulk.TransportBulkAction;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
@@ -50,9 +51,10 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.internal.DocumentParsingProvider;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
+import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptModule;
@@ -87,9 +89,9 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.function.IntConsumer;
 import java.util.function.LongSupplier;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils.executeAndAssertSuccessful;
@@ -151,7 +153,8 @@ public void testIngestPlugin() {
             List.of(DUMMY_PLUGIN),
             client,
             null,
-            DocumentParsingProvider.EMPTY_INSTANCE
+            DocumentParsingProvider.EMPTY_INSTANCE,
+            FailureStoreMetrics.NOOP
         );
         Map<String, Processor.Factory> factories = ingestService.getProcessorFactories();
         assertTrue(factories.containsKey("foo"));
@@ -171,7 +174,8 @@ public void testIngestPluginDuplicate() {
                 List.of(DUMMY_PLUGIN, DUMMY_PLUGIN),
                 client,
                 null,
-                DocumentParsingProvider.EMPTY_INSTANCE
+                DocumentParsingProvider.EMPTY_INSTANCE,
+                FailureStoreMetrics.NOOP
             )
         );
         assertTrue(e.getMessage(), e.getMessage().contains("already registered"));
@@ -188,7 +192,8 @@ public void testExecuteIndexPipelineDoesNotExist() {
             List.of(DUMMY_PLUGIN),
             client,
             null,
-            DocumentParsingProvider.EMPTY_INSTANCE
+            DocumentParsingProvider.EMPTY_INSTANCE,
+            FailureStoreMetrics.NOOP
         );
         final IndexRequest indexRequest = new IndexRequest("_index").id("_id")
             .source(Map.of())
@@ -1175,20 +1180,19 @@ public void testExecuteBulkRequestCallsDocumentSizeObserver() {
         AtomicInteger parsedValueWasUsed = new AtomicInteger(0);
         DocumentParsingProvider documentParsingProvider = new DocumentParsingProvider() {
             @Override
-            public <T> DocumentSizeObserver newDocumentSizeObserver(DocWriteRequest<T> request) {
-                return new DocumentSizeObserver() {
+            public <T> XContentMeteringParserDecorator newMeteringParserDecorator(DocWriteRequest<T> request) {
+                return new XContentMeteringParserDecorator() {
                     @Override
-                    public XContentParser wrapParser(XContentParser xContentParser) {
-                        wrappedObserverWasUsed.incrementAndGet();
-                        return xContentParser;
+                    public ParsedDocument.DocumentSize meteredDocumentSize() {
+                        parsedValueWasUsed.incrementAndGet();
+                        return new ParsedDocument.DocumentSize(0, 0);
                     }
 
                     @Override
-                    public long normalisedBytesParsed() {
-                        parsedValueWasUsed.incrementAndGet();
-                        return 0;
+                    public XContentParser decorate(XContentParser xContentParser) {
+                        wrappedObserverWasUsed.incrementAndGet();
+                        return xContentParser;
                     }
-
                 };
             }
         };
@@ -1665,7 +1669,7 @@ public void testExecuteFailureRedirection() throws Exception {
             .setFinalPipeline("_id2");
         doThrow(new RuntimeException()).when(processor)
             .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any());
-        final Predicate<String> redirectCheck = (idx) -> indexRequest.index().equals(idx);
+        final Function<String, Boolean> redirectCheck = (idx) -> indexRequest.index().equals(idx);
         @SuppressWarnings("unchecked")
         final TriConsumer<Integer, String, Exception> redirectHandler = mock(TriConsumer.class);
         @SuppressWarnings("unchecked")
@@ -1722,7 +1726,7 @@ public void testExecuteFailureRedirectionWithNestedOnFailure() throws Exception
             .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any());
         doThrow(new RuntimeException()).when(processor)
             .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any());
-        final Predicate<String> redirectPredicate = (idx) -> indexRequest.index().equals(idx);
+        final Function<String, Boolean> redirectCheck = (idx) -> indexRequest.index().equals(idx);
         @SuppressWarnings("unchecked")
         final TriConsumer<Integer, String, Exception> redirectHandler = mock(TriConsumer.class);
         @SuppressWarnings("unchecked")
@@ -1733,7 +1737,7 @@ public void testExecuteFailureRedirectionWithNestedOnFailure() throws Exception
             1,
             List.of(indexRequest),
             indexReq -> {},
-            redirectPredicate,
+            redirectCheck,
             redirectHandler,
             failureHandler,
             completionHandler,
@@ -1826,9 +1830,9 @@ public void testBulkRequestExecution() throws Exception {
         for (int i = 0; i < numRequest; i++) {
             IndexRequest indexRequest = new IndexRequest("_index").id("_id").setPipeline(pipelineId).setFinalPipeline("_none");
             indexRequest.source(xContentType, "field1", "value1");
-            boolean shouldListExecutedPipelines = randomBoolean();
-            executedPipelinesExpected.add(shouldListExecutedPipelines);
-            indexRequest.setListExecutedPipelines(shouldListExecutedPipelines);
+            boolean shouldListExecutedPiplines = randomBoolean();
+            executedPipelinesExpected.add(shouldListExecutedPiplines);
+            indexRequest.setListExecutedPipelines(shouldListExecutedPiplines);
             bulkRequest.add(indexRequest);
         }
 
@@ -2320,7 +2324,8 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
             List.of(testPlugin),
             client,
             null,
-            DocumentParsingProvider.EMPTY_INSTANCE
+            DocumentParsingProvider.EMPTY_INSTANCE,
+            FailureStoreMetrics.NOOP
         );
         ingestService.addIngestClusterStateListener(ingestClusterStateListener);
 
@@ -2675,7 +2680,8 @@ private void testUpdatingPipeline(String pipelineString) throws Exception {
             List.of(DUMMY_PLUGIN),
             client,
             null,
-            DocumentParsingProvider.EMPTY_INSTANCE
+            DocumentParsingProvider.EMPTY_INSTANCE,
+            FailureStoreMetrics.NOOP
         );
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, clusterState));
 
@@ -2974,7 +2980,8 @@ public Map<String, Processor.Factory> getProcessors(final Processor.Parameters p
             }),
             client,
             null,
-            documentParsingProvider
+            documentParsingProvider,
+            FailureStoreMetrics.NOOP
         );
         if (randomBoolean()) {
             /*
diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java
index cfbdbc3792082..d9058e83acfe0 100644
--- a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.test.ESTestCase;
@@ -16,7 +17,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.function.LongSupplier;
 
@@ -32,12 +32,12 @@ public class PipelineProcessorTests extends ESTestCase {
     public void testExecutesPipeline() throws Exception {
         String pipelineId = "pipeline";
         IngestService ingestService = createIngestService();
-        CompletableFuture<IngestDocument> invoked = new CompletableFuture<>();
+        PlainActionFuture<IngestDocument> invoked = new PlainActionFuture<>();
         IngestDocument testIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         Pipeline pipeline = new Pipeline(pipelineId, null, null, null, new CompoundProcessor(new Processor() {
             @Override
-            public IngestDocument execute(final IngestDocument ingestDocument) throws Exception {
-                invoked.complete(ingestDocument);
+            public IngestDocument execute(final IngestDocument ingestDocument) {
+                invoked.onResponse(ingestDocument);
                 return ingestDocument;
             }
 
@@ -61,7 +61,7 @@ public String getDescription() {
         Map<String, Object> config = new HashMap<>();
         config.put("name", pipelineId);
         factory.create(Map.of(), null, null, config).execute(testIngestDocument, (result, e) -> {});
-        assertIngestDocument(testIngestDocument, invoked.get());
+        assertIngestDocument(testIngestDocument, safeGet(invoked));
     }
 
     public void testThrowsOnMissingPipeline() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java
index 30145ab37c322..18f66676cfd1f 100644
--- a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.action.bulk.FailureStoreMetrics;
 import org.elasticsearch.action.bulk.SimulateBulkRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -115,11 +116,23 @@ private static IngestService createWithProcessors(Map<String, Processor.Factory>
         ThreadPool threadPool = mock(ThreadPool.class);
         when(threadPool.generic()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE);
         when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE);
-        return new IngestService(mock(ClusterService.class), threadPool, null, null, null, List.of(new IngestPlugin() {
+        var ingestPlugin = new IngestPlugin() {
             @Override
             public Map<String, Processor.Factory> getProcessors(final Processor.Parameters parameters) {
                 return processors;
             }
-        }), client, null, DocumentParsingProvider.EMPTY_INSTANCE);
+        };
+        return new IngestService(
+            mock(ClusterService.class),
+            threadPool,
+            null,
+            null,
+            null,
+            List.of(ingestPlugin),
+            client,
+            null,
+            DocumentParsingProvider.EMPTY_INSTANCE,
+            FailureStoreMetrics.NOOP
+        );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/injection/InjectorTests.java b/server/src/test/java/org/elasticsearch/injection/InjectorTests.java
new file mode 100644
index 0000000000000..025596e640896
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/injection/InjectorTests.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.injection;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+import java.util.Set;
+
+public class InjectorTests extends ESTestCase {
+
+    public record First() {}
+
+    public record Second(First first) {}
+
+    public record Third(First first, Second second) {}
+
+    public record ExistingInstances(First first, Second second) {}
+
+    public void testMultipleResultsMap() {
+        Injector injector = Injector.create().addClasses(List.of(Service1.class, Component3.class));
+        var resultMap = injector.inject(List.of(Service1.class, Component3.class));
+        assertEquals(Set.of(Service1.class, Component3.class), resultMap.keySet());
+        Service1 service1 = (Service1) resultMap.get(Service1.class);
+        Component3 component3 = (Component3) resultMap.get(Component3.class);
+        assertSame(service1, component3.service1());
+    }
+
+    /**
+     * In most cases, if there are two objects that are instances of a class, that's ambiguous.
+     * However, if a concrete (non-abstract) superclass is configured directly, that is not ambiguous:
+     * the instance of that superclass takes precedence over any instances of any subclasses.
+     */
+    public void testConcreteSubclass() {
+        MethodHandles.lookup();
+        assertEquals(
+            Superclass.class,
+            Injector.create()
+                .addClasses(List.of(Superclass.class, Subclass.class)) // Superclass first
+                .inject(List.of(Superclass.class))
+                .get(Superclass.class)
+                .getClass()
+        );
+        MethodHandles.lookup();
+        assertEquals(
+            Superclass.class,
+            Injector.create()
+                .addClasses(List.of(Subclass.class, Superclass.class)) // Subclass first
+                .inject(List.of(Superclass.class))
+                .get(Superclass.class)
+                .getClass()
+        );
+        MethodHandles.lookup();
+        assertEquals(
+            Superclass.class,
+            Injector.create()
+                .addClasses(List.of(Subclass.class))
+                .inject(List.of(Superclass.class)) // Superclass is not mentioned until here
+                .get(Superclass.class)
+                .getClass()
+        );
+    }
+
+    //
+    // Sad paths
+    //
+
+    public void testBadInterfaceClass() {
+        assertThrows(IllegalStateException.class, () -> {
+            MethodHandles.lookup();
+            Injector.create().addClass(Listener.class).inject(List.of());
+        });
+    }
+
+    public void testBadUnknownType() {
+        // Injector knows only about Component4, discovers Listener, but can't find any subtypes
+        MethodHandles.lookup();
+        Injector injector = Injector.create().addClass(Component4.class);
+
+        assertThrows(IllegalStateException.class, () -> injector.inject(List.of()));
+    }
+
+    public void testBadCircularDependency() {
+        assertThrows(IllegalStateException.class, () -> {
+            MethodHandles.lookup();
+            Injector injector = Injector.create();
+            injector.addClasses(List.of(Circular1.class, Circular2.class)).inject(List.of());
+        });
+    }
+
+    /**
+     * For this one, we don't explicitly tell the injector about the classes involved in the cycle;
+     * it finds them on its own.
+     */
+    public void testBadCircularDependencyViaParameter() {
+        record UsesCircular1(Circular1 circular1) {}
+        assertThrows(IllegalStateException.class, () -> {
+            MethodHandles.lookup();
+            Injector.create().addClass(UsesCircular1.class).inject(List.of());
+        });
+    }
+
+    public void testBadCircularDependencyViaSupertype() {
+        interface Service1 {}
+        record Service2(Service1 service1) {}
+        record Service3(Service2 service2) implements Service1 {}
+        assertThrows(IllegalStateException.class, () -> {
+            MethodHandles.lookup();
+            Injector injector = Injector.create();
+            injector.addClasses(List.of(Service2.class, Service3.class)).inject(List.of());
+        });
+    }
+
+    // Common injectable things
+
+    public record Service1() {}
+
+    public interface Listener {}
+
+    public record Component1() implements Listener {}
+
+    public record Component2(Component1 component1) {}
+
+    public record Component3(Service1 service1) {}
+
+    public record Component4(Listener listener) {}
+
+    public record GoodService(List<Component1> components) {}
+
+    public record BadService(List<Component1> components) {
+        public BadService {
+            // Shouldn't be using the component list here!
+            assert components.isEmpty() == false;
+        }
+    }
+
+    public record MultiService(List<Component1> component1s, List<Component2> component2s) {}
+
+    public record Circular1(Circular2 service2) {}
+
+    public record Circular2(Circular1 service2) {}
+
+    public static class Superclass {}
+
+    public static class Subclass extends Superclass {}
+
+}
diff --git a/server/src/test/java/org/elasticsearch/lucene/spatial/CartesianPointDocValuesQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/spatial/CartesianPointDocValuesQueryTests.java
new file mode 100644
index 0000000000000..ff9074dba52eb
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/lucene/spatial/CartesianPointDocValuesQueryTests.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.lucene.spatial;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.XYDocValuesField;
+import org.apache.lucene.document.XYPointField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.SerialMergeScheduler;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.search.CheckHits;
+import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.geo.ShapeTestUtils;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class CartesianPointDocValuesQueryTests extends ESTestCase {
+
+    private static final String FIELD_NAME = "field";
+
+    public void testIndexSimpleShapes() throws Exception {
+        IndexWriterConfig iwc = newIndexWriterConfig();
+        // Else seeds may not reproduce:
+        iwc.setMergeScheduler(new SerialMergeScheduler());
+        // Else we can get O(N^2) merging:
+        iwc.setMaxBufferedDocs(10);
+        Directory dir = newDirectory();
+        // RandomIndexWriter is too slow here:
+        IndexWriter w = new IndexWriter(dir, iwc);
+        final int numDocs = randomIntBetween(10, 1000);
+        for (int id = 0; id < numDocs; id++) {
+            Document doc = new Document();
+            Point point = ShapeTestUtils.randomPoint();
+            doc.add(new XYPointField(FIELD_NAME, (float) point.getX(), (float) point.getY()));
+            doc.add(new XYDocValuesField(FIELD_NAME, (float) point.getX(), (float) point.getY()));
+            w.addDocument(doc);
+        }
+
+        if (random().nextBoolean()) {
+            w.forceMerge(1);
+        }
+        final IndexReader r = DirectoryReader.open(w);
+        w.close();
+
+        IndexSearcher s = newSearcher(r);
+        for (int i = 0; i < 25; i++) {
+            Geometry geometry = ShapeTestUtils.randomGeometry(false);
+            for (ShapeRelation relation : ShapeRelation.values()) {
+                Query indexQuery = XYQueriesUtils.toXYPointQuery(geometry, FIELD_NAME, relation, true, false);
+                Query docValQuery = XYQueriesUtils.toXYPointQuery(geometry, FIELD_NAME, relation, false, true);
+                assertQueries(s, indexQuery, docValQuery, numDocs);
+            }
+        }
+        IOUtils.close(r, dir);
+    }
+
+    public void testIndexMultiShapes() throws Exception {
+        IndexWriterConfig iwc = newIndexWriterConfig();
+        // Else seeds may not reproduce:
+        iwc.setMergeScheduler(new SerialMergeScheduler());
+        // Else we can get O(N^2) merging:
+        iwc.setMaxBufferedDocs(10);
+        Directory dir = newDirectory();
+        // RandomIndexWriter is too slow here:
+        IndexWriter w = new IndexWriter(dir, iwc);
+        final int numDocs = randomIntBetween(10, 100);
+        CartesianShapeIndexer indexer = new CartesianShapeIndexer(FIELD_NAME);
+        for (int id = 0; id < numDocs; id++) {
+            Document doc = new Document();
+            for (int i = 0; i < randomIntBetween(1, 5); i++) {
+                Point point = ShapeTestUtils.randomPoint();
+                doc.add(new XYPointField(FIELD_NAME, (float) point.getX(), (float) point.getY()));
+                doc.add(new XYDocValuesField(FIELD_NAME, (float) point.getX(), (float) point.getY()));
+                w.addDocument(doc);
+            }
+            w.addDocument(doc);
+        }
+
+        if (random().nextBoolean()) {
+            w.forceMerge(1);
+        }
+        final IndexReader r = DirectoryReader.open(w);
+        w.close();
+
+        IndexSearcher s = newSearcher(r);
+        for (int i = 0; i < 25; i++) {
+            Geometry geometry = ShapeTestUtils.randomGeometry(false);
+            for (ShapeRelation relation : ShapeRelation.values()) {
+                Query indexQuery = XYQueriesUtils.toXYPointQuery(geometry, FIELD_NAME, relation, true, false);
+                Query docValQuery = XYQueriesUtils.toXYPointQuery(geometry, FIELD_NAME, relation, false, true);
+                assertQueries(s, indexQuery, docValQuery, numDocs);
+            }
+        }
+        IOUtils.close(r, dir);
+    }
+
+    private void assertQueries(IndexSearcher s, Query indexQuery, Query docValQuery, int numDocs) throws IOException {
+        assertEquals(s.count(indexQuery), s.count(docValQuery));
+        CheckHits.checkEqual(docValQuery, s.search(indexQuery, numDocs).scoreDocs, s.search(docValQuery, numDocs).scoreDocs);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/lucene/spatial/CartesianShapeDocValuesQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/spatial/CartesianShapeDocValuesQueryTests.java
index 4ce3d87d6420d..e98b9016cca1c 100644
--- a/server/src/test/java/org/elasticsearch/lucene/spatial/CartesianShapeDocValuesQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/lucene/spatial/CartesianShapeDocValuesQueryTests.java
@@ -11,7 +11,6 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.ShapeField;
 import org.apache.lucene.document.XYShape;
-import org.apache.lucene.geo.XYGeometry;
 import org.apache.lucene.geo.XYPolygon;
 import org.apache.lucene.geo.XYRectangle;
 import org.apache.lucene.index.DirectoryReader;
@@ -19,6 +18,7 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.NoMergeScheduler;
 import org.apache.lucene.index.SerialMergeScheduler;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
@@ -26,10 +26,12 @@
 import org.apache.lucene.tests.search.CheckHits;
 import org.apache.lucene.tests.search.QueryUtils;
 import org.elasticsearch.common.geo.LuceneGeometriesUtils;
+import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.geo.ShapeTestUtils;
 import org.elasticsearch.geo.XShapeTestUtil;
 import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.index.mapper.ShapeIndexer;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -54,6 +56,40 @@ public void testEqualsAndHashcode() {
         QueryUtils.checkUnequal(q1, q4);
     }
 
+    public void testEmptySegment() throws Exception {
+        IndexWriterConfig iwc = newIndexWriterConfig();
+        // No merges
+        iwc.setMergeScheduler(NoMergeScheduler.INSTANCE);
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, iwc);
+        ShapeIndexer indexer = new CartesianShapeIndexer(FIELD_NAME);
+        Geometry geometry = new org.elasticsearch.geometry.Point(0, 0);
+        Document document = new Document();
+        List<IndexableField> fields = indexer.indexShape(geometry);
+        for (IndexableField field : fields) {
+            document.add(field);
+        }
+        BinaryShapeDocValuesField docVal = new BinaryShapeDocValuesField(FIELD_NAME, CoordinateEncoder.CARTESIAN);
+        docVal.add(fields, geometry);
+        document.add(docVal);
+        w.addDocument(document);
+        w.flush();
+        // add empty segment
+        w.addDocument(new Document());
+        w.flush();
+        final IndexReader r = DirectoryReader.open(w);
+        w.close();
+
+        IndexSearcher s = newSearcher(r);
+        XYRectangle rectangle = new XYRectangle(-10, 10, -10, 10);
+        for (ShapeField.QueryRelation relation : ShapeField.QueryRelation.values()) {
+            Query indexQuery = XYShape.newGeometryQuery(FIELD_NAME, relation, rectangle);
+            Query docValQuery = new CartesianShapeDocValuesQuery(FIELD_NAME, relation, rectangle);
+            assertQueries(s, indexQuery, docValQuery, 1);
+        }
+        IOUtils.close(r, dir);
+    }
+
     public void testIndexSimpleShapes() throws Exception {
         IndexWriterConfig iwc = newIndexWriterConfig();
         // Else seeds may not reproduce:
@@ -92,10 +128,10 @@ public void testIndexSimpleShapes() throws Exception {
 
         IndexSearcher s = newSearcher(r);
         for (int i = 0; i < 25; i++) {
-            XYGeometry[] geometries = randomLuceneQueryGeometries();
-            for (ShapeField.QueryRelation relation : ShapeField.QueryRelation.values()) {
-                Query indexQuery = XYShape.newGeometryQuery(FIELD_NAME, relation, geometries);
-                Query docValQuery = new CartesianShapeDocValuesQuery(FIELD_NAME, relation, geometries);
+            Geometry geometry = ShapeTestUtils.randomGeometry(false);
+            for (ShapeRelation relation : ShapeRelation.values()) {
+                Query indexQuery = XYQueriesUtils.toXYShapeQuery(geometry, FIELD_NAME, relation, true, false);
+                Query docValQuery = XYQueriesUtils.toXYShapeQuery(geometry, FIELD_NAME, relation, false, true);
                 assertQueries(s, indexQuery, docValQuery, numDocs);
             }
         }
@@ -134,10 +170,10 @@ public void testIndexMultiShapes() throws Exception {
 
         IndexSearcher s = newSearcher(r);
         for (int i = 0; i < 25; i++) {
-            XYGeometry[] geometries = randomLuceneQueryGeometries();
-            for (ShapeField.QueryRelation relation : ShapeField.QueryRelation.values()) {
-                Query indexQuery = XYShape.newGeometryQuery(FIELD_NAME, relation, geometries);
-                Query docValQuery = new CartesianShapeDocValuesQuery(FIELD_NAME, relation, geometries);
+            Geometry geometry = ShapeTestUtils.randomGeometry(false);
+            for (ShapeRelation relation : ShapeRelation.values()) {
+                Query indexQuery = XYQueriesUtils.toXYShapeQuery(geometry, FIELD_NAME, relation, true, false);
+                Query docValQuery = XYQueriesUtils.toXYShapeQuery(geometry, FIELD_NAME, relation, false, true);
                 assertQueries(s, indexQuery, docValQuery, numDocs);
             }
         }
@@ -148,22 +184,4 @@ private void assertQueries(IndexSearcher s, Query indexQuery, Query docValQuery,
         assertEquals(s.count(indexQuery), s.count(docValQuery));
         CheckHits.checkEqual(docValQuery, s.search(indexQuery, numDocs).scoreDocs, s.search(docValQuery, numDocs).scoreDocs);
     }
-
-    private XYGeometry[] randomLuceneQueryGeometries() {
-        int numGeom = randomIntBetween(1, 3);
-        XYGeometry[] geometries = new XYGeometry[numGeom];
-        for (int i = 0; i < numGeom; i++) {
-            geometries[i] = randomLuceneQueryGeometry();
-        }
-        return geometries;
-    }
-
-    private XYGeometry randomLuceneQueryGeometry() {
-        return switch (randomInt(3)) {
-            case 0 -> LuceneGeometriesUtils.toXYPolygon(ShapeTestUtils.randomPolygon(false));
-            case 1 -> LuceneGeometriesUtils.toXYCircle(ShapeTestUtils.randomCircle(false));
-            case 2 -> LuceneGeometriesUtils.toXYPoint(ShapeTestUtils.randomPoint(false));
-            default -> XShapeTestUtil.nextBox();
-        };
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/lucene/spatial/LatLonShapeDocValuesQueryTests.java b/server/src/test/java/org/elasticsearch/lucene/spatial/LatLonShapeDocValuesQueryTests.java
index 99fab30e3ade2..e00b7fa4736b3 100644
--- a/server/src/test/java/org/elasticsearch/lucene/spatial/LatLonShapeDocValuesQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/lucene/spatial/LatLonShapeDocValuesQueryTests.java
@@ -20,6 +20,7 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.NoMergeScheduler;
 import org.apache.lucene.index.SerialMergeScheduler;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
@@ -56,6 +57,40 @@ public void testEqualsAndHashcode() {
         QueryUtils.checkUnequal(q1, q4);
     }
 
+    public void testEmptySegment() throws Exception {
+        IndexWriterConfig iwc = newIndexWriterConfig();
+        // No merges
+        iwc.setMergeScheduler(NoMergeScheduler.INSTANCE);
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, iwc);
+        GeoShapeIndexer indexer = new GeoShapeIndexer(Orientation.CCW, FIELD_NAME);
+        Geometry geometry = new org.elasticsearch.geometry.Point(0, 0);
+        Document document = new Document();
+        List<IndexableField> fields = indexer.indexShape(geometry);
+        for (IndexableField field : fields) {
+            document.add(field);
+        }
+        BinaryShapeDocValuesField docVal = new BinaryShapeDocValuesField(FIELD_NAME, CoordinateEncoder.GEO);
+        docVal.add(fields, geometry);
+        document.add(docVal);
+        w.addDocument(document);
+        w.flush();
+        // add empty segment
+        w.addDocument(new Document());
+        w.flush();
+        final IndexReader r = DirectoryReader.open(w);
+        w.close();
+
+        IndexSearcher s = newSearcher(r);
+        Rectangle rectangle = new Rectangle(-10, 10, -10, 10);
+        for (ShapeField.QueryRelation relation : ShapeField.QueryRelation.values()) {
+            Query indexQuery = LatLonShape.newGeometryQuery(FIELD_NAME, relation, rectangle);
+            Query docValQuery = new LatLonShapeDocValuesQuery(FIELD_NAME, relation, rectangle);
+            assertQueries(s, indexQuery, docValQuery, 1);
+        }
+        IOUtils.close(r, dir);
+    }
+
     public void testIndexSimpleShapes() throws Exception {
         IndexWriterConfig iwc = newIndexWriterConfig();
         // Else seeds may not reproduce:
diff --git a/server/src/test/java/org/elasticsearch/node/NodeTests.java b/server/src/test/java/org/elasticsearch/node/NodeTests.java
index b36cafd694378..35a515e0521a1 100644
--- a/server/src/test/java/org/elasticsearch/node/NodeTests.java
+++ b/server/src/test/java/org/elasticsearch/node/NodeTests.java
@@ -641,7 +641,7 @@ public Optional<PersistedClusterStateServiceFactory> getPersistedClusterStateSer
                         nodeEnvironment,
                         namedXContentRegistry,
                         clusterSettings,
-                        threadPool::relativeTimeInMillis
+                        threadPool.relativeTimeInMillisSupplier()
                     )
             );
         }
diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
index bf436b235d93b..6406ee640380b 100644
--- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
+++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
@@ -30,12 +30,12 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.Lifecycle;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata.Assignment;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask;
 import org.elasticsearch.plugins.ActionPlugin;
diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
index 83cb189415f7e..59e0b955d1cff 100644
--- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
@@ -41,7 +41,6 @@
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository;
-import org.elasticsearch.snapshots.SnapshotDeleteListener;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.test.ClusterServiceUtils;
@@ -454,9 +453,10 @@ public void deleteSnapshots(
             Collection<SnapshotId> snapshotIds,
             long repositoryDataGeneration,
             IndexVersion minimumNodeVersion,
-            SnapshotDeleteListener listener
+            ActionListener<RepositoryData> repositoryDataUpdateListener,
+            Runnable onCompletion
         ) {
-            listener.onFailure(new UnsupportedOperationException());
+            repositoryDataUpdateListener.onFailure(new UnsupportedOperationException());
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunnerTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunnerTests.java
index 6bc6f6e0b4a99..6ac73fb043f93 100644
--- a/server/src/test/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunnerTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunnerTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.engine.Engine;
@@ -31,12 +32,14 @@
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.lessThan;
 
 public class ShardSnapshotTaskRunnerTests extends ESTestCase {
 
@@ -110,13 +113,17 @@ public static SnapshotShardContext dummyContext() {
     }
 
     public static SnapshotShardContext dummyContext(final SnapshotId snapshotId, final long startTime) {
-        IndexId indexId = new IndexId(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
-        ShardId shardId = new ShardId(indexId.getName(), indexId.getId(), 1);
-        IndexSettings indexSettings = new IndexSettings(
+        return dummyContext(snapshotId, startTime, randomIdentifier(), 1);
+    }
+
+    public static SnapshotShardContext dummyContext(final SnapshotId snapshotId, final long startTime, String indexName, int shardIndex) {
+        final var indexId = new IndexId(indexName, UUIDs.randomBase64UUID());
+        final var shardId = new ShardId(indexId.getName(), UUIDs.randomBase64UUID(), shardIndex);
+        final var indexSettings = new IndexSettings(
             IndexMetadata.builder(indexId.getName()).settings(indexSettings(IndexVersion.current(), 1, 0)).build(),
             Settings.EMPTY
         );
-        Store dummyStore = new Store(shardId, indexSettings, new ByteBuffersDirectory(), new DummyShardLock(shardId));
+        final var dummyStore = new Store(shardId, indexSettings, new ByteBuffersDirectory(), new DummyShardLock(shardId));
         return new SnapshotShardContext(
             dummyStore,
             null,
@@ -151,43 +158,78 @@ public void testShardSnapshotTaskRunner() throws Exception {
     }
 
     public void testCompareToShardSnapshotTask() {
-        ShardSnapshotTaskRunner workers = new ShardSnapshotTaskRunner(1, executor, context -> {}, (context, fileInfo) -> {});
-        SnapshotId s1 = new SnapshotId("s1", "s1-uuid");
-        SnapshotId s2 = new SnapshotId("s2", "s2-uuid");
-        SnapshotId s3 = new SnapshotId("s3", "s3-uuid");
-        ActionListener<Void> listener = ActionListener.noop();
-        final long s1StartTime = threadPool.absoluteTimeInMillis();
-        final long s2StartTime = s1StartTime + randomLongBetween(1, 1000);
-        SnapshotShardContext s1Context = dummyContext(s1, s1StartTime);
-        SnapshotShardContext s2Context = dummyContext(s2, s2StartTime);
-        SnapshotShardContext s3Context = dummyContext(s3, s2StartTime);
-        // Shard snapshot and file snapshot tasks for earlier snapshots have higher priority
-        assertThat(
-            workers.new ShardSnapshotTask(s1Context).compareTo(
-                workers.new FileSnapshotTask(s2Context, ShardSnapshotTaskRunnerTests::dummyFileInfo, listener)
-            ),
-            lessThan(0)
-        );
-        assertThat(
-            workers.new FileSnapshotTask(s1Context, ShardSnapshotTaskRunnerTests::dummyFileInfo, listener).compareTo(
-                workers.new ShardSnapshotTask(s2Context)
-            ),
-            lessThan(0)
-        );
-        // Two tasks with the same start time and of the same type are ordered by snapshot UUID
-        assertThat(workers.new ShardSnapshotTask(s2Context).compareTo(workers.new ShardSnapshotTask(s3Context)), lessThan(0));
-        assertThat(
-            workers.new FileSnapshotTask(s2Context, ShardSnapshotTaskRunnerTests::dummyFileInfo, listener).compareTo(
-                workers.new ShardSnapshotTask(s3Context)
-            ),
-            lessThan(0)
-        );
-        // Shard snapshot task has a higher priority over file snapshot within the same snapshot
-        assertThat(
-            workers.new ShardSnapshotTask(s1Context).compareTo(
-                workers.new FileSnapshotTask(s1Context, ShardSnapshotTaskRunnerTests::dummyFileInfo, listener)
-            ),
-            lessThan(0)
-        );
+
+        record CapturedTask(SnapshotShardContext context, @Nullable BlobStoreIndexShardSnapshot.FileInfo fileInfo) {
+            CapturedTask(SnapshotShardContext context) {
+                this(context, null);
+            }
+        }
+
+        final List<CapturedTask> tasksInExpectedOrder = new ArrayList<>();
+
+        // first snapshot, one shard, one file, but should execute the shard-level task before the file task
+        final var earlyStartTime = randomLongBetween(1L, Long.MAX_VALUE - 1000);
+        final var s1Context = dummyContext(new SnapshotId(randomIdentifier(), randomUUID()), earlyStartTime);
+        tasksInExpectedOrder.add(new CapturedTask(s1Context));
+        tasksInExpectedOrder.add(new CapturedTask(s1Context, dummyFileInfo()));
+
+        // second snapshot, also one shard and one file, starts later than the first
+        final var laterStartTime = randomLongBetween(earlyStartTime + 1, Long.MAX_VALUE);
+        final var s2Context = dummyContext(new SnapshotId(randomIdentifier(), "early-uuid"), laterStartTime);
+        tasksInExpectedOrder.add(new CapturedTask(s2Context));
+        tasksInExpectedOrder.add(new CapturedTask(s2Context, dummyFileInfo()));
+
+        // third snapshot, starts at the same time as the second but has a later UUID
+        final var snapshotId3 = new SnapshotId(randomIdentifier(), "later-uuid");
+
+        // the third snapshot has three shards, and their respective tasks should execute in shard-id then index-name order:
+        final var s3ContextShard1 = dummyContext(snapshotId3, laterStartTime, "early-index-name", 0);
+        final var s3ContextShard2 = dummyContext(snapshotId3, laterStartTime, "later-index-name", 0);
+        final var s3ContextShard3 = dummyContext(snapshotId3, laterStartTime, randomIdentifier(), 1);
+
+        tasksInExpectedOrder.add(new CapturedTask(s3ContextShard1));
+        tasksInExpectedOrder.add(new CapturedTask(s3ContextShard2));
+        tasksInExpectedOrder.add(new CapturedTask(s3ContextShard3));
+
+        tasksInExpectedOrder.add(new CapturedTask(s3ContextShard1, dummyFileInfo()));
+        tasksInExpectedOrder.add(new CapturedTask(s3ContextShard2, dummyFileInfo()));
+        tasksInExpectedOrder.add(new CapturedTask(s3ContextShard3, dummyFileInfo()));
+
+        final var readyLatch = new CountDownLatch(1);
+        final var startLatch = new CountDownLatch(1);
+        final var doneLatch = new CountDownLatch(tasksInExpectedOrder.size() + 1);
+
+        final List<CapturedTask> tasksInExecutionOrder = new ArrayList<>();
+        final var runner = new ShardSnapshotTaskRunner(1, executor, context -> {
+            tasksInExecutionOrder.add(new CapturedTask(context, null));
+            readyLatch.countDown();
+            safeAwait(startLatch);
+            doneLatch.countDown();
+        }, (context, fileInfo) -> {
+            tasksInExecutionOrder.add(new CapturedTask(context, fileInfo));
+            doneLatch.countDown();
+        });
+
+        // prime the pipeline by executing a dummy task and waiting for it to block the executor, so that the rest of the tasks are sorted
+        // by the underlying PriorityQueue before any of them start to execute
+        runner.enqueueShardSnapshot(dummyContext(new SnapshotId(randomIdentifier(), UUIDs.randomBase64UUID()), randomNonNegativeLong()));
+        safeAwait(readyLatch);
+        tasksInExecutionOrder.clear(); // remove the dummy task
+
+        // submit the tasks in random order
+        for (final var task : shuffledList(tasksInExpectedOrder)) {
+            if (task.fileInfo() == null) {
+                runner.enqueueShardSnapshot(task.context());
+            } else {
+                runner.enqueueFileSnapshot(task.context(), task::fileInfo, ActionListener.noop());
+            }
+        }
+
+        // allow the tasks to execute
+        startLatch.countDown();
+        safeAwait(doneLatch);
+
+        // finally verify that they executed in the order we expected
+        assertEquals(tasksInExpectedOrder, tasksInExecutionOrder);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java b/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
index 90d9f1bcac879..a1898082418e5 100644
--- a/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
@@ -155,17 +155,17 @@ public void testInvalidHeaderValueEmpty() {
         expectThrows(IllegalArgumentException.class, () -> DeprecationRestHandler.requireValidHeader(blank));
     }
 
-    public void testSupportsContentStreamTrue() {
-        when(handler.supportsContentStream()).thenReturn(true);
+    public void testSupportsBulkContentTrue() {
+        when(handler.supportsBulkContent()).thenReturn(true);
         assertTrue(
-            new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsContentStream()
+            new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsBulkContent()
         );
     }
 
-    public void testSupportsContentStreamFalse() {
-        when(handler.supportsContentStream()).thenReturn(false);
+    public void testSupportsBulkContentFalse() {
+        when(handler.supportsBulkContent()).thenReturn(false);
         assertFalse(
-            new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsContentStream()
+            new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsBulkContent()
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index 67f42e6cf1808..391b2a08021fd 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -598,7 +598,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c
             }
 
             @Override
-            public boolean supportsContentStream() {
+            public boolean supportsBulkContent() {
                 return true;
             }
         });
@@ -637,7 +637,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c
             }
 
             @Override
-            public boolean supportsContentStream() {
+            public boolean supportsBulkContent() {
                 return true;
             }
         });
@@ -662,7 +662,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c
             }
 
             @Override
-            public boolean supportsContentStream() {
+            public boolean supportsBulkContent() {
                 return true;
             }
         });
@@ -688,7 +688,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c
             }
 
             @Override
-            public boolean supportsContentStream() {
+            public boolean supportsBulkContent() {
                 return true;
             }
         });
@@ -713,7 +713,7 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c
             }
 
             @Override
-            public boolean supportsContentStream() {
+            public boolean supportsBulkContent() {
                 return true;
             }
         });
diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
index 59ab7ec719cf4..1270c23227756 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
@@ -74,7 +75,8 @@ public void stubValidateQueryAction() {
         final TransportAction<? extends ActionRequest, ? extends ActionResponse> transportAction = new TransportAction<>(
             ValidateQueryAction.NAME,
             new ActionFilters(Collections.emptySet()),
-            taskManager
+            taskManager,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
         ) {
             @Override
             protected void doExecute(Task task, ActionRequest request, ActionListener<ActionResponse> listener) {}
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 4a047e4f696b3..612039685821a 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -456,10 +456,10 @@ public void testGetStoredScript() throws Exception {
             )
             .build();
 
-        assertEquals("abc", ScriptService.getStoredScript(cs, new GetStoredScriptRequest("_id")).getSource());
+        assertEquals("abc", ScriptService.getStoredScript(cs, new GetStoredScriptRequest(TEST_REQUEST_TIMEOUT, "_id")).getSource());
 
         cs = ClusterState.builder(new ClusterName("_name")).build();
-        assertNull(ScriptService.getStoredScript(cs, new GetStoredScriptRequest("_id")));
+        assertNull(ScriptService.getStoredScript(cs, new GetStoredScriptRequest(TEST_REQUEST_TIMEOUT, "_id")));
     }
 
     public void testMaxSizeLimit() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java
new file mode 100644
index 0000000000000..d748ad0f1569d
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/script/ScriptTermStatsTests.java
@@ -0,0 +1,358 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matcher;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ScriptTermStatsTests extends ESTestCase {
+    public void testMatchedTermsCount() throws IOException {
+        // Returns number of matched terms for each doc.
+        assertAllDocs(
+            Set.of(new Term("field", "foo"), new Term("field", "bar")),
+            ScriptTermStats::matchedTermsCount,
+            Map.of("doc-1", equalTo(2), "doc-2", equalTo(2), "doc-3", equalTo(1))
+        );
+
+        // Partial match
+        assertAllDocs(
+            Set.of(new Term("field", "foo"), new Term("field", "baz")),
+            ScriptTermStats::matchedTermsCount,
+            Map.of("doc-1", equalTo(1), "doc-2", equalTo(1), "doc-3", equalTo(0))
+        );
+
+        // Always returns 0 when no term is provided.
+        assertAllDocs(
+            Set.of(),
+            ScriptTermStats::matchedTermsCount,
+            Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(0)))
+        );
+
+        // Always Returns 0 when none of the provided term has a match.
+        assertAllDocs(
+            randomTerms(),
+            ScriptTermStats::matchedTermsCount,
+            Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(0)))
+        );
+
+        // Always returns 0 when using a non-existing field
+        assertAllDocs(
+            Set.of(new Term("field-that-does-not-exists", "foo"), new Term("field-that-does-not-exists", "bar")),
+            ScriptTermStats::matchedTermsCount,
+            Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(0)))
+        );
+    }
+
+    public void testDocFreq() throws IOException {
+        // Single term
+        {
+            StatsSummary expected = new StatsSummary(1, 2, 2, 2);
+            assertAllDocs(
+                Set.of(new Term("field", "foo")),
+                ScriptTermStats::docFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+
+        // Multiple terms
+        {
+            StatsSummary expected = new StatsSummary(2, 5, 2, 3);
+            assertAllDocs(
+                Set.of(new Term("field", "foo"), new Term("field", "bar")),
+                ScriptTermStats::docFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+
+        // With missing terms
+        {
+            StatsSummary expected = new StatsSummary(2, 2, 0, 2);
+            assertAllDocs(
+                Set.of(new Term("field", "foo"), new Term("field", "baz")),
+                ScriptTermStats::docFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+
+        // When no term is provided.
+        {
+            StatsSummary expected = new StatsSummary();
+            assertAllDocs(
+                Set.of(),
+                ScriptTermStats::docFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+
+        // When using a non-existing field
+        {
+            StatsSummary expected = new StatsSummary(2, 0, 0, 0);
+            assertAllDocs(
+                Set.of(new Term("non-existing-field", "foo"), new Term("non-existing-field", "baz")),
+                ScriptTermStats::docFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+    }
+
+    public void testTotalTermFreq() throws IOException {
+        // Single term
+        {
+            StatsSummary expected = new StatsSummary(1, 3, 3, 3);
+            assertAllDocs(
+                Set.of(new Term("field", "foo")),
+                ScriptTermStats::totalTermFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+
+        // Multiple terms
+        {
+            StatsSummary expected = new StatsSummary(2, 6, 3, 3);
+            assertAllDocs(
+                Set.of(new Term("field", "foo"), new Term("field", "bar")),
+                ScriptTermStats::totalTermFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+
+        // With missing terms
+        {
+            StatsSummary expected = new StatsSummary(2, 3, 0, 3);
+            assertAllDocs(
+                Set.of(new Term("field", "foo"), new Term("field", "baz")),
+                ScriptTermStats::totalTermFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+
+        // When no term is provided.
+        {
+            StatsSummary expected = new StatsSummary();
+            assertAllDocs(
+                Set.of(),
+                ScriptTermStats::totalTermFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+
+        // When using a non-existing field
+        {
+            StatsSummary expected = new StatsSummary(2, 0, 0, 0);
+            assertAllDocs(
+                Set.of(new Term("non-existing-field", "foo"), new Term("non-existing-field", "baz")),
+                ScriptTermStats::totalTermFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+    }
+
+    public void testTermFreq() throws IOException {
+        // Single term
+        {
+
+            assertAllDocs(
+                Set.of(new Term("field", "foo")),
+                ScriptTermStats::termFreq,
+                Map.ofEntries(
+                    Map.entry("doc-1", equalTo(new StatsSummary(1, 1, 1, 1))),
+                    Map.entry("doc-2", equalTo(new StatsSummary(1, 2, 2, 2))),
+                    Map.entry("doc-3", equalTo(new StatsSummary(1, 0, 0, 0)))
+                )
+            );
+        }
+
+        // Multiple terms
+        {
+            StatsSummary expected = new StatsSummary(2, 6, 3, 3);
+            assertAllDocs(
+                Set.of(new Term("field", "foo"), new Term("field", "bar")),
+                ScriptTermStats::termFreq,
+                Map.ofEntries(
+                    Map.entry("doc-1", equalTo(new StatsSummary(2, 2, 1, 1))),
+                    Map.entry("doc-2", equalTo(new StatsSummary(2, 3, 1, 2))),
+                    Map.entry("doc-3", equalTo(new StatsSummary(2, 1, 0, 1)))
+                )
+            );
+        }
+
+        // With missing terms
+        {
+            assertAllDocs(
+                Set.of(new Term("field", "foo"), new Term("field", "baz")),
+                ScriptTermStats::termFreq,
+                Map.ofEntries(
+                    Map.entry("doc-1", equalTo(new StatsSummary(2, 1, 0, 1))),
+                    Map.entry("doc-2", equalTo(new StatsSummary(2, 2, 0, 2))),
+                    Map.entry("doc-3", equalTo(new StatsSummary(2, 0, 0, 0)))
+                )
+            );
+        }
+
+        // When no term is provided.
+        {
+            StatsSummary expected = new StatsSummary();
+            assertAllDocs(
+                Set.of(),
+                ScriptTermStats::termFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+
+        // When using a non-existing field
+        {
+            StatsSummary expected = new StatsSummary(2, 0, 0, 0);
+            assertAllDocs(
+                Set.of(new Term("non-existing-field", "foo"), new Term("non-existing-field", "baz")),
+                ScriptTermStats::termFreq,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+    }
+
+    public void testTermPositions() throws IOException {
+        // Single term
+        {
+
+            assertAllDocs(
+                Set.of(new Term("field", "foo")),
+                ScriptTermStats::termPositions,
+                Map.ofEntries(
+                    Map.entry("doc-1", equalTo(new StatsSummary(1, 1, 1, 1))),
+                    Map.entry("doc-2", equalTo(new StatsSummary(2, 3, 1, 2))),
+                    Map.entry("doc-3", equalTo(new StatsSummary()))
+                )
+            );
+        }
+
+        // Multiple terms
+        {
+            StatsSummary expected = new StatsSummary(2, 6, 3, 3);
+            assertAllDocs(
+                Set.of(new Term("field", "foo"), new Term("field", "bar")),
+                ScriptTermStats::termPositions,
+                Map.ofEntries(
+                    Map.entry("doc-1", equalTo(new StatsSummary(2, 3, 1, 2))),
+                    Map.entry("doc-2", equalTo(new StatsSummary(3, 6, 1, 3))),
+                    Map.entry("doc-3", equalTo(new StatsSummary(1, 1, 1, 1)))
+                )
+            );
+        }
+
+        // With missing terms
+        {
+            assertAllDocs(
+                Set.of(new Term("field", "foo"), new Term("field", "baz")),
+                ScriptTermStats::termPositions,
+                Map.ofEntries(
+                    Map.entry("doc-1", equalTo(new StatsSummary(1, 1, 1, 1))),
+                    Map.entry("doc-2", equalTo(new StatsSummary(2, 3, 1, 2))),
+                    Map.entry("doc-3", equalTo(new StatsSummary()))
+                )
+            );
+        }
+
+        // When no term is provided.
+        {
+            StatsSummary expected = new StatsSummary();
+            assertAllDocs(
+                Set.of(),
+                ScriptTermStats::termPositions,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+
+        // When using a non-existing field
+        {
+            StatsSummary expected = new StatsSummary();
+            assertAllDocs(
+                Set.of(new Term("non-existing-field", "foo"), new Term("non-existing-field", "bar")),
+                ScriptTermStats::termPositions,
+                Stream.of("doc-1", "doc-2", "doc-3").collect(Collectors.toMap(Function.identity(), k -> equalTo(expected)))
+            );
+        }
+    }
+
+    private void withIndexSearcher(CheckedConsumer<IndexSearcher, IOException> consummer) throws IOException {
+        try (Directory dir = new ByteBuffersDirectory()) {
+            IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+
+            Document doc = new Document();
+            doc.add(new TextField("id", "doc-1", Field.Store.YES));
+            doc.add(new TextField("field", "foo bar", Field.Store.YES));
+            w.addDocument(doc);
+
+            doc = new Document();
+            doc.add(new TextField("id", "doc-2", Field.Store.YES));
+            doc.add(new TextField("field", "foo foo bar", Field.Store.YES));
+            w.addDocument(doc);
+
+            doc = new Document();
+            doc.add(new TextField("id", "doc-3", Field.Store.YES));
+            doc.add(new TextField("field", "bar", Field.Store.YES));
+            w.addDocument(doc);
+
+            try (IndexReader r = DirectoryReader.open(w)) {
+                w.close();
+                consummer.accept(newSearcher(r));
+            }
+        }
+    }
+
+    private <T> void assertAllDocs(Set<Term> terms, Function<ScriptTermStats, T> function, Map<String, Matcher<T>> expectedValues)
+        throws IOException {
+        withIndexSearcher(searcher -> {
+            for (LeafReaderContext leafReaderContext : searcher.getLeafContexts()) {
+                IndexReader reader = leafReaderContext.reader();
+                DocIdSetIterator docIdSetIterator = DocIdSetIterator.all(reader.maxDoc());
+                ScriptTermStats termStats = new ScriptTermStats(searcher, leafReaderContext, docIdSetIterator::docID, terms);
+                while (docIdSetIterator.nextDoc() <= reader.maxDoc()) {
+                    String docId = reader.document(docIdSetIterator.docID()).get("id");
+                    if (expectedValues.containsKey(docId)) {
+                        assertThat(function.apply(termStats), expectedValues.get(docId));
+                    }
+                }
+            }
+        });
+    }
+
+    private Set<Term> randomTerms() {
+        Predicate<Term> isReservedTerm = term -> List.of("foo", "bar").contains(term.text());
+        Supplier<Term> termSupplier = () -> randomValueOtherThanMany(
+            isReservedTerm,
+            () -> new Term("field", randomAlphaOfLengthBetween(1, 5))
+        );
+        return randomSet(1, randomIntBetween(1, 10), termSupplier);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/script/StatsSummaryTests.java b/server/src/test/java/org/elasticsearch/script/StatsSummaryTests.java
new file mode 100644
index 0000000000000..4c3d3ea4b7595
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/script/StatsSummaryTests.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+public class StatsSummaryTests extends ESTestCase {
+    public void tesEmpty() {
+        StatsSummary accumulator = new StatsSummary();
+        assertThat(accumulator.getMin(), equalTo(0.0));
+        assertThat(accumulator.getMax(), equalTo(0.0));
+        assertThat(accumulator.getSum(), equalTo(0.0));
+        assertThat(accumulator.getAverage(), equalTo(0.0));
+    }
+
+    public void testGivenPositiveValues() {
+        StatsSummary accumulator = new StatsSummary();
+
+        for (int i = 1; i <= 10; i++) {
+            accumulator.accept(i);
+        }
+
+        assertThat(accumulator.getMin(), equalTo(1.0));
+        assertThat(accumulator.getMax(), equalTo(10.0));
+        assertThat(accumulator.getSum(), equalTo(55.0));
+        assertThat(accumulator.getAverage(), equalTo(5.5));
+    }
+
+    public void testGivenNegativeValues() {
+        StatsSummary accumulator = new StatsSummary();
+
+        for (int i = 1; i <= 10; i++) {
+            accumulator.accept(-1 * i);
+        }
+
+        assertThat(accumulator.getMin(), equalTo(-10.0));
+        assertThat(accumulator.getMax(), equalTo(-1.0));
+        assertThat(accumulator.getSum(), equalTo(-55.0));
+        assertThat(accumulator.getAverage(), equalTo(-5.5));
+    }
+
+    public void testReset() {
+        StatsSummary accumulator = new StatsSummary();
+        randomDoubles(randomIntBetween(1, 20)).forEach(accumulator);
+        assertThat(accumulator, not(equalTo(new StatsSummary())));
+
+        accumulator.reset();
+        assertThat(accumulator, equalTo(new StatsSummary()));
+        assertThat(accumulator.getMin(), equalTo(0.0));
+        assertThat(accumulator.getMax(), equalTo(0.0));
+        assertThat(accumulator.getSum(), equalTo(0.0));
+        assertThat(accumulator.getAverage(), equalTo(0.0));
+    }
+
+    public void testEqualsAndHashCode() {
+        StatsSummary stats1 = new StatsSummary();
+        StatsSummary stats2 = new StatsSummary();
+
+        // Empty accumulators are equals.
+        assertThat(stats1, equalTo(stats2));
+        assertThat(stats1.hashCode(), equalTo(stats2.hashCode()));
+
+        // Accumulators with same values are equals
+        randomDoubles(randomIntBetween(0, 20)).forEach(stats1.andThen(stats2));
+        assertThat(stats1, equalTo(stats2));
+        assertThat(stats1.hashCode(), equalTo(stats2.hashCode()));
+
+        // Accumulators with different values are not equals
+        randomDoubles(randomIntBetween(0, 20)).forEach(stats1);
+        randomDoubles(randomIntBetween(0, 20)).forEach(stats2);
+        assertThat(stats1, not(equalTo(stats2)));
+        assertThat(stats1.hashCode(), not(equalTo(stats2.hashCode())));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 7ddcc88facb2a..2617f82b09f08 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -111,9 +111,9 @@
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rank.RankBuilder;
+import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.search.rank.RankShardResult;
 import org.elasticsearch.search.rank.TestRankBuilder;
-import org.elasticsearch.search.rank.TestRankDoc;
 import org.elasticsearch.search.rank.TestRankShardResult;
 import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
@@ -504,9 +504,9 @@ public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
                                         // we know we have just 1 query, so return all the docs from it
                                         return new TestRankShardResult(
                                             Arrays.stream(rankResults.get(0).scoreDocs)
-                                                .map(x -> new TestRankDoc(x.doc, x.score, x.shardIndex))
+                                                .map(x -> new RankDoc(x.doc, x.score, x.shardIndex))
                                                 .limit(rankWindowSize())
-                                                .toArray(TestRankDoc[]::new)
+                                                .toArray(RankDoc[]::new)
                                         );
                                     }
                                 };
@@ -553,7 +553,7 @@ public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId)
             queryResult = (QuerySearchResult) queryPhaseResults.get();
 
             // these are the matched docs from the query phase
-            final TestRankDoc[] queryRankDocs = ((TestRankShardResult) queryResult.getRankShardResult()).testRankDocs;
+            final RankDoc[] queryRankDocs = ((TestRankShardResult) queryResult.getRankShardResult()).testRankDocs;
 
             // assume that we have cut down to these from the coordinator node as the top-docs to run the rank feature phase upon
             List<Integer> topRankWindowSizeDocs = randomNonEmptySubsetOf(Arrays.stream(queryRankDocs).map(x -> x.doc).toList());
@@ -709,18 +709,18 @@ public ScoreDoc[] rankQueryPhaseResults(
                                             List<QuerySearchResult> querySearchResults,
                                             SearchPhaseController.TopDocsStats topDocStats
                                         ) {
-                                            List<TestRankDoc> rankDocs = new ArrayList<>();
+                                            List<RankDoc> rankDocs = new ArrayList<>();
                                             for (int i = 0; i < querySearchResults.size(); i++) {
                                                 QuerySearchResult querySearchResult = querySearchResults.get(i);
                                                 TestRankShardResult shardResult = (TestRankShardResult) querySearchResult
                                                     .getRankShardResult();
-                                                for (TestRankDoc trd : shardResult.testRankDocs) {
+                                                for (RankDoc trd : shardResult.testRankDocs) {
                                                     trd.shardIndex = i;
                                                     rankDocs.add(trd);
                                                 }
                                             }
-                                            rankDocs.sort(Comparator.comparing((TestRankDoc doc) -> doc.score).reversed());
-                                            TestRankDoc[] topResults = rankDocs.stream().limit(rankWindowSize).toArray(TestRankDoc[]::new);
+                                            rankDocs.sort(Comparator.comparing((RankDoc doc) -> doc.score).reversed());
+                                            RankDoc[] topResults = rankDocs.stream().limit(rankWindowSize).toArray(RankDoc[]::new);
                                             topDocStats.fetchHits = topResults.length;
                                             return topResults;
                                         }
@@ -741,9 +741,9 @@ public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
                                             // we know we have just 1 query, so return all the docs from it
                                             return new TestRankShardResult(
                                                 Arrays.stream(rankResults.get(0).scoreDocs)
-                                                    .map(x -> new TestRankDoc(x.doc, x.score, x.shardIndex))
+                                                    .map(x -> new RankDoc(x.doc, x.score, x.shardIndex))
                                                     .limit(rankWindowSize())
-                                                    .toArray(TestRankDoc[]::new)
+                                                    .toArray(RankDoc[]::new)
                                             );
                                         }
                                     };
@@ -868,9 +868,9 @@ public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
                                         // we know we have just 1 query, so return all the docs from it
                                         return new TestRankShardResult(
                                             Arrays.stream(rankResults.get(0).scoreDocs)
-                                                .map(x -> new TestRankDoc(x.doc, x.score, x.shardIndex))
+                                                .map(x -> new RankDoc(x.doc, x.score, x.shardIndex))
                                                 .limit(rankWindowSize())
-                                                .toArray(TestRankDoc[]::new)
+                                                .toArray(RankDoc[]::new)
                                         );
                                     }
                                 };
@@ -969,18 +969,18 @@ public ScoreDoc[] rankQueryPhaseResults(
                                             List<QuerySearchResult> querySearchResults,
                                             SearchPhaseController.TopDocsStats topDocStats
                                         ) {
-                                            List<TestRankDoc> rankDocs = new ArrayList<>();
+                                            List<RankDoc> rankDocs = new ArrayList<>();
                                             for (int i = 0; i < querySearchResults.size(); i++) {
                                                 QuerySearchResult querySearchResult = querySearchResults.get(i);
                                                 TestRankShardResult shardResult = (TestRankShardResult) querySearchResult
                                                     .getRankShardResult();
-                                                for (TestRankDoc trd : shardResult.testRankDocs) {
+                                                for (RankDoc trd : shardResult.testRankDocs) {
                                                     trd.shardIndex = i;
                                                     rankDocs.add(trd);
                                                 }
                                             }
-                                            rankDocs.sort(Comparator.comparing((TestRankDoc doc) -> doc.score).reversed());
-                                            TestRankDoc[] topResults = rankDocs.stream().limit(rankWindowSize).toArray(TestRankDoc[]::new);
+                                            rankDocs.sort(Comparator.comparing((RankDoc doc) -> doc.score).reversed());
+                                            RankDoc[] topResults = rankDocs.stream().limit(rankWindowSize).toArray(RankDoc[]::new);
                                             topDocStats.fetchHits = topResults.length;
                                             return topResults;
                                         }
@@ -1001,9 +1001,9 @@ public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
                                             // we know we have just 1 query, so return all the docs from it
                                             return new TestRankShardResult(
                                                 Arrays.stream(rankResults.get(0).scoreDocs)
-                                                    .map(x -> new TestRankDoc(x.doc, x.score, x.shardIndex))
+                                                    .map(x -> new RankDoc(x.doc, x.score, x.shardIndex))
                                                     .limit(rankWindowSize())
-                                                    .toArray(TestRankDoc[]::new)
+                                                    .toArray(RankDoc[]::new)
                                             );
                                         }
                                     };
@@ -1097,18 +1097,18 @@ public ScoreDoc[] rankQueryPhaseResults(
                                             List<QuerySearchResult> querySearchResults,
                                             SearchPhaseController.TopDocsStats topDocStats
                                         ) {
-                                            List<TestRankDoc> rankDocs = new ArrayList<>();
+                                            List<RankDoc> rankDocs = new ArrayList<>();
                                             for (int i = 0; i < querySearchResults.size(); i++) {
                                                 QuerySearchResult querySearchResult = querySearchResults.get(i);
                                                 TestRankShardResult shardResult = (TestRankShardResult) querySearchResult
                                                     .getRankShardResult();
-                                                for (TestRankDoc trd : shardResult.testRankDocs) {
+                                                for (RankDoc trd : shardResult.testRankDocs) {
                                                     trd.shardIndex = i;
                                                     rankDocs.add(trd);
                                                 }
                                             }
-                                            rankDocs.sort(Comparator.comparing((TestRankDoc doc) -> doc.score).reversed());
-                                            TestRankDoc[] topResults = rankDocs.stream().limit(rankWindowSize).toArray(TestRankDoc[]::new);
+                                            rankDocs.sort(Comparator.comparing((RankDoc doc) -> doc.score).reversed());
+                                            RankDoc[] topResults = rankDocs.stream().limit(rankWindowSize).toArray(RankDoc[]::new);
                                             topDocStats.fetchHits = topResults.length;
                                             return topResults;
                                         }
@@ -1129,9 +1129,9 @@ public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
                                             // we know we have just 1 query, so return all the docs from it
                                             return new TestRankShardResult(
                                                 Arrays.stream(rankResults.get(0).scoreDocs)
-                                                    .map(x -> new TestRankDoc(x.doc, x.score, x.shardIndex))
+                                                    .map(x -> new RankDoc(x.doc, x.score, x.shardIndex))
                                                     .limit(rankWindowSize())
-                                                    .toArray(TestRankDoc[]::new)
+                                                    .toArray(RankDoc[]::new)
                                             );
                                         }
                                     };
@@ -1548,7 +1548,7 @@ public void testMaxOpenScrollContexts() throws Exception {
 
         ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
         clearScrollRequest.setScrollIds(clearScrollIds);
-        client().clearScroll(clearScrollRequest);
+        client().clearScroll(clearScrollRequest).get();
 
         for (int i = 0; i < clearScrollIds.size(); i++) {
             client().prepareSearch("index").setSize(1).setScroll(TimeValue.timeValueMinutes(1)).get().decRef();
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java
index ade47e248a73c..364355ac81054 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java
@@ -358,7 +358,7 @@ public void collect(int doc, long bucket) throws IOException {
                     }
                     assertEquals(size, Math.min(queue.size(), expected.length - pos));
                     int ptr = pos + ((int) queue.size() - 1);
-                    pos += queue.size();
+                    pos += Math.toIntExact(queue.size());
                     last = null;
                     while (queue.size() > pos) {
                         CompositeKey key = queue.toCompositeKey(queue.pop());
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java
index a6c95efef9c46..209f67b1bb504 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java
@@ -134,6 +134,11 @@ public boolean needs_score() {
                     return true;
                 }
 
+                @Override
+                public boolean needs_termStats() {
+                    return false;
+                }
+
                 @Override
                 public ScoreScript newInstance(DocReader reader) throws IOException {
                     return new ScoreScript(params, lookup, reader) {
diff --git a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
index b1db177f6ccdd..cb0b69e5ef2ac 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
@@ -27,9 +27,9 @@
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.search.rank.RankShardResult;
 import org.elasticsearch.search.rank.TestRankBuilder;
-import org.elasticsearch.search.rank.TestRankDoc;
 import org.elasticsearch.search.rank.TestRankShardResult;
 import org.elasticsearch.search.suggest.SuggestTests;
 import org.elasticsearch.test.ESTestCase;
@@ -80,9 +80,9 @@ private static QuerySearchResult createTestInstance() throws Exception {
         result.from(randomInt());
         if (randomBoolean()) {
             int queryCount = randomIntBetween(2, 4);
-            TestRankDoc[] docs = new TestRankDoc[randomIntBetween(5, 20)];
+            RankDoc[] docs = new RankDoc[randomIntBetween(5, 20)];
             for (int di = 0; di < docs.length; ++di) {
-                docs[di] = new TestRankDoc(di, -1, queryCount);
+                docs[di] = new RankDoc(di, -1, queryCount);
             }
             result.setRankShardResult(new TestRankShardResult(docs));
         }
diff --git a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java
index 6fff108cfb5ce..177968b9a1326 100644
--- a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java
@@ -15,9 +15,14 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.lucene.search.Queries;
@@ -26,13 +31,16 @@
 import org.elasticsearch.script.DocReader;
 import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptTermStats;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
+import org.mockito.ArgumentCaptor;
 
 import java.io.IOException;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -40,6 +48,8 @@
 import static org.hamcrest.collection.IsArrayWithSize.arrayWithSize;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class ScriptScoreQueryTests extends ESTestCase {
@@ -62,7 +72,7 @@ public void initSearcher() throws IOException {
         w.commit();
         reader = DirectoryReader.open(w);
         searcher = newSearcher(reader);
-        leafReaderContext = reader.leaves().get(0);
+        leafReaderContext = searcher.getTopReaderContext().leaves().get(0);
     }
 
     @After
@@ -74,22 +84,13 @@ public void closeAllTheReaders() throws IOException {
 
     public void testExplain() throws IOException {
         Script script = new Script("script using explain");
-        ScoreScript.LeafFactory factory = newFactory(script, true, explanation -> {
+        ScoreScript.LeafFactory factory = newFactory(script, true, false, explanation -> {
             assertNotNull(explanation);
             explanation.set("this explains the score");
             return 1.0;
         });
 
-        ScriptScoreQuery query = new ScriptScoreQuery(
-            Queries.newMatchAllQuery(),
-            script,
-            factory,
-            lookup,
-            null,
-            "index",
-            0,
-            IndexVersion.current()
-        );
+        ScriptScoreQuery query = createScriptScoreQuery(Queries.newMatchAllQuery(), script, factory);
         Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);
         Explanation explanation = weight.explain(leafReaderContext, 0);
         assertNotNull(explanation);
@@ -99,18 +100,9 @@ public void testExplain() throws IOException {
 
     public void testExplainDefault() throws IOException {
         Script script = new Script("script without setting explanation");
-        ScoreScript.LeafFactory factory = newFactory(script, true, explanation -> 1.5);
+        ScoreScript.LeafFactory factory = newFactory(script, true, false, explanation -> 1.5);
 
-        ScriptScoreQuery query = new ScriptScoreQuery(
-            Queries.newMatchAllQuery(),
-            script,
-            factory,
-            lookup,
-            null,
-            "index",
-            0,
-            IndexVersion.current()
-        );
+        ScriptScoreQuery query = createScriptScoreQuery(Queries.newMatchAllQuery(), script, factory);
         Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);
         Explanation explanation = weight.explain(leafReaderContext, 0);
         assertNotNull(explanation);
@@ -124,18 +116,9 @@ public void testExplainDefault() throws IOException {
 
     public void testExplainDefaultNoScore() throws IOException {
         Script script = new Script("script without setting explanation and no score");
-        ScoreScript.LeafFactory factory = newFactory(script, false, explanation -> 2.0);
+        ScoreScript.LeafFactory factory = newFactory(script, false, false, explanation -> 2.0);
 
-        ScriptScoreQuery query = new ScriptScoreQuery(
-            Queries.newMatchAllQuery(),
-            script,
-            factory,
-            lookup,
-            null,
-            "index",
-            0,
-            IndexVersion.current()
-        );
+        ScriptScoreQuery query = createScriptScoreQuery(Queries.newMatchAllQuery(), script, factory);
         Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);
         Explanation explanation = weight.explain(leafReaderContext, 0);
         assertNotNull(explanation);
@@ -148,43 +131,91 @@ public void testExplainDefaultNoScore() throws IOException {
 
     public void testScriptScoreErrorOnNegativeScore() {
         Script script = new Script("script that returns a negative score");
-        ScoreScript.LeafFactory factory = newFactory(script, false, explanation -> -1000.0);
-        ScriptScoreQuery query = new ScriptScoreQuery(
-            Queries.newMatchAllQuery(),
-            script,
-            factory,
-            lookup,
-            null,
-            "index",
-            0,
-            IndexVersion.current()
-        );
+        ScoreScript.LeafFactory factory = newFactory(script, false, false, explanation -> -1000.0);
+        ScriptScoreQuery query = createScriptScoreQuery(Queries.newMatchAllQuery(), script, factory);
+
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> searcher.search(query, 1));
         assertTrue(e.getMessage().contains("Must be a non-negative score!"));
     }
 
+    public void testScriptTermStatsAvailable() throws IOException {
+        Script script = new Script("termStats script without setting explanation");
+        ScoreScript scoreScriptMock = mock(ScoreScript.class);
+        ScoreScript.LeafFactory factory = newFactory(false, true, (lookup, docReader) -> scoreScriptMock);
+
+        ScriptScoreQuery query = createScriptScoreQuery(
+            new BooleanQuery.Builder().add(new TermQuery(new Term("field", "text")), BooleanClause.Occur.MUST)
+                .add(new TermQuery(new Term("field", "missing")), BooleanClause.Occur.SHOULD)
+                .build(),
+            script,
+            factory
+        );
+
+        query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f).scorer(leafReaderContext);
+
+        ArgumentCaptor<ScriptTermStats> scriptTermStats = ArgumentCaptor.forClass(ScriptTermStats.class);
+        verify(scoreScriptMock)._setTermStats(scriptTermStats.capture());
+        assertThat(scriptTermStats.getValue().uniqueTermsCount(), equalTo(2));
+    }
+
+    public void testScriptTermStatsNotAvailable() throws IOException {
+        Script script = new Script("termStats script without setting explanation");
+        ScoreScript scoreScriptMock = mock(ScoreScript.class);
+        ScoreScript.LeafFactory factory = newFactory(false, false, (lookup, docReader) -> scoreScriptMock);
+
+        ScriptScoreQuery query = createScriptScoreQuery(
+            new BooleanQuery.Builder().add(new TermQuery(new Term("field", "text")), BooleanClause.Occur.MUST)
+                .add(new TermQuery(new Term("field", "missing")), BooleanClause.Occur.SHOULD)
+                .build(),
+            script,
+            factory
+        );
+
+        query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f).scorer(leafReaderContext);
+        verify(scoreScriptMock, never())._setTermStats(any());
+    }
+
+    private ScriptScoreQuery createScriptScoreQuery(Query subQuery, Script script, ScoreScript.LeafFactory factory) {
+        return new ScriptScoreQuery(subQuery, script, factory, lookup, null, "index", 0, IndexVersion.current());
+    }
+
     private ScoreScript.LeafFactory newFactory(
         Script script,
         boolean needsScore,
+        boolean needsTermStats,
         Function<ScoreScript.ExplanationHolder, Double> function
+    ) {
+        return newFactory(needsScore, needsTermStats, (lookup, docReader) -> new ScoreScript(script.getParams(), lookup, docReader) {
+            @Override
+            public double execute(ExplanationHolder explanation) {
+                return function.apply(explanation);
+            }
+        });
+    }
+
+    private ScoreScript.LeafFactory newFactory(
+        boolean needsScore,
+        boolean needsTermStats,
+        BiFunction<SearchLookup, DocReader, ScoreScript> scopreScriptProvider
     ) {
         SearchLookup lookup = mock(SearchLookup.class);
         LeafSearchLookup leafLookup = mock(LeafSearchLookup.class);
         when(lookup.getLeafSearchLookup(any())).thenReturn(leafLookup);
+
         return new ScoreScript.LeafFactory() {
             @Override
             public boolean needs_score() {
                 return needsScore;
             }
 
+            @Override
+            public boolean needs_termStats() {
+                return needsTermStats;
+            }
+
             @Override
             public ScoreScript newInstance(DocReader docReader) {
-                return new ScoreScript(script.getParams(), lookup, docReader) {
-                    @Override
-                    public double execute(ExplanationHolder explanation) {
-                        return function.apply(explanation);
-                    }
-                };
+                return scopreScriptProvider.apply(lookup, docReader);
             }
         };
     }
diff --git a/server/src/test/java/org/elasticsearch/search/rank/RankDocTests.java b/server/src/test/java/org/elasticsearch/search/rank/RankDocTests.java
new file mode 100644
index 0000000000000..5e7ac2957c250
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/rank/RankDocTests.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.rank;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+
+public class RankDocTests extends AbstractWireSerializingTestCase<RankDoc> {
+
+    static RankDoc createTestRankDoc() {
+        RankDoc rankDoc = new RankDoc(randomNonNegativeInt(), randomFloat(), randomIntBetween(0, 1));
+        rankDoc.rank = randomNonNegativeInt();
+        return rankDoc;
+    }
+
+    @Override
+    protected Writeable.Reader<RankDoc> instanceReader() {
+        return RankDoc::new;
+    }
+
+    @Override
+    protected RankDoc createTestInstance() {
+        return createTestRankDoc();
+    }
+
+    @Override
+    protected RankDoc mutateInstance(RankDoc instance) throws IOException {
+        RankDoc mutated = new RankDoc(instance.doc, instance.score, instance.shardIndex);
+        mutated.rank = instance.rank;
+        if (frequently()) {
+            mutated.doc = randomNonNegativeInt();
+        }
+        if (frequently()) {
+            mutated.score = randomFloat();
+        }
+        if (frequently()) {
+            mutated.shardIndex = randomNonNegativeInt();
+        }
+        if (frequently()) {
+            mutated.rank = randomNonNegativeInt();
+        }
+        return mutated;
+    }
+
+    public void testExplain() {
+        RankDoc instance = createTestRankDoc();
+        assertEquals(instance.explain().toString(), instance.explain().toString());
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java
index de35d765a1551..afa6ff89a79e9 100644
--- a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java
+++ b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -18,6 +19,9 @@
 import org.elasticsearch.index.query.Rewriteable;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder;
+import org.elasticsearch.search.vectors.ExactKnnQueryBuilder;
 import org.elasticsearch.test.AbstractXContentTestCase;
 import org.elasticsearch.usage.SearchUsage;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -102,6 +106,34 @@ public void testRewrite() throws IOException {
         }
     }
 
+    public void testIsCompound() {
+        KnnRetrieverBuilder knnRetriever = createRandomKnnRetrieverBuilder();
+        assertFalse(knnRetriever.isCompound());
+    }
+
+    public void testTopDocsQuery() {
+        KnnRetrieverBuilder knnRetriever = createRandomKnnRetrieverBuilder();
+        knnRetriever.rankDocs = new RankDoc[] {
+            new RankDoc(0, randomFloat(), 0),
+            new RankDoc(10, randomFloat(), 0),
+            new RankDoc(20, randomFloat(), 1),
+            new RankDoc(25, randomFloat(), 1), };
+        final int preFilters = knnRetriever.preFilterQueryBuilders.size();
+        QueryBuilder topDocsQuery = knnRetriever.topDocsQuery();
+        assertNotNull(topDocsQuery);
+        assertThat(topDocsQuery, instanceOf(BoolQueryBuilder.class));
+        assertThat(((BoolQueryBuilder) topDocsQuery).filter().size(), equalTo(1 + preFilters));
+        assertThat(((BoolQueryBuilder) topDocsQuery).filter().get(0), instanceOf(RankDocsQueryBuilder.class));
+        for (int i = 0; i < preFilters; i++) {
+            assertThat(
+                ((BoolQueryBuilder) topDocsQuery).filter().get(i + 1),
+                instanceOf(knnRetriever.preFilterQueryBuilders.get(i).getClass())
+            );
+        }
+        assertThat(((BoolQueryBuilder) topDocsQuery).should().size(), equalTo(1));
+        assertThat(((BoolQueryBuilder) topDocsQuery).should().get(0), instanceOf(ExactKnnQueryBuilder.class));
+    }
+
     @Override
     protected boolean supportsUnknownFields() {
         return false;
diff --git a/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java
new file mode 100644
index 0000000000000..59f5ddf0d87ca
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.retriever;
+
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.DisMaxQueryBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.RandomQueryBuilder;
+import org.elasticsearch.index.query.Rewriteable;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder;
+import org.elasticsearch.search.retriever.rankdoc.RankDocsSortBuilder;
+import org.elasticsearch.search.sort.ScoreSortBuilder;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.search.vectors.KnnSearchBuilderTests.randomVector;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.mockito.Mockito.mock;
+
+public class RankDocsRetrieverBuilderTests extends ESTestCase {
+
+    private Supplier<RankDoc[]> rankDocsSupplier() {
+        final int rankDocsCount = randomIntBetween(0, 10);
+        final int shardIndex = 0;
+        RankDoc[] rankDocs = new RankDoc[rankDocsCount];
+        int docId = 0;
+        for (int i = 0; i < rankDocsCount; i++) {
+            RankDoc testRankDoc = new RankDoc(docId, randomFloat(), shardIndex);
+            docId += randomInt(100);
+            rankDocs[i] = testRankDoc;
+        }
+        return () -> rankDocs;
+    }
+
+    private List<RetrieverBuilder> innerRetrievers() {
+        List<RetrieverBuilder> retrievers = new ArrayList<>();
+        int numRetrievers = randomIntBetween(1, 10);
+        for (int i = 0; i < numRetrievers; i++) {
+            if (randomBoolean()) {
+                StandardRetrieverBuilder standardRetrieverBuilder = new StandardRetrieverBuilder();
+                standardRetrieverBuilder.queryBuilder = RandomQueryBuilder.createQuery(random());
+                if (randomBoolean()) {
+                    standardRetrieverBuilder.preFilterQueryBuilders = preFilters();
+                }
+                retrievers.add(standardRetrieverBuilder);
+            } else {
+                KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(
+                    randomAlphaOfLength(10),
+                    randomVector(randomInt(10)),
+                    null,
+                    randomInt(10),
+                    randomIntBetween(10, 100),
+                    randomFloat()
+                );
+                if (randomBoolean()) {
+                    knnRetrieverBuilder.preFilterQueryBuilders = preFilters();
+                }
+                knnRetrieverBuilder.rankDocs = rankDocsSupplier().get();
+                retrievers.add(knnRetrieverBuilder);
+            }
+        }
+        return retrievers;
+    }
+
+    private List<QueryBuilder> preFilters() {
+        List<QueryBuilder> preFilters = new ArrayList<>();
+        int numPreFilters = randomInt(10);
+        for (int i = 0; i < numPreFilters; i++) {
+            preFilters.add(RandomQueryBuilder.createQuery(random()));
+        }
+        return preFilters;
+    }
+
+    private RankDocsRetrieverBuilder createRandomRankDocsRetrieverBuilder() {
+        return new RankDocsRetrieverBuilder(randomInt(100), innerRetrievers(), rankDocsSupplier(), preFilters());
+    }
+
+    public void testExtractToSearchSourceBuilder() {
+        RankDocsRetrieverBuilder retriever = createRandomRankDocsRetrieverBuilder();
+        SearchSourceBuilder source = new SearchSourceBuilder();
+        if (randomBoolean()) {
+            source.aggregation(new TermsAggregationBuilder("name").field("field"));
+        }
+        retriever.extractToSearchSourceBuilder(source, randomBoolean());
+        assertThat(source.sorts().size(), equalTo(2));
+        assertThat(source.sorts().get(0), instanceOf(RankDocsSortBuilder.class));
+        assertThat(source.sorts().get(1), instanceOf(ScoreSortBuilder.class));
+        assertThat(source.query(), instanceOf(BoolQueryBuilder.class));
+        BoolQueryBuilder bq = (BoolQueryBuilder) source.query();
+        if (source.aggregations() != null) {
+            assertThat(bq.must().size(), equalTo(0));
+            assertThat(bq.should().size(), greaterThanOrEqualTo(1));
+            assertThat(bq.should().get(0), instanceOf(RankDocsQueryBuilder.class));
+            assertNotNull(source.postFilter());
+            assertThat(source.postFilter(), instanceOf(RankDocsQueryBuilder.class));
+        } else {
+            assertThat(bq.must().size(), equalTo(1));
+            assertThat(bq.must().get(0), instanceOf(RankDocsQueryBuilder.class));
+            assertNull(source.postFilter());
+        }
+        assertThat(bq.filter().size(), equalTo(retriever.preFilterQueryBuilders.size()));
+    }
+
+    public void testTopDocsQuery() {
+        RankDocsRetrieverBuilder retriever = createRandomRankDocsRetrieverBuilder();
+        QueryBuilder topDocs = retriever.topDocsQuery();
+        assertNotNull(topDocs);
+        assertThat(topDocs, instanceOf(DisMaxQueryBuilder.class));
+        assertThat(((DisMaxQueryBuilder) topDocs).innerQueries(), hasSize(retriever.sources.size()));
+    }
+
+    public void testRewrite() throws IOException {
+        RankDocsRetrieverBuilder retriever = createRandomRankDocsRetrieverBuilder();
+        boolean compoundAdded = false;
+        if (randomBoolean()) {
+            compoundAdded = true;
+            retriever.sources.add(new TestRetrieverBuilder("compound_retriever") {
+                @Override
+                public boolean isCompound() {
+                    return true;
+                }
+            });
+        }
+        SearchSourceBuilder source = new SearchSourceBuilder().retriever(retriever);
+        QueryRewriteContext queryRewriteContext = mock(QueryRewriteContext.class);
+        if (compoundAdded) {
+            expectThrows(AssertionError.class, () -> Rewriteable.rewrite(source, queryRewriteContext));
+        } else {
+            SearchSourceBuilder rewrittenSource = Rewriteable.rewrite(source, queryRewriteContext);
+            assertNull(rewrittenSource.retriever());
+            assertTrue(rewrittenSource.knnSearch().isEmpty());
+            assertThat(
+                rewrittenSource.query(),
+                anyOf(instanceOf(BoolQueryBuilder.class), instanceOf(MatchAllQueryBuilder.class), instanceOf(MatchNoneQueryBuilder.class))
+            );
+            if (rewrittenSource.query() instanceof BoolQueryBuilder) {
+                BoolQueryBuilder bq = (BoolQueryBuilder) rewrittenSource.query();
+                assertThat(bq.filter().size(), equalTo(retriever.preFilterQueryBuilders.size()));
+                // we don't have any aggregations so the RankDocs query is set as a must clause
+                assertThat(bq.must().size(), equalTo(1));
+                assertThat(bq.must().get(0), instanceOf(RankDocsQueryBuilder.class));
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderErrorTests.java b/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderErrorTests.java
index 97f7c018974fa..b4eb1e31176bc 100644
--- a/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderErrorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderErrorTests.java
@@ -36,7 +36,7 @@ public void testRetrieverExtractionErrors() throws IOException {
         ) {
             SearchSourceBuilder ssb = new SearchSourceBuilder();
             ssb.parseXContent(parser, true, nf -> true);
-            ActionRequestValidationException iae = ssb.validate(null, false);
+            ActionRequestValidationException iae = ssb.validate(null, false, false);
             assertNotNull(iae);
             assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [query]"));
         }
@@ -50,7 +50,7 @@ public void testRetrieverExtractionErrors() throws IOException {
         ) {
             SearchSourceBuilder ssb = new SearchSourceBuilder();
             ssb.parseXContent(parser, true, nf -> true);
-            ActionRequestValidationException iae = ssb.validate(null, false);
+            ActionRequestValidationException iae = ssb.validate(null, false, false);
             assertNotNull(iae);
             assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [knn]"));
         }
@@ -58,7 +58,7 @@ public void testRetrieverExtractionErrors() throws IOException {
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"search_after\": [1], \"retriever\":{\"standard\":{}}}")) {
             SearchSourceBuilder ssb = new SearchSourceBuilder();
             ssb.parseXContent(parser, true, nf -> true);
-            ActionRequestValidationException iae = ssb.validate(null, false);
+            ActionRequestValidationException iae = ssb.validate(null, false, false);
             assertNotNull(iae);
             assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [search_after]"));
 
@@ -67,7 +67,7 @@ public void testRetrieverExtractionErrors() throws IOException {
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"terminate_after\": 1, \"retriever\":{\"standard\":{}}}")) {
             SearchSourceBuilder ssb = new SearchSourceBuilder();
             ssb.parseXContent(parser, true, nf -> true);
-            ActionRequestValidationException iae = ssb.validate(null, false);
+            ActionRequestValidationException iae = ssb.validate(null, false, false);
             assertNotNull(iae);
             assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [terminate_after]"));
         }
@@ -75,7 +75,7 @@ public void testRetrieverExtractionErrors() throws IOException {
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"sort\": [\"field\"], \"retriever\":{\"standard\":{}}}")) {
             SearchSourceBuilder ssb = new SearchSourceBuilder();
             ssb.parseXContent(parser, true, nf -> true);
-            ActionRequestValidationException iae = ssb.validate(null, false);
+            ActionRequestValidationException iae = ssb.validate(null, false, false);
             assertNotNull(iae);
             assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [sort]"));
         }
@@ -88,7 +88,7 @@ public void testRetrieverExtractionErrors() throws IOException {
         ) {
             SearchSourceBuilder ssb = new SearchSourceBuilder();
             ssb.parseXContent(parser, true, nf -> true);
-            ActionRequestValidationException iae = ssb.validate(null, false);
+            ActionRequestValidationException iae = ssb.validate(null, false, false);
             assertNotNull(iae);
             assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [rescore]"));
         }
@@ -96,7 +96,7 @@ public void testRetrieverExtractionErrors() throws IOException {
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"min_score\": 2, \"retriever\":{\"standard\":{}}}")) {
             SearchSourceBuilder ssb = new SearchSourceBuilder();
             ssb.parseXContent(parser, true, nf -> true);
-            ActionRequestValidationException iae = ssb.validate(null, false);
+            ActionRequestValidationException iae = ssb.validate(null, false, false);
             assertNotNull(iae);
             assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [min_score]"));
         }
@@ -109,7 +109,7 @@ public void testRetrieverExtractionErrors() throws IOException {
         ) {
             SearchSourceBuilder ssb = new SearchSourceBuilder();
             ssb.parseXContent(parser, true, nf -> true);
-            ActionRequestValidationException iae = ssb.validate(null, false);
+            ActionRequestValidationException iae = ssb.validate(null, false, false);
             assertNotNull(iae);
             assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [query, terminate_after, min_score]"));
         }
diff --git a/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java
index cd0d8f8d50c1e..166b07e23446c 100644
--- a/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java
+++ b/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java
@@ -166,6 +166,37 @@ public void testRewrite() throws IOException {
         }
     }
 
+    public void testIsCompound() {
+        StandardRetrieverBuilder standardRetriever = createTestInstance();
+        assertFalse(standardRetriever.isCompound());
+    }
+
+    public void testTopDocsQuery() throws IOException {
+        StandardRetrieverBuilder standardRetriever = createTestInstance();
+        final int preFilters = standardRetriever.preFilterQueryBuilders.size();
+        if (standardRetriever.queryBuilder == null) {
+            if (preFilters > 0) {
+                expectThrows(IllegalArgumentException.class, standardRetriever::topDocsQuery);
+            }
+        } else {
+            QueryBuilder topDocsQuery = standardRetriever.topDocsQuery();
+            assertNotNull(topDocsQuery);
+            if (preFilters > 0) {
+                assertThat(topDocsQuery, instanceOf(BoolQueryBuilder.class));
+                assertThat(((BoolQueryBuilder) topDocsQuery).filter().size(), equalTo(1 + preFilters));
+                assertThat(((BoolQueryBuilder) topDocsQuery).filter().get(0), instanceOf(standardRetriever.queryBuilder.getClass()));
+                for (int i = 0; i < preFilters; i++) {
+                    assertThat(
+                        ((BoolQueryBuilder) topDocsQuery).filter().get(i + 1),
+                        instanceOf(standardRetriever.preFilterQueryBuilders.get(i).getClass())
+                    );
+                }
+            } else {
+                assertThat(topDocsQuery, instanceOf(standardRetriever.queryBuilder.getClass()));
+            }
+        }
+    }
+
     private static void assertEqualQueryOrMatchAllNone(QueryBuilder actual, QueryBuilder expected) {
         assertThat(actual, anyOf(instanceOf(MatchAllQueryBuilder.class), instanceOf(MatchNoneQueryBuilder.class), equalTo(expected)));
     }
diff --git a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java
new file mode 100644
index 0000000000000..5ff0ec2017e96
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.retriever.rankdoc;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.test.AbstractQueryTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class RankDocsQueryBuilderTests extends AbstractQueryTestCase<RankDocsQueryBuilder> {
+
+    private RankDoc[] generateRandomRankDocs() {
+        int totalDocs = randomIntBetween(0, 10);
+        RankDoc[] rankDocs = new RankDoc[totalDocs];
+        int currentDoc = 0;
+        for (int i = 0; i < totalDocs; i++) {
+            RankDoc rankDoc = new RankDoc(currentDoc, randomFloat(), randomIntBetween(0, 2));
+            rankDocs[i] = rankDoc;
+            currentDoc += randomIntBetween(0, 100);
+        }
+        return rankDocs;
+    }
+
+    @Override
+    protected RankDocsQueryBuilder doCreateTestQueryBuilder() {
+        RankDoc[] rankDocs = generateRandomRankDocs();
+        return new RankDocsQueryBuilder(rankDocs);
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(RankDocsQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException {
+        assertTrue(query instanceof RankDocsQuery);
+        RankDocsQuery rankDocsQuery = (RankDocsQuery) query;
+        assertArrayEquals(queryBuilder.rankDocs(), rankDocsQuery.rankDocs());
+    }
+
+    /**
+     * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader}
+     */
+    @Override
+    public void testToQuery() throws IOException {
+        try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
+            iw.addDocument(new Document());
+            try (IndexReader reader = iw.getReader()) {
+                SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader));
+                RankDocsQueryBuilder queryBuilder = createTestQueryBuilder();
+                Query query = queryBuilder.doToQuery(context);
+
+                assertTrue(query instanceof RankDocsQuery);
+                RankDocsQuery rankDocsQuery = (RankDocsQuery) query;
+
+                int shardIndex = context.getShardRequestIndex();
+                int expectedDocs = (int) Arrays.stream(queryBuilder.rankDocs()).filter(x -> x.shardIndex == shardIndex).count();
+                assertEquals(expectedDocs, rankDocsQuery.rankDocs().length);
+            }
+        }
+    }
+
+    /**
+     * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader}
+     */
+    @Override
+    public void testCacheability() throws IOException {
+        try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
+            iw.addDocument(new Document());
+            try (IndexReader reader = iw.getReader()) {
+                SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader));
+                RankDocsQueryBuilder queryBuilder = createTestQueryBuilder();
+                QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new SearchExecutionContext(context));
+                assertNotNull(rewriteQuery.toQuery(context));
+                assertTrue("query should be cacheable: " + queryBuilder.toString(), context.isCacheable());
+            }
+        }
+    }
+
+    /**
+     * Overridden to ensure that {@link SearchExecutionContext} has a non-null {@link IndexReader}
+     */
+    @Override
+    public void testMustRewrite() throws IOException {
+        try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
+            iw.addDocument(new Document());
+            try (IndexReader reader = iw.getReader()) {
+                SearchExecutionContext context = createSearchExecutionContext(newSearcher(reader));
+                context.setAllowUnmappedFields(true);
+                RankDocsQueryBuilder queryBuilder = createTestQueryBuilder();
+                queryBuilder.toQuery(context);
+            }
+        }
+    }
+
+    @Override
+    public void testFromXContent() throws IOException {
+        // no-op since RankDocsQueryBuilder is an internal only API
+    }
+
+    @Override
+    public void testUnknownField() throws IOException {
+        // no-op since RankDocsQueryBuilder is agnostic to unknown fields and an internal only API
+    }
+
+    @Override
+    public void testValidOutput() throws IOException {
+        // no-op since RankDocsQueryBuilder is an internal only API
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilderTests.java
new file mode 100644
index 0000000000000..ba7728c5dc622
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilderTests.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.retriever.rankdoc;
+
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.search.sort.AbstractSortTestCase;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class RankDocsSortBuilderTests extends AbstractSortTestCase<RankDocsSortBuilder> {
+
+    @Override
+    protected RankDocsSortBuilder createTestItem() {
+        return randomRankDocsSortBuulder();
+    }
+
+    private RankDocsSortBuilder randomRankDocsSortBuulder() {
+        RankDoc[] rankDocs = randomRankDocs(randomInt(100));
+        return new RankDocsSortBuilder(rankDocs);
+    }
+
+    private RankDoc[] randomRankDocs(int totalDocs) {
+        RankDoc[] rankDocs = new RankDoc[totalDocs];
+        for (int i = 0; i < totalDocs; i++) {
+            rankDocs[i] = new RankDoc(randomNonNegativeInt(), randomFloat(), randomIntBetween(0, 1));
+            rankDocs[i].rank = i + 1;
+        }
+        return rankDocs;
+    }
+
+    @Override
+    protected RankDocsSortBuilder mutate(RankDocsSortBuilder original) throws IOException {
+        RankDocsSortBuilder mutated = new RankDocsSortBuilder(original);
+        mutated.rankDocs(randomRankDocs(original.rankDocs().length + randomInt(100)));
+        return mutated;
+    }
+
+    @Override
+    public void testFromXContent() throws IOException {
+        // no-op
+    }
+
+    @Override
+    protected RankDocsSortBuilder fromXContent(XContentParser parser, String fieldName) throws IOException {
+        throw new UnsupportedOperationException(
+            "{" + RankDocsSortBuilder.class.getSimpleName() + "} does not support parsing from XContent"
+        );
+    }
+
+    @Override
+    protected void sortFieldAssertions(RankDocsSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException {
+        assertThat(builder.order(), equalTo(SortOrder.ASC));
+        assertThat(sortField, instanceOf(RankDocsSortField.class));
+        assertThat(sortField.getField(), equalTo(RankDocsSortField.NAME));
+        assertThat(sortField.getType(), equalTo(SortField.Type.CUSTOM));
+        assertThat(sortField.getReverse(), equalTo(false));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java
index f0899384dbc5e..565a09dbff0d3 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
+import org.elasticsearch.index.query.InnerHitsRewriteContext;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -306,6 +307,22 @@ private void assertBWCSerialization(QueryBuilder newQuery, QueryBuilder bwcQuery
         }
     }
 
+    public void testRewriteForInnerHits() throws IOException {
+        SearchExecutionContext context = createSearchExecutionContext();
+        InnerHitsRewriteContext innerHitsRewriteContext = new InnerHitsRewriteContext(context.getParserConfig(), System::currentTimeMillis);
+        KnnVectorQueryBuilder queryBuilder = createTestQueryBuilder();
+        queryBuilder.boost(randomFloat());
+        queryBuilder.queryName(randomAlphaOfLength(10));
+        QueryBuilder rewritten = queryBuilder.rewrite(innerHitsRewriteContext);
+        assertTrue(rewritten instanceof ExactKnnQueryBuilder);
+        ExactKnnQueryBuilder exactKnnQueryBuilder = (ExactKnnQueryBuilder) rewritten;
+        assertEquals(queryBuilder.queryVector(), exactKnnQueryBuilder.getQuery());
+        assertEquals(queryBuilder.getFieldName(), exactKnnQueryBuilder.getField());
+        assertEquals(queryBuilder.boost(), exactKnnQueryBuilder.boost(), 0.0001f);
+        assertEquals(queryBuilder.queryName(), exactKnnQueryBuilder.queryName());
+        assertEquals(queryBuilder.getVectorSimilarity(), exactKnnQueryBuilder.vectorSimilarity());
+    }
+
     public void testRewriteWithQueryVectorBuilder() throws Exception {
         int dims = randomInt(1024);
         float[] expectedArray = new float[dims];
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilderTests.java
index 5f4fb61718a7e..c302dc7bd63c9 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilderTests.java
@@ -53,12 +53,12 @@ protected ExactKnnQueryBuilder doCreateTestQueryBuilder() {
         for (int i = 0; i < VECTOR_DIMENSION; i++) {
             query[i] = randomFloat();
         }
-        return new ExactKnnQueryBuilder(query, VECTOR_FIELD);
+        return new ExactKnnQueryBuilder(VectorData.fromFloats(query), VECTOR_FIELD, randomBoolean() ? randomFloat() : null);
     }
 
     @Override
     public void testValidOutput() {
-        ExactKnnQueryBuilder query = new ExactKnnQueryBuilder(new float[] { 1.0f, 2.0f, 3.0f }, "field");
+        ExactKnnQueryBuilder query = new ExactKnnQueryBuilder(VectorData.fromFloats(new float[] { 1.0f, 2.0f, 3.0f }), "field", null);
         String expected = """
             {
               "exact_knn" : {
@@ -71,10 +71,29 @@ public void testValidOutput() {
               }
             }""";
         assertEquals(expected, query.toString());
+        query = new ExactKnnQueryBuilder(VectorData.fromFloats(new float[] { 1.0f, 2.0f, 3.0f }), "field", 1f);
+        expected = """
+            {
+              "exact_knn" : {
+                "query" : [
+                  1.0,
+                  2.0,
+                  3.0
+                ],
+                "field" : "field",
+                "similarity" : 1.0
+              }
+            }""";
+        assertEquals(expected, query.toString());
     }
 
     @Override
     protected void doAssertLuceneQuery(ExactKnnQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException {
+        if (queryBuilder.vectorSimilarity() != null) {
+            assertTrue(query instanceof VectorSimilarityQuery);
+            VectorSimilarityQuery vectorSimilarityQuery = (VectorSimilarityQuery) query;
+            query = vectorSimilarityQuery.getInnerKnnQuery();
+        }
         assertTrue(query instanceof DenseVectorQuery.Floats);
         DenseVectorQuery.Floats denseVectorQuery = (DenseVectorQuery.Floats) query;
         assertEquals(VECTOR_FIELD, denseVectorQuery.field);
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java
index a558081c2d16f..c09ed24668963 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilderTests.java
@@ -56,7 +56,8 @@ protected KnnScoreDocQueryBuilder doCreateTestQueryBuilder() {
         return new KnnScoreDocQueryBuilder(
             scoreDocs.toArray(new ScoreDoc[0]),
             randomBoolean() ? "field" : null,
-            randomBoolean() ? VectorData.fromFloats(randomVector(10)) : null
+            randomBoolean() ? VectorData.fromFloats(randomVector(10)) : null,
+            randomBoolean() ? randomFloat() : null
         );
     }
 
@@ -65,7 +66,8 @@ public void testValidOutput() {
         KnnScoreDocQueryBuilder query = new KnnScoreDocQueryBuilder(
             new ScoreDoc[] { new ScoreDoc(0, 4.25f), new ScoreDoc(5, 1.6f) },
             "field",
-            VectorData.fromFloats(new float[] { 1.0f, 2.0f })
+            VectorData.fromFloats(new float[] { 1.0f, 2.0f }),
+            null
         );
         String expected = """
             {
@@ -155,7 +157,8 @@ public void testRewriteToMatchNone() throws IOException {
         KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder(
             new ScoreDoc[0],
             randomBoolean() ? "field" : null,
-            randomBoolean() ? VectorData.fromFloats(randomVector(10)) : null
+            randomBoolean() ? VectorData.fromFloats(randomVector(10)) : null,
+            randomBoolean() ? randomFloat() : null
         );
         QueryRewriteContext context = randomBoolean()
             ? new InnerHitsRewriteContext(createSearchExecutionContext().getParserConfig(), System::currentTimeMillis)
@@ -169,7 +172,8 @@ public void testRewriteForInnerHits() throws IOException {
         KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder(
             new ScoreDoc[] { new ScoreDoc(0, 4.25f), new ScoreDoc(5, 1.6f) },
             randomAlphaOfLength(10),
-            VectorData.fromFloats(randomVector(10))
+            VectorData.fromFloats(randomVector(10)),
+            randomBoolean() ? randomFloat() : null
         );
         queryBuilder.boost(randomFloat());
         queryBuilder.queryName(randomAlphaOfLength(10));
@@ -180,6 +184,7 @@ public void testRewriteForInnerHits() throws IOException {
         assertEquals(queryBuilder.fieldName(), exactKnnQueryBuilder.getField());
         assertEquals(queryBuilder.boost(), exactKnnQueryBuilder.boost(), 0.0001f);
         assertEquals(queryBuilder.queryName(), exactKnnQueryBuilder.queryName());
+        assertEquals(queryBuilder.vectorSimilarity(), exactKnnQueryBuilder.vectorSimilarity());
     }
 
     @Override
@@ -221,7 +226,8 @@ public void testScoreDocQueryWeightCount() throws IOException {
                 KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder(
                     scoreDocs,
                     "field",
-                    VectorData.fromFloats(randomVector(10))
+                    VectorData.fromFloats(randomVector(10)),
+                    null
                 );
                 Query query = queryBuilder.doToQuery(context);
                 final Weight w = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0f);
@@ -268,7 +274,8 @@ public void testScoreDocQuery() throws IOException {
                 KnnScoreDocQueryBuilder queryBuilder = new KnnScoreDocQueryBuilder(
                     scoreDocs,
                     "field",
-                    VectorData.fromFloats(randomVector(10))
+                    VectorData.fromFloats(randomVector(10)),
+                    null
                 );
                 final Query query = queryBuilder.doToQuery(context);
                 final Weight w = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0f);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/RegisteredPolicySnapshotsSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/RegisteredPolicySnapshotsSerializationTests.java
new file mode 100644
index 0000000000000..4de6b0c901f1a
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/snapshots/RegisteredPolicySnapshotsSerializationTests.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RegisteredPolicySnapshotsSerializationTests extends AbstractChunkedSerializingTestCase<RegisteredPolicySnapshots> {
+    public void testMaybeAdd() {
+        {
+            RegisteredPolicySnapshots.Builder builder = new RegisteredPolicySnapshots.Builder(RegisteredPolicySnapshots.EMPTY);
+            var snap = new SnapshotId(randomAlphaOfLength(10), randomUUID());
+
+            builder.maybeAdd(null, snap);
+            builder.maybeAdd(Map.of(), snap);
+            builder.maybeAdd(Map.of("not_policy", "policy-10"), snap);
+            builder.maybeAdd(Map.of(SnapshotsService.POLICY_ID_METADATA_FIELD, 5), snap);
+
+            // immutable map in Map.of doesn't allows nulls
+            var meta = new HashMap<String, Object>();
+            meta.put(SnapshotsService.POLICY_ID_METADATA_FIELD, null);
+            builder.maybeAdd(meta, snap);
+
+            RegisteredPolicySnapshots registered = builder.build();
+            assertTrue(registered.getSnapshots().isEmpty());
+        }
+
+        {
+            RegisteredPolicySnapshots.Builder builder = new RegisteredPolicySnapshots.Builder(RegisteredPolicySnapshots.EMPTY);
+            var snap = new SnapshotId(randomAlphaOfLength(10), randomUUID());
+            builder.maybeAdd(Map.of(SnapshotsService.POLICY_ID_METADATA_FIELD, "cheddar"), snap);
+            RegisteredPolicySnapshots registered = builder.build();
+            assertEquals(List.of(new RegisteredPolicySnapshots.PolicySnapshot("cheddar", snap)), registered.getSnapshots());
+        }
+    }
+
+    @Override
+    protected RegisteredPolicySnapshots doParseInstance(XContentParser parser) throws IOException {
+        return RegisteredPolicySnapshots.parse(parser);
+    }
+
+    @Override
+    protected Writeable.Reader<RegisteredPolicySnapshots> instanceReader() {
+        return RegisteredPolicySnapshots::new;
+    }
+
+    @Override
+    protected RegisteredPolicySnapshots createTestInstance() {
+        return randomRegisteredPolicySnapshots();
+    }
+
+    @Override
+    protected RegisteredPolicySnapshots mutateInstance(RegisteredPolicySnapshots instance) throws IOException {
+        if (instance.getSnapshots().isEmpty()) {
+            return new RegisteredPolicySnapshots(List.of(randomPolicySnapshot()));
+        }
+
+        final int randIndex = between(0, instance.getSnapshots().size() - 1);
+        final RegisteredPolicySnapshots.PolicySnapshot policySnapshot = instance.getSnapshots().get(randIndex);
+
+        String policy = policySnapshot.getPolicy();
+        String snapshotName = policySnapshot.getSnapshotId().getName();
+        String snapshotUUID = policySnapshot.getSnapshotId().getUUID();
+
+        switch (between(0, 2)) {
+            case 0 -> {
+                policy = randomValueOtherThan(policy, this::randomPolicy);
+            }
+            case 1 -> {
+                snapshotName = randomValueOtherThan(snapshotName, ESTestCase::randomIdentifier);
+            }
+            case 2 -> {
+                snapshotUUID = randomValueOtherThan(snapshotName, ESTestCase::randomUUID);
+            }
+            default -> throw new AssertionError("failure, got illegal switch case");
+        }
+
+        List<RegisteredPolicySnapshots.PolicySnapshot> newSnapshots = new ArrayList<>(instance.getSnapshots());
+        newSnapshots.set(randIndex, new RegisteredPolicySnapshots.PolicySnapshot(policy, new SnapshotId(snapshotName, snapshotUUID)));
+        return new RegisteredPolicySnapshots(newSnapshots);
+    }
+
+    private RegisteredPolicySnapshots randomRegisteredPolicySnapshots() {
+        final List<RegisteredPolicySnapshots.PolicySnapshot> snapshots = new ArrayList<>();
+        for (int i = 0; i < randomIntBetween(0, 20); i++) {
+            snapshots.add(randomPolicySnapshot());
+        }
+        return new RegisteredPolicySnapshots(snapshots);
+    }
+
+    private String randomPolicy() {
+        return "policy-" + randomIntBetween(0, 20);
+    }
+
+    private RegisteredPolicySnapshots.PolicySnapshot randomPolicySnapshot() {
+        SnapshotId snapshotId = new SnapshotId(randomIdentifier(), randomUUID());
+        return new RegisteredPolicySnapshots.PolicySnapshot(randomPolicy(), snapshotId);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java
index 0d0293b962609..726d8fce4fc44 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.repositories.RepositoriesService;
@@ -159,7 +160,12 @@ public void testPrefixNotChanged() {
     public void testRefreshRepositoryUuidsDoesNothingIfDisabled() {
         final RepositoriesService repositoriesService = mock(RepositoriesService.class);
         final AtomicBoolean called = new AtomicBoolean();
-        RestoreService.refreshRepositoryUuids(false, repositoriesService, () -> assertTrue(called.compareAndSet(false, true)));
+        RestoreService.refreshRepositoryUuids(
+            false,
+            repositoriesService,
+            () -> assertTrue(called.compareAndSet(false, true)),
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
+        );
         assertTrue(called.get());
         verifyNoMoreInteractions(repositoriesService);
     }
@@ -209,7 +215,12 @@ public void testRefreshRepositoryUuidsRefreshesAsNeeded() {
         final RepositoriesService repositoriesService = mock(RepositoriesService.class);
         when(repositoriesService.getRepositories()).thenReturn(repositories);
         final AtomicBoolean completed = new AtomicBoolean();
-        RestoreService.refreshRepositoryUuids(true, repositoriesService, () -> assertTrue(completed.compareAndSet(false, true)));
+        RestoreService.refreshRepositoryUuids(
+            true,
+            repositoriesService,
+            () -> assertTrue(completed.compareAndSet(false, true)),
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
+        );
         assertTrue(completed.get());
         assertThat(pendingRefreshes, empty());
         finalAssertions.forEach(Runnable::run);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index 54051f8311967..f5e69a65a6d06 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -45,6 +45,7 @@
 import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.bulk.FailureStoreMetrics;
 import org.elasticsearch.action.bulk.TransportBulkAction;
 import org.elasticsearch.action.bulk.TransportShardBulkAction;
 import org.elasticsearch.action.index.IndexRequest;
@@ -195,6 +196,7 @@
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.usage.UsageService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.junit.After;
 import org.junit.Before;
@@ -1353,7 +1355,7 @@ public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(
             if (randomBoolean()) {
                 final var snapshotName = "snapshot-" + i;
                 testListener = testListener.andThen(
-                    (stepListener, v) -> scheduleNow(
+                    stepListener -> scheduleNow(
                         ActionRunnable.wrap(
                             stepListener,
                             l -> client.admin()
@@ -1367,7 +1369,7 @@ public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(
                 );
             } else {
                 final var cloneName = "clone-" + i;
-                testListener = testListener.andThen((stepListener, v) -> scheduleNow(ActionRunnable.wrap(stepListener, l -> {
+                testListener = testListener.andThen(stepListener -> scheduleNow(ActionRunnable.wrap(stepListener, l -> {
                     // The clone API only responds when the clone is complete, but we only want to wait until the clone starts so we watch
                     // the cluster state instead.
                     ClusterServiceUtils.addTemporaryStateListener(
@@ -1390,7 +1392,7 @@ public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(
             }
         }
 
-        testListener = testListener.andThen((l, ignored) -> scheduleNow(() -> {
+        testListener = testListener.andThen(l -> scheduleNow(() -> {
             // Once all snapshots & clones have started, drop the data node and wait for all snapshot activity to complete
             testClusterNodes.disconnectNode(testClusterNodes.randomDataNodeSafe());
             ClusterServiceUtils.addTemporaryStateListener(masterClusterService, cs -> SnapshotsInProgress.get(cs).isEmpty()).addListener(l);
@@ -1444,7 +1446,7 @@ public void testFullSnapshotUnassignedShards() {
 
             // Take the snapshot to check the reaction to having unassigned shards
             .<Void>andThen(
-                (l, ignored) -> client().admin()
+                l -> client().admin()
                     .cluster()
                     .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repoName, randomIdentifier())
                     .setWaitForCompletion(randomBoolean())
@@ -1495,7 +1497,7 @@ public void testSnapshotNameAlreadyInUseExceptionLogging() {
         final var testListener = createRepoAndIndex(repoName, "index", between(1, 2))
             // take snapshot once
             .<CreateSnapshotResponse>andThen(
-                (l, ignored) -> client().admin()
+                l -> client().admin()
                     .cluster()
                     .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repoName, snapshotName)
                     .setWaitForCompletion(true)
@@ -1503,7 +1505,7 @@ public void testSnapshotNameAlreadyInUseExceptionLogging() {
             )
             // take snapshot again
             .<CreateSnapshotResponse>andThen(
-                (l, ignored) -> client().admin()
+                l -> client().admin()
                     .cluster()
                     .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repoName, snapshotName)
                     .setWaitForCompletion(randomBoolean())
@@ -1522,7 +1524,7 @@ public void onFailure(Exception e) {
             )
             // attempt to clone snapshot
             .<AcknowledgedResponse>andThen(
-                (l, ignored) -> client().admin()
+                l -> client().admin()
                     .cluster()
                     .prepareCloneSnapshot(TEST_REQUEST_TIMEOUT, repoName, snapshotName, snapshotName)
                     .setIndices("*")
@@ -1581,7 +1583,7 @@ public void testIndexNotFoundExceptionLogging() {
             )
             // take snapshot of index that does not exist
             .<CreateSnapshotResponse>andThen(
-                (l, ignored) -> client().admin()
+                l -> client().admin()
                     .cluster()
                     .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repoName, randomIdentifier())
                     .setIndices(indexName)
@@ -1633,7 +1635,7 @@ public void testIllegalArgumentExceptionLogging() {
             )
             // attempt to take snapshot with illegal config ('none' is allowed as a feature state iff it's the only one in the list)
             .<CreateSnapshotResponse>andThen(
-                (l, ignored) -> client().admin()
+                l -> client().admin()
                     .cluster()
                     .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repoName, randomIdentifier())
                     .setFeatureStates("none", "none")
@@ -1816,10 +1818,7 @@ private void scheduleNow(Runnable runnable) {
 
     private static Settings defaultIndexSettings(int shards) {
         // TODO: randomize replica count settings once recovery operations aren't blocking anymore
-        return Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            .build();
+        return indexSettings(shards, 0).build();
     }
 
     private static <T> void continueOrDie(SubscribableListener<T> listener, CheckedConsumer<T, Exception> onResponse) {
@@ -2061,6 +2060,8 @@ private final class TestClusterNode {
 
             private final BigArrays bigArrays;
 
+            private final UsageService usageService;
+
             private Coordinator coordinator;
 
             TestClusterNode(DiscoveryNode node, TransportInterceptorFactory transportInterceptorFactory) throws IOException {
@@ -2071,6 +2072,7 @@ private final class TestClusterNode {
                 masterService = new FakeThreadPoolMasterService(node.getName(), threadPool, deterministicTaskQueue::scheduleNow);
                 final Settings settings = environment.settings();
                 client = new NodeClient(settings, threadPool);
+                this.usageService = new UsageService();
                 final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
                 clusterService = new ClusterService(
                     settings,
@@ -2398,14 +2400,16 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() {
                             Collections.emptyList(),
                             client,
                             null,
-                            DocumentParsingProvider.EMPTY_INSTANCE
+                            DocumentParsingProvider.EMPTY_INSTANCE,
+                            FailureStoreMetrics.NOOP
                         ),
                         mockFeatureService,
                         client,
                         actionFilters,
                         indexNameExpressionResolver,
                         new IndexingPressure(settings),
-                        EmptySystemIndices.INSTANCE
+                        EmptySystemIndices.INSTANCE,
+                        FailureStoreMetrics.NOOP
                     )
                 );
                 final TransportShardBulkAction transportShardBulkAction = new TransportShardBulkAction(
@@ -2486,7 +2490,8 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() {
                         EmptySystemIndices.INSTANCE.getExecutorSelector(),
                         new SearchTransportAPMMetrics(TelemetryProvider.NOOP.getMeterRegistry()),
                         new SearchResponseMetrics(TelemetryProvider.NOOP.getMeterRegistry()),
-                        client
+                        client,
+                        usageService
                     )
                 );
                 actions.put(
diff --git a/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java b/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java
index 05150cd5dd362..d15eae47968e4 100644
--- a/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java
+++ b/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.network.CloseableChannel;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
@@ -335,7 +336,12 @@ public void testRegisterAndExecuteStartsAndStopsTracing() {
 
         final Task task = taskManager.registerAndExecute(
             "testType",
-            new TransportAction<ActionRequest, ActionResponse>("actionName", new ActionFilters(Set.of()), taskManager) {
+            new TransportAction<ActionRequest, ActionResponse>(
+                "actionName",
+                new ActionFilters(Set.of()),
+                taskManager,
+                EsExecutors.DIRECT_EXECUTOR_SERVICE
+            ) {
                 @Override
                 protected void doExecute(Task task, ActionRequest request, ActionListener<ActionResponse> listener) {
                     listener.onResponse(new ActionResponse() {
diff --git a/server/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java
index 6be78f27135a5..7ece91c48da6f 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java
@@ -34,7 +34,7 @@ public void testRejectedExecutionCounter() throws InterruptedException {
             .put("thread_pool." + threadPoolName + ".queue_size", queueSize)
             .build();
         try {
-            threadPool = new ThreadPool(nodeSettings, MeterRegistry.NOOP);
+            threadPool = new ThreadPool(nodeSettings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
 
             // these tasks will consume the thread pool causing further
             // submissions to queue
diff --git a/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
index 9a0c5c4b75d54..8c2ed2cc5bf00 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
@@ -425,7 +425,7 @@ public void runScalingThreadPoolTest(final Settings settings, final BiConsumer<C
         try {
             final String test = Thread.currentThread().getStackTrace()[2].getMethodName();
             final Settings nodeSettings = Settings.builder().put(settings).put("node.name", test).build();
-            threadPool = new ThreadPool(nodeSettings, MeterRegistry.NOOP);
+            threadPool = new ThreadPool(nodeSettings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
             final ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
             consumer.accept(clusterSettings, threadPool);
         } finally {
diff --git a/server/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java b/server/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java
index 8e10fb7c43b77..dcec47156cbad 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java
@@ -52,7 +52,8 @@ public class ScheduleWithFixedDelayTests extends ESTestCase {
     public void setup() {
         threadPool = new ThreadPool(
             Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "fixed delay tests").build(),
-            MeterRegistry.NOOP
+            MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders()
         );
     }
 
@@ -255,7 +256,8 @@ public void testOnRejectionCausesCancellation() throws Exception {
         terminate(threadPool);
         threadPool = new ThreadPool(
             Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "fixed delay tests").build(),
-            MeterRegistry.NOOP
+            MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders()
         ) {
             @Override
             public ScheduledCancellable schedule(Runnable command, TimeValue delay, Executor executor) {
diff --git a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
index 46c551987e325..d4f47dccc2504 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
@@ -85,7 +85,7 @@ public void testThatToXContentWritesOutUnboundedCorrectly() throws Exception {
 
     public void testThatNegativeSettingAllowsToStart() throws InterruptedException {
         Settings settings = Settings.builder().put("node.name", "write").put("thread_pool.write.queue_size", "-1").build();
-        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
         assertThat(threadPool.info("write").getQueueSize(), is(nullValue()));
         terminate(threadPool);
     }
diff --git a/server/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java b/server/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java
index b68f3ef76bbac..eca55dfe9a681 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java
@@ -39,7 +39,8 @@ public void testCorrectThreadPoolTypePermittedInSettings() {
                     .put("node.name", "testCorrectThreadPoolTypePermittedInSettings")
                     .put("thread_pool." + threadPoolName + ".type", correctThreadPoolType.getType())
                     .build(),
-                MeterRegistry.NOOP
+                MeterRegistry.NOOP,
+                new DefaultBuiltInExecutorBuilders()
             );
             assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), correctThreadPoolType);
         } finally {
@@ -60,7 +61,8 @@ public void testWriteThreadPoolsMaxSize() throws InterruptedException {
                         .put("node.name", "testIndexingThreadPoolsMaxSize")
                         .put("thread_pool." + Names.WRITE + ".size", tooBig)
                         .build(),
-                    MeterRegistry.NOOP
+                    MeterRegistry.NOOP,
+                    new DefaultBuiltInExecutorBuilders()
                 );
             } finally {
                 terminateThreadPoolIfNeeded(tp);
@@ -93,7 +95,7 @@ public void testFixedExecutorType() throws InterruptedException {
                 .put("node.name", "testFixedExecutorType")
                 .put("thread_pool." + threadPoolName + ".size", expectedSize)
                 .build();
-            threadPool = new ThreadPool(nodeSettings, MeterRegistry.NOOP);
+            threadPool = new ThreadPool(nodeSettings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
             assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class));
 
             assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED);
@@ -117,7 +119,7 @@ public void testScalingExecutorType() throws InterruptedException {
                 .put("thread_pool." + threadPoolName + ".max", 10)
                 .put("node.name", "testScalingExecutorType")
                 .build();
-            threadPool = new ThreadPool(nodeSettings, MeterRegistry.NOOP);
+            threadPool = new ThreadPool(nodeSettings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
             final int expectedMinimum = "generic".equals(threadPoolName) ? 4 : 1;
             assertThat(info(threadPool, threadPoolName).getMin(), equalTo(expectedMinimum));
             assertThat(info(threadPool, threadPoolName).getMax(), equalTo(10));
@@ -138,7 +140,7 @@ public void testShutdownNowInterrupts() throws Exception {
                 .put("thread_pool." + threadPoolName + ".queue_size", 1000)
                 .put("node.name", "testShutdownNowInterrupts")
                 .build();
-            threadPool = new ThreadPool(nodeSettings, MeterRegistry.NOOP);
+            threadPool = new ThreadPool(nodeSettings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
             assertEquals(info(threadPool, threadPoolName).getQueueSize().singles(), 1000L);
 
             final CountDownLatch shutDownLatch = new CountDownLatch(1);
@@ -185,6 +187,7 @@ public void testCustomThreadPool() throws Exception {
             threadPool = new ThreadPool(
                 Settings.builder().put("node.name", "testCustomThreadPool").build(),
                 MeterRegistry.NOOP,
+                new DefaultBuiltInExecutorBuilders(),
                 scaling,
                 fixed
             );
diff --git a/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java b/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java
index e97fb3220923d..27874d4311cd2 100644
--- a/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLog;
 import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -68,7 +69,7 @@ public class ClusterConnectionManagerTests extends ESTestCase {
     @Before
     public void createConnectionManager() {
         Settings settings = Settings.builder().put("node.name", ClusterConnectionManagerTests.class.getSimpleName()).build();
-        threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
         transport = mock(Transport.class);
         connectionManager = new ClusterConnectionManager(settings, transport, threadPool.getThreadContext());
         TimeValue oneSecond = new TimeValue(1000);
diff --git a/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json b/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json
new file mode 100644
index 0000000000000..fe9c77cb2a183
--- /dev/null
+++ b/server/src/test/resources/org/elasticsearch/action/admin/cluster/stats/telemetry_test.json
@@ -0,0 +1,67 @@
+{
+  "_search" : {
+    "total" : 10,
+    "success" : 20,
+    "skipped" : 5,
+    "took" : {
+      "max" : 1991,
+      "avg" : 1496,
+      "p90" : 1895
+    },
+    "took_mrt_true" : {
+      "max" : 9983,
+      "avg" : 7476,
+      "p90" : 9471
+    },
+    "took_mrt_false" : {
+      "max" : 19967,
+      "avg" : 14952,
+      "p90" : 18943
+    },
+    "remotes_per_search_max" : 6,
+    "remotes_per_search_avg" : 7.89,
+    "failure_reasons" : {
+      "reason1" : 1,
+      "reason2" : 2,
+      "unknown" : 3
+    },
+    "features" : {
+      "async" : 10,
+      "mrt" : 20,
+      "wildcard" : 30
+    },
+    "clients" : {
+      "kibana" : 40,
+      "other" : 500
+    },
+    "clusters" : {
+      "(local)" : {
+        "total" : 12,
+        "skipped" : 0,
+        "took" : {
+          "max" : 3983,
+          "avg" : 2992,
+          "p90" : 3791
+        }
+      },
+      "remote1" : {
+        "total" : 100,
+        "skipped" : 22,
+        "took" : {
+          "max" : 3983,
+          "avg" : 2992,
+          "p90" : 3791
+        }
+      },
+      "remote2" : {
+        "total" : 300,
+        "skipped" : 42,
+        "took" : {
+          "max" : 995327,
+          "avg" : 747531,
+          "p90" : 946175
+        }
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/test/external-modules/apm-integration/build.gradle b/test/external-modules/apm-integration/build.gradle
index 3ae62c72968b3..98090f33ee2c7 100644
--- a/test/external-modules/apm-integration/build.gradle
+++ b/test/external-modules/apm-integration/build.gradle
@@ -22,4 +22,5 @@ tasks.named('javaRestTest').configure {
 
 dependencies {
   clusterModules project(':modules:apm')
+  implementation project(':libs:elasticsearch-logging')
 }
diff --git a/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java b/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java
index 9980c0a25a5dd..e8d8e20727c0a 100644
--- a/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java
+++ b/test/external-modules/apm-integration/src/javaRestTest/java/org/elasticsearch/test/apmintegration/MetricsApmIT.java
@@ -89,10 +89,17 @@ public void testApmIntegration() throws Exception {
                 var metricset = (Map<String, Object>) apmMessage.get("metricset");
                 var samples = (Map<String, Object>) metricset.get("samples");
 
-                samples.entrySet().forEach(sampleEntry -> {
-                    var assertion = sampleAssertions.get(sampleEntry.getKey());// sample name
-                    if (assertion != null && assertion.test((Map<String, Object>) sampleEntry.getValue())) {// sample object
-                        sampleAssertions.remove(sampleEntry.getKey());
+                samples.forEach((key, value) -> {
+                    var assertion = sampleAssertions.get(key);// sample name
+                    if (assertion != null) {
+                        logger.info("Matched {}", key);
+                        var sampleObject = (Map<String, Object>) value;
+                        if (assertion.test(sampleObject)) {// sample object
+                            logger.info("{} assertion PASSED", key);
+                            sampleAssertions.remove(key);
+                        } else {
+                            logger.error("{} assertion FAILED: {}", key, sampleObject.get("value"));
+                        }
                     }
                 });
             }
@@ -106,10 +113,8 @@ public void testApmIntegration() throws Exception {
 
         client().performRequest(new Request("GET", "/_use_apm_metrics"));
 
-        assertTrue(
-            "Timeout when waiting for assertions to complete. Remaining assertions to match: " + sampleAssertions,
-            finished.await(30, TimeUnit.SECONDS)
-        );
+        var completed = finished.await(30, TimeUnit.SECONDS);
+        assertTrue("Timeout when waiting for assertions to complete. Remaining assertions to match: " + sampleAssertions, completed);
     }
 
     private <T> Map.Entry<String, Predicate<Map<String, Object>>> assertion(
diff --git a/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/TestApmIntegrationRestHandler.java b/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/TestApmIntegrationRestHandler.java
index b4f6708cc71e8..e667fa5ddcbfb 100644
--- a/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/TestApmIntegrationRestHandler.java
+++ b/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/TestApmIntegrationRestHandler.java
@@ -22,7 +22,7 @@
 
 public class TestApmIntegrationRestHandler extends BaseRestHandler {
 
-    private SetOnce<TestMeterUsages> testMeterUsages = new SetOnce<>();
+    private final SetOnce<TestMeterUsages> testMeterUsages = new SetOnce<>();
 
     TestApmIntegrationRestHandler() {}
 
diff --git a/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/TestMeterUsages.java b/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/TestMeterUsages.java
index 9c23ce371e044..07d43090f70ab 100644
--- a/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/TestMeterUsages.java
+++ b/test/external-modules/apm-integration/src/main/java/org/elasticsearch/test/apmintegration/TestMeterUsages.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.test.apmintegration;
 
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.telemetry.metric.DoubleCounter;
 import org.elasticsearch.telemetry.metric.DoubleHistogram;
 import org.elasticsearch.telemetry.metric.DoubleWithAttributes;
@@ -20,6 +22,8 @@
 
 public class TestMeterUsages {
 
+    private static final Logger logger = LogManager.getLogger(TestMeterUsages.class);
+
     private final DoubleCounter doubleCounter;
     private final DoubleCounter longCounter;
     private final DoubleHistogram doubleHistogram;
@@ -32,14 +36,30 @@ public TestMeterUsages(MeterRegistry meterRegistry) {
         this.longCounter = meterRegistry.registerDoubleCounter("es.test.double_counter.total", "test", "unit");
         this.doubleHistogram = meterRegistry.registerDoubleHistogram("es.test.double_histogram.histogram", "test", "unit");
         this.longHistogram = meterRegistry.registerLongHistogram("es.test.long_histogram.histogram", "test", "unit");
-        meterRegistry.registerDoubleGauge("es.test.double_gauge.current", "test", "unit", doubleWithAttributes::get);
-        meterRegistry.registerLongGauge("es.test.long_gauge.current", "test", "unit", longWithAttributes::get);
-
-        meterRegistry.registerLongAsyncCounter("es.test.async_long_counter.total", "test", "unit", longWithAttributes::get);
-        meterRegistry.registerDoubleAsyncCounter("es.test.async_double_counter.total", "test", "unit", doubleWithAttributes::get);
+        meterRegistry.registerDoubleGauge("es.test.double_gauge.current", "test", "unit", () -> {
+            var value = doubleWithAttributes.get();
+            logger.info("[es.test.double_gauge.current] callback with value [{}]", value);
+            return value;
+        });
+        meterRegistry.registerLongGauge("es.test.long_gauge.current", "test", "unit", () -> {
+            var value = longWithAttributes.get();
+            logger.info("[es.test.long_gauge.current] callback with value [{}]", value);
+            return value;
+        });
+        meterRegistry.registerLongAsyncCounter("es.test.async_long_counter.total", "test", "unit", () -> {
+            var value = longWithAttributes.get();
+            logger.info("[es.test.async_long_counter.total] callback with value [{}]", value);
+            return value;
+        });
+        meterRegistry.registerDoubleAsyncCounter("es.test.async_double_counter.total", "test", "unit", () -> {
+            var value = doubleWithAttributes.get();
+            logger.info("[es.test.async_double_counter.total] callback with value [{}]", value);
+            return value;
+        });
     }
 
     public void testUponRequest() {
+        logger.info("setting counters");
         doubleCounter.increment();
         longCounter.increment();
         doubleHistogram.record(1.0);
@@ -48,6 +68,7 @@ public void testUponRequest() {
         longHistogram.record(2);
 
         // triggers gauges and async counters
+        logger.info("setting async counters");
         doubleWithAttributes.set(new DoubleWithAttributes(1.0, Map.of()));
         longWithAttributes.set(new LongWithAttributes(1, Map.of()));
     }
diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
index 45ef737dc6095..97b40dfeee52a 100644
--- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
+++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java
@@ -162,7 +162,8 @@ public void testSortByManyLongsTooMuchMemoryAsync() throws IOException {
             matchesMap().entry("status", 429)
                 .entry(
                     "error",
-                    matchesMap().entry("bytes_wanted", greaterThan(1000))
+                    matchesMap().extraOk()
+                        .entry("bytes_wanted", greaterThan(1000))
                         .entry("reason", matchesRegex("\\[request] Data too large, data for \\[topn] would .+"))
                         .entry("durability", "TRANSIENT")
                         .entry("type", "circuit_breaking_exception")
diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java
index 2e80bdecbad8d..b9f7d7044a958 100644
--- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java
+++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpFixture.java
@@ -8,43 +8,258 @@
 package fixture.azure;
 
 import com.sun.net.httpserver.HttpServer;
+import com.sun.net.httpserver.HttpsConfigurator;
+import com.sun.net.httpserver.HttpsServer;
 
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ssl.KeyStoreUtil;
+import org.elasticsearch.common.ssl.PemUtils;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.test.ESTestCase;
 import org.junit.rules.ExternalResource;
 
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.SecureRandom;
+import java.security.cert.Certificate;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Predicate;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+
+import static org.elasticsearch.test.ESTestCase.assertThat;
+import static org.elasticsearch.test.ESTestCase.fail;
+import static org.hamcrest.Matchers.hasSize;
 
 public class AzureHttpFixture extends ExternalResource {
 
-    private final boolean enabled;
+    private final Protocol protocol;
     private final String account;
     private final String container;
+    private final String clientId;
+    private final String tenantId;
+    private final Predicate<String> authHeaderPredicate;
+
     private HttpServer server;
+    private HttpServer metadataServer;
+    private HttpServer oauthTokenServiceServer;
+
+    // JWT-looking value for workload identity authentication -- the exact value has no inherent meaning
+    private final String federatedToken = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9";
+
+    public enum Protocol {
+        NONE,
+        HTTP,
+        HTTPS
+    }
+
+    /**
+     * @param  account The name of the Azure Blob Storage account against which the request should be authorized..
+     * @return a predicate that matches the {@code Authorization} HTTP header that the Azure SDK sends when using shared key auth (i.e.
+     *         using a key or SAS token).
+     * @see <a href="https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key">Azure docs on shared key auth</a>
+     */
+    public static Predicate<String> sharedKeyForAccountPredicate(String account) {
+        return new Predicate<>() {
+            @Override
+            public boolean test(String s) {
+                return s.startsWith("SharedKey " + account + ":");
+            }
+
+            @Override
+            public String toString() {
+                return "SharedKey[" + account + "]";
+            }
+        };
+    }
+
+    /**
+     * A sentinel value for {@code #authHeaderPredicate} which indicates that requests should be authorized by a bearer token provided
+     * by the managed identity service.
+     */
+    public static final Predicate<String> MANAGED_IDENTITY_BEARER_TOKEN_PREDICATE = s -> fail(null, "should not be called");
+
+    /**
+     * A sentinel value for {@code #authHeaderPredicate} which indicates that requests should be authorized by a bearer token provided
+     * by the workload identity service.
+     */
+    public static final Predicate<String> WORK_IDENTITY_BEARER_TOKEN_PREDICATE = s -> fail(null, "should not be called");
+
+    /**
+     * @param bearerToken The bearer token to accept
+     * @return a predicate that matches the {@code Authorization} HTTP header that the Azure SDK sends when using bearer token auth (i.e.
+     *         using Managed Identity)
+     */
+    private static Predicate<String> bearerTokenPredicate(String bearerToken) {
+        return new Predicate<>() {
+            @Override
+            public boolean test(String s) {
+                return s.equals("Bearer " + bearerToken);
+            }
 
-    public AzureHttpFixture(boolean enabled, String account, String container) {
-        this.enabled = enabled;
+            @Override
+            public String toString() {
+                return "Bearer[" + bearerToken + "]";
+            }
+        };
+    }
+
+    public AzureHttpFixture(
+        Protocol protocol,
+        String account,
+        String container,
+        @Nullable String rawTenantId,
+        @Nullable String rawClientId,
+        Predicate<String> authHeaderPredicate
+    ) {
+        final var tenantId = Strings.hasText(rawTenantId) ? rawTenantId : null;
+        final var clientId = Strings.hasText(rawClientId) ? rawClientId : null;
+
+        if ((clientId == null) != (tenantId == null)) {
+            fail(null, "either both [tenantId] and [clientId] must be set or neither must be set");
+        }
+        if (tenantId != null) {
+            if (protocol != Protocol.HTTPS) {
+                fail(null, "when [tenantId] is set, protocol must be HTTPS");
+            }
+        }
+        this.protocol = protocol;
         this.account = account;
         this.container = container;
+        this.tenantId = tenantId;
+        this.clientId = clientId;
+        this.authHeaderPredicate = authHeaderPredicate;
+    }
+
+    private String scheme() {
+        return switch (protocol) {
+            case NONE -> fail(null, "fixture is disabled");
+            case HTTP -> "http";
+            case HTTPS -> "https";
+        };
     }
 
     public String getAddress() {
-        return "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort() + "/" + account;
+        return scheme() + "://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort() + "/" + account;
+    }
+
+    public String getOAuthTokenServiceAddress() {
+        return scheme()
+            + "://"
+            + oauthTokenServiceServer.getAddress().getHostString()
+            + ":"
+            + oauthTokenServiceServer.getAddress().getPort()
+            + "/";
+    }
+
+    public String getMetadataAddress() {
+        return "http://" + metadataServer.getAddress().getHostString() + ":" + metadataServer.getAddress().getPort() + "/";
+    }
+
+    public String getFederatedToken() {
+        return federatedToken;
     }
 
     @Override
-    protected void before() throws IOException {
-        if (enabled) {
-            this.server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
-            server.createContext("/" + account, new AzureHttpHandler(account, container));
-            server.start();
+    protected void before() {
+        try {
+            // Set different bearer tokens for managed and workload identity to ensure that we are using the expected one for access
+            final var managedIdentityBearerToken = ESTestCase.randomIdentifier();
+            final var workloadIdentityBearerToken = ESTestCase.randomIdentifier();
+
+            final Predicate<String> actualAuthHeaderPredicate;
+            if (authHeaderPredicate == MANAGED_IDENTITY_BEARER_TOKEN_PREDICATE) {
+                actualAuthHeaderPredicate = bearerTokenPredicate(managedIdentityBearerToken);
+            } else if (authHeaderPredicate == WORK_IDENTITY_BEARER_TOKEN_PREDICATE) {
+                actualAuthHeaderPredicate = bearerTokenPredicate(workloadIdentityBearerToken);
+            } else {
+                actualAuthHeaderPredicate = authHeaderPredicate;
+            }
+
+            if (protocol != Protocol.NONE) {
+                this.metadataServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+                metadataServer.createContext("/", new AzureMetadataServiceHttpHandler(managedIdentityBearerToken));
+                metadataServer.start();
+            }
+
+            switch (protocol) {
+                case NONE -> {
+                }
+                case HTTP -> {
+                    server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+                    server.createContext("/" + account, new AzureHttpHandler(account, container, actualAuthHeaderPredicate));
+                    server.start();
+
+                    oauthTokenServiceServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+                    oauthTokenServiceServer.createContext(
+                        "/",
+                        new AzureOAuthTokenServiceHttpHandler(workloadIdentityBearerToken, federatedToken, tenantId, clientId)
+                    );
+                    oauthTokenServiceServer.start();
+                }
+                case HTTPS -> {
+                    final var tmpdir = ESTestCase.createTempDir();
+                    final var certificates = PemUtils.readCertificates(List.of(copyResource(tmpdir, "azure-http-fixture.pem")));
+                    assertThat(certificates, hasSize(1));
+                    final SSLContext sslContext = SSLContext.getInstance("TLS");
+                    sslContext.init(
+                        new KeyManager[] {
+                            KeyStoreUtil.createKeyManager(
+                                new Certificate[] { certificates.get(0) },
+                                PemUtils.readPrivateKey(copyResource(tmpdir, "azure-http-fixture.key"), () -> null),
+                                null
+                            ) },
+                        null,
+                        new SecureRandom()
+                    );
+                    {
+                        final var httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+                        this.server = httpsServer;
+                        httpsServer.setHttpsConfigurator(new HttpsConfigurator(sslContext));
+                        httpsServer.createContext("/" + account, new AzureHttpHandler(account, container, actualAuthHeaderPredicate));
+                        httpsServer.start();
+                    }
+                    {
+                        final var httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+                        this.oauthTokenServiceServer = httpsServer;
+                        httpsServer.setHttpsConfigurator(new HttpsConfigurator(sslContext));
+                        httpsServer.createContext(
+                            "/",
+                            new AzureOAuthTokenServiceHttpHandler(workloadIdentityBearerToken, federatedToken, tenantId, clientId)
+                        );
+                        httpsServer.start();
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new AssertionError("unexpected", e);
+        }
+    }
+
+    private Path copyResource(Path tmpdir, String name) throws IOException {
+        try (
+            var stream = Objects.requireNonNullElseGet(
+                getClass().getResourceAsStream(name),
+                () -> ESTestCase.fail(null, "resource [%s] not found", name)
+            )
+        ) {
+            final var path = tmpdir.resolve(name);
+            Files.write(path, stream.readAllBytes());
+            return path;
         }
     }
 
     @Override
     protected void after() {
-        if (enabled) {
+        if (protocol != Protocol.NONE) {
             server.stop(0);
+            metadataServer.stop(0);
+            oauthTokenServiceServer.stop(0);
         }
     }
 }
diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
index e89dd10081a95..d46afdcf93cd2 100644
--- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
+++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
@@ -15,9 +15,12 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.RestUtils;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -32,6 +35,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -45,15 +49,67 @@ public class AzureHttpHandler implements HttpHandler {
     private final Map<String, BytesReference> blobs;
     private final String account;
     private final String container;
+    private final Predicate<String> authHeaderPredicate;
 
-    public AzureHttpHandler(final String account, final String container) {
+    public AzureHttpHandler(final String account, final String container, @Nullable Predicate<String> authHeaderPredicate) {
         this.account = Objects.requireNonNull(account);
         this.container = Objects.requireNonNull(container);
+        this.authHeaderPredicate = authHeaderPredicate;
         this.blobs = new ConcurrentHashMap<>();
     }
 
+    private static List<String> getAuthHeader(HttpExchange exchange) {
+        return exchange.getRequestHeaders().get("Authorization");
+    }
+
+    private boolean isValidAuthHeader(HttpExchange exchange) {
+        if (authHeaderPredicate == null) {
+            return true;
+        }
+
+        final var authHeader = getAuthHeader(exchange);
+        if (authHeader == null) {
+            return false;
+        }
+
+        if (authHeader.size() != 1) {
+            return false;
+        }
+
+        return authHeaderPredicate.test(authHeader.get(0));
+    }
+
     @Override
     public void handle(final HttpExchange exchange) throws IOException {
+        if (isValidAuthHeader(exchange) == false) {
+            try (exchange; var builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+                builder.startObject();
+                builder.field("method", exchange.getRequestMethod());
+                builder.field("uri", exchange.getRequestURI().toString());
+                builder.field("predicate", authHeaderPredicate.toString());
+                builder.field("authorization", Objects.toString(getAuthHeader(exchange)));
+                builder.startObject("headers");
+                for (final var header : exchange.getRequestHeaders().entrySet()) {
+                    if (header.getValue() == null) {
+                        builder.nullField(header.getKey());
+                    } else {
+                        builder.startArray(header.getKey());
+                        for (final var value : header.getValue()) {
+                            builder.value(value);
+                        }
+                        builder.endArray();
+                    }
+                }
+                builder.endObject();
+                builder.endObject();
+                final var responseBytes = BytesReference.bytes(builder);
+                exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8");
+                exchange.sendResponseHeaders(RestStatus.FORBIDDEN.getStatus(), responseBytes.length());
+                responseBytes.writeTo(exchange.getResponseBody());
+                return;
+            }
+        }
+
         final String request = exchange.getRequestMethod() + " " + exchange.getRequestURI().toString();
         if (request.startsWith("GET") || request.startsWith("HEAD") || request.startsWith("DELETE")) {
             int read = exchange.getRequestBody().read();
diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureMetadataServiceHttpHandler.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureMetadataServiceHttpHandler.java
new file mode 100644
index 0000000000000..18f87fc7ae5cc
--- /dev/null
+++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureMetadataServiceHttpHandler.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package fixture.azure;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Emulates the instance metadata service that runs on Azure
+ */
+@SuppressForbidden(reason = "Uses a HttpServer to emulate an Azure endpoint")
+public class AzureMetadataServiceHttpHandler implements HttpHandler {
+    private static final Logger logger = LogManager.getLogger(AzureMetadataServiceHttpHandler.class);
+
+    private final String bearerToken;
+
+    public AzureMetadataServiceHttpHandler(String bearerToken) {
+        this.bearerToken = bearerToken;
+    }
+
+    @Override
+    public void handle(HttpExchange exchange) throws IOException {
+        if ("GET".equals(exchange.getRequestMethod())
+            && "/metadata/identity/oauth2/token".equals(exchange.getRequestURI().getPath())
+            && "api-version=2018-02-01&resource=https://storage.azure.com".equals(exchange.getRequestURI().getQuery())) {
+            AzureOAuthTokenServiceHttpHandler.respondWithValidAccessToken(exchange, bearerToken);
+            return;
+        }
+
+        final var msgBuilder = new StringWriter();
+        msgBuilder.append("method: ").append(exchange.getRequestMethod()).append(System.lineSeparator());
+        msgBuilder.append("uri: ").append(exchange.getRequestURI().toString()).append(System.lineSeparator());
+        msgBuilder.append("headers:").append(System.lineSeparator());
+        for (final var header : exchange.getRequestHeaders().entrySet()) {
+            msgBuilder.append("- ").append(header.getKey()).append(System.lineSeparator());
+            for (final var value : header.getValue()) {
+                msgBuilder.append("  - ").append(value).append(System.lineSeparator());
+            }
+        }
+        final var msg = msgBuilder.toString();
+        logger.info("{}", msg);
+        final var responseBytes = msg.getBytes(StandardCharsets.UTF_8);
+        exchange.sendResponseHeaders(200, responseBytes.length);
+        new BytesArray(responseBytes).writeTo(exchange.getResponseBody());
+        exchange.close();
+    }
+}
diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureOAuthTokenServiceHttpHandler.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureOAuthTokenServiceHttpHandler.java
new file mode 100644
index 0000000000000..52002b2b86147
--- /dev/null
+++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureOAuthTokenServiceHttpHandler.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package fixture.azure;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.rest.RestUtils;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+
+@SuppressForbidden(reason = "Uses a HttpServer to emulate an Azure endpoint")
+public class AzureOAuthTokenServiceHttpHandler implements HttpHandler {
+    private static final Logger logger = LogManager.getLogger(AzureOAuthTokenServiceHttpHandler.class);
+    private static final String EXPECTED_SCOPE = "https://storage.azure.com/.default";
+
+    private final String bearerToken;
+    private final String federatedToken;
+    @Nullable
+    private final String tenantId;
+    @Nullable
+    private final String clientId;
+
+    public AzureOAuthTokenServiceHttpHandler(
+        String bearerToken,
+        String federatedToken,
+        @Nullable String tenantId,
+        @Nullable String clientId
+    ) {
+        this.bearerToken = bearerToken;
+        this.federatedToken = federatedToken;
+        this.tenantId = tenantId;
+        this.clientId = clientId;
+    }
+
+    @Override
+    public void handle(HttpExchange exchange) throws IOException {
+        if (tenantId == null) {
+            throw new AssertionError("OAuth token service handler should not be called if [tenantId] is null");
+        }
+        if (clientId == null) {
+            throw new AssertionError("OAuth token service handler should not be called if [clientId] is null");
+        }
+
+        if ("POST".equals(exchange.getRequestMethod())
+            && ("/" + tenantId + "/oauth2/v2.0/token").equals(exchange.getRequestURI().getPath())) {
+            final String requestBody = Streams.copyToString(new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8));
+
+            final Map<String, String> params = new HashMap<>();
+            RestUtils.decodeQueryString(requestBody, 0, params);
+
+            if (clientId.equals(params.get("client_id"))
+                && federatedToken.equals(params.get("client_assertion"))
+                && EXPECTED_SCOPE.equals(params.get("scope"))) {
+                respondWithValidAccessToken(exchange, bearerToken);
+                return;
+            }
+
+            logger.error(
+                "Request body did not contain expected [client_id], [client_assertion], or [scope]. Request data: [{}]",
+                requestBody
+            );
+            // fall through to further logging and dummy response
+
+        }
+
+        final var msgBuilder = new StringWriter();
+        msgBuilder.append("method: ").append(exchange.getRequestMethod()).append(System.lineSeparator());
+        msgBuilder.append("uri: ").append(exchange.getRequestURI().toString()).append(System.lineSeparator());
+        msgBuilder.append("headers:").append(System.lineSeparator());
+        for (final var header : exchange.getRequestHeaders().entrySet()) {
+            msgBuilder.append("- ").append(header.getKey()).append(System.lineSeparator());
+            for (final var value : header.getValue()) {
+                msgBuilder.append("  - ").append(value).append(System.lineSeparator());
+            }
+        }
+        final var msg = msgBuilder.toString();
+        logger.info("{}", msg);
+        final var responseBytes = msg.getBytes(StandardCharsets.UTF_8);
+        exchange.sendResponseHeaders(200, responseBytes.length);
+        new BytesArray(responseBytes).writeTo(exchange.getResponseBody());
+        exchange.close();
+    }
+
+    static void respondWithValidAccessToken(HttpExchange exchange, String bearerToken) throws IOException {
+        try (exchange; var accessTokenXContent = accessTokenXContent(bearerToken)) {
+            final var responseBytes = BytesReference.bytes(accessTokenXContent);
+            exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8");
+            exchange.sendResponseHeaders(200, responseBytes.length());
+            responseBytes.writeTo(exchange.getResponseBody());
+        }
+    }
+
+    private static XContentBuilder accessTokenXContent(String bearerToken) throws IOException {
+        final var xcb = XContentBuilder.builder(XContentType.JSON.xContent());
+        xcb.startObject();
+        xcb.field("access_token", bearerToken);
+        xcb.field("client_id", UUIDs.randomBase64UUID());
+        xcb.field("expires_in", 86400L);
+        xcb.field("expires_on", System.currentTimeMillis() / 1000L + 86400L);
+        xcb.field("ext_expires_in", 86400L);
+        xcb.field("not_before", System.currentTimeMillis() / 1000L);
+        xcb.field("resource", "https://storage.azure.com");
+        xcb.field("token_type", "Bearer");
+        xcb.endObject();
+        return xcb;
+    }
+}
diff --git a/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.key b/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.key
new file mode 100644
index 0000000000000..29efd3a6913a0
--- /dev/null
+++ b/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCswbyZDtaghZXs
+Phs1+lqCnq5HmRT2P6Drrs9bJlABeql29IhzdHOGLr+lTMhKOUpHuphgC31qbf/G
+vQLS65qdOzTjNfLv93+Jj0gp4S7Q6eRZvn1ihUgzECHazTYwIlzVs4sFPm5i2fQb
+DK7W6zQm+h9r6GjCYj01OeIAe7rbRI9Ar+svuHGfZnaQHzLZlfYkkM2bCaXBgKWV
+wmEUmwMW+IMOPCrVm+gk1MDbGnu9KtY/LqrJcddsqOdkK8qJ0Lpchg3zlP4qIzbm
+WRyTUIy1USbcazjuC/vMmN4fr/Xr0Jrhi4Rw8l2LGdyA8qnqtKYTqMzo3uv1ESlE
+R8EAZDUbAgMBAAECggEAY8lYRdSTVo8y5Q2OrCQa6b38jvC2cfKY4enMbir4JZKT
+lllzA7VtEUGpgzKRsoXbCQmYAEpCvBojlskQe4KJgW50gxVjaQa9zVhM55vhbdzc
+AJaOWD0CUjRsSbUlKrJ+ixW1JGdGXaTlYkZ2K0AalLT/N1Y8RKN4FWmEyKCvcvz4
+0XzOIVmG+HqcNURamXTxMKbj1yzi5goue2/iP2kMFo8sHxRsGvvV4PWo6JrktE2y
+47oiH42lpSIcpLSE8z/csLbMTw/Q/pPQAYqyvEJHU22uhac1XmMqFHWNSpQZq6gr
+46t8YQ3FJSN8UrZf1h1mdvLlK/sgPEvCQa6TrCq4GQKBgQDbl0M/4gJZhgpvBuCC
+aratamWcFoa/pu1/JoQhPXLv6uGwB/cFhJUVr0ZoI5KPFJr9SG4kQ/eEPywkY2qT
+mGPVlVmGOeJa1VK8TRUTzkEFTYWytUepACM2//LiWvzABciO8SxWgNZrmUKghQTN
+d989b8edy0ti6y7lHpkTyawVXQKBgQDJZo7X6K+6cyuXORApv98cU5C3vEVtBp/c
+QfU/rRj/YXAbYFhKIS5rF/Gfg2YoDa5YguMxGsIPzYwdTI5gGGgSgolapz3fr22q
+edCPaFg8qO64pIii+Ar4lx4k1IyNtpJ+nvlam7sI9yGzksrVazsWhpaSKX8xGd7r
+9ZSr/c8U1wKBgGWl+pJay52nR7MnasvUHCXgR5LedpfG7M9cA/PjHw5iGwDCXx2l
+xuFX1m6kcNZcwnYWji2pbK1CFOvvPUl/VE9tKBjTOK21a+wQfn5BjqWmwgn8kmRv
+1N1D06nmVnOI+dL5Xv3X++mo80ec66E1KRimYq/viEEM/xM+e7vGMitdAoGAUAUe
+pix+fa8615fFk0D37bJKIqZ8Uyg5pfLS9ZzZ/MYDG+14xuNOJSDbUMyNb0aYSfSf
+PihqiIrbq9x6CTZJS2lwF4Oxcsmp4f0KX6BOxrM8PkKpQ08YVNL+GBYXTksHA6Y4
+XsbXVmWSj124l3lGfdm1w5cXQTQNPWVSz89FUvsCgYEArl27gbXndTLpZ4dNkeBS
+0JMe84nPPrrwHbNwcqkoDiux5Ln+AZE8jYSlp4kUfd1A7XDTxXWXIxeD0YLiabf3
+5+/QzJ6j1qi77JoyadomnL7CFI5f2FotKt029PeVxAUOohao94p8J5OuaRMPvkGC
+CNhjfRAIBhfm9kdyjPmwVLU=
+-----END PRIVATE KEY-----
diff --git a/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.pem b/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.pem
new file mode 100644
index 0000000000000..b291aaa9362de
--- /dev/null
+++ b/test/fixtures/azure-fixture/src/main/resources/fixture/azure/azure-http-fixture.pem
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDRTCCAi2gAwIBAgIVAJpxxIbXWyvdd6/rIFXPgWe6fyvTMA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMB4XDTE4MTIyMTA3NDY1NVoXDTQ2MDUwNzA3NDY1NVowGTEXMBUG
+A1UEAxMObGRhcC10ZXN0LWNhc2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQCswbyZDtaghZXsPhs1+lqCnq5HmRT2P6Drrs9bJlABeql29IhzdHOGLr+l
+TMhKOUpHuphgC31qbf/GvQLS65qdOzTjNfLv93+Jj0gp4S7Q6eRZvn1ihUgzECHa
+zTYwIlzVs4sFPm5i2fQbDK7W6zQm+h9r6GjCYj01OeIAe7rbRI9Ar+svuHGfZnaQ
+HzLZlfYkkM2bCaXBgKWVwmEUmwMW+IMOPCrVm+gk1MDbGnu9KtY/LqrJcddsqOdk
+K8qJ0Lpchg3zlP4qIzbmWRyTUIy1USbcazjuC/vMmN4fr/Xr0Jrhi4Rw8l2LGdyA
+8qnqtKYTqMzo3uv1ESlER8EAZDUbAgMBAAGjaTBnMB0GA1UdDgQWBBQaiCDScfBa
+jHOSk04XOymffbLBxTAfBgNVHSMEGDAWgBROJaHRWe17um5rqqYn10aqedr55DAa
+BgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwCQYDVR0TBAIwADANBgkqhkiG9w0B
+AQsFAAOCAQEAXBovNqVg+VQ1LR0PfEMpbgbQlekky8qY2y1tz7J0ntGepAq+Np6n
+7J9En6ty1ELZUvgPUCF2btQqZbv8uyHz/C+rojKC5xzHN5qbZ31o5/0I/kNase1Z
+NbXuNJe3wAXuz+Mj5rtuOGZvlFsbtocuoydVYOclfqjUXcoZtqCcRamSvye7vGl2
+CHPqDi0uK8d75nE9Jrnmz/BNNV7CjPg636PJmCUrLL21+t69ZFL1eGAFtLBmmjcw
+cMkyv9bJirjZbjt/9UB+fW9XzV3RVLAzfrIHtToupXmWc4+hTOnlbKfFwqB9fa7Y
+XcCfGrZoJg9di1HbJrSJmv5QgRTM+/zkrA==
+-----END CERTIFICATE-----
diff --git a/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java b/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java
index 9a5205f66d1f4..5932890dd8459 100644
--- a/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java
+++ b/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/EnterpriseGeoIpHttpFixture.java
@@ -32,7 +32,6 @@
 public class EnterpriseGeoIpHttpFixture extends ExternalResource {
 
     private final Path source;
-    private final boolean enabled;
     private final String[] databaseTypes;
     private HttpServer server;
 
@@ -40,8 +39,7 @@ public class EnterpriseGeoIpHttpFixture extends ExternalResource {
      * The values in databaseTypes must be in DatabaseConfiguration.MAXMIND_NAMES, and must be one of the databases copied in the
      * copyFiles method of thisi class.
      */
-    public EnterpriseGeoIpHttpFixture(boolean enabled, String... databaseTypes) {
-        this.enabled = enabled;
+    public EnterpriseGeoIpHttpFixture(String... databaseTypes) {
         this.databaseTypes = databaseTypes;
         try {
             this.source = Files.createTempDirectory("source");
@@ -56,28 +54,26 @@ public String getAddress() {
 
     @Override
     protected void before() throws Throwable {
-        if (enabled) {
-            copyFiles();
-            this.server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+        copyFiles();
+        this.server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
 
-            // for expediency reasons, it is handy to have this test fixture be able to serve the dual purpose of actually stubbing
-            // out the download protocol for downloading files from maxmind (see the looped context creation after this stanza), as
-            // we as to serve an empty response for the geoip.elastic.co service here
-            this.server.createContext("/", exchange -> {
-                String response = "[]"; // an empty json array
-                exchange.sendResponseHeaders(200, response.length());
-                try (OutputStream os = exchange.getResponseBody()) {
-                    os.write(response.getBytes(StandardCharsets.UTF_8));
-                }
-            });
-
-            // register the file types for the download fixture
-            for (String databaseType : databaseTypes) {
-                createContextForEnterpriseDatabase(databaseType);
+        // for expediency reasons, it is handy to have this test fixture be able to serve the dual purpose of actually stubbing
+        // out the download protocol for downloading files from maxmind (see the looped context creation after this stanza), as
+        // we as to serve an empty response for the geoip.elastic.co service here
+        this.server.createContext("/", exchange -> {
+            String response = "[]"; // an empty json array
+            exchange.sendResponseHeaders(200, response.length());
+            try (OutputStream os = exchange.getResponseBody()) {
+                os.write(response.getBytes(StandardCharsets.UTF_8));
             }
+        });
 
-            server.start();
+        // register the file types for the download fixture
+        for (String databaseType : databaseTypes) {
+            createContextForEnterpriseDatabase(databaseType);
         }
+
+        server.start();
     }
 
     private void createContextForEnterpriseDatabase(String databaseType) {
@@ -108,9 +104,7 @@ private void createContextForEnterpriseDatabase(String databaseType) {
 
     @Override
     protected void after() {
-        if (enabled) {
-            server.stop(0);
-        }
+        server.stop(0);
     }
 
     private void copyFiles() throws Exception {
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index bd4acf4e51505..2c7acb06c0c57 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -52,7 +52,7 @@ def hdfs2Jar = tasks.register("hdfs2jar", ShadowJar) {
   configurations << project.configurations.hdfs2
 }
 
-tasks.withType(ShadowJar) {
+tasks.withType(ShadowJar).configureEach {
   dependencies {
 //    exclude(dependency('commons-io:commons-io:2.8.0'))
     exclude(dependency("com.carrotsearch.randomizedtesting:randomizedtesting-runner:.*"))
diff --git a/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java b/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java
index fa75b57ea87a6..14357b6d47bbc 100644
--- a/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java
+++ b/test/fixtures/krb5kdc-fixture/src/main/java/org/elasticsearch/test/fixtures/krb5kdc/Krb5kDcContainer.java
@@ -22,6 +22,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -74,6 +75,7 @@ public Krb5kDcContainer(ProvisioningId provisioningId) {
         this.provisioningId = provisioningId;
         withNetwork(Network.newNetwork());
         addExposedPorts(88, 4444);
+        withStartupTimeout(Duration.ofMinutes(2));
         withCreateContainerCmdModifier(cmd -> {
             // Add previously exposed ports and UDP port
             List<ExposedPort> exposedPorts = new ArrayList<>();
diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java
index 7f363fe0b87c3..447e225005b58 100644
--- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java
+++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java
@@ -61,6 +61,7 @@ public class S3HttpHandler implements HttpHandler {
 
     private final String bucket;
     private final String path;
+    private final String basePrefix;
 
     private final ConcurrentMap<String, BytesReference> blobs = new ConcurrentHashMap<>();
     private final ConcurrentMap<String, MultipartUpload> uploads = new ConcurrentHashMap<>();
@@ -71,6 +72,7 @@ public S3HttpHandler(final String bucket) {
 
     public S3HttpHandler(final String bucket, @Nullable final String basePath) {
         this.bucket = Objects.requireNonNull(bucket);
+        this.basePrefix = Objects.requireNonNullElse(basePath, "");
         this.path = bucket + (basePath != null && basePath.isEmpty() == false ? "/" + basePath : "");
     }
 
@@ -96,7 +98,9 @@ public void handle(final HttpExchange exchange) throws IOException {
                 } else {
                     exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1);
                 }
-            } else if (Regex.simpleMatch("GET /" + bucket + "/?uploads&prefix=*", request)) {
+            } else if (isListMultipartUploadsRequest(request)) {
+                assert request.contains("prefix=" + basePrefix) : basePrefix + " vs " + request;
+
                 final Map<String, String> params = new HashMap<>();
                 RestUtils.decodeQueryString(request, request.indexOf('?') + 1, params);
                 final var prefix = params.get("prefix");
@@ -329,6 +333,11 @@ public void handle(final HttpExchange exchange) throws IOException {
         }
     }
 
+    private boolean isListMultipartUploadsRequest(String request) {
+        return Regex.simpleMatch("GET /" + bucket + "/?uploads&prefix=*", request)
+            || Regex.simpleMatch("GET /" + bucket + "/?uploads&max-uploads=*&prefix=*", request);
+    }
+
     public Map<String, BytesReference> blobs() {
         return blobs;
     }
diff --git a/test/framework/build.gradle b/test/framework/build.gradle
index 4d598a00de7b6..c8d4aba10b478 100644
--- a/test/framework/build.gradle
+++ b/test/framework/build.gradle
@@ -31,6 +31,9 @@ dependencies {
   api 'org.objenesis:objenesis:3.3'
 
   api "org.elasticsearch:mocksocket:${versions.mocksocket}"
+
+  testImplementation project(':x-pack:plugin:mapper-unsigned-long')
+  testImplementation project(":modules:mapper-extras")
 }
 
 sourceSets {
diff --git a/test/framework/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/StoredScriptIntegTestUtils.java b/test/framework/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/StoredScriptIntegTestUtils.java
new file mode 100644
index 0000000000000..3036658de1661
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/StoredScriptIntegTestUtils.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.storedscripts;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.XContentType;
+
+import static org.elasticsearch.test.ESTestCase.TEST_REQUEST_TIMEOUT;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+public class StoredScriptIntegTestUtils {
+    private StoredScriptIntegTestUtils() {/* no instances */}
+
+    public static void putJsonStoredScript(String id, String jsonContent) {
+        putJsonStoredScript(id, new BytesArray(jsonContent));
+    }
+
+    public static void putJsonStoredScript(String id, BytesReference jsonContent) {
+        assertAcked(
+            ESIntegTestCase.safeExecute(
+                TransportPutStoredScriptAction.TYPE,
+                new PutStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).id(id).content(jsonContent, XContentType.JSON)
+            )
+        );
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/TestPlainActionFuture.java b/test/framework/src/main/java/org/elasticsearch/action/support/TestPlainActionFuture.java
index 0264920c9d017..e6729a20d93e3 100644
--- a/test/framework/src/main/java/org/elasticsearch/action/support/TestPlainActionFuture.java
+++ b/test/framework/src/main/java/org/elasticsearch/action/support/TestPlainActionFuture.java
@@ -13,7 +13,7 @@
  */
 public class TestPlainActionFuture<T> extends PlainActionFuture<T> {
     @Override
-    boolean allowedExecutors(Thread thread1, Thread thread2) {
+    boolean allowedExecutors(Thread blockedThread, Thread completingThread) {
         return true;
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueue.java b/test/framework/src/main/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueue.java
index 1fac5a9917807..ddf94bc838c4e 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueue.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueue.java
@@ -339,7 +339,7 @@ public String toString() {
 
             @Override
             public long relativeTimeInNanos() {
-                throw new AssertionError("DeterministicTaskQueue does not support nanosecond-precision timestamps");
+                return TimeValue.timeValueMillis(currentTimeMillis).nanos();
             }
 
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
index 19fded1ea90e8..8412e9e250885 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
@@ -99,7 +99,6 @@
 import org.elasticsearch.index.translog.TranslogDeletionPolicy;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -250,7 +249,7 @@ public static EngineConfig copy(EngineConfig config, LongSupplier globalCheckpoi
             config.getMergePolicy(),
             config.getAnalyzer(),
             config.getSimilarity(),
-            config.getCodecService(),
+            config.getCodecProvider(),
             config.getEventListener(),
             config.getQueryCache(),
             config.getQueryCachingPolicy(),
@@ -282,7 +281,7 @@ public EngineConfig copy(EngineConfig config, Analyzer analyzer) {
             config.getMergePolicy(),
             analyzer,
             config.getSimilarity(),
-            config.getCodecService(),
+            config.getCodecProvider(),
             config.getEventListener(),
             config.getQueryCache(),
             config.getQueryCachingPolicy(),
@@ -314,7 +313,7 @@ public EngineConfig copy(EngineConfig config, MergePolicy mergePolicy) {
             mergePolicy,
             config.getAnalyzer(),
             config.getSimilarity(),
-            config.getCodecService(),
+            config.getCodecProvider(),
             config.getEventListener(),
             config.getQueryCache(),
             config.getQueryCachingPolicy(),
@@ -429,7 +428,7 @@ protected static ParsedDocument testParsedDocument(
             source,
             XContentType.JSON,
             mappingUpdate,
-            DocumentSizeObserver.EMPTY_INSTANCE
+            ParsedDocument.DocumentSize.UNKNOWN
         );
     }
 
@@ -670,7 +669,7 @@ public static InternalEngine createInternalEngine(
         if (localCheckpointTrackerSupplier == null) {
             return new InternalTestEngine(config) {
                 @Override
-                IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException {
+                protected IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException {
                     return (indexWriterFactory != null)
                         ? indexWriterFactory.createWriter(directory, iwc)
                         : super.createWriter(directory, iwc);
@@ -686,7 +685,7 @@ protected long doGenerateSeqNoForOperation(final Operation operation) {
         } else {
             return new InternalTestEngine(config, IndexWriter.MAX_DOCS, localCheckpointTrackerSupplier) {
                 @Override
-                IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException {
+                protected IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException {
                     return (indexWriterFactory != null)
                         ? indexWriterFactory.createWriter(directory, iwc)
                         : super.createWriter(directory, iwc);
@@ -1401,12 +1400,7 @@ public static void assertAtMostOneLuceneDocumentPerSequenceNumber(IndexSettings
 
     public static MapperService createMapperService() throws IOException {
         IndexMetadata indexMetadata = IndexMetadata.builder("test")
-            .settings(
-                Settings.builder()
-                    .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
-                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
-            )
+            .settings(indexSettings(1, 1).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
             .putMapping("{\"properties\": {}}")
             .build();
         MapperService mapperService = MapperTestUtils.newMapperService(
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java
index 6abe923851318..2f452161b10ca 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/KeywordFieldSyntheticSourceSupport.java
@@ -21,8 +21,6 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.hamcrest.Matchers.equalTo;
-
 public class KeywordFieldSyntheticSourceSupport implements MapperTestCase.SyntheticSourceSupport {
     private final Integer ignoreAbove;
     private final boolean allIgnored;
@@ -128,11 +126,6 @@ private void mapping(XContentBuilder b) throws IOException {
 
     @Override
     public List<MapperTestCase.SyntheticSourceInvalidExample> invalidExample() throws IOException {
-        return List.of(
-            new MapperTestCase.SyntheticSourceInvalidExample(
-                equalTo("field [field] of type [keyword] doesn't support synthetic source because it declares a normalizer"),
-                b -> b.field("type", "keyword").field("normalizer", "lowercase")
-            )
-        );
+        return List.of();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
index b5a42efd67088..7c11e7446e5c5 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
@@ -61,7 +61,7 @@
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.TelemetryPlugin;
-import org.elasticsearch.plugins.internal.DocumentSizeObserver;
+import org.elasticsearch.plugins.internal.XContentMeteringParserDecorator;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptContext;
@@ -383,7 +383,7 @@ protected static SourceToParse source(
             XContentType.JSON,
             routing,
             dynamicTemplates,
-            DocumentSizeObserver.EMPTY_INSTANCE
+            XContentMeteringParserDecorator.NOOP
         );
     }
 
@@ -426,8 +426,13 @@ protected static XContentBuilder topMapping(CheckedConsumer<XContentBuilder, IOE
     }
 
     protected static XContentBuilder mappingNoSubobjects(CheckedConsumer<XContentBuilder, IOException> buildFields) throws IOException {
+        return mappingWithSubobjects(buildFields, "false");
+    }
+
+    protected static XContentBuilder mappingWithSubobjects(CheckedConsumer<XContentBuilder, IOException> buildFields, String subobjects)
+        throws IOException {
         return topMapping(xContentBuilder -> {
-            xContentBuilder.field("subobjects", false);
+            xContentBuilder.field("subobjects", subobjects);
             xContentBuilder.startObject("properties");
             buildFields.accept(xContentBuilder);
             xContentBuilder.endObject();
@@ -783,7 +788,8 @@ protected TriFunction<MappedFieldType, Supplier<SearchLookup>, MappedFieldType.F
     }
 
     protected RandomIndexWriter indexWriterForSyntheticSource(Directory directory) throws IOException {
-        return new RandomIndexWriter(random(), directory);
+        // MockAnalyzer (rarely) produces random payloads that lead to failures during assertReaderEquals.
+        return new RandomIndexWriter(random(), directory, new StandardAnalyzer());
     }
 
     protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer<XContentBuilder, IOException> build) throws IOException {
diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index 6157d6997641d..13bdf2edb0f24 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -128,10 +128,7 @@ protected IndexMetadata buildIndexMetadata(int replicas, String mappings) throws
     }
 
     protected IndexMetadata buildIndexMetadata(int replicas, Settings indexSettings, String mappings) {
-        Settings settings = Settings.builder()
-            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas)
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+        Settings settings = indexSettings(1, replicas).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
             .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000))
             .put(indexSettings)
             .build();
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index 4d15bf9e3f943..bebaeced797db 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -301,10 +301,7 @@ protected IndexShard newShard(
         final IndexingOperationListener... listeners
     ) throws IOException {
         assert shardRouting.initializing() : shardRouting;
-        Settings indexSettings = Settings.builder()
-            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+        Settings indexSettings = indexSettings(1, 0).put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
             .put(
                 IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(),
                 randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000)
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
index 26e887338158d..92ce7e083df3e 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.repositories.ShardGenerations;
 import org.elasticsearch.repositories.ShardSnapshotResult;
 import org.elasticsearch.repositories.SnapshotShardContext;
-import org.elasticsearch.snapshots.SnapshotDeleteListener;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 
@@ -110,9 +109,10 @@ public void deleteSnapshots(
         Collection<SnapshotId> snapshotIds,
         long repositoryDataGeneration,
         IndexVersion minimumNodeVersion,
-        SnapshotDeleteListener listener
+        ActionListener<RepositoryData> repositoryDataUpdateListener,
+        Runnable onCompletion
     ) {
-        listener.onFailure(new UnsupportedOperationException());
+        repositoryDataUpdateListener.onFailure(new UnsupportedOperationException());
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshotsIntegritySuppressor.java b/test/framework/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshotsIntegritySuppressor.java
new file mode 100644
index 0000000000000..511116d9b2125
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshotsIntegritySuppressor.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.snapshots.blobstore;
+
+import org.elasticsearch.core.Releasable;
+
+/**
+ * Test utility class to suppress assertions about the integrity of the contents of a blobstore repository, in order to verify the
+ * production behaviour on encountering invalid data.
+ */
+public class BlobStoreIndexShardSnapshotsIntegritySuppressor implements Releasable {
+
+    public BlobStoreIndexShardSnapshotsIntegritySuppressor() {
+        BlobStoreIndexShardSnapshots.INTEGRITY_ASSERTIONS_ENABLED = false;
+    }
+
+    @Override
+    public void close() {
+        BlobStoreIndexShardSnapshots.INTEGRITY_ASSERTIONS_ENABLED = true;
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java
index c6c9f5b727980..16ddcb750dd6b 100644
--- a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java
@@ -107,12 +107,7 @@ protected void checkTransientErrorsDuringRecoveryAreRetried(String recoveryActio
         assertThat(response.isTimedOut(), is(false));
 
         indicesAdmin().prepareCreate(indexName)
-            .setSettings(
-                Settings.builder()
-                    .put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "blue")
-                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-            )
+            .setSettings(indexSettings(1, 0).put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "blue"))
             .get();
 
         List<IndexRequestBuilder> requests = new ArrayList<>();
@@ -213,12 +208,7 @@ public void checkDisconnectsWhileRecovering(String recoveryActionToBlock) throws
         assertThat(response.isTimedOut(), is(false));
 
         indicesAdmin().prepareCreate(indexName)
-            .setSettings(
-                Settings.builder()
-                    .put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "blue")
-                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-            )
+            .setSettings(indexSettings(1, 0).put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "blue"))
             .get();
 
         List<IndexRequestBuilder> requests = new ArrayList<>();
@@ -314,12 +304,7 @@ public void checkDisconnectsDuringRecovery(boolean useSnapshotBasedRecoveries) t
         final String redNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "red").put(nodeSettings).build());
 
         indicesAdmin().prepareCreate(indexName)
-            .setSettings(
-                Settings.builder()
-                    .put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "blue")
-                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-            )
+            .setSettings(indexSettings(1, 0).put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "blue"))
             .get();
 
         List<IndexRequestBuilder> requests = new ArrayList<>();
diff --git a/test/framework/src/main/java/org/elasticsearch/common/inject/ModuleTestCase.java b/test/framework/src/main/java/org/elasticsearch/injection/guice/ModuleTestCase.java
similarity index 89%
rename from test/framework/src/main/java/org/elasticsearch/common/inject/ModuleTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/injection/guice/ModuleTestCase.java
index 6f378f263490c..0d48435a53644 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/inject/ModuleTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/injection/guice/ModuleTestCase.java
@@ -5,12 +5,12 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.common.inject;
+package org.elasticsearch.injection.guice;
 
-import org.elasticsearch.common.inject.spi.Element;
-import org.elasticsearch.common.inject.spi.Elements;
-import org.elasticsearch.common.inject.spi.InstanceBinding;
-import org.elasticsearch.common.inject.spi.ProviderInstanceBinding;
+import org.elasticsearch.injection.guice.spi.Element;
+import org.elasticsearch.injection.guice.spi.Elements;
+import org.elasticsearch.injection.guice.spi.InstanceBinding;
+import org.elasticsearch.injection.guice.spi.ProviderInstanceBinding;
 import org.elasticsearch.test.ESTestCase;
 
 import java.lang.annotation.Annotation;
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java
new file mode 100644
index 0000000000000..ed2d968a78818
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGenerator.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.fields.TopLevelObjectFieldDataGenerator;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Entry point of data generation logic.
+ * Every instance of generator generates a random mapping and a document generation routine
+ * that produces randomly generated documents valid for this mapping.
+ */
+public class DataGenerator {
+    private final TopLevelObjectFieldDataGenerator topLevelGenerator;
+
+    public DataGenerator(DataGeneratorSpecification specification) {
+        this.topLevelGenerator = new TopLevelObjectFieldDataGenerator(specification);
+    }
+
+    /**
+     * Writes a fully built mapping document (enclosed in a top-level object) to a provided builder.
+     * @param mapping destination
+     * @throws IOException
+     */
+    public void writeMapping(XContentBuilder mapping) throws IOException {
+        mapping.startObject().field("_doc");
+        topLevelGenerator.mappingWriter(Map.of()).accept(mapping);
+        mapping.endObject();
+    }
+
+    /**
+     * Writes a fully built mapping document (enclosed in a top-level object) to a provided builder.
+     * Allows customizing parameters of top level object mapper.
+     * @param mapping destination
+     * @param customMappingParameters writer of custom mapping parameters of top level object mapping
+     * @throws IOException
+     */
+    public void writeMapping(XContentBuilder mapping, Map<String, Object> customMappingParameters) throws IOException {
+        mapping.startObject().field("_doc");
+        topLevelGenerator.mappingWriter(customMappingParameters).accept(mapping);
+        mapping.endObject();
+    }
+
+    /**
+     * Generates a document and writes it to a provided builder. New document is generated every time.
+     * @param document
+     * @throws IOException
+     */
+    public void generateDocument(XContentBuilder document) throws IOException {
+        topLevelGenerator.fieldValueGenerator(b -> {}).accept(document);
+    }
+
+    /**
+     * Generates a document and writes it to a provided builder. New document is generated every time.
+     * Supports appending custom content to generated document (e.g. a custom generated field).
+     * @param document
+     * @param customDocumentModifications
+     * @throws IOException
+     */
+    public void generateDocument(XContentBuilder document, CheckedConsumer<XContentBuilder, IOException> customDocumentModifications)
+        throws IOException {
+        topLevelGenerator.fieldValueGenerator(customDocumentModifications).accept(document);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java
new file mode 100644
index 0000000000000..04c30ae484f44
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSpecification.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler;
+import org.elasticsearch.logsdb.datageneration.fields.PredefinedField;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Allows configuring behavior of {@link  DataGenerator}.
+ * @param dataSource source of generated data
+ * @param maxFieldCountPerLevel maximum number of fields that an individual object in mapping has.
+ *                              Applies to subobjects.
+ * @param maxObjectDepth maximum depth of nested objects
+ * @param nestedFieldsLimit how many total nested fields can be present in a produced mapping
+ * @param fullyDynamicMapping if the mapping is fully dynamic, meaning none of the fields are mapped (essentially mapping is empty)
+ * @param predefinedFields predefined fields that must be present in mapping and documents. Only top level fields are supported.
+ */
+public record DataGeneratorSpecification(
+    DataSource dataSource,
+    int maxFieldCountPerLevel,
+    int maxObjectDepth,
+    int nestedFieldsLimit,
+    boolean fullyDynamicMapping,
+    List<PredefinedField> predefinedFields
+) {
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static DataGeneratorSpecification buildDefault() {
+        return builder().build();
+    }
+
+    public static class Builder {
+        private List<DataSourceHandler> dataSourceHandlers;
+        private int maxFieldCountPerLevel;
+        private int maxObjectDepth;
+        private int nestedFieldsLimit;
+        private boolean fullyDynamicMapping;
+        private List<PredefinedField> predefinedFields;
+
+        public Builder() {
+            this.dataSourceHandlers = new ArrayList<>();
+            // Simply sufficiently big numbers to get some permutations
+            this.maxFieldCountPerLevel = 50;
+            this.maxObjectDepth = 2;
+            // Default value of index.mapping.nested_fields.limit
+            this.nestedFieldsLimit = 50;
+            fullyDynamicMapping = false;
+            this.predefinedFields = new ArrayList<>();
+        }
+
+        public Builder withDataSourceHandlers(Collection<DataSourceHandler> handlers) {
+            this.dataSourceHandlers.addAll(handlers);
+            return this;
+        }
+
+        public Builder withMaxFieldCountPerLevel(int maxFieldCountPerLevel) {
+            this.maxFieldCountPerLevel = maxFieldCountPerLevel;
+            return this;
+        }
+
+        public Builder withMaxObjectDepth(int maxObjectDepth) {
+            this.maxObjectDepth = maxObjectDepth;
+            return this;
+        }
+
+        public Builder withNestedFieldsLimit(int nestedFieldsLimit) {
+            this.nestedFieldsLimit = nestedFieldsLimit;
+            return this;
+        }
+
+        public Builder withFullyDynamicMapping(boolean fullyDynamicMapping) {
+            this.fullyDynamicMapping = fullyDynamicMapping;
+            return this;
+        }
+
+        public Builder withPredefinedFields(List<PredefinedField> predefinedFields) {
+            this.predefinedFields = predefinedFields;
+            return this;
+        }
+
+        public DataGeneratorSpecification build() {
+            return new DataGeneratorSpecification(
+                new DataSource(dataSourceHandlers),
+                maxFieldCountPerLevel,
+                maxObjectDepth,
+                nestedFieldsLimit,
+                fullyDynamicMapping,
+                predefinedFields
+            );
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java
new file mode 100644
index 0000000000000..b53794fdd1961
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Entity responsible for generating a valid randomized mapping for a field
+ * and a generator of field values valid for this mapping.
+ *
+ * Generator is expected to produce the same mapping per instance of generator.
+ * Function returned by {@link FieldDataGenerator#fieldValueGenerator() } is expected
+ * to produce a randomized value each time.
+ */
+public interface FieldDataGenerator {
+    CheckedConsumer<XContentBuilder, IOException> mappingWriter();
+
+    CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator();
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java
new file mode 100644
index 0000000000000..4ed36ea685238
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.FloatFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.HalfFloatFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.IntegerFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.KeywordFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.LongFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.ScaledFloatFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.ShortFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.UnsignedLongFieldDataGenerator;
+
+/**
+ * Lists all leaf field types that are supported for data generation.
+ */
+public enum FieldType {
+    KEYWORD,
+    LONG,
+    UNSIGNED_LONG,
+    INTEGER,
+    SHORT,
+    BYTE,
+    DOUBLE,
+    FLOAT,
+    HALF_FLOAT,
+    SCALED_FLOAT;
+
+    public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
+        return switch (this) {
+            case KEYWORD -> new KeywordFieldDataGenerator(fieldName, dataSource);
+            case LONG -> new LongFieldDataGenerator(fieldName, dataSource);
+            case UNSIGNED_LONG -> new UnsignedLongFieldDataGenerator(fieldName, dataSource);
+            case INTEGER -> new IntegerFieldDataGenerator(fieldName, dataSource);
+            case SHORT -> new ShortFieldDataGenerator(fieldName, dataSource);
+            case BYTE -> new ByteFieldDataGenerator(fieldName, dataSource);
+            case DOUBLE -> new DoubleFieldDataGenerator(fieldName, dataSource);
+            case FLOAT -> new FloatFieldDataGenerator(fieldName, dataSource);
+            case HALF_FLOAT -> new HalfFloatFieldDataGenerator(fieldName, dataSource);
+            case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(fieldName, dataSource);
+        };
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java
new file mode 100644
index 0000000000000..d550b1ce757a1
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.datasource;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * This class handles any decision performed during data generation that changes the output.
+ * For example: generating a random number, array of random size, mapping parameter.
+ * <p>
+ * Goals of this abstraction are:
+ * <ul>
+ * <li> to be able to easily add new types of decisions/generators </li>
+ * <li> to decouple different types of decisions from each other, adding new data type should be an isolated additive change </li>
+ * <li> to allow overriding only small specific subset of behavior (e.g. for testing purposes) </li>
+ * </ul>
+ */
+public class DataSource {
+    private List<DataSourceHandler> handlers;
+
+    public DataSource(Collection<DataSourceHandler> additionalHandlers) {
+        this.handlers = new ArrayList<>();
+
+        this.handlers.addAll(additionalHandlers);
+
+        this.handlers.add(new DefaultPrimitiveTypesHandler());
+        this.handlers.add(new DefaultWrappersHandler());
+        this.handlers.add(new DefaultObjectGenerationHandler());
+        this.handlers.add(new DefaultMappingParametersHandler());
+    }
+
+    public <T extends DataSourceResponse> T get(DataSourceRequest<T> request) {
+        for (var handler : handlers) {
+            var response = request.accept(handler);
+            if (response != null) {
+                return response;
+            }
+        }
+
+        throw new IllegalStateException("Request is not supported by data source");
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java
new file mode 100644
index 0000000000000..0cb6ff33c612b
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.datasource;
+
+public interface DataSourceHandler {
+    default DataSourceResponse.LongGenerator handle(DataSourceRequest.LongGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.UnsignedLongGenerator handle(DataSourceRequest.UnsignedLongGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.IntegerGenerator handle(DataSourceRequest.IntegerGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.ShortGenerator handle(DataSourceRequest.ShortGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.ByteGenerator handle(DataSourceRequest.ByteGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.DoubleGenerator handle(DataSourceRequest.DoubleGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.FloatGenerator handle(DataSourceRequest.FloatGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.HalfFloatGenerator handle(DataSourceRequest.HalfFloatGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.StringGenerator handle(DataSourceRequest.StringGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) {
+        return null;
+    }
+
+    default DataSourceResponse.ArrayWrapper handle(DataSourceRequest.ArrayWrapper request) {
+        return null;
+    }
+
+    default DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFieldGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectArrayGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
+        return null;
+    }
+
+    default DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) {
+        return null;
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java
new file mode 100644
index 0000000000000..fadf51ee3ea10
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.datasource;
+
+import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping;
+
+public interface DataSourceRequest<TResponse extends DataSourceResponse> {
+    TResponse accept(DataSourceHandler handler);
+
+    record LongGenerator() implements DataSourceRequest<DataSourceResponse.LongGenerator> {
+        public DataSourceResponse.LongGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record UnsignedLongGenerator() implements DataSourceRequest<DataSourceResponse.UnsignedLongGenerator> {
+        public DataSourceResponse.UnsignedLongGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record IntegerGenerator() implements DataSourceRequest<DataSourceResponse.IntegerGenerator> {
+        public DataSourceResponse.IntegerGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record ShortGenerator() implements DataSourceRequest<DataSourceResponse.ShortGenerator> {
+        public DataSourceResponse.ShortGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record ByteGenerator() implements DataSourceRequest<DataSourceResponse.ByteGenerator> {
+        public DataSourceResponse.ByteGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record DoubleGenerator() implements DataSourceRequest<DataSourceResponse.DoubleGenerator> {
+        public DataSourceResponse.DoubleGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record FloatGenerator() implements DataSourceRequest<DataSourceResponse.FloatGenerator> {
+        public DataSourceResponse.FloatGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record HalfFloatGenerator() implements DataSourceRequest<DataSourceResponse.HalfFloatGenerator> {
+        public DataSourceResponse.HalfFloatGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record StringGenerator() implements DataSourceRequest<DataSourceResponse.StringGenerator> {
+        public DataSourceResponse.StringGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record NullWrapper() implements DataSourceRequest<DataSourceResponse.NullWrapper> {
+        public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record ArrayWrapper() implements DataSourceRequest<DataSourceResponse.ArrayWrapper> {
+        public DataSourceResponse.ArrayWrapper accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record ChildFieldGenerator(DataGeneratorSpecification specification)
+        implements
+            DataSourceRequest<DataSourceResponse.ChildFieldGenerator> {
+        public DataSourceResponse.ChildFieldGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record FieldTypeGenerator(DynamicMapping dynamicMapping) implements DataSourceRequest<DataSourceResponse.FieldTypeGenerator> {
+        public DataSourceResponse.FieldTypeGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record ObjectArrayGenerator() implements DataSourceRequest<DataSourceResponse.ObjectArrayGenerator> {
+        public DataSourceResponse.ObjectArrayGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record LeafMappingParametersGenerator(String fieldName, FieldType fieldType)
+        implements
+            DataSourceRequest<DataSourceResponse.LeafMappingParametersGenerator> {
+        public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
+    record ObjectMappingParametersGenerator(boolean isNested)
+        implements
+            DataSourceRequest<DataSourceResponse.ObjectMappingParametersGenerator> {
+        public DataSourceResponse.ObjectMappingParametersGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java
new file mode 100644
index 0000000000000..9093799ade41d
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.datasource;
+
+import org.elasticsearch.logsdb.datageneration.FieldType;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+public interface DataSourceResponse {
+    record LongGenerator(Supplier<Long> generator) implements DataSourceResponse {}
+
+    record UnsignedLongGenerator(Supplier<Object> generator) implements DataSourceResponse {}
+
+    record IntegerGenerator(Supplier<Integer> generator) implements DataSourceResponse {}
+
+    record ShortGenerator(Supplier<Short> generator) implements DataSourceResponse {}
+
+    record ByteGenerator(Supplier<Byte> generator) implements DataSourceResponse {}
+
+    record DoubleGenerator(Supplier<Double> generator) implements DataSourceResponse {}
+
+    record FloatGenerator(Supplier<Float> generator) implements DataSourceResponse {}
+
+    record HalfFloatGenerator(Supplier<Float> generator) implements DataSourceResponse {}
+
+    record StringGenerator(Supplier<String> generator) implements DataSourceResponse {}
+
+    record NullWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
+
+    record ArrayWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
+
+    interface ChildFieldGenerator extends DataSourceResponse {
+        int generateChildFieldCount();
+
+        boolean generateDynamicSubObject();
+
+        boolean generateNestedSubObject();
+
+        boolean generateRegularSubObject();
+
+        String generateFieldName();
+    }
+
+    record FieldTypeGenerator(Supplier<FieldTypeInfo> generator) implements DataSourceResponse {
+        public record FieldTypeInfo(FieldType fieldType, boolean dynamic) {}
+    }
+
+    record ObjectArrayGenerator(Supplier<Optional<Integer>> lengthGenerator) implements DataSourceResponse {}
+
+    record LeafMappingParametersGenerator(Supplier<Map<String, Object>> mappingGenerator) implements DataSourceResponse {}
+
+    record ObjectMappingParametersGenerator(Supplier<Map<String, Object>> mappingGenerator) implements DataSourceResponse {}
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java
new file mode 100644
index 0000000000000..e57257f69da20
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.datasource;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class DefaultMappingParametersHandler implements DataSourceHandler {
+    @Override
+    public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
+        return new DataSourceResponse.LeafMappingParametersGenerator(switch (request.fieldType()) {
+            case KEYWORD -> keywordMapping();
+            case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT -> numberMapping();
+            case UNSIGNED_LONG -> unsignedLongMapping();
+            case SCALED_FLOAT -> scaledFloatMapping();
+        });
+    }
+
+    // TODO enable doc_values: false
+    // It is disabled because it hits a bug in synthetic source.
+    private Supplier<Map<String, Object>> keywordMapping() {
+        return () -> Map.of("store", ESTestCase.randomBoolean(), "index", ESTestCase.randomBoolean());
+    }
+
+    private Supplier<Map<String, Object>> numberMapping() {
+        return () -> Map.of(
+            "store",
+            ESTestCase.randomBoolean(),
+            "index",
+            ESTestCase.randomBoolean(),
+            "doc_values",
+            ESTestCase.randomBoolean()
+        );
+    }
+
+    private Supplier<Map<String, Object>> unsignedLongMapping() {
+        return () -> Map.of("store", ESTestCase.randomBoolean(), "index", ESTestCase.randomBoolean());
+    }
+
+    private Supplier<Map<String, Object>> scaledFloatMapping() {
+        return () -> {
+            var scalingFactor = ESTestCase.randomFrom(10, 1000, 100000, 100.5);
+            return Map.of("scaling_factor", scalingFactor, "store", ESTestCase.randomBoolean(), "index", ESTestCase.randomBoolean());
+        };
+    }
+
+    @Override
+    public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) {
+        if (request.isNested()) {
+            return new DataSourceResponse.ObjectMappingParametersGenerator(() -> {
+                var parameters = new HashMap<String, Object>();
+                if (ESTestCase.randomBoolean()) {
+                    parameters.put("dynamic", ESTestCase.randomFrom("true", "false", "strict"));
+                }
+
+                return parameters;
+            });
+        }
+
+        return new DataSourceResponse.ObjectMappingParametersGenerator(() -> {
+            var parameters = new HashMap<String, Object>();
+            if (ESTestCase.randomBoolean()) {
+                parameters.put("dynamic", ESTestCase.randomFrom("true", "false", "strict", "runtime"));
+            }
+            if (ESTestCase.randomBoolean()) {
+                parameters.put("enabled", ESTestCase.randomFrom("true", "false"));
+            }
+
+            return parameters;
+        });
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java
new file mode 100644
index 0000000000000..4ad9ac61a9158
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.datasource;
+
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween;
+import static org.elasticsearch.test.ESTestCase.randomDouble;
+import static org.elasticsearch.test.ESTestCase.randomIntBetween;
+
+public class DefaultObjectGenerationHandler implements DataSourceHandler {
+    @Override
+    public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFieldGenerator request) {
+        return new DataSourceResponse.ChildFieldGenerator() {
+            @Override
+            public int generateChildFieldCount() {
+                return ESTestCase.randomIntBetween(0, request.specification().maxFieldCountPerLevel());
+            }
+
+            @Override
+            public boolean generateDynamicSubObject() {
+                // Using a static 5% change, this is just a chosen value that can be tweaked.
+                return randomDouble() <= 0.05;
+            }
+
+            @Override
+            public boolean generateNestedSubObject() {
+                // Using a static 5% change, this is just a chosen value that can be tweaked.
+                return randomDouble() <= 0.05;
+            }
+
+            @Override
+            public boolean generateRegularSubObject() {
+                // Using a static 5% change, this is just a chosen value that can be tweaked.
+                return randomDouble() <= 0.05;
+            }
+
+            @Override
+            public String generateFieldName() {
+                return randomAlphaOfLengthBetween(1, 10);
+            }
+        };
+    }
+
+    // UNSIGNED_LONG is excluded because it is mapped as long
+    // and values larger than long fail to parse.
+    private static final Set<FieldType> EXCLUDED_FROM_DYNAMIC_MAPPING = Set.of(FieldType.UNSIGNED_LONG);
+
+    @Override
+    public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) {
+        Supplier<DataSourceResponse.FieldTypeGenerator.FieldTypeInfo> generator = switch (request.dynamicMapping()) {
+            case FORBIDDEN -> () -> generateFieldTypeInfo(false);
+            case FORCED -> () -> generateFieldTypeInfo(true);
+            case SUPPORTED -> () -> generateFieldTypeInfo(ESTestCase.randomBoolean());
+        };
+
+        return new DataSourceResponse.FieldTypeGenerator(generator);
+    }
+
+    private static DataSourceResponse.FieldTypeGenerator.FieldTypeInfo generateFieldTypeInfo(boolean isDynamic) {
+        var excluded = isDynamic ? EXCLUDED_FROM_DYNAMIC_MAPPING : Set.of();
+
+        var fieldType = ESTestCase.randomValueOtherThanMany(excluded::contains, () -> ESTestCase.randomFrom(FieldType.values()));
+
+        return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(fieldType, isDynamic);
+    }
+
+    @Override
+    public DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectArrayGenerator request) {
+        return new DataSourceResponse.ObjectArrayGenerator(() -> {
+            if (ESTestCase.randomBoolean()) {
+                return Optional.of(randomIntBetween(0, 5));
+            }
+
+            return Optional.empty();
+        });
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultPrimitiveTypesHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultPrimitiveTypesHandler.java
new file mode 100644
index 0000000000000..2514cbd51e21c
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultPrimitiveTypesHandler.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.datasource;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.math.BigInteger;
+
+public class DefaultPrimitiveTypesHandler implements DataSourceHandler {
+    @Override
+    public DataSourceResponse.LongGenerator handle(DataSourceRequest.LongGenerator request) {
+        return new DataSourceResponse.LongGenerator(ESTestCase::randomLong);
+    }
+
+    @Override
+    public DataSourceResponse.UnsignedLongGenerator handle(DataSourceRequest.UnsignedLongGenerator request) {
+        return new DataSourceResponse.UnsignedLongGenerator(() -> new BigInteger(64, ESTestCase.random()));
+    }
+
+    @Override
+    public DataSourceResponse.IntegerGenerator handle(DataSourceRequest.IntegerGenerator request) {
+        return new DataSourceResponse.IntegerGenerator(ESTestCase::randomInt);
+    }
+
+    @Override
+    public DataSourceResponse.ShortGenerator handle(DataSourceRequest.ShortGenerator request) {
+        return new DataSourceResponse.ShortGenerator(ESTestCase::randomShort);
+    }
+
+    @Override
+    public DataSourceResponse.ByteGenerator handle(DataSourceRequest.ByteGenerator request) {
+        return new DataSourceResponse.ByteGenerator(ESTestCase::randomByte);
+    }
+
+    @Override
+    public DataSourceResponse.DoubleGenerator handle(DataSourceRequest.DoubleGenerator request) {
+        return new DataSourceResponse.DoubleGenerator(ESTestCase::randomDouble);
+    }
+
+    @Override
+    public DataSourceResponse.FloatGenerator handle(DataSourceRequest.FloatGenerator request) {
+        return new DataSourceResponse.FloatGenerator(ESTestCase::randomFloat);
+    }
+
+    @Override
+    public DataSourceResponse.HalfFloatGenerator handle(DataSourceRequest.HalfFloatGenerator request) {
+        // All floats are accepted but stored precision is reduced.
+        return new DataSourceResponse.HalfFloatGenerator(ESTestCase::randomFloat);
+    }
+
+    @Override
+    public DataSourceResponse.StringGenerator handle(DataSourceRequest.StringGenerator request) {
+        return new DataSourceResponse.StringGenerator(() -> ESTestCase.randomAlphaOfLengthBetween(0, 50));
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java
new file mode 100644
index 0000000000000..57af9786f200b
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.datasource;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+
+public class DefaultWrappersHandler implements DataSourceHandler {
+    @Override
+    public DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper ignored) {
+        return new DataSourceResponse.NullWrapper(injectNulls());
+    }
+
+    @Override
+    public DataSourceResponse.ArrayWrapper handle(DataSourceRequest.ArrayWrapper ignored) {
+        return new DataSourceResponse.ArrayWrapper(wrapInArray());
+    }
+
+    private static Function<Supplier<Object>, Supplier<Object>> injectNulls() {
+        return (values) -> () -> ESTestCase.randomBoolean() ? null : values.get();
+    }
+
+    private static Function<Supplier<Object>, Supplier<Object>> wrapInArray() {
+        return (values) -> () -> {
+            if (ESTestCase.randomBoolean()) {
+                var size = ESTestCase.randomIntBetween(0, 5);
+                return IntStream.range(0, size).mapToObj((i) -> values.get()).toList();
+            }
+
+            return values.get();
+        };
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java
new file mode 100644
index 0000000000000..ef83ced13f0dc
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields;
+
+import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+
+class Context {
+    private final DataGeneratorSpecification specification;
+
+    private final DataSourceResponse.ChildFieldGenerator childFieldGenerator;
+    private final DataSourceResponse.ObjectArrayGenerator objectArrayGenerator;
+    private final int objectDepth;
+    // We don't need atomicity, but we need to pass counter by reference to accumulate total value from sub-objects.
+    private final AtomicInteger nestedFieldsCount;
+    private final DynamicMapping parentDynamicMapping;
+
+    Context(DataGeneratorSpecification specification, DynamicMapping parentDynamicMapping) {
+        this(specification, 0, new AtomicInteger(0), parentDynamicMapping);
+    }
+
+    private Context(
+        DataGeneratorSpecification specification,
+        int objectDepth,
+        AtomicInteger nestedFieldsCount,
+        DynamicMapping parentDynamicMapping
+    ) {
+        this.specification = specification;
+        this.childFieldGenerator = specification.dataSource().get(new DataSourceRequest.ChildFieldGenerator(specification));
+        this.objectArrayGenerator = specification.dataSource().get(new DataSourceRequest.ObjectArrayGenerator());
+        this.objectDepth = objectDepth;
+        this.nestedFieldsCount = nestedFieldsCount;
+        this.parentDynamicMapping = parentDynamicMapping;
+    }
+
+    public DataGeneratorSpecification specification() {
+        return specification;
+    }
+
+    public DataSourceResponse.ChildFieldGenerator childFieldGenerator() {
+        return childFieldGenerator;
+    }
+
+    public DataSourceResponse.FieldTypeGenerator fieldTypeGenerator(DynamicMapping dynamicMapping) {
+        return specification.dataSource().get(new DataSourceRequest.FieldTypeGenerator(dynamicMapping));
+    }
+
+    public Context subObject(DynamicMapping dynamicMapping) {
+        return new Context(specification, objectDepth + 1, nestedFieldsCount, dynamicMapping);
+    }
+
+    public Context nestedObject(DynamicMapping dynamicMapping) {
+        nestedFieldsCount.incrementAndGet();
+        return new Context(specification, objectDepth + 1, nestedFieldsCount, dynamicMapping);
+    }
+
+    public boolean shouldAddDynamicObjectField(DynamicMapping dynamicMapping) {
+        if (objectDepth >= specification.maxObjectDepth() || dynamicMapping == DynamicMapping.FORBIDDEN) {
+            return false;
+        }
+
+        return childFieldGenerator.generateDynamicSubObject();
+    }
+
+    public boolean shouldAddObjectField() {
+        if (objectDepth >= specification.maxObjectDepth() || parentDynamicMapping == DynamicMapping.FORCED) {
+            return false;
+        }
+
+        return childFieldGenerator.generateRegularSubObject();
+    }
+
+    public boolean shouldAddNestedField() {
+        if (objectDepth >= specification.maxObjectDepth()
+            || nestedFieldsCount.get() >= specification.nestedFieldsLimit()
+            || parentDynamicMapping == DynamicMapping.FORCED) {
+            return false;
+        }
+
+        return childFieldGenerator.generateNestedSubObject();
+    }
+
+    public Optional<Integer> generateObjectArray() {
+        if (objectDepth == 0) {
+            return Optional.empty();
+        }
+
+        return objectArrayGenerator.lengthGenerator().get();
+    }
+
+    public DynamicMapping determineDynamicMapping(Map<String, Object> mappingParameters) {
+        if (parentDynamicMapping == DynamicMapping.FORCED) {
+            return DynamicMapping.FORCED;
+        }
+
+        var dynamicParameter = mappingParameters.get("dynamic");
+        // Inherited from parent
+        if (dynamicParameter == null) {
+            return parentDynamicMapping;
+        }
+
+        return dynamicParameter.equals("strict") ? DynamicMapping.FORBIDDEN : DynamicMapping.SUPPORTED;
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/DynamicMapping.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/DynamicMapping.java
new file mode 100644
index 0000000000000..ea74080c03610
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/DynamicMapping.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields;
+
+public enum DynamicMapping {
+    SUPPORTED,
+    FORBIDDEN,
+    FORCED
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java
new file mode 100644
index 0000000000000..e8f9724fee269
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Generic generator for any type of object field (e.g. "object", "nested").
+ */
+public class GenericSubObjectFieldDataGenerator {
+    private final Context context;
+
+    GenericSubObjectFieldDataGenerator(Context context) {
+        this.context = context;
+    }
+
+    List<ChildField> generateChildFields(DynamicMapping dynamicMapping) {
+        var existingFieldNames = new HashSet<String>();
+        // no child fields is legal
+        var childFieldsCount = context.childFieldGenerator().generateChildFieldCount();
+        var result = new ArrayList<ChildField>(childFieldsCount);
+
+        for (int i = 0; i < childFieldsCount; i++) {
+            var fieldName = generateFieldName(existingFieldNames);
+
+            if (context.shouldAddDynamicObjectField(dynamicMapping)) {
+                result.add(new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(DynamicMapping.FORCED)), true));
+            } else if (context.shouldAddObjectField()) {
+                result.add(new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(dynamicMapping)), false));
+            } else if (context.shouldAddNestedField()) {
+                result.add(new ChildField(fieldName, new NestedFieldDataGenerator(context.nestedObject(dynamicMapping)), false));
+            } else {
+                var fieldTypeInfo = context.fieldTypeGenerator(dynamicMapping).generator().get();
+                var generator = fieldTypeInfo.fieldType().generator(fieldName, context.specification().dataSource());
+                result.add(new ChildField(fieldName, generator, fieldTypeInfo.dynamic()));
+            }
+        }
+
+        return result;
+    }
+
+    List<ChildField> generateChildFields(List<PredefinedField> predefinedFields) {
+        return predefinedFields.stream()
+            .map(pf -> new ChildField(pf.name(), pf.generator(context.specification().dataSource()), false))
+            .toList();
+    }
+
+    static void writeChildFieldsMapping(XContentBuilder mapping, List<ChildField> childFields) throws IOException {
+        for (var childField : childFields) {
+            if (childField.dynamic() == false) {
+                mapping.field(childField.fieldName);
+                childField.generator.mappingWriter().accept(mapping);
+            }
+        }
+    }
+
+    static void writeObjectsData(XContentBuilder document, Context context, CheckedConsumer<XContentBuilder, IOException> objectWriter)
+        throws IOException {
+        var optionalLength = context.generateObjectArray();
+        if (optionalLength.isPresent()) {
+            int size = optionalLength.get();
+
+            document.startArray();
+            for (int i = 0; i < size; i++) {
+                objectWriter.accept(document);
+            }
+            document.endArray();
+        } else {
+            objectWriter.accept(document);
+        }
+    }
+
+    static void writeSingleObject(XContentBuilder document, Iterable<ChildField> childFields) throws IOException {
+        document.startObject();
+        writeChildFieldsData(document, childFields);
+        document.endObject();
+    }
+
+    static void writeChildFieldsData(XContentBuilder document, Iterable<ChildField> childFields) throws IOException {
+        for (var childField : childFields) {
+            document.field(childField.fieldName);
+            childField.generator.fieldValueGenerator().accept(document);
+        }
+    }
+
+    private String generateFieldName(Set<String> existingFields) {
+        var fieldName = context.childFieldGenerator().generateFieldName();
+        while (existingFields.contains(fieldName)) {
+            fieldName = context.childFieldGenerator().generateFieldName();
+        }
+        existingFields.add(fieldName);
+
+        return fieldName;
+    }
+
+    record ChildField(String fieldName, FieldDataGenerator generator, boolean dynamic) {}
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java
new file mode 100644
index 0000000000000..5b9891ab15920
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class NestedFieldDataGenerator implements FieldDataGenerator {
+    private final Context context;
+    private final Map<String, Object> mappingParameters;
+    private final List<GenericSubObjectFieldDataGenerator.ChildField> childFields;
+
+    NestedFieldDataGenerator(Context context) {
+        this.context = context;
+
+        this.mappingParameters = context.specification()
+            .dataSource()
+            .get(new DataSourceRequest.ObjectMappingParametersGenerator(true))
+            .mappingGenerator()
+            .get();
+        var dynamicMapping = context.determineDynamicMapping(mappingParameters);
+
+        var genericGenerator = new GenericSubObjectFieldDataGenerator(context);
+        this.childFields = genericGenerator.generateChildFields(dynamicMapping);
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject();
+
+            b.field("type", "nested");
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.startObject("properties");
+            GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, childFields);
+            b.endObject();
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        CheckedConsumer<XContentBuilder, IOException> objectWriter = object -> GenericSubObjectFieldDataGenerator.writeSingleObject(
+            object,
+            childFields
+        );
+        return b -> GenericSubObjectFieldDataGenerator.writeObjectsData(b, context, objectWriter);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java
new file mode 100644
index 0000000000000..51732b71b73d0
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class ObjectFieldDataGenerator implements FieldDataGenerator {
+    private final Context context;
+    private final Map<String, Object> mappingParameters;
+    private final List<GenericSubObjectFieldDataGenerator.ChildField> childFields;
+
+    ObjectFieldDataGenerator(Context context) {
+        this.context = context;
+
+        this.mappingParameters = context.specification()
+            .dataSource()
+            .get(new DataSourceRequest.ObjectMappingParametersGenerator(false))
+            .mappingGenerator()
+            .get();
+        var dynamicMapping = context.determineDynamicMapping(mappingParameters);
+
+        var genericGenerator = new GenericSubObjectFieldDataGenerator(context);
+        this.childFields = genericGenerator.generateChildFields(dynamicMapping);
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject();
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.startObject("properties");
+            GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, childFields);
+            b.endObject();
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        CheckedConsumer<XContentBuilder, IOException> objectWriter = object -> GenericSubObjectFieldDataGenerator.writeSingleObject(
+            object,
+            childFields
+        );
+        return b -> GenericSubObjectFieldDataGenerator.writeObjectsData(b, context, objectWriter);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java
new file mode 100644
index 0000000000000..81b73e4407bd7
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields;
+
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+
+public interface PredefinedField {
+    String name();
+
+    FieldDataGenerator generator(DataSource dataSource);
+
+    record WithType(String fieldName, FieldType fieldType) implements PredefinedField {
+        @Override
+        public String name() {
+            return fieldName;
+        }
+
+        @Override
+        public FieldDataGenerator generator(DataSource dataSource) {
+            return fieldType().generator(fieldName, dataSource);
+        }
+    }
+
+    record WithGenerator(String fieldName, FieldDataGenerator generator) implements PredefinedField {
+        @Override
+        public String name() {
+            return fieldName;
+        }
+
+        @Override
+        public FieldDataGenerator generator(DataSource dataSource) {
+            return generator;
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java
new file mode 100644
index 0000000000000..f2c209fb73862
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class TopLevelObjectFieldDataGenerator {
+    private final Context context;
+    private final Map<String, Object> mappingParameters;
+
+    // Child fields of top level object that were explicitly requested, they have predefined name and type.
+    private final List<GenericSubObjectFieldDataGenerator.ChildField> predefinedFields;
+    // Child fields of top level object that are generated and merged with predefined fields.
+    private final List<GenericSubObjectFieldDataGenerator.ChildField> generatedChildFields;
+
+    public TopLevelObjectFieldDataGenerator(DataGeneratorSpecification specification) {
+        DynamicMapping dynamicMapping;
+        if (specification.fullyDynamicMapping()) {
+            dynamicMapping = DynamicMapping.FORCED;
+            this.mappingParameters = Map.of();
+        } else {
+            this.mappingParameters = new HashMap<>(
+                specification.dataSource().get(new DataSourceRequest.ObjectMappingParametersGenerator(false)).mappingGenerator().get()
+            );
+            // Top-level object can't be disabled because @timestamp is a required field in data streams.
+            this.mappingParameters.remove("enabled");
+
+            dynamicMapping = mappingParameters.getOrDefault("dynamic", "true").equals("strict")
+                ? DynamicMapping.FORBIDDEN
+                : DynamicMapping.SUPPORTED;
+        }
+        this.context = new Context(specification, dynamicMapping);
+        var genericGenerator = new GenericSubObjectFieldDataGenerator(context);
+
+        this.predefinedFields = genericGenerator.generateChildFields(specification.predefinedFields());
+        this.generatedChildFields = genericGenerator.generateChildFields(dynamicMapping);
+    }
+
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter(Map<String, Object> customMappingParameters) {
+        return b -> {
+            b.startObject();
+
+            var mergedParameters = Stream.of(this.mappingParameters, customMappingParameters)
+                .flatMap(map -> map.entrySet().stream())
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (l, r) -> r));
+
+            for (var entry : mergedParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.startObject("properties");
+            GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, predefinedFields);
+            GenericSubObjectFieldDataGenerator.writeChildFieldsMapping(b, generatedChildFields);
+            b.endObject();
+
+            b.endObject();
+        };
+    }
+
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator(
+        CheckedConsumer<XContentBuilder, IOException> customDocumentModification
+    ) {
+        CheckedConsumer<XContentBuilder, IOException> objectWriter = b -> {
+            b.startObject();
+
+            customDocumentModification.accept(b);
+            GenericSubObjectFieldDataGenerator.writeChildFieldsData(b, predefinedFields);
+            GenericSubObjectFieldDataGenerator.writeChildFieldsData(b, generatedChildFields);
+
+            b.endObject();
+        };
+        return b -> GenericSubObjectFieldDataGenerator.writeObjectsData(b, context, objectWriter);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java
new file mode 100644
index 0000000000000..d7461294456e9
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields.leaf;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class ByteFieldDataGenerator implements FieldDataGenerator {
+    private final Supplier<Object> valueGenerator;
+    private final Map<String, Object> mappingParameters;
+
+    public ByteFieldDataGenerator(String fieldName, DataSource dataSource) {
+        var bytes = dataSource.get(new DataSourceRequest.ByteGenerator());
+        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
+        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+
+        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> bytes.generator().get());
+        this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.BYTE))
+            .mappingGenerator()
+            .get();
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject().field("type", "byte");
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        return b -> b.value(valueGenerator.get());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java
new file mode 100644
index 0000000000000..98d588754ef93
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields.leaf;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class DoubleFieldDataGenerator implements FieldDataGenerator {
+    private final Supplier<Object> valueGenerator;
+    private final Map<String, Object> mappingParameters;
+
+    public DoubleFieldDataGenerator(String fieldName, DataSource dataSource) {
+        var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator());
+        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
+        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+
+        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get());
+        this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.DOUBLE))
+            .mappingGenerator()
+            .get();
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject().field("type", "double");
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        return b -> b.value(valueGenerator.get());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java
new file mode 100644
index 0000000000000..438617ee3c38d
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields.leaf;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class FloatFieldDataGenerator implements FieldDataGenerator {
+    private final Supplier<Object> valueGenerator;
+    private final Map<String, Object> mappingParameters;
+
+    public FloatFieldDataGenerator(String fieldName, DataSource dataSource) {
+        var floats = dataSource.get(new DataSourceRequest.FloatGenerator());
+        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
+        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+
+        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> floats.generator().get());
+        this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.FLOAT))
+            .mappingGenerator()
+            .get();
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject().field("type", "float");
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        return b -> b.value(valueGenerator.get());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java
new file mode 100644
index 0000000000000..b21467a2afdcc
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields.leaf;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class HalfFloatFieldDataGenerator implements FieldDataGenerator {
+    private final Supplier<Object> valueGenerator;
+    private final Map<String, Object> mappingParameters;
+
+    public HalfFloatFieldDataGenerator(String fieldName, DataSource dataSource) {
+        var halfFloats = dataSource.get(new DataSourceRequest.HalfFloatGenerator());
+        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
+        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+
+        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> halfFloats.generator().get());
+        this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.HALF_FLOAT))
+            .mappingGenerator()
+            .get();
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject().field("type", "half_float");
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        return b -> b.value(valueGenerator.get());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java
new file mode 100644
index 0000000000000..06f5f12fceac3
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields.leaf;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class IntegerFieldDataGenerator implements FieldDataGenerator {
+    private final Supplier<Object> valueGenerator;
+    private final Map<String, Object> mappingParameters;
+
+    public IntegerFieldDataGenerator(String fieldName, DataSource dataSource) {
+        var ints = dataSource.get(new DataSourceRequest.IntegerGenerator());
+        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
+        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+
+        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> ints.generator().get());
+        this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.INTEGER))
+            .mappingGenerator()
+            .get();
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject().field("type", "integer");
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        return b -> b.value(valueGenerator.get());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java
new file mode 100644
index 0000000000000..a33b4ce1b2365
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields.leaf;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class KeywordFieldDataGenerator implements FieldDataGenerator {
+    private final Supplier<Object> valueGenerator;
+    private final Map<String, Object> mappingParameters;
+
+    public KeywordFieldDataGenerator(String fieldName, DataSource dataSource) {
+        var strings = dataSource.get(new DataSourceRequest.StringGenerator());
+        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
+        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+
+        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> strings.generator().get());
+        this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.KEYWORD))
+            .mappingGenerator()
+            .get();
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject().field("type", "keyword");
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        return b -> b.value(valueGenerator.get());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java
new file mode 100644
index 0000000000000..6f6fc5db253aa
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields.leaf;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class LongFieldDataGenerator implements FieldDataGenerator {
+    private final Supplier<Object> valueGenerator;
+    private final Map<String, Object> mappingParameters;
+
+    public LongFieldDataGenerator(String fieldName, DataSource dataSource) {
+        var longs = dataSource.get(new DataSourceRequest.LongGenerator());
+        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
+        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+
+        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> longs.generator().get());
+        this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.LONG))
+            .mappingGenerator()
+            .get();
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject().field("type", "long");
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        return b -> b.value(valueGenerator.get());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java
new file mode 100644
index 0000000000000..fb50933f74d70
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields.leaf;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class ScaledFloatFieldDataGenerator implements FieldDataGenerator {
+    private final Supplier<Object> valueGenerator;
+    private final Map<String, Object> mappingParameters;
+
+    public ScaledFloatFieldDataGenerator(String fieldName, DataSource dataSource) {
+        var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator());
+        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
+        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+
+        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get());
+        this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.SCALED_FLOAT))
+            .mappingGenerator()
+            .get();
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject().field("type", "scaled_float");
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        return b -> b.value(valueGenerator.get());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java
new file mode 100644
index 0000000000000..43d390c0857b9
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields.leaf;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class ShortFieldDataGenerator implements FieldDataGenerator {
+    private final Supplier<Object> valueGenerator;
+    private final Map<String, Object> mappingParameters;
+
+    public ShortFieldDataGenerator(String fieldName, DataSource dataSource) {
+        var shorts = dataSource.get(new DataSourceRequest.ShortGenerator());
+        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
+        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+
+        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> shorts.generator().get());
+        this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.SHORT))
+            .mappingGenerator()
+            .get();
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject().field("type", "short");
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        return b -> b.value(valueGenerator.get());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java
new file mode 100644
index 0000000000000..7e833d6486e52
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration.fields.leaf;
+
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class UnsignedLongFieldDataGenerator implements FieldDataGenerator {
+    private final Supplier<Object> valueGenerator;
+    private final Map<String, Object> mappingParameters;
+
+    public UnsignedLongFieldDataGenerator(String fieldName, DataSource dataSource) {
+        var unsignedLongs = dataSource.get(new DataSourceRequest.UnsignedLongGenerator());
+        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
+        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+
+        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> unsignedLongs.generator().get());
+        this.mappingParameters = dataSource.get(new DataSourceRequest.LeafMappingParametersGenerator(fieldName, FieldType.UNSIGNED_LONG))
+            .mappingGenerator()
+            .get();
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> mappingWriter() {
+        return b -> {
+            b.startObject().field("type", "unsigned_long");
+
+            for (var entry : mappingParameters.entrySet()) {
+                b.field(entry.getKey(), entry.getValue());
+            }
+
+            b.endObject();
+        };
+    }
+
+    @Override
+    public CheckedConsumer<XContentBuilder, IOException> fieldValueGenerator() {
+        return b -> b.value(valueGenerator.get());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionUtils.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionUtils.java
new file mode 100644
index 0000000000000..3670013f571e0
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionUtils.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories.blobstore;
+
+import org.apache.lucene.tests.mockfile.ExtrasFS;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Base64;
+
+import static org.elasticsearch.test.ESTestCase.between;
+import static org.elasticsearch.test.ESTestCase.randomBoolean;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
+import static org.elasticsearch.test.ESTestCase.randomValueOtherThan;
+import static org.elasticsearch.test.ESTestCase.randomValueOtherThanMany;
+
+public class BlobStoreCorruptionUtils {
+    private static final Logger logger = LogManager.getLogger(BlobStoreCorruptionUtils.class);
+
+    public static Path corruptRandomFile(Path repositoryRootPath) throws IOException {
+        final var corruptedFileType = getRandomCorruptibleFileType();
+        final var corruptedFile = getRandomFileToCorrupt(repositoryRootPath, corruptedFileType);
+        if (randomBoolean()) {
+            logger.info("--> deleting [{}]", corruptedFile);
+            Files.delete(corruptedFile);
+        } else {
+            corruptFileContents(corruptedFile);
+        }
+        return corruptedFile;
+    }
+
+    public static void corruptFileContents(Path fileToCorrupt) throws IOException {
+        final var oldFileContents = Files.readAllBytes(fileToCorrupt);
+        logger.info("--> contents of [{}] before corruption: [{}]", fileToCorrupt, Base64.getEncoder().encodeToString(oldFileContents));
+        final byte[] newFileContents = new byte[randomBoolean() ? oldFileContents.length : between(0, oldFileContents.length)];
+        System.arraycopy(oldFileContents, 0, newFileContents, 0, newFileContents.length);
+        if (newFileContents.length == oldFileContents.length) {
+            final var corruptionPosition = between(0, newFileContents.length - 1);
+            newFileContents[corruptionPosition] = randomValueOtherThan(oldFileContents[corruptionPosition], ESTestCase::randomByte);
+            logger.info(
+                "--> updating byte at position [{}] from [{}] to [{}]",
+                corruptionPosition,
+                oldFileContents[corruptionPosition],
+                newFileContents[corruptionPosition]
+            );
+        } else {
+            logger.info("--> truncating file from length [{}] to length [{}]", oldFileContents.length, newFileContents.length);
+        }
+        Files.write(fileToCorrupt, newFileContents);
+        logger.info("--> contents of [{}] after corruption: [{}]", fileToCorrupt, Base64.getEncoder().encodeToString(newFileContents));
+    }
+
+    public static RepositoryFileType getRandomCorruptibleFileType() {
+        return randomValueOtherThanMany(
+            // these blob types do not have reliable corruption detection, so we must skip them
+            t -> t == RepositoryFileType.ROOT_INDEX_N || t == RepositoryFileType.ROOT_INDEX_LATEST,
+            () -> randomFrom(RepositoryFileType.values())
+        );
+    }
+
+    public static Path getRandomFileToCorrupt(Path repositoryRootPath, RepositoryFileType corruptedFileType) throws IOException {
+        final var corruptibleFiles = new ArrayList<Path>();
+        Files.walkFileTree(repositoryRootPath, new SimpleFileVisitor<>() {
+            @Override
+            public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) throws IOException {
+                if (ExtrasFS.isExtra(filePath.getFileName().toString()) == false
+                    && RepositoryFileType.getRepositoryFileType(repositoryRootPath, filePath) == corruptedFileType) {
+                    corruptibleFiles.add(filePath);
+                }
+                return super.visitFile(filePath, attrs);
+            }
+        });
+        return randomFrom(corruptibleFiles);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/RepositoryFileType.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/RepositoryFileType.java
new file mode 100644
index 0000000000000..8d20cce33bbb4
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/RepositoryFileType.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.repositories.blobstore;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
+
+import java.nio.file.Path;
+import java.util.regex.Pattern;
+
+/**
+ * The types of blobs in a {@link BlobStoreRepository}.
+ */
+public enum RepositoryFileType {
+
+    ROOT_INDEX_N("index-NUM"),
+    ROOT_INDEX_LATEST("index.latest"),
+    SNAPSHOT_INFO("snap-UUID.dat"),
+    GLOBAL_METADATA("meta-UUID.dat"),
+    INDEX_METADATA("indices/UUID/meta-SHORTUUID.dat"),
+    SHARD_GENERATION("indices/UUID/NUM/index-UUID"),
+    SHARD_SNAPSHOT_INFO("indices/UUID/NUM/snap-UUID.dat"),
+    SHARD_DATA("indices/UUID/NUM/__UUID"),
+    // NB no support for legacy names (yet)
+    ;
+
+    private final Pattern pattern;
+
+    RepositoryFileType(String regex) {
+        pattern = Pattern.compile(
+            "^("
+                + regex
+                    // decimal numbers
+                    .replace("NUM", "(0|[1-9][0-9]*)")
+                    // 15-byte UUIDS from TimeBasedUUIDGenerator
+                    .replace("SHORTUUID", "[0-9a-zA-Z_-]{" + UUIDs.TIME_BASED_UUID_STRING_LENGTH + "}")
+                    // 16-byte UUIDs from RandomBasedUUIDGenerator
+                    .replace("UUID", "[0-9a-zA-Z_-]{" + UUIDs.RANDOM_BASED_UUID_STRING_LENGTH + "}")
+                + ")$"
+        );
+    }
+
+    public static RepositoryFileType getRepositoryFileType(Path repositoryRoot, Path blobPath) {
+        final var relativePath = repositoryRoot.relativize(blobPath).toString().replace(repositoryRoot.getFileSystem().getSeparator(), "/");
+        for (final var repositoryFileType : RepositoryFileType.values()) {
+            if (repositoryFileType.pattern.matcher(relativePath).matches()) {
+                return repositoryFileType;
+            }
+        }
+        throw new IllegalArgumentException(
+            Strings.format("[%s] is not the path of a known blob type within [%s]", relativePath, repositoryRoot)
+        );
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index 2eb1811c12691..995481b31243e 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -739,6 +739,11 @@ public boolean needs_score() {
                     return true;
                 }
 
+                @Override
+                public boolean needs_termStats() {
+                    return false;
+                }
+
                 @Override
                 public ScoreScript newInstance(DocReader docReader) throws IOException {
                     Scorable[] scorerHolder = new Scorable[1];
diff --git a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankDoc.java b/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankDoc.java
deleted file mode 100644
index f2f3cb82d203f..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankDoc.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.search.rank;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-
-import java.io.IOException;
-
-public class TestRankDoc extends RankDoc {
-
-    public TestRankDoc(int doc, float score, int shardIndex) {
-        super(doc, score, shardIndex);
-    }
-
-    public TestRankDoc(StreamInput in) throws IOException {
-        super(in);
-    }
-
-    @Override
-    public void doWriteTo(StreamOutput out) throws IOException {
-        // do nothing
-    }
-
-    @Override
-    public boolean doEquals(RankDoc rd) {
-        return true;
-    }
-
-    @Override
-    public int doHashCode() {
-        return 0;
-    }
-
-    @Override
-    public String getWriteableName() {
-        return "test_rank_doc";
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankShardResult.java b/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankShardResult.java
index ab66d021497d5..6c1faaf8d2abf 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankShardResult.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankShardResult.java
@@ -17,14 +17,14 @@
 
 public class TestRankShardResult implements RankShardResult {
 
-    public final TestRankDoc[] testRankDocs;
+    public final RankDoc[] testRankDocs;
 
-    public TestRankShardResult(TestRankDoc[] testRankDocs) {
+    public TestRankShardResult(RankDoc[] testRankDocs) {
         this.testRankDocs = testRankDocs;
     }
 
     public TestRankShardResult(StreamInput in) throws IOException {
-        testRankDocs = in.readArray(TestRankDoc::new, TestRankDoc[]::new);
+        testRankDocs = in.readArray(RankDoc::new, RankDoc[]::new);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/search/retriever/TestRetrieverBuilder.java b/test/framework/src/main/java/org/elasticsearch/search/retriever/TestRetrieverBuilder.java
index 40cc1890f69ed..fcc01adf1815a 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/retriever/TestRetrieverBuilder.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/retriever/TestRetrieverBuilder.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.search.retriever;
 
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
@@ -65,6 +66,11 @@ public TestRetrieverBuilder(String value) {
         this.value = value;
     }
 
+    @Override
+    public QueryBuilder topDocsQuery() {
+        throw new UnsupportedOperationException("only used for parsing tests");
+    }
+
     @Override
     public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
         throw new UnsupportedOperationException("only used for parsing tests");
diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
index 1b49209b49c7f..1656a09daa123 100644
--- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -288,7 +288,7 @@ public static void failReadsAllDataNodes(String repository) {
         }
     }
 
-    public static void waitForBlockOnAnyDataNode(String repository) throws InterruptedException {
+    public static void waitForBlockOnAnyDataNode(String repository) {
         final boolean blocked = waitUntil(() -> {
             for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) {
                 MockRepository mockRepository = (MockRepository) repositoriesService.repository(repository);
@@ -475,13 +475,13 @@ protected SnapshotInfo createSnapshot(String repositoryName, String snapshot, Li
         return createSnapshot(repositoryName, snapshot, indices, Collections.singletonList(NO_FEATURE_STATES_VALUE));
     }
 
-    protected void createIndexWithRandomDocs(String indexName, int docCount) throws InterruptedException {
+    protected void createIndexWithRandomDocs(String indexName, int docCount) {
         createIndex(indexName);
         ensureGreen();
         indexRandomDocs(indexName, docCount);
     }
 
-    protected void indexRandomDocs(String index, int numdocs) throws InterruptedException {
+    protected void indexRandomDocs(String index, int numdocs) {
         logger.info("--> indexing [{}] documents into [{}]", numdocs, index);
         IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs];
         for (int i = 0; i < builders.length; i++) {
diff --git a/test/framework/src/main/java/org/elasticsearch/telemetry/RecordingInstruments.java b/test/framework/src/main/java/org/elasticsearch/telemetry/RecordingInstruments.java
index 35417c16e7e1c..49e667bb74e5b 100644
--- a/test/framework/src/main/java/org/elasticsearch/telemetry/RecordingInstruments.java
+++ b/test/framework/src/main/java/org/elasticsearch/telemetry/RecordingInstruments.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.telemetry.metric.LongUpDownCounter;
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
 
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
@@ -53,7 +54,7 @@ public String getName() {
         }
     }
 
-    protected interface NumberWithAttributesObserver extends Supplier<Tuple<Number, Map<String, Object>>> {
+    protected interface NumberWithAttributesObserver extends Supplier<Collection<Tuple<Number, Map<String, Object>>>> {
 
     }
 
@@ -74,7 +75,7 @@ public void run() {
                     return;
                 }
                 var observation = observer.get();
-                call(observation.v1(), observation.v2());
+                observation.forEach(o -> call(o.v1(), o.v2()));
             }
         }
 
@@ -109,10 +110,10 @@ public void incrementBy(double inc, Map<String, Object> attributes) {
     }
 
     public static class RecordingDoubleGauge extends CallbackRecordingInstrument implements DoubleGauge {
-        public RecordingDoubleGauge(String name, Supplier<DoubleWithAttributes> observer, MetricRecorder<Instrument> recorder) {
+        public RecordingDoubleGauge(String name, Supplier<Collection<DoubleWithAttributes>> observer, MetricRecorder<Instrument> recorder) {
             super(name, () -> {
                 var observation = observer.get();
-                return new Tuple<>(observation.value(), observation.attributes());
+                return observation.stream().map(o -> new Tuple<>((Number) o.value(), o.attributes())).toList();
             }, recorder);
         }
     }
@@ -172,10 +173,14 @@ public void incrementBy(long inc, Map<String, Object> attributes) {
 
     public static class RecordingAsyncLongCounter extends CallbackRecordingInstrument implements LongAsyncCounter {
 
-        public RecordingAsyncLongCounter(String name, Supplier<LongWithAttributes> observer, MetricRecorder<Instrument> recorder) {
+        public RecordingAsyncLongCounter(
+            String name,
+            Supplier<Collection<LongWithAttributes>> observer,
+            MetricRecorder<Instrument> recorder
+        ) {
             super(name, () -> {
                 var observation = observer.get();
-                return new Tuple<>(observation.value(), observation.attributes());
+                return observation.stream().map(o -> new Tuple<>((Number) o.value(), o.attributes())).toList();
             }, recorder);
         }
 
@@ -183,10 +188,14 @@ public RecordingAsyncLongCounter(String name, Supplier<LongWithAttributes> obser
 
     public static class RecordingAsyncDoubleCounter extends CallbackRecordingInstrument implements DoubleAsyncCounter {
 
-        public RecordingAsyncDoubleCounter(String name, Supplier<DoubleWithAttributes> observer, MetricRecorder<Instrument> recorder) {
+        public RecordingAsyncDoubleCounter(
+            String name,
+            Supplier<Collection<DoubleWithAttributes>> observer,
+            MetricRecorder<Instrument> recorder
+        ) {
             super(name, () -> {
                 var observation = observer.get();
-                return new Tuple<>(observation.value(), observation.attributes());
+                return observation.stream().map(o -> new Tuple<>((Number) o.value(), o.attributes())).toList();
             }, recorder);
         }
 
@@ -194,10 +203,10 @@ public RecordingAsyncDoubleCounter(String name, Supplier<DoubleWithAttributes> o
 
     public static class RecordingLongGauge extends CallbackRecordingInstrument implements LongGauge {
 
-        public RecordingLongGauge(String name, Supplier<LongWithAttributes> observer, MetricRecorder<Instrument> recorder) {
+        public RecordingLongGauge(String name, Supplier<Collection<LongWithAttributes>> observer, MetricRecorder<Instrument> recorder) {
             super(name, () -> {
                 var observation = observer.get();
-                return new Tuple<>(observation.value(), observation.attributes());
+                return observation.stream().map(o -> new Tuple<>((Number) o.value(), o.attributes())).toList();
             }, recorder);
         }
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/telemetry/RecordingMeterRegistry.java b/test/framework/src/main/java/org/elasticsearch/telemetry/RecordingMeterRegistry.java
index 33693c297f166..392445aa77a8f 100644
--- a/test/framework/src/main/java/org/elasticsearch/telemetry/RecordingMeterRegistry.java
+++ b/test/framework/src/main/java/org/elasticsearch/telemetry/RecordingMeterRegistry.java
@@ -23,6 +23,8 @@
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.function.Supplier;
 
 /**
@@ -71,6 +73,16 @@ protected DoubleUpDownCounter buildDoubleUpDownCounter(String name, String descr
 
     @Override
     public DoubleGauge registerDoubleGauge(String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+        return registerDoublesGauge(name, description, unit, () -> Collections.singleton(observer.get()));
+    }
+
+    @Override
+    public DoubleGauge registerDoublesGauge(
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<DoubleWithAttributes>> observer
+    ) {
         DoubleGauge instrument = buildDoubleGauge(name, description, unit, observer);
         recorder.register(instrument, InstrumentType.fromInstrument(instrument), name, description, unit);
         return instrument;
@@ -81,7 +93,12 @@ public DoubleGauge getDoubleGauge(String name) {
         return (DoubleGauge) recorder.getInstrument(InstrumentType.DOUBLE_GAUGE, name);
     }
 
-    protected DoubleGauge buildDoubleGauge(String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+    protected DoubleGauge buildDoubleGauge(
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<DoubleWithAttributes>> observer
+    ) {
         return new RecordingInstruments.RecordingDoubleGauge(name, observer, recorder);
     }
 
@@ -110,6 +127,16 @@ public LongCounter registerLongCounter(String name, String description, String u
 
     @Override
     public LongAsyncCounter registerLongAsyncCounter(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+        return registerLongsAsyncCounter(name, description, unit, () -> Collections.singleton(observer.get()));
+    }
+
+    @Override
+    public LongAsyncCounter registerLongsAsyncCounter(
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<LongWithAttributes>> observer
+    ) {
         LongAsyncCounter instrument = new RecordingInstruments.RecordingAsyncLongCounter(name, observer, recorder);
         recorder.register(instrument, InstrumentType.fromInstrument(instrument), name, description, unit);
         return instrument;
@@ -126,6 +153,16 @@ public DoubleAsyncCounter registerDoubleAsyncCounter(
         String description,
         String unit,
         Supplier<DoubleWithAttributes> observer
+    ) {
+        return registerDoublesAsyncCounter(name, description, unit, () -> Collections.singleton(observer.get()));
+    }
+
+    @Override
+    public DoubleAsyncCounter registerDoublesAsyncCounter(
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<DoubleWithAttributes>> observer
     ) {
         DoubleAsyncCounter instrument = new RecordingInstruments.RecordingAsyncDoubleCounter(name, observer, recorder);
         recorder.register(instrument, InstrumentType.fromInstrument(instrument), name, description, unit);
@@ -165,6 +202,11 @@ protected LongUpDownCounter buildLongUpDownCounter(String name, String descripti
 
     @Override
     public LongGauge registerLongGauge(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+        return registerLongsGauge(name, description, unit, () -> Collections.singleton(observer.get()));
+    }
+
+    @Override
+    public LongGauge registerLongsGauge(String name, String description, String unit, Supplier<Collection<LongWithAttributes>> observer) {
         LongGauge instrument = buildLongGauge(name, description, unit, observer);
         recorder.register(instrument, InstrumentType.fromInstrument(instrument), name, description, unit);
         return instrument;
@@ -175,7 +217,7 @@ public LongGauge getLongGauge(String name) {
         return (LongGauge) recorder.getInstrument(InstrumentType.LONG_GAUGE, name);
     }
 
-    protected LongGauge buildLongGauge(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+    protected LongGauge buildLongGauge(String name, String description, String unit, Supplier<Collection<LongWithAttributes>> observer) {
         return new RecordingInstruments.RecordingLongGauge(name, observer, recorder);
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
index 271df2a971fb1..1634572e0b6b7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
@@ -59,6 +59,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardLongFieldRange;
 import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.indices.DateFieldRangeInfo;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
@@ -617,18 +618,19 @@ QueryRewriteContext createQueryRewriteContext() {
                 null,
                 () -> true,
                 scriptService,
-                createMockResolvedIndices()
+                createMockResolvedIndices(),
+                null
             );
         }
 
         CoordinatorRewriteContext createCoordinatorContext(DateFieldMapper.DateFieldType dateFieldType, long min, long max) {
-            return new CoordinatorRewriteContext(
-                parserConfiguration,
-                this.client,
-                () -> nowInMillis,
+            DateFieldRangeInfo timestampFieldInfo = new DateFieldRangeInfo(
+                dateFieldType,
                 IndexLongFieldRange.NO_SHARDS.extendWithShardRange(0, 1, ShardLongFieldRange.of(min, max)),
-                dateFieldType
+                dateFieldType,
+                IndexLongFieldRange.NO_SHARDS.extendWithShardRange(0, 1, ShardLongFieldRange.of(min, max))
             );
+            return new CoordinatorRewriteContext(parserConfiguration, this.client, () -> nowInMillis, timestampFieldInfo);
         }
 
         DataRewriteContext createDataContext() {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java
index 82d7f98f34301..b4503a69acca3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java
@@ -25,11 +25,18 @@ public abstract class AbstractWireSerializingTestCase<T extends Writeable> exten
      */
     protected abstract Writeable.Reader<T> instanceReader();
 
+    /**
+     * Returns a {@link Writeable.Writer} that will be used to serialize the instance
+     */
+    protected Writeable.Writer<T> instanceWriter() {
+        return StreamOutput::writeWriteable;
+    }
+
     /**
      * Copy the {@link Writeable} by round tripping it through {@linkplain StreamInput} and {@linkplain StreamOutput}.
      */
     @Override
     protected final T copyInstance(T instance, TransportVersion version) throws IOException {
-        return copyWriteable(instance, getNamedWriteableRegistry(), instanceReader(), version);
+        return copyInstance(instance, getNamedWriteableRegistry(), instanceWriter(), instanceReader(), version);
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ByteSizeEqualsMatcher.java b/test/framework/src/main/java/org/elasticsearch/test/ByteSizeEqualsMatcher.java
new file mode 100644
index 0000000000000..172d5f2076a0f
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/ByteSizeEqualsMatcher.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.hamcrest.Description;
+import org.hamcrest.TypeSafeMatcher;
+
+/**
+ * Equality matcher for {@link ByteSizeValue} that has a nice description of failures.
+ */
+public class ByteSizeEqualsMatcher extends TypeSafeMatcher<ByteSizeValue> {
+    public static ByteSizeEqualsMatcher byteSizeEquals(ByteSizeValue expected) {
+        return new ByteSizeEqualsMatcher(expected);
+    }
+
+    private final ByteSizeValue expected;
+
+    private ByteSizeEqualsMatcher(ByteSizeValue expected) {
+        this.expected = expected;
+    }
+
+    @Override
+    protected boolean matchesSafely(ByteSizeValue byteSizeValue) {
+        return expected.equals(byteSizeValue);
+    }
+
+    @Override
+    public void describeTo(Description description) {
+        description.appendValue(expected.toString()).appendText(" (").appendValue(expected.getBytes()).appendText(" bytes)");
+    }
+
+    @Override
+    protected void describeMismatchSafely(ByteSizeValue item, Description mismatchDescription) {
+        mismatchDescription.appendValue(item.toString()).appendText(" (").appendValue(item.getBytes()).appendText(" bytes)");
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index db6fc9ea696d5..4bdbc81bcc3f0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -21,8 +21,10 @@
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.tests.util.LuceneTestCase;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequest;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
@@ -56,6 +58,7 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.RefCountingListener;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
@@ -82,6 +85,8 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -94,14 +99,14 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.gateway.PersistedClusterStateService;
@@ -119,6 +124,7 @@
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.indices.IndicesRequestCache;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.monitor.jvm.HotThreads;
 import org.elasticsearch.node.NodeMocksPlugin;
@@ -177,7 +183,6 @@
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.Callable;
-import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
@@ -203,7 +208,6 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
 import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
@@ -663,6 +667,15 @@ public static Client dataNodeClient() {
         return client;
     }
 
+    /**
+     * Execute the given {@link ActionRequest} using the given {@link ActionType} and a default node client, wait for it to complete with
+     * a timeout of {@link #SAFE_AWAIT_TIMEOUT}, and then return the result. An exceptional response, timeout or interrupt triggers a test
+     * failure.
+     */
+    public static <T extends ActionResponse> T safeExecute(ActionType<T> action, ActionRequest request) {
+        return safeExecute(client(), action, request);
+    }
+
     public static Iterable<Client> clients() {
         return cluster().getClients();
     }
@@ -943,6 +956,13 @@ public ClusterHealthStatus ensureYellow(String... indices) {
         return ensureColor(ClusterHealthStatus.YELLOW, TimeValue.timeValueSeconds(30), false, indices);
     }
 
+    /**
+     * Ensures the cluster has a red state via the cluster health API.
+     */
+    public ClusterHealthStatus ensureRed(String... indices) {
+        return ensureColor(ClusterHealthStatus.RED, TimeValue.timeValueSeconds(30), false, indices);
+    }
+
     /**
      * Ensures the cluster has a yellow state via the cluster health API and ensures the that cluster has no initializing shards
      * for the given indices
@@ -1120,12 +1140,11 @@ public void waitForDocs(final long numDocs, final BackgroundIndexer indexer) thr
 
             if (lastKnownCount >= numDocs) {
                 try {
-                    long count = SearchResponseUtils.getTotalHitsValue(
-                        prepareSearch().setTrackTotalHits(true).setSize(0).setQuery(matchAllQuery())
-                    );
-                    if (count == lastKnownCount) {
-                        // no progress - try to refresh for the next time
+                    long count = getTotalHitsAllIndices();
+                    if (count < lastKnownCount) {
+                        // not caught up - try to refresh
                         indicesAdmin().prepareRefresh().get();
+                        count = getTotalHitsAllIndices();
                     }
                     lastKnownCount = count;
                 } catch (Exception e) { // count now acts like search and barfs if all shards failed...
@@ -1146,6 +1165,10 @@ public void waitForDocs(final long numDocs, final BackgroundIndexer indexer) thr
         }, maxWaitTimeMs, TimeUnit.MILLISECONDS);
     }
 
+    private static long getTotalHitsAllIndices() {
+        return SearchResponseUtils.getTotalHitsValue(prepareSearch().setTrackTotalHits(true).setSize(0).setQuery(matchAllQuery()));
+    }
+
     public static SearchRequestBuilder prepareSearch(String... indices) {
         return client().prepareSearch(indices);
     }
@@ -1176,23 +1199,19 @@ public static List<PersistentTasksCustomMetadata.PersistentTask<?>> findTasks(Cl
     @Nullable
     public static DiscoveryNode waitAndGetHealthNode(InternalTestCluster internalCluster) {
         DiscoveryNode[] healthNode = new DiscoveryNode[1];
-        try {
-            waitUntil(() -> {
-                ClusterState state = internalCluster.client()
-                    .admin()
-                    .cluster()
-                    .prepareState()
-                    .clear()
-                    .setMetadata(true)
-                    .setNodes(true)
-                    .get()
-                    .getState();
-                healthNode[0] = HealthNode.findHealthNode(state);
-                return healthNode[0] != null;
-            }, 15, TimeUnit.SECONDS);
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-        }
+        waitUntil(() -> {
+            ClusterState state = internalCluster.client()
+                .admin()
+                .cluster()
+                .prepareState()
+                .clear()
+                .setMetadata(true)
+                .setNodes(true)
+                .get()
+                .getState();
+            healthNode[0] = HealthNode.findHealthNode(state);
+            return healthNode[0] != null;
+        }, 15, TimeUnit.SECONDS);
         return healthNode[0];
     }
 
@@ -1215,48 +1234,74 @@ protected void ensureClusterSizeConsistency() {
      */
     protected void ensureClusterStateConsistency() throws IOException {
         if (cluster() != null && cluster().size() > 0) {
-            final NamedWriteableRegistry namedWriteableRegistry = cluster().getNamedWriteableRegistry();
-            final Client masterClient = client();
-            ClusterState masterClusterState = masterClient.admin().cluster().prepareState().all().get().getState();
-            byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState);
-            // remove local node reference
-            masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null, namedWriteableRegistry);
-            Map<String, Object> masterStateMap = convertToMap(masterClusterState);
-            int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length;
-            String masterId = masterClusterState.nodes().getMasterNodeId();
-            for (Client client : cluster().getClients()) {
-                ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState();
-                byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState);
-                // remove local node reference
-                localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null, namedWriteableRegistry);
-                final Map<String, Object> localStateMap = convertToMap(localClusterState);
-                final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length;
-                // Check that the non-master node has the same version of the cluster state as the master and
-                // that the master node matches the master (otherwise there is no requirement for the cluster state to match)
-                if (masterClusterState.version() == localClusterState.version()
-                    && masterId.equals(localClusterState.nodes().getMasterNodeId())) {
-                    try {
-                        assertEquals("cluster state UUID does not match", masterClusterState.stateUUID(), localClusterState.stateUUID());
-                        // We cannot compare serialization bytes since serialization order of maps is not guaranteed
-                        // but we can compare serialization sizes - they should be the same
-                        assertEquals("cluster state size does not match", masterClusterStateSize, localClusterStateSize);
-                        // Compare JSON serialization
-                        assertNull(
-                            "cluster state JSON serialization does not match",
-                            differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap)
-                        );
-                    } catch (final AssertionError error) {
-                        logger.error(
-                            "Cluster state from master:\n{}\nLocal cluster state:\n{}",
-                            masterClusterState.toString(),
-                            localClusterState.toString()
-                        );
-                        throw error;
-                    }
-                }
-            }
+            doEnsureClusterStateConsistency(cluster().getNamedWriteableRegistry());
         }
+    }
 
+    protected final void doEnsureClusterStateConsistency(NamedWriteableRegistry namedWriteableRegistry) {
+        final PlainActionFuture<Void> future = new PlainActionFuture<>();
+        final List<SubscribableListener<ClusterStateResponse>> localStates = new ArrayList<>(cluster().size());
+        for (Client client : cluster().getClients()) {
+            localStates.add(SubscribableListener.newForked(l -> client.admin().cluster().prepareState().all().setLocal(true).execute(l)));
+        }
+        try (RefCountingListener refCountingListener = new RefCountingListener(future)) {
+            SubscribableListener.<ClusterStateResponse>newForked(l -> client().admin().cluster().prepareState().all().execute(l))
+                .andThenAccept(masterStateResponse -> {
+                    byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterStateResponse.getState());
+                    // remove local node reference
+                    final ClusterState masterClusterState = ClusterState.Builder.fromBytes(
+                        masterClusterStateBytes,
+                        null,
+                        namedWriteableRegistry
+                    );
+                    Map<String, Object> masterStateMap = convertToMap(masterClusterState);
+                    int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length;
+                    String masterId = masterClusterState.nodes().getMasterNodeId();
+                    for (SubscribableListener<ClusterStateResponse> localStateListener : localStates) {
+                        localStateListener.andThenAccept(localClusterStateResponse -> {
+                            byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterStateResponse.getState());
+                            // remove local node reference
+                            final ClusterState localClusterState = ClusterState.Builder.fromBytes(
+                                localClusterStateBytes,
+                                null,
+                                namedWriteableRegistry
+                            );
+                            final Map<String, Object> localStateMap = convertToMap(localClusterState);
+                            final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length;
+                            // Check that the non-master node has the same version of the cluster state as the master and
+                            // that the master node matches the master (otherwise there is no requirement for the cluster state to
+                            // match)
+                            if (masterClusterState.version() == localClusterState.version()
+                                && masterId.equals(localClusterState.nodes().getMasterNodeId())) {
+                                try {
+                                    assertEquals(
+                                        "cluster state UUID does not match",
+                                        masterClusterState.stateUUID(),
+                                        localClusterState.stateUUID()
+                                    );
+                                    // We cannot compare serialization bytes since serialization order of maps is not guaranteed
+                                    // but we can compare serialization sizes - they should be the same
+                                    assertEquals("cluster state size does not match", masterClusterStateSize, localClusterStateSize);
+                                    // Compare JSON serialization
+                                    assertNull(
+                                        "cluster state JSON serialization does not match",
+                                        differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap)
+                                    );
+                                } catch (final AssertionError error) {
+                                    logger.error(
+                                        "Cluster state from master:\n{}\nLocal cluster state:\n{}",
+                                        masterClusterState.toString(),
+                                        localClusterState.toString()
+                                    );
+                                    throw error;
+                                }
+                            }
+                        }).addListener(refCountingListener.acquire());
+                    }
+                })
+                .addListener(refCountingListener.acquire());
+        }
+        safeGet(future);
     }
 
     protected void ensureClusterStateCanBeReadByNodeTool() throws IOException {
@@ -1598,7 +1643,7 @@ protected static IndicesAdminClient indicesAdmin() {
         return admin().indices();
     }
 
-    public void indexRandom(boolean forceRefresh, String index, int numDocs) throws InterruptedException {
+    public void indexRandom(boolean forceRefresh, String index, int numDocs) {
         IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
         for (int i = 0; i < builders.length; i++) {
             builders[i] = prepareIndex(index).setSource("field", "value");
@@ -1609,11 +1654,11 @@ public void indexRandom(boolean forceRefresh, String index, int numDocs) throws
     /**
      * Convenience method that forwards to {@link #indexRandom(boolean, List)}.
      */
-    public void indexRandom(boolean forceRefresh, IndexRequestBuilder... builders) throws InterruptedException {
+    public void indexRandom(boolean forceRefresh, IndexRequestBuilder... builders) {
         indexRandom(forceRefresh, Arrays.asList(builders));
     }
 
-    public void indexRandom(boolean forceRefresh, boolean dummyDocuments, IndexRequestBuilder... builders) throws InterruptedException {
+    public void indexRandom(boolean forceRefresh, boolean dummyDocuments, IndexRequestBuilder... builders) {
         indexRandom(forceRefresh, dummyDocuments, Arrays.asList(builders));
     }
 
@@ -1632,7 +1677,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, IndexReque
      * @param builders     the documents to index.
      * @see #indexRandom(boolean, boolean, java.util.List)
      */
-    public void indexRandom(boolean forceRefresh, List<IndexRequestBuilder> builders) throws InterruptedException {
+    public void indexRandom(boolean forceRefresh, List<IndexRequestBuilder> builders) {
         indexRandom(forceRefresh, forceRefresh, builders);
     }
 
@@ -1648,7 +1693,7 @@ public void indexRandom(boolean forceRefresh, List<IndexRequestBuilder> builders
      *                       all documents are indexed. This is useful to produce deleted documents on the server side.
      * @param builders       the documents to index.
      */
-    public void indexRandom(boolean forceRefresh, boolean dummyDocuments, List<IndexRequestBuilder> builders) throws InterruptedException {
+    public void indexRandom(boolean forceRefresh, boolean dummyDocuments, List<IndexRequestBuilder> builders) {
         indexRandom(forceRefresh, dummyDocuments, true, builders);
     }
 
@@ -1665,8 +1710,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, List<Index
      * @param maybeFlush     if {@code true} this method may randomly execute full flushes after index operations.
      * @param builders       the documents to index.
      */
-    public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean maybeFlush, List<IndexRequestBuilder> builders)
-        throws InterruptedException {
+    public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean maybeFlush, List<IndexRequestBuilder> builders) {
         Random random = random();
         Set<String> indices = new HashSet<>();
         builders = new ArrayList<>(builders);
@@ -1688,7 +1732,6 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma
             }
         }
         Collections.shuffle(builders, random());
-        final CopyOnWriteArrayList<Tuple<IndexRequestBuilder, Exception>> errors = new CopyOnWriteArrayList<>();
         List<CountDownLatch> inFlightAsyncOperations = new ArrayList<>();
         // If you are indexing just a few documents then frequently do it one at a time. If many then frequently in bulk.
         final String[] indicesArray = indices.toArray(new String[] {});
@@ -1697,7 +1740,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma
                 logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), true, false);
                 for (IndexRequestBuilder indexRequestBuilder : builders) {
                     indexRequestBuilder.execute(
-                        new PayloadLatchedActionListener<>(indexRequestBuilder, newLatch(inFlightAsyncOperations), errors)
+                        new LatchedActionListener<DocWriteResponse>(newLatch(inFlightAsyncOperations)).delegateResponse((l, e) -> fail(e))
                     );
                     postIndexAsyncActions(indicesArray, inFlightAsyncOperations, maybeFlush);
                 }
@@ -1724,19 +1767,8 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma
             }
         }
         for (CountDownLatch operation : inFlightAsyncOperations) {
-            operation.await();
-        }
-        final List<Exception> actualErrors = new ArrayList<>();
-        for (Tuple<IndexRequestBuilder, Exception> tuple : errors) {
-            Throwable t = ExceptionsHelper.unwrapCause(tuple.v2());
-            if (t instanceof EsRejectedExecutionException) {
-                logger.debug("Error indexing doc: " + t.getMessage() + ", reindexing.");
-                tuple.v1().get(); // re-index if rejected
-            } else {
-                actualErrors.add(tuple.v2());
-            }
+            safeAwait(operation);
         }
-        assertThat(actualErrors, emptyIterable());
         if (bogusIds.isEmpty() == false) {
             // delete the bogus types again - it might trigger merges or at least holes in the segments and enforces deleted docs!
             for (List<String> doc : bogusIds) {
@@ -1792,8 +1824,7 @@ private static CountDownLatch newLatch(List<CountDownLatch> latches) {
     /**
      * Maybe refresh, force merge, or flush then always make sure there aren't too many in flight async operations.
      */
-    private void postIndexAsyncActions(String[] indices, List<CountDownLatch> inFlightAsyncOperations, boolean maybeFlush)
-        throws InterruptedException {
+    private void postIndexAsyncActions(String[] indices, List<CountDownLatch> inFlightAsyncOperations, boolean maybeFlush) {
         if (rarely()) {
             if (rarely()) {
                 indicesAdmin().prepareRefresh(indices)
@@ -1813,7 +1844,7 @@ private void postIndexAsyncActions(String[] indices, List<CountDownLatch> inFlig
         }
         while (inFlightAsyncOperations.size() > MAX_IN_FLIGHT_ASYNC_INDEXES) {
             int waitFor = between(0, inFlightAsyncOperations.size() - 1);
-            inFlightAsyncOperations.remove(waitFor).await();
+            safeAwait(inFlightAsyncOperations.remove(waitFor));
         }
     }
 
@@ -1910,23 +1941,6 @@ protected void addError(Exception e) {}
 
     }
 
-    private class PayloadLatchedActionListener<Response, T> extends LatchedActionListener<Response> {
-        private final CopyOnWriteArrayList<Tuple<T, Exception>> errors;
-        private final T builder;
-
-        PayloadLatchedActionListener(T builder, CountDownLatch latch, CopyOnWriteArrayList<Tuple<T, Exception>> errors) {
-            super(latch);
-            this.errors = errors;
-            this.builder = builder;
-        }
-
-        @Override
-        protected void addError(Exception e) {
-            errors.add(new Tuple<>(builder, e));
-        }
-
-    }
-
     /**
      * Clears the given scroll Ids
      */
@@ -2340,6 +2354,13 @@ public final void cleanUpCluster() throws Exception {
         }
     }
 
+    @Override
+    protected boolean enableBigArraysReleasedCheck() {
+        // checking that all big arrays have been released makes little sense for a still-running cluster, see comments in
+        // #ensureAllArraysAreReleased for details
+        return isSuiteScopedTest(getTestClass()) == false;
+    }
+
     @AfterClass
     public static void afterClass() throws Exception {
         try {
@@ -2348,6 +2369,7 @@ public static void afterClass() throws Exception {
             } else {
                 INSTANCE.printTestMessage("cleaning up after");
                 INSTANCE.afterInternal(true);
+                MockBigArrays.ensureAllArraysAreReleased();
                 checkStaticState();
             }
         } finally {
@@ -2543,4 +2565,41 @@ public Settings onNodeStopped(String nodeName) throws Exception {
             }
         });
     }
+
+    /**
+     * Allocate the entire capacity of a circuit breaker on a specific node
+     *
+     * @param targetNode The node on which to allocate
+     * @param breakerName The circuit breaker to allocate
+     * @return A {@link Releasable} which will de-allocate the amount allocated
+     */
+    protected static Releasable fullyAllocateCircuitBreakerOnNode(String targetNode, String breakerName) {
+        final var circuitBreaker = internalCluster().getInstance(CircuitBreakerService.class, targetNode).getBreaker(breakerName);
+        final long totalAllocated = fullyAllocate(circuitBreaker);
+        return () -> circuitBreaker.addWithoutBreaking(-totalAllocated);
+    }
+
+    /**
+     * Fully allocate a circuit breaker
+     *
+     * @param circuitBreaker The circuit breaker to allocate
+     * @return the amount of bytes allocated
+     */
+    private static long fullyAllocate(CircuitBreaker circuitBreaker) {
+        long allocationSize = 1;
+        long totalAllocated = 0;
+        while (true) {
+            try {
+                circuitBreaker.addEstimateBytesAndMaybeBreak(allocationSize, "test");
+                totalAllocated += allocationSize;
+            } catch (CircuitBreakingException e) {
+                circuitBreaker.addWithoutBreaking(allocationSize);
+                totalAllocated += allocationSize;
+                break;
+            }
+            allocationSize <<= 1;
+            assert 0 <= allocationSize;
+        }
+        return totalAllocated;
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index 7fdc5765a90e8..f0eb33445db20 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -9,6 +9,9 @@
 
 import com.carrotsearch.randomizedtesting.RandomizedContext;
 
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
@@ -96,7 +99,7 @@ protected void startNode(long seed) throws Exception {
         indicesAdmin().preparePutTemplate("one_shard_index_template")
             .setPatterns(Collections.singletonList("*"))
             .setOrder(0)
-            .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0))
+            .setSettings(indexSettings(1, 0))
             .get();
         indicesAdmin().preparePutTemplate("random-soft-deletes-template")
             .setPatterns(Collections.singletonList("*"))
@@ -133,11 +136,9 @@ public void tearDown() throws Exception {
         ensureNoInitializingShards();
         ensureAllFreeContextActionsAreConsumed();
 
-        SearchService searchService = getInstanceFromNode(SearchService.class);
-        assertThat(searchService.getActiveContexts(), equalTo(0));
-        assertThat(searchService.getOpenScrollContexts(), equalTo(0));
+        ensureAllContextsReleased(getInstanceFromNode(SearchService.class));
         super.tearDown();
-        var deleteDataStreamsRequest = new DeleteDataStreamAction.Request("*");
+        var deleteDataStreamsRequest = new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, "*");
         deleteDataStreamsRequest.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN);
         try {
             assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, deleteDataStreamsRequest));
@@ -294,6 +295,15 @@ public Client client() {
         return wrapClient(NODE.client());
     }
 
+    /**
+     * Execute the given {@link ActionRequest} using the given {@link ActionType} and the default node client, wait for it to complete with
+     * a timeout of {@link #SAFE_AWAIT_TIMEOUT}, and then return the result. An exceptional response, timeout or interrupt triggers a test
+     * failure.
+     */
+    public <T extends ActionResponse> T safeExecute(ActionType<T> action, ActionRequest request) {
+        return safeExecute(client(), action, request);
+    }
+
     /**
      * Returns an admin client.
      */
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index dc8a1dc29d233..58487d6552bcd 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -42,11 +42,15 @@
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.RequestBuilder;
 import org.elasticsearch.action.support.ActionTestUtils;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.TestPlainActionFuture;
 import org.elasticsearch.bootstrap.BootstrapForTesting;
+import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.client.internal.Requests;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -115,6 +119,7 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.MockSearchService;
+import org.elasticsearch.search.SearchService;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
 import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter;
 import org.elasticsearch.threadpool.ExecutorBuilder;
@@ -154,6 +159,7 @@
 import java.lang.invoke.MethodHandles;
 import java.math.BigInteger;
 import java.net.InetAddress;
+import java.net.URI;
 import java.net.UnknownHostException;
 import java.nio.file.Path;
 import java.security.NoSuchAlgorithmException;
@@ -207,6 +213,7 @@
 import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.in;
 import static org.hamcrest.Matchers.startsWith;
 
 /**
@@ -577,8 +584,15 @@ protected boolean enableWarningsCheck() {
         return true;
     }
 
+    protected boolean enableBigArraysReleasedCheck() {
+        return true;
+    }
+
     @After
     public final void after() throws Exception {
+        if (enableBigArraysReleasedCheck()) {
+            MockBigArrays.ensureAllArraysAreReleased();
+        }
         checkStaticState();
         // We check threadContext != null rather than enableWarningsCheck()
         // because after methods are still called in the event that before
@@ -765,8 +779,6 @@ public void log(StatusData data) {
 
     // separate method so that this can be checked again after suite scoped cluster is shut down
     protected static void checkStaticState() throws Exception {
-        MockBigArrays.ensureAllArraysAreReleased();
-
         // ensure no one changed the status logger level on us
         assertThat(StatusLogger.getLogger().getLevel(), equalTo(Level.WARN));
         synchronized (statusData) {
@@ -1409,9 +1421,8 @@ public static void assertBusy(CheckedRunnable<Exception> codeBlock, long maxWait
      *
      * @param breakSupplier determines whether to return immediately or continue waiting.
      * @return the last value returned by <code>breakSupplier</code>
-     * @throws InterruptedException if any sleep calls were interrupted.
      */
-    public static boolean waitUntil(BooleanSupplier breakSupplier) throws InterruptedException {
+    public static boolean waitUntil(BooleanSupplier breakSupplier) {
         return waitUntil(breakSupplier, 10, TimeUnit.SECONDS);
     }
 
@@ -1427,9 +1438,8 @@ public static boolean waitUntil(BooleanSupplier breakSupplier) throws Interrupte
      * @param maxWaitTime the maximum amount of time to wait
      * @param unit the unit of tie for <code>maxWaitTime</code>
      * @return the last value returned by <code>breakSupplier</code>
-     * @throws InterruptedException if any sleep calls were interrupted.
      */
-    public static boolean waitUntil(BooleanSupplier breakSupplier, long maxWaitTime, TimeUnit unit) throws InterruptedException {
+    public static boolean waitUntil(BooleanSupplier breakSupplier, long maxWaitTime, TimeUnit unit) {
         long maxTimeInMillis = TimeUnit.MILLISECONDS.convert(maxWaitTime, unit);
         long timeInMillis = 1;
         long sum = 0;
@@ -1437,12 +1447,12 @@ public static boolean waitUntil(BooleanSupplier breakSupplier, long maxWaitTime,
             if (breakSupplier.getAsBoolean()) {
                 return true;
             }
-            Thread.sleep(timeInMillis);
+            safeSleep(timeInMillis);
             sum += timeInMillis;
             timeInMillis = Math.min(AWAIT_BUSY_THRESHOLD, timeInMillis * 2);
         }
         timeInMillis = maxTimeInMillis - sum;
-        Thread.sleep(Math.max(timeInMillis, 0));
+        safeSleep(Math.max(timeInMillis, 0));
         return breakSupplier.getAsBoolean();
     }
 
@@ -1476,10 +1486,24 @@ public Path getDataPath(String relativePath) {
         // we override LTC behavior here: wrap even resources with mockfilesystems,
         // because some code is buggy when it comes to multiple nio.2 filesystems
         // (e.g. FileSystemUtils, and likely some tests)
+        return getResourceDataPath(getClass(), relativePath);
+    }
+
+    public static Path getResourceDataPath(Class<?> clazz, String relativePath) {
+        final var resource = Objects.requireNonNullElseGet(
+            clazz.getResource(relativePath),
+            () -> fail(null, "resource not found: [%s][%s]", clazz.getCanonicalName(), relativePath)
+        );
+        final URI uri;
         try {
-            return PathUtils.get(getClass().getResource(relativePath).toURI()).toAbsolutePath().normalize();
+            uri = resource.toURI();
         } catch (Exception e) {
-            throw new RuntimeException("resource not found: " + relativePath, e);
+            return fail(null, "resource URI not found: [%s][%s]", clazz.getCanonicalName(), relativePath);
+        }
+        try {
+            return PathUtils.get(uri).toAbsolutePath().normalize();
+        } catch (Exception e) {
+            return fail(e, "resource path not found: %s", uri);
         }
     }
 
@@ -2303,6 +2327,15 @@ public static <T> T safeAwait(CheckedConsumer<ActionListener<T>, ?> consumer) {
         return safeAwait(SubscribableListener.newForked(consumer));
     }
 
+    /**
+     * Execute the given {@link ActionRequest} using the given {@link ActionType} and the given {@link ElasticsearchClient}, wait for
+     * it to complete with a timeout of {@link #SAFE_AWAIT_TIMEOUT}, and then return the result. An exceptional response, timeout or
+     * interrupt triggers a test failure.
+     */
+    public static <T extends ActionResponse> T safeExecute(ElasticsearchClient client, ActionType<T> action, ActionRequest request) {
+        return safeAwait(l -> client.execute(action, request, l));
+    }
+
     /**
      * Wait for the successful completion of the given {@link Future}, with a timeout of {@link #SAFE_AWAIT_TIMEOUT}, preserving the
      * thread's interrupt status flag and converting all exceptions into an {@link AssertionError} to trigger a test failure.
@@ -2471,7 +2504,7 @@ public static <T extends Throwable> T expectThrows(Class<T> expectedType, Reques
      * Same as {@link #runInParallel(int, IntConsumer)} but also attempts to start all tasks at the same time by blocking execution on a
      * barrier until all threads are started and ready to execute their task.
      */
-    public static void startInParallel(int numberOfTasks, IntConsumer taskFactory) throws InterruptedException {
+    public static void startInParallel(int numberOfTasks, IntConsumer taskFactory) {
         final CyclicBarrier barrier = new CyclicBarrier(numberOfTasks);
         runInParallel(numberOfTasks, i -> {
             safeAwait(barrier);
@@ -2485,7 +2518,7 @@ public static void startInParallel(int numberOfTasks, IntConsumer taskFactory) t
      * @param numberOfTasks number of tasks to run in parallel
      * @param taskFactory task factory
      */
-    public static void runInParallel(int numberOfTasks, IntConsumer taskFactory) throws InterruptedException {
+    public static void runInParallel(int numberOfTasks, IntConsumer taskFactory) {
         final ArrayList<Future<?>> futures = new ArrayList<>(numberOfTasks);
         final Thread[] threads = new Thread[numberOfTasks - 1];
         for (int i = 0; i < numberOfTasks; i++) {
@@ -2500,19 +2533,40 @@ public static void runInParallel(int numberOfTasks, IntConsumer taskFactory) thr
                 threads[i].start();
             }
         }
-        for (Thread thread : threads) {
-            thread.join();
-        }
         Exception e = null;
-        for (Future<?> future : futures) {
-            try {
-                future.get();
-            } catch (Exception ex) {
-                e = ExceptionsHelper.useOrSuppress(e, ex);
+        try {
+            for (Thread thread : threads) {
+                // no sense in waiting for the rest of the threads, nor any futures, if interrupted, just bail out and fail
+                thread.join();
+            }
+            for (Future<?> future : futures) {
+                try {
+                    future.get();
+                } catch (InterruptedException interruptedException) {
+                    // no sense in waiting for the rest of the futures if interrupted, just bail out and fail
+                    Thread.currentThread().interrupt();
+                    throw interruptedException;
+                } catch (Exception executionException) {
+                    e = ExceptionsHelper.useOrSuppress(e, executionException);
+                }
             }
+        } catch (InterruptedException interruptedException) {
+            Thread.currentThread().interrupt();
+            e = ExceptionsHelper.useOrSuppress(e, interruptedException);
         }
         if (e != null) {
             throw new AssertionError(e);
         }
     }
+
+    public static void ensureAllContextsReleased(SearchService searchService) {
+        try {
+            assertBusy(() -> {
+                assertThat(searchService.getActiveContexts(), equalTo(0));
+                assertThat(searchService.getOpenScrollContexts(), equalTo(0));
+            });
+        } catch (Exception e) {
+            throw new AssertionError("Failed to verify search contexts", e);
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java b/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java
index 0db85e4e67711..e0c2456db144e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java
@@ -14,16 +14,20 @@
 import org.apache.lucene.tests.index.FieldFilterLeafReader;
 
 import java.io.IOException;
-import java.util.Collections;
+import java.util.Set;
 
 public class FieldMaskingReader extends FilterDirectoryReader {
-    private final String field;
+    private final Set<String> fields;
 
     public FieldMaskingReader(String field, DirectoryReader in) throws IOException {
+        this(Set.of(field), in);
+    }
+
+    public FieldMaskingReader(Set<String> fields, DirectoryReader in) throws IOException {
         super(in, new FilterDirectoryReader.SubReaderWrapper() {
             @Override
             public LeafReader wrap(LeafReader reader) {
-                return new FilterLeafReader(new FieldFilterLeafReader(reader, Collections.singleton(field), true)) {
+                return new FilterLeafReader(new FieldFilterLeafReader(reader, fields, true)) {
 
                     // FieldFilterLeafReader does not forward cache helpers
                     // since it considers it is illegal because of the fact
@@ -43,13 +47,13 @@ public CacheHelper getCoreCacheHelper() {
                 };
             }
         });
-        this.field = field;
+        this.fields = fields;
 
     }
 
     @Override
     protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
-        return new FieldMaskingReader(field, in);
+        return new FieldMaskingReader(fields, in);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java b/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java
index a9250a6b327d2..212dcc7dc95c3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java
@@ -8,13 +8,13 @@
 package org.elasticsearch.test;
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.injection.guice.AbstractModule;
 
 import java.util.Arrays;
 import java.util.HashSet;
@@ -49,10 +49,8 @@ public static IndexSettings newIndexSettings(Index index, Settings settings, Set
     }
 
     public static IndexSettings newIndexSettings(Index index, Settings indexSetting, Settings nodeSettings, Setting<?>... setting) {
-        Settings build = Settings.builder()
+        Settings build = ESTestCase.indexSettings(1, 1)
             .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(indexSetting)
             .build();
         IndexMetadata metadata = IndexMetadata.builder(index.getName())
@@ -67,10 +65,8 @@ public static IndexSettings newIndexSettings(Index index, Settings indexSetting,
     }
 
     public static IndexSettings newIndexSettings(Index index, Settings settings, IndexScopedSettings indexScopedSettings) {
-        Settings build = Settings.builder()
+        Settings build = ESTestCase.indexSettings(1, 1)
             .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(settings)
             .build();
         IndexMetadata metadata = IndexMetadata.builder(index.getName())
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index af37fb6feefbd..332df7123fd1b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -1744,11 +1744,7 @@ private synchronized void startAndPublishNodesAndClients(List<NodeAndClient> nod
                 .filter(nac -> nodes.containsKey(nac.name) == false) // filter out old masters
                 .count();
             rebuildUnicastHostFiles(nodeAndClients); // ensure that new nodes can find the existing nodes when they start
-            try {
-                runInParallel(nodeAndClients.size(), i -> nodeAndClients.get(i).startNode());
-            } catch (InterruptedException e) {
-                throw new AssertionError("interrupted while starting nodes", e);
-            }
+            runInParallel(nodeAndClients.size(), i -> nodeAndClients.get(i).startNode());
             nodeAndClients.forEach(this::publishNode);
 
             if (autoManageMasterNodes && newMasters > 0) {
@@ -2545,15 +2541,7 @@ public void assertRequestsFinished() {
 
     private void assertSearchContextsReleased() {
         for (NodeAndClient nodeAndClient : nodes.values()) {
-            SearchService searchService = getInstance(SearchService.class, nodeAndClient.name);
-            try {
-                assertBusy(() -> {
-                    assertThat(searchService.getActiveContexts(), equalTo(0));
-                    assertThat(searchService.getOpenScrollContexts(), equalTo(0));
-                });
-            } catch (Exception e) {
-                throw new AssertionError("Failed to verify search contexts", e);
-            }
+            ESTestCase.ensureAllContextsReleased(getInstance(SearchService.class, nodeAndClient.name));
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/MapMatcher.java b/test/framework/src/main/java/org/elasticsearch/test/MapMatcher.java
index 7a788eaacc6d4..b702809de5bed 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/MapMatcher.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/MapMatcher.java
@@ -14,6 +14,8 @@
 import org.hamcrest.StringDescription;
 import org.hamcrest.TypeSafeMatcher;
 
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -292,17 +294,24 @@ static void describeEntryUnexepectedButOk(Object value, Description description)
     }
 
     static void describeEntryValue(int keyWidth, Matcher<?> matcher, Object v, Description description) {
-        if (v instanceof Map && matcher instanceof MapMatcher) {
-            ((MapMatcher) matcher).describePotentialMismatch(keyWidth + INDENT, (Map<?, ?>) v, description);
+        if (v instanceof Map && matcher instanceof MapMatcher mm) {
+            mm.describePotentialMismatch(keyWidth + INDENT, (Map<?, ?>) v, description);
             return;
         }
-        if (v instanceof List && matcher instanceof ListMatcher) {
-            ((ListMatcher) matcher).describePotentialMismatch(keyWidth + INDENT, (List<?>) v, description);
+        if (v instanceof List && matcher instanceof ListMatcher lm) {
+            lm.describePotentialMismatch(keyWidth + INDENT, (List<?>) v, description);
             return;
         }
         if (false == matcher.matches(v)) {
-            description.appendText("expected ").appendDescriptionOf(matcher).appendText(" but ");
-            matcher.describeMismatch(v, description);
+            try {
+                description.appendText("expected ").appendDescriptionOf(matcher).appendText(" but ");
+                matcher.describeMismatch(v, description);
+            } catch (Exception e) {
+                description.appendText("error describing ");
+                StringWriter trace = new StringWriter();
+                e.printStackTrace(new PrintWriter(trace));
+                description.appendValue(trace);
+            }
             return;
         }
         description.appendValue(v);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
index 73acdb6e19d4b..67ff67ee6fe05 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
@@ -10,6 +10,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComponentTemplateAction;
@@ -19,25 +20,26 @@
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.RefCountingListener;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.IndexTemplateMissingException;
 import org.elasticsearch.repositories.RepositoryMissingException;
+import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Random;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.ESTestCase.safeAwait;
 
 /**
  * Base test cluster that exposes the basis to run tests against any elasticsearch cluster, whose layout
@@ -69,14 +71,99 @@ public void beforeTest(Random randomGenerator) throws IOException, InterruptedEx
      * Wipes any data that a test can leave behind: indices, templates (except exclude templates) and repositories
      */
     public void wipe(Set<String> excludeTemplates) {
-        // First delete data streams, because composable index templates can't be deleted if these templates are still used by data streams.
-        wipeAllDataStreams();
-        wipeAllComposableIndexTemplates(excludeTemplates);
-        wipeAllComponentTemplates(excludeTemplates);
-
-        wipeIndices("_all");
-        wipeAllTemplates(excludeTemplates);
-        wipeRepositories();
+        if (size() == 0) {
+            return;
+        }
+        safeAwait((ActionListener<Void> done) -> {
+            try (RefCountingListener listeners = new RefCountingListener(done)) {
+                wipeAllTemplates(excludeTemplates, listeners);
+                // First delete data streams, because composable index templates can't be deleted if these templates are still used by data
+                // streams.
+                SubscribableListener
+
+                    .<AcknowledgedResponse>newForked(
+                        l -> client().execute(
+                            DeleteDataStreamAction.INSTANCE,
+                            new DeleteDataStreamAction.Request(ESTestCase.TEST_REQUEST_TIMEOUT, "*").indicesOptions(
+                                IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN
+                            ),
+                            l.delegateResponse((ll, e) -> {
+                                // Ignore if action isn't registered, because data streams is a module and
+                                // if the delete action isn't registered then there no data streams to delete.
+                                if (e.getMessage().startsWith("failed to find action") == false) {
+                                    ll.onFailure(e);
+                                } else {
+                                    ll.onResponse(AcknowledgedResponse.TRUE);
+                                }
+                            })
+                        )
+                    )
+                    .andThenAccept(ElasticsearchAssertions::assertAcked)
+                    .andThenAccept(v -> {
+                        SubscribableListener.<Void>newForked(ll -> wipeIndicesAsync(new String[] { "_all" }, ll))
+                            .andThen(this::wipeRepositories)
+                            .addListener(listeners.acquire());
+
+                        deleteTemplates(excludeTemplates, listeners.acquire());
+                    })
+                    .addListener(listeners.acquire());
+            }
+        });
+    }
+
+    private void deleteTemplates(Set<String> excludeTemplates, ActionListener<Void> listener) {
+        final SubscribableListener<GetComposableIndexTemplateAction.Response> getComposableTemplates = SubscribableListener.newForked(
+            l -> client().execute(GetComposableIndexTemplateAction.INSTANCE, new GetComposableIndexTemplateAction.Request("*"), l)
+        );
+
+        final SubscribableListener<GetComponentTemplateAction.Response> getComponentTemplates = SubscribableListener.newForked(
+            l -> client().execute(GetComponentTemplateAction.INSTANCE, new GetComponentTemplateAction.Request("*"), l)
+        );
+
+        SubscribableListener
+
+            // dummy start step for symmetry
+            .newSucceeded(null)
+
+            // delete composable templates
+            .<GetComposableIndexTemplateAction.Response>andThen(getComposableTemplates::addListener)
+            .<AcknowledgedResponse>andThen((l, r) -> {
+                var templates = r.indexTemplates()
+                    .keySet()
+                    .stream()
+                    .filter(template -> excludeTemplates.contains(template) == false)
+                    .toArray(String[]::new);
+                if (templates.length == 0) {
+                    l.onResponse(AcknowledgedResponse.TRUE);
+                } else {
+                    var request = new TransportDeleteComposableIndexTemplateAction.Request(templates);
+                    client().execute(TransportDeleteComposableIndexTemplateAction.TYPE, request, l);
+                }
+            })
+            .andThenAccept(ElasticsearchAssertions::assertAcked)
+
+            // then delete component templates
+            .<GetComponentTemplateAction.Response>andThen(getComponentTemplates::addListener)
+            .<AcknowledgedResponse>andThen((l, response) -> {
+                var componentTemplates = response.getComponentTemplates()
+                    .keySet()
+                    .stream()
+                    .filter(template -> excludeTemplates.contains(template) == false)
+                    .toArray(String[]::new);
+                if (componentTemplates.length == 0) {
+                    l.onResponse(AcknowledgedResponse.TRUE);
+                } else {
+                    client().execute(
+                        TransportDeleteComponentTemplateAction.TYPE,
+                        new TransportDeleteComponentTemplateAction.Request(componentTemplates),
+                        l
+                    );
+                }
+            })
+            .andThenAccept(ElasticsearchAssertions::assertAcked)
+
+            // and finish
+            .addListener(listener);
     }
 
     /**
@@ -136,52 +223,88 @@ public void assertAfterTest() throws Exception {
      * all indices are removed.
      */
     public void wipeIndices(String... indices) {
+        safeAwait((ActionListener<Void> l) -> wipeIndicesAsync(indices, l));
+    }
+
+    private void wipeIndicesAsync(String[] indices, ActionListener<Void> listener) {
         assert indices != null && indices.length > 0;
-        if (size() > 0) {
-            try {
-                // include wiping hidden indices!
-                assertAcked(
-                    client().admin()
-                        .indices()
-                        .prepareDelete(indices)
-                        .setIndicesOptions(IndicesOptions.fromOptions(false, true, true, true, true, false, false, true, false))
-                );
-            } catch (IndexNotFoundException e) {
-                // ignore
-            } catch (IllegalArgumentException e) {
-                // Happens if `action.destructive_requires_name` is set to true
-                // which is the case in the CloseIndexDisableCloseAllTests
-                if ("_all".equals(indices[0])) {
-                    ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
-                    ArrayList<String> concreteIndices = new ArrayList<>();
-                    for (IndexMetadata indexMetadata : clusterStateResponse.getState().metadata()) {
-                        concreteIndices.add(indexMetadata.getIndex().getName());
-                    }
-                    if (concreteIndices.isEmpty() == false) {
-                        assertAcked(client().admin().indices().prepareDelete(concreteIndices.toArray(new String[0])));
-                    }
-                }
+        SubscribableListener
+
+            .<AcknowledgedResponse>newForked(
+                l -> client().admin()
+                    .indices()
+                    .prepareDelete(indices)
+                    .setIndicesOptions(
+                        // include wiping hidden indices!
+                        IndicesOptions.fromOptions(false, true, true, true, true, false, false, true, false)
+                    )
+                    .execute(l.delegateResponse((ll, exception) -> handleWipeIndicesFailure(exception, "_all".equals(indices[0]), ll)))
+            )
+            .andThenAccept(ElasticsearchAssertions::assertAcked)
+            .addListener(listener);
+    }
+
+    private void handleWipeIndicesFailure(Exception exception, boolean wipingAllIndices, ActionListener<AcknowledgedResponse> listener) {
+        Throwable unwrapped = ExceptionsHelper.unwrap(exception, IndexNotFoundException.class, IllegalArgumentException.class);
+        if (unwrapped instanceof IndexNotFoundException) {
+            // ignore
+            listener.onResponse(AcknowledgedResponse.TRUE);
+        } else if (unwrapped instanceof IllegalArgumentException) {
+            // Happens if `action.destructive_requires_name` is set to true
+            // which is the case in the CloseIndexDisableCloseAllTests
+            if (wipingAllIndices) {
+                SubscribableListener
+
+                    .<ClusterStateResponse>newForked(l -> client().admin().cluster().prepareState().execute(l))
+                    .<AcknowledgedResponse>andThen((l, clusterStateResponse) -> {
+                        ArrayList<String> concreteIndices = new ArrayList<>();
+                        for (IndexMetadata indexMetadata : clusterStateResponse.getState().metadata()) {
+                            concreteIndices.add(indexMetadata.getIndex().getName());
+                        }
+                        if (concreteIndices.isEmpty() == false) {
+                            client().admin().indices().prepareDelete(concreteIndices.toArray(Strings.EMPTY_ARRAY)).execute(l);
+                        } else {
+                            l.onResponse(AcknowledgedResponse.TRUE);
+                        }
+                    })
+                    .addListener(listener);
+            } else {
+                // TODO: this is clearly wrong but at least
+                // org.elasticsearch.xpack.watcher.test.integration.BootStrapTests.testTriggeredWatchLoading depends on this
+                // quietly passing when it tries to delete an alias instead of its backing indices
+                listener.onResponse(AcknowledgedResponse.TRUE);
             }
+        } else {
+            listener.onFailure(exception);
         }
     }
 
     /**
      * Removes all templates, except the templates defined in the exclude
      */
-    public void wipeAllTemplates(Set<String> exclude) {
-        if (size() > 0) {
-            GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
-            for (IndexTemplateMetadata indexTemplate : response.getIndexTemplates()) {
-                if (exclude.contains(indexTemplate.getName())) {
-                    continue;
-                }
-                try {
-                    client().admin().indices().prepareDeleteTemplate(indexTemplate.getName()).get();
-                } catch (IndexTemplateMissingException e) {
-                    // ignore
+    private void wipeAllTemplates(Set<String> exclude, RefCountingListener listeners) {
+        SubscribableListener
+
+            .<GetIndexTemplatesResponse>newForked(l -> client().admin().indices().prepareGetTemplates().execute(l))
+            .andThenAccept(response -> {
+                for (IndexTemplateMetadata indexTemplate : response.getIndexTemplates()) {
+                    if (exclude.contains(indexTemplate.getName())) {
+                        continue;
+                    }
+                    client().admin()
+                        .indices()
+                        .prepareDeleteTemplate(indexTemplate.getName())
+                        .execute(listeners.<AcknowledgedResponse>acquire(ElasticsearchAssertions::assertAcked).delegateResponse((l, e) -> {
+                            if (e instanceof IndexTemplateMissingException) {
+                                // ignore
+                                l.onResponse(AcknowledgedResponse.TRUE);
+                            } else {
+                                l.onFailure(e);
+                            }
+                        }));
                 }
-            }
-        }
+            })
+            .addListener(listeners.acquire());
     }
 
     /**
@@ -207,91 +330,24 @@ public void wipeTemplates(String... templates) {
     /**
      * Deletes repositories, supports wildcard notation.
      */
-    public void wipeRepositories(String... repositories) {
-        if (size() > 0) {
-            // if nothing is provided, delete all
-            if (repositories.length == 0) {
-                repositories = new String[] { "*" };
-            }
-            final var future = new PlainActionFuture<Void>();
-            try (var listeners = new RefCountingListener(future)) {
-                for (String repository : repositories) {
-                    ActionListener.run(
-                        listeners.acquire(),
-                        l -> client().admin()
-                            .cluster()
-                            .prepareDeleteRepository(ESTestCase.TEST_REQUEST_TIMEOUT, ESTestCase.TEST_REQUEST_TIMEOUT, repository)
-                            .execute(new ActionListener<>() {
-                                @Override
-                                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
-                                    l.onResponse(null);
-                                }
-
-                                @Override
-                                public void onFailure(Exception e) {
-                                    if (e instanceof RepositoryMissingException) {
-                                        // ignore
-                                        l.onResponse(null);
-                                    } else {
-                                        l.onFailure(e);
-                                    }
-                                }
-                            })
-                    );
-                }
-            }
-            future.actionGet(30, TimeUnit.SECONDS);
-        }
-    }
-
-    public void wipeAllDataStreams() {
-        if (size() > 0) {
-            var request = new DeleteDataStreamAction.Request("*");
-            request.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN);
-            try {
-                assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, request).actionGet());
-            } catch (IllegalStateException e) {
-                // Ignore if action isn't registered, because data streams is a module and
-                // if the delete action isn't registered then there no data streams to delete.
-                if (e.getMessage().startsWith("failed to find action") == false) {
-                    throw e;
-                }
-            }
-        }
-    }
-
-    public void wipeAllComposableIndexTemplates(Set<String> excludeTemplates) {
-        if (size() > 0) {
-            var templates = client().execute(GetComposableIndexTemplateAction.INSTANCE, new GetComposableIndexTemplateAction.Request("*"))
-                .actionGet()
-                .indexTemplates()
-                .keySet()
-                .stream()
-                .filter(template -> excludeTemplates.contains(template) == false)
-                .toArray(String[]::new);
-
-            if (templates.length != 0) {
-                var request = new TransportDeleteComposableIndexTemplateAction.Request(templates);
-                assertAcked(client().execute(TransportDeleteComposableIndexTemplateAction.TYPE, request).actionGet());
-            }
-        }
-    }
-
-    public void wipeAllComponentTemplates(Set<String> excludeTemplates) {
-        if (size() > 0) {
-            var templates = client().execute(GetComponentTemplateAction.INSTANCE, new GetComponentTemplateAction.Request("*"))
-                .actionGet()
-                .getComponentTemplates()
-                .keySet()
-                .stream()
-                .filter(template -> excludeTemplates.contains(template) == false)
-                .toArray(String[]::new);
-
-            if (templates.length != 0) {
-                var request = new TransportDeleteComponentTemplateAction.Request(templates);
-                assertAcked(client().execute(TransportDeleteComponentTemplateAction.TYPE, request).actionGet());
-            }
-        }
+    private void wipeRepositories(ActionListener<Void> listener) {
+        SubscribableListener
+
+            .<AcknowledgedResponse>newForked(
+                l -> client().admin()
+                    .cluster()
+                    .prepareDeleteRepository(ESTestCase.TEST_REQUEST_TIMEOUT, ESTestCase.TEST_REQUEST_TIMEOUT, "*")
+                    .execute(l.delegateResponse((ll, e) -> {
+                        if (e instanceof RepositoryMissingException) {
+                            // ignore
+                            l.onResponse(AcknowledgedResponse.TRUE);
+                        } else {
+                            l.onFailure(e);
+                        }
+                    }))
+            )
+            .andThenAccept(ElasticsearchAssertions::assertAcked)
+            .addListener(listener);
     }
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestTrustStore.java b/test/framework/src/main/java/org/elasticsearch/test/TestTrustStore.java
new file mode 100644
index 0000000000000..b3c2da5dcec10
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestTrustStore.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.CheckedSupplier;
+import org.elasticsearch.common.ssl.KeyStoreUtil;
+import org.junit.rules.ExternalResource;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateFactory;
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.lucene.tests.util.LuceneTestCase.createTempDir;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestTrustStore extends ExternalResource {
+
+    private final CheckedSupplier<InputStream, IOException> pemStreamSupplier;
+
+    public TestTrustStore(CheckedSupplier<InputStream, IOException> pemStreamSupplier) {
+        this.pemStreamSupplier = pemStreamSupplier;
+    }
+
+    private Path trustStorePath;
+
+    public Path getTrustStorePath() {
+        // TODO when https://github.com/elastic/elasticsearch/issues/111532 addressed we should be able to use this in FIPS mode too
+        assertFalse("Tests in FIPS mode cannot supply a custom trust store", ESTestCase.inFipsJvm());
+        return Objects.requireNonNullElseGet(trustStorePath, () -> ESTestCase.fail(null, "trust store not created"));
+    }
+
+    @Override
+    protected void before() {
+        final var tmpDir = createTempDir();
+        final var tmpTrustStorePath = tmpDir.resolve("trust-store.jks");
+        try (var pemStream = pemStreamSupplier.get(); var jksStream = Files.newOutputStream(tmpTrustStorePath)) {
+            final List<Certificate> certificates = CertificateFactory.getInstance("X.509")
+                .generateCertificates(pemStream)
+                .stream()
+                .map(i -> (Certificate) i)
+                .toList();
+            final var trustStore = KeyStoreUtil.buildTrustStore(certificates, "jks");
+            trustStore.store(jksStream, new char[0]);
+            trustStorePath = tmpTrustStorePath;
+        } catch (Exception e) {
+            throw new AssertionError("unexpected", e);
+        }
+    }
+
+    @Override
+    protected void after() {
+        assertTrue(trustStorePath + " should still exist at teardown", Files.exists(trustStorePath));
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
index a591fc86979c3..0cb3b754fb633 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
@@ -178,11 +178,13 @@ private ReproduceErrorMessageBuilder appendESProperties() {
             if (System.getProperty("tests.jvm.argline") != null && System.getProperty("tests.jvm.argline").isEmpty() == false) {
                 appendOpt("tests.jvm.argline", "\"" + System.getProperty("tests.jvm.argline") + "\"");
             }
+            if (Boolean.parseBoolean(System.getProperty("build.snapshot", "true")) == false) {
+                appendOpt("license.key", "x-pack/license-tools/src/test/resources/public.key");
+            }
             appendOpt("tests.locale", Locale.getDefault().toLanguageTag());
             appendOpt("tests.timezone", TimeZone.getDefault().getID());
             appendOpt("tests.distribution", System.getProperty("tests.distribution"));
             appendOpt("runtime.java", Integer.toString(Runtime.version().feature()));
-            appendOpt("license.key", System.getProperty("licence.key"));
             appendOpt(ESTestCase.FIPS_SYSPROP, System.getProperty(ESTestCase.FIPS_SYSPROP));
             return this;
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 9e88d66c430e7..8cafd73858c74 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -44,7 +44,6 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.client.WarningsHandler;
-import org.elasticsearch.cluster.ClusterFeatures;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -307,11 +306,11 @@ protected static Optional<Boolean> clusterHasCapability(
     }
 
     protected static boolean clusterHasFeature(String featureId) {
-        return testFeatureService.clusterHasFeature(featureId);
+        return testFeatureService.clusterHasFeature(featureId, false);
     }
 
     protected static boolean clusterHasFeature(NodeFeature feature) {
-        return testFeatureService.clusterHasFeature(feature.id());
+        return testFeatureService.clusterHasFeature(feature.id(), false);
     }
 
     protected static boolean testFeatureServiceInitialized() {
@@ -418,11 +417,7 @@ protected final TestFeatureService createTestFeatureService(
                 RestTestLegacyFeatures.class.getCanonicalName()
             );
         }
-        return new ESRestTestFeatureService(
-            additionalTestOnlyHistoricalFeatures(),
-            semanticNodeVersions,
-            ClusterFeatures.calculateAllNodeFeatures(clusterStateFeatures.values())
-        );
+        return new ESRestTestFeatureService(additionalTestOnlyHistoricalFeatures(), semanticNodeVersions, clusterStateFeatures.values());
     }
 
     protected static boolean has(ProductFeature feature) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java
index 92d72afbf9d52..aa04855c03626 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.test.rest;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.SuppressForbidden;
@@ -28,13 +27,13 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.BiConsumer;
+import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -48,11 +47,12 @@ class ESRestTestFeatureService implements TestFeatureService {
      */
     private static final Pattern VERSION_FEATURE_PATTERN = Pattern.compile("gte_v(\\d+\\.\\d+\\.\\d+)");
 
-    private final Set<String> allSupportedFeatures;
     private final Set<String> knownHistoricalFeatureNames;
-    private final Version version;
+    private final Collection<Version> nodeVersions;
+    private final Collection<Set<String>> nodeFeatures;
+    private final Collection<Set<String>> nodeHistoricalFeatures;
 
-    ESRestTestFeatureService(List<FeatureSpecification> featureSpecs, Collection<Version> nodeVersions, Set<String> clusterStateFeatures) {
+    ESRestTestFeatureService(List<FeatureSpecification> featureSpecs, Set<Version> nodeVersions, Collection<Set<String>> nodeFeatures) {
         List<FeatureSpecification> specs = new ArrayList<>(featureSpecs);
         specs.add(new RestTestLegacyFeatures());
         if (MetadataHolder.HISTORICAL_FEATURES != null) {
@@ -65,17 +65,26 @@ class ESRestTestFeatureService implements TestFeatureService {
                 featureData.getNodeFeatures().keySet()
             );
         this.knownHistoricalFeatureNames = featureData.getHistoricalFeatures().lastEntry().getValue();
-        this.version = nodeVersions.stream().min(Comparator.naturalOrder()).orElse(Version.CURRENT);
-        this.allSupportedFeatures = Sets.union(clusterStateFeatures, featureData.getHistoricalFeatures().floorEntry(version).getValue());
+        this.nodeVersions = nodeVersions;
+        this.nodeFeatures = nodeFeatures;
+        this.nodeHistoricalFeatures = nodeVersions.stream()
+            .map(featureData.getHistoricalFeatures()::floorEntry)
+            .map(Map.Entry::getValue)
+            .toList();
     }
 
     public static boolean hasFeatureMetadata() {
         return MetadataHolder.HISTORICAL_FEATURES != null;
     }
 
+    private static <T> boolean checkCollection(Collection<T> coll, Predicate<T> pred, boolean any) {
+        return any ? coll.stream().anyMatch(pred) : coll.isEmpty() == false && coll.stream().allMatch(pred);
+    }
+
     @Override
-    public boolean clusterHasFeature(String featureId) {
-        if (allSupportedFeatures.contains(featureId)) {
+    public boolean clusterHasFeature(String featureId, boolean any) {
+        if (checkCollection(nodeFeatures, s -> s.contains(featureId), any)
+            || checkCollection(nodeHistoricalFeatures, s -> s.contains(featureId), any)) {
             return true;
         }
         if (MetadataHolder.FEATURE_NAMES.contains(featureId) || knownHistoricalFeatureNames.contains(featureId)) {
@@ -86,7 +95,7 @@ public boolean clusterHasFeature(String featureId) {
         Matcher matcher = VERSION_FEATURE_PATTERN.matcher(featureId);
         if (matcher.matches()) {
             Version extractedVersion = Version.fromString(matcher.group(1));
-            return version.onOrAfter(extractedVersion);
+            return checkCollection(nodeVersions, v -> v.onOrAfter(extractedVersion), any);
         }
 
         if (hasFeatureMetadata()) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/TestFeatureService.java b/test/framework/src/main/java/org/elasticsearch/test/rest/TestFeatureService.java
index 8b3ba874e1678..e7bb23a1edb83 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/TestFeatureService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/TestFeatureService.java
@@ -9,7 +9,20 @@
 package org.elasticsearch.test.rest;
 
 public interface TestFeatureService {
-    TestFeatureService ALL_FEATURES = feature -> true;
+    TestFeatureService ALL_FEATURES = (feature, any) -> true;
 
-    boolean clusterHasFeature(String featureId);
+    /**
+     * Returns {@code true} if all nodes in the cluster have feature {@code featureId}
+     */
+    default boolean clusterHasFeature(String featureId) {
+        return clusterHasFeature(featureId, false);
+    }
+
+    /**
+     * @param featureId The feature to check
+     * @param any
+     *         {@code true} if it should check if any node has the feature,
+     *         {@code false} if it should check if all nodes have the feature
+     */
+    boolean clusterHasFeature(String featureId, boolean any);
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java b/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java
index e2fa31c31a46f..8d467799f2007 100644
--- a/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java
+++ b/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java
@@ -22,7 +22,12 @@ public TestThreadPool(String name, ExecutorBuilder<?>... customBuilders) {
     }
 
     public TestThreadPool(String name, Settings settings, ExecutorBuilder<?>... customBuilders) {
-        super(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), name).put(settings).build(), MeterRegistry.NOOP, customBuilders);
+        super(
+            Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), name).put(settings).build(),
+            MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders(),
+            customBuilders
+        );
     }
 
     @Override
diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java
new file mode 100644
index 0000000000000..81d6028182d21
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorSnapshotTests.java
@@ -0,0 +1,263 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class DataGeneratorSnapshotTests extends ESTestCase {
+    public void testSnapshot() throws Exception {
+        var dataGenerator = new DataGenerator(
+            DataGeneratorSpecification.builder()
+                .withDataSourceHandlers(List.of(new DataSourceOverrides()))
+                .withMaxFieldCountPerLevel(5)
+                .withMaxObjectDepth(2)
+                .build()
+        );
+
+        var mapping = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint();
+        dataGenerator.writeMapping(mapping);
+
+        var document = XContentBuilder.builder(XContentType.JSON.xContent()).prettyPrint();
+        dataGenerator.generateDocument(document);
+
+        var expectedMapping = """
+            {
+              "_doc" : {
+                "dynamic" : "false",
+                "properties" : {
+                  "f1" : {
+                    "dynamic" : "false",
+                    "properties" : {
+                      "f2" : {
+                        "dynamic" : "false",
+                        "properties" : {
+                          "f3" : {
+                            "type" : "keyword",
+                            "store" : "true"
+                          },
+                          "f4" : {
+                            "type" : "long",
+                            "index" : "false"
+                          }
+                        }
+                      },
+                      "f5" : {
+                        "dynamic" : "false",
+                        "properties" : {
+                          "f6" : {
+                            "type" : "keyword",
+                            "store" : "true"
+                          },
+                          "f7" : {
+                            "type" : "long",
+                            "index" : "false"
+                          }
+                        }
+                      }
+                    }
+                  },
+                  "f8" : {
+                    "type" : "nested",
+                    "dynamic" : "false",
+                    "properties" : {
+                      "f9" : {
+                        "type" : "nested",
+                        "dynamic" : "false",
+                        "properties" : {
+                          "f10" : {
+                            "type" : "keyword",
+                            "store" : "true"
+                          },
+                          "f11" : {
+                            "type" : "long",
+                            "index" : "false"
+                          }
+                        }
+                      },
+                      "f12" : {
+                        "type" : "keyword",
+                        "store" : "true"
+                      }
+                    }
+                  }
+                }
+              }
+            }""";
+
+        var expectedDocument = """
+            {
+              "f1" : [
+                {
+                  "f2" : {
+                    "f3" : [
+                      null,
+                      "string1"
+                    ],
+                    "f4" : 0
+                  },
+                  "f5" : {
+                    "f6" : "string2",
+                    "f7" : null
+                  }
+                },
+                {
+                  "f2" : {
+                    "f3" : [
+                      "string3",
+                      "string4"
+                    ],
+                    "f4" : 1
+                  },
+                  "f5" : {
+                    "f6" : null,
+                    "f7" : 2
+                  }
+                }
+              ],
+              "f8" : {
+                "f9" : {
+                  "f10" : [
+                    "string5",
+                    "string6"
+                  ],
+                  "f11" : null
+                },
+                "f12" : "string7"
+              }
+            }""";
+
+        assertEquals(expectedMapping, Strings.toString(mapping));
+        assertEquals(expectedDocument, Strings.toString(document));
+    }
+
+    private static class DataSourceOverrides implements DataSourceHandler {
+        private long longValue = 0;
+        private long generatedStrings = 0;
+        private int generateNullChecks = 0;
+        private int generateArrayChecks = 0;
+        private boolean producedObjectArray = false;
+        private FieldType fieldType = FieldType.KEYWORD;
+        private final StaticChildFieldGenerator childFieldGenerator = new StaticChildFieldGenerator();
+
+        @Override
+        public DataSourceResponse.LongGenerator handle(DataSourceRequest.LongGenerator request) {
+            return new DataSourceResponse.LongGenerator(() -> longValue++);
+        }
+
+        @Override
+        public DataSourceResponse.StringGenerator handle(DataSourceRequest.StringGenerator request) {
+            return new DataSourceResponse.StringGenerator(() -> "string" + (generatedStrings++ + 1));
+        }
+
+        @Override
+        public DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) {
+            return new DataSourceResponse.NullWrapper((values) -> () -> generateNullChecks++ % 4 == 0 ? null : values.get());
+        }
+
+        @Override
+        public DataSourceResponse.ArrayWrapper handle(DataSourceRequest.ArrayWrapper request) {
+
+            return new DataSourceResponse.ArrayWrapper((values) -> () -> {
+                if (generateArrayChecks++ % 4 == 0) {
+                    // we have nulls so can't use List.of
+                    return new Object[] { values.get(), values.get() };
+                }
+
+                return values.get();
+            });
+        }
+
+        @Override
+        public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFieldGenerator request) {
+            return childFieldGenerator;
+        }
+
+        @Override
+        public DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectArrayGenerator request) {
+            return new DataSourceResponse.ObjectArrayGenerator(() -> {
+                if (producedObjectArray == false) {
+                    producedObjectArray = true;
+                    return Optional.of(2);
+                }
+
+                return Optional.empty();
+            });
+        }
+
+        @Override
+        public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) {
+            return new DataSourceResponse.FieldTypeGenerator(() -> {
+                if (fieldType == FieldType.KEYWORD) {
+                    fieldType = FieldType.LONG;
+                    return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.KEYWORD, false);
+                }
+
+                fieldType = FieldType.KEYWORD;
+                return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG, false);
+            });
+        }
+
+        @Override
+        public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
+            if (request.fieldType() == FieldType.KEYWORD) {
+                return new DataSourceResponse.LeafMappingParametersGenerator(() -> Map.of("store", "true"));
+            }
+
+            if (request.fieldType() == FieldType.LONG) {
+                return new DataSourceResponse.LeafMappingParametersGenerator(() -> Map.of("index", "false"));
+            }
+
+            return null;
+        }
+
+        @Override
+        public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) {
+            return new DataSourceResponse.ObjectMappingParametersGenerator(() -> Map.of("dynamic", "false"));
+        }
+    }
+
+    private static class StaticChildFieldGenerator implements DataSourceResponse.ChildFieldGenerator {
+        private int generatedFields = 0;
+
+        @Override
+        public int generateChildFieldCount() {
+            return 2;
+        }
+
+        @Override
+        public boolean generateDynamicSubObject() {
+            return false;
+        }
+
+        @Override
+        public boolean generateNestedSubObject() {
+            return generatedFields > 6 && generatedFields < 12;
+        }
+
+        @Override
+        public boolean generateRegularSubObject() {
+            return generatedFields < 6;
+        }
+
+        @Override
+        public String generateFieldName() {
+            return "f" + (generatedFields++ + 1);
+        }
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java
new file mode 100644
index 0000000000000..e8535263d387c
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGeneratorTests.java
@@ -0,0 +1,204 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.index.mapper.MapperServiceTestCase;
+import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
+import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+
+public class DataGeneratorTests extends ESTestCase {
+    public void testDataGeneratorSanity() throws IOException {
+        var dataGenerator = new DataGenerator(DataGeneratorSpecification.buildDefault());
+
+        var mapping = XContentBuilder.builder(XContentType.JSON.xContent());
+        dataGenerator.writeMapping(mapping);
+
+        for (int i = 0; i < 1000; i++) {
+            var document = XContentBuilder.builder(XContentType.JSON.xContent());
+            dataGenerator.generateDocument(document);
+        }
+    }
+
+    public void testDataGeneratorProducesValidMappingAndDocument() throws IOException {
+        // Make sure objects, nested objects and all field types are covered.
+        var testChildFieldGenerator = new DataSourceResponse.ChildFieldGenerator() {
+            private boolean dynamicSubObjectCovered = false;
+            private boolean subObjectCovered = false;
+            private boolean nestedCovered = false;
+            private int generatedFields = 0;
+
+            @Override
+            public int generateChildFieldCount() {
+                // Make sure to generate enough fields to go through all field types.
+                return 20;
+            }
+
+            @Override
+            public boolean generateDynamicSubObject() {
+                if (dynamicSubObjectCovered == false) {
+                    dynamicSubObjectCovered = true;
+                    return true;
+                }
+
+                return false;
+            }
+
+            @Override
+            public boolean generateNestedSubObject() {
+                if (nestedCovered == false) {
+                    nestedCovered = true;
+                    return true;
+                }
+
+                return false;
+            }
+
+            @Override
+            public boolean generateRegularSubObject() {
+                if (subObjectCovered == false) {
+                    subObjectCovered = true;
+                    return true;
+                }
+
+                return false;
+            }
+
+            @Override
+            public String generateFieldName() {
+                return "f" + generatedFields++;
+            }
+        };
+
+        var dataSourceOverride = new DataSourceHandler() {
+            private int generatedFields = 0;
+
+            @Override
+            public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFieldGenerator request) {
+                return testChildFieldGenerator;
+            }
+
+            @Override
+            public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) {
+                if (request.dynamicMapping() == DynamicMapping.FORBIDDEN || request.dynamicMapping() == DynamicMapping.SUPPORTED) {
+                    return new DataSourceResponse.FieldTypeGenerator(
+                        () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(
+                            FieldType.values()[generatedFields++ % FieldType.values().length],
+                            false
+                        )
+                    );
+                }
+
+                return new DataSourceResponse.FieldTypeGenerator(() -> {
+                    var fieldType = FieldType.values()[generatedFields++ % FieldType.values().length];
+                    // Does not really work with dynamic mapping.
+                    if (fieldType == FieldType.UNSIGNED_LONG) {
+                        fieldType = FieldType.values()[generatedFields++ % FieldType.values().length];
+                    }
+
+                    return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(fieldType, true);
+                });
+            }
+        };
+
+        var dataGenerator = new DataGenerator(
+            DataGeneratorSpecification.builder().withDataSourceHandlers(List.of(dataSourceOverride)).build()
+        );
+
+        var mapping = XContentBuilder.builder(XContentType.JSON.xContent());
+        dataGenerator.writeMapping(mapping);
+
+        var mappingService = new MapperServiceTestCase() {
+            @Override
+            protected Collection<? extends Plugin> getPlugins() {
+                return List.of(new UnsignedLongMapperPlugin(), new MapperExtrasPlugin());
+            }
+        }.createMapperService(mapping);
+
+        var document = XContentBuilder.builder(XContentType.JSON.xContent());
+        dataGenerator.generateDocument(document);
+
+        mappingService.documentMapper().parse(new SourceToParse("1", BytesReference.bytes(document), XContentType.JSON));
+    }
+
+    public void testDataGeneratorStressTest() throws IOException {
+        // Let's generate 125000 fields to test an extreme case (2 levels of objects + 1 leaf level with 50 fields per object).
+        var testChildFieldGenerator = new DataSourceResponse.ChildFieldGenerator() {
+            private int generatedFields = 0;
+
+            @Override
+            public int generateChildFieldCount() {
+                return 50;
+            }
+
+            @Override
+            public boolean generateDynamicSubObject() {
+                return false;
+            }
+
+            @Override
+            public boolean generateNestedSubObject() {
+                return false;
+            }
+
+            @Override
+            public boolean generateRegularSubObject() {
+                return true;
+            }
+
+            @Override
+            public String generateFieldName() {
+                return "f" + generatedFields++;
+            }
+        };
+
+        var dataSourceOverride = new DataSourceHandler() {
+            @Override
+            public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFieldGenerator request) {
+                return testChildFieldGenerator;
+            }
+
+            @Override
+            public DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectArrayGenerator request) {
+                return new DataSourceResponse.ObjectArrayGenerator(Optional::empty);
+            }
+
+            @Override
+            public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) {
+                return new DataSourceResponse.FieldTypeGenerator(
+                    () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG, false)
+                );
+            }
+        };
+
+        var dataGenerator = new DataGenerator(
+            DataGeneratorSpecification.builder().withDataSourceHandlers(List.of(dataSourceOverride)).withMaxObjectDepth(2).build()
+        );
+
+        var mapping = XContentBuilder.builder(XContentType.JSON.xContent());
+        dataGenerator.writeMapping(mapping);
+
+        var document = XContentBuilder.builder(XContentType.JSON.xContent());
+        dataGenerator.generateDocument(document);
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DefaultWrappersHandlerTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DefaultWrappersHandlerTests.java
new file mode 100644
index 0000000000000..1ac6d117f0931
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DefaultWrappersHandlerTests.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logsdb.datageneration;
+
+import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.logsdb.datageneration.datasource.DefaultWrappersHandler;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+import java.util.function.Supplier;
+
+public class DefaultWrappersHandlerTests extends ESTestCase {
+    public void testSanity() {
+        var sut = new DefaultWrappersHandler();
+
+        Supplier<Object> values = () -> 100;
+        var nulls = sut.handle(new DataSourceRequest.NullWrapper());
+        var arrays = sut.handle(new DataSourceRequest.ArrayWrapper());
+
+        var valuesWithNullsAndWrappedInArray = arrays.wrapper().compose(nulls.wrapper()).apply(values);
+
+        var value = valuesWithNullsAndWrappedInArray.get();
+
+        if (value instanceof List<?> list) {
+            assertTrue(list.size() <= 5);
+            for (var item : list) {
+                if (item instanceof Integer intValue) {
+                    assertEquals(Integer.valueOf(100), intValue);
+                } else {
+                    assertNull(item);
+                }
+            }
+        } else if (value instanceof Integer intValue) {
+            assertEquals(Integer.valueOf(100), intValue);
+        } else {
+            assertNull(value);
+        }
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/MapMatcherTests.java b/test/framework/src/test/java/org/elasticsearch/test/MapMatcherTests.java
index 48c9fcab3898a..3822c0d93d28d 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/MapMatcherTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/MapMatcherTests.java
@@ -11,8 +11,10 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.json.JsonXContent;
+import org.hamcrest.Description;
 import org.hamcrest.Matcher;
 import org.hamcrest.StringDescription;
+import org.hamcrest.TypeSafeMatcher;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -24,7 +26,9 @@
 import static org.elasticsearch.test.ListMatcher.matchesList;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
+import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
@@ -395,6 +399,45 @@ public void testSubMapDescribeTo() {
               baz: <0>"""));
     }
 
+    public void testSubMatcherDescribeFails() {
+        assertMismatch(Map.of("foo", 2.0, "bar", 2), matchesMap().entry("foo", new TypeSafeMatcher<Object>() {
+            @Override
+            public void describeTo(Description description) {
+                throw new IllegalStateException("intentional failure");
+            }
+
+            @Override
+            protected boolean matchesSafely(Object o) {
+                return false;
+            }
+        }).entry("bar", 2), both(containsString("""
+            a map containing
+            foo: expected error describing <java.lang.IllegalStateException: intentional failure""")).and(containsString("""
+            bar: <2>""")));
+    }
+
+    public void testSubMatcherMismatchFails() {
+        assertMismatch(Map.of("foo", 2.0, "bar", 2), matchesMap().entry("foo", new TypeSafeMatcher<Object>() {
+            @Override
+            protected void describeMismatchSafely(Object item, Description mismatchDescription) {
+                throw new IllegalStateException("intentional failure");
+            }
+
+            @Override
+            public void describeTo(Description description) {
+                description.appendValue("foo");
+            }
+
+            @Override
+            protected boolean matchesSafely(Object o) {
+                return false;
+            }
+        }).entry("bar", 2), both(containsString("""
+            a map containing
+            foo: expected "foo" but error describing <java.lang.IllegalStateException: intentional failure""")).and(containsString("""
+            bar: <2>""")));
+    }
+
     static <T> void assertMismatch(T v, Matcher<? super T> matcher, Matcher<String> mismatchDescriptionMatcher) {
         assertMap(v, not(matcher));
         StringDescription description = new StringDescription();
diff --git a/test/immutable-collections-patch/build.gradle b/test/immutable-collections-patch/build.gradle
index 2d42215b3e02c..c3354e189847d 100644
--- a/test/immutable-collections-patch/build.gradle
+++ b/test/immutable-collections-patch/build.gradle
@@ -35,7 +35,7 @@ generatePatch.configure {
     executable = "${BuildParams.runtimeJavaHome}/bin/java" + (OS.current() == OS.WINDOWS ? '.exe' : '')
   } else {
     javaLauncher = javaToolchains.launcherFor {
-      languageVersion = JavaLanguageVersion.of(BuildParams.runtimeJavaVersion.majorVersion)
+      languageVersion = JavaLanguageVersion.of(VersionProperties.bundledJdkMajorVersion)
       vendor = VersionProperties.bundledJdkVendor == "openjdk" ?
         JvmVendorSpec.ORACLE :
         JvmVendorSpec.matching(VersionProperties.bundledJdkVendor)
diff --git a/test/logger-usage/build.gradle b/test/logger-usage/build.gradle
index 4404fb6fd11b0..31b02eed31e11 100644
--- a/test/logger-usage/build.gradle
+++ b/test/logger-usage/build.gradle
@@ -9,9 +9,9 @@
 apply plugin: 'elasticsearch.java'
 
 dependencies {
-  api 'org.ow2.asm:asm:9.2'
-  api 'org.ow2.asm:asm-tree:9.2'
-  api 'org.ow2.asm:asm-analysis:9.2'
+  api 'org.ow2.asm:asm:9.7'
+  api 'org.ow2.asm:asm-tree:9.7'
+  api 'org.ow2.asm:asm-analysis:9.7'
   api "org.apache.logging.log4j:log4j-api:${versions.log4j}"
   testImplementation project(":test:framework")
 }
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java
index 0101c76b21f76..b8251d2006eeb 100644
--- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java
@@ -720,6 +720,11 @@ private Map<String, String> getEnvironmentVariables() {
             // Windows requires this as it defaults to `c:\windows` despite ES_TMPDIR
             environment.put("TMP", workingDir.resolve("tmp").toString());
 
+            environment = environment.entrySet()
+                .stream()
+                .map(p -> Map.entry(p.getKey(), p.getValue().replace("${ES_PATH_CONF}", configDir.toString())))
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
             String featureFlagProperties = "";
             if (spec.getFeatures().isEmpty() == false && distributionDescriptor.isSnapshot() == false) {
                 featureFlagProperties = spec.getFeatures()
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java
index 5292d917df630..a2d7fe11698e8 100644
--- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java
@@ -39,7 +39,7 @@ public class DefaultLocalClusterHandle implements LocalClusterHandle {
     public static final AtomicInteger NEXT_DEBUG_PORT = new AtomicInteger(5007);
 
     private static final Logger LOGGER = LogManager.getLogger(DefaultLocalClusterHandle.class);
-    private static final Duration CLUSTER_UP_TIMEOUT = Duration.ofSeconds(30);
+    private static final Duration CLUSTER_UP_TIMEOUT = Duration.ofMinutes(5);
 
     public final ForkJoinPool executor = new ForkJoinPool(
         Math.max(Runtime.getRuntime().availableProcessors(), 4),
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/WaitForHttpResource.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/WaitForHttpResource.java
index 0c839aeddd189..e897dca20b0a5 100644
--- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/WaitForHttpResource.java
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/WaitForHttpResource.java
@@ -56,7 +56,20 @@ public class WaitForHttpResource {
     private String password;
 
     public WaitForHttpResource(String protocol, String host, int numberOfNodes) throws MalformedURLException {
-        this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow"));
+        this(
+            new URL(
+                protocol
+                    + "://"
+                    + host
+                    + "/_cluster/health"
+                    + "?wait_for_nodes=>="
+                    + numberOfNodes
+                    + "&wait_for_status=yellow"
+                    + "&wait_for_events=LANGUID"
+                    + "&wait_for_no_initializing_shards"
+                    + "&wait_for_no_relocating_shards"
+            )
+        );
     }
 
     public WaitForHttpResource(URL url) {
diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
index 4954065369ad9..1f4a4dcf9dbfa 100644
--- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
+++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
@@ -15,10 +15,12 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.client.Node;
 import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction;
 import org.elasticsearch.test.rest.Stash;
 import org.elasticsearch.test.rest.TestFeatureService;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
@@ -198,7 +200,7 @@ private HttpEntity createEntity(List<Map<String, Object>> bodies, Map<String, St
                 for (int i = bytesRef.offset; i < bytesRef.length; i++) {
                     bytes[position++] = bytesRef.bytes[i];
                 }
-                bytes[position++] = xContentType.xContent().streamSeparator();
+                bytes[position++] = xContentType.xContent().bulkSeparator();
             }
             return new ByteArrayEntity(bytes, ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8));
         }
@@ -276,12 +278,18 @@ public ClientYamlTestCandidate getClientYamlTestCandidate() {
         return clientYamlTestCandidate;
     }
 
-    public boolean clusterHasFeature(String featureId) {
-        return testFeatureService.clusterHasFeature(featureId);
+    public boolean clusterHasFeature(String featureId, boolean any) {
+        return testFeatureService.clusterHasFeature(featureId, any);
     }
 
-    public Optional<Boolean> clusterHasCapabilities(String method, String path, String parametersString, String capabilitiesString) {
-        Map<String, String> params = Maps.newMapWithExpectedSize(5);
+    public Optional<Boolean> clusterHasCapabilities(
+        String method,
+        String path,
+        String parametersString,
+        String capabilitiesString,
+        boolean any
+    ) {
+        Map<String, String> params = Maps.newMapWithExpectedSize(6);
         params.put("method", method);
         params.put("path", path);
         if (Strings.hasLength(parametersString)) {
@@ -291,8 +299,46 @@ public Optional<Boolean> clusterHasCapabilities(String method, String path, Stri
             params.put("capabilities", capabilitiesString);
         }
         params.put("error_trace", "false"); // disable error trace
+
+        if (clusterHasFeature(RestNodesCapabilitiesAction.LOCAL_ONLY_CAPABILITIES.id(), false) == false) {
+            // can only check the whole cluster
+            if (any) {
+                logger.warn(
+                    "Cluster does not support checking individual nodes for capabilities,"
+                        + "check for [{} {}?{} {}] may be incorrect in mixed-version clusters",
+                    method,
+                    path,
+                    parametersString,
+                    capabilitiesString
+                );
+            }
+            return checkCapability(NodeSelector.ANY, params);
+        } else {
+            // check each node individually - we can actually check any here
+            params.put("local_only", "true");   // we're calling each node individually
+
+            // individually call each node, so we can control whether we do an 'any' or 'all' check
+            List<Node> nodes = clientYamlTestClient.getRestClient(NodeSelector.ANY).getNodes();
+
+            for (Node n : nodes) {
+                Optional<Boolean> nodeResult = checkCapability(new SpecificNodeSelector(n), params);
+                if (nodeResult.isEmpty()) {
+                    return Optional.empty();
+                } else if (any == nodeResult.get()) {
+                    // either any == true and node has cap,
+                    // or any == false (ie all) and this node does not have cap
+                    return nodeResult;
+                }
+            }
+
+            // if we got here, either any is true and no node has it, or any == false and all nodes have it
+            return Optional.of(any == false);
+        }
+    }
+
+    private Optional<Boolean> checkCapability(NodeSelector nodeSelector, Map<String, String> params) {
         try {
-            ClientYamlTestResponse resp = callApi("capabilities", params, emptyList(), emptyMap());
+            ClientYamlTestResponse resp = callApi("capabilities", params, emptyList(), emptyMap(), nodeSelector);
             // anything other than 200 should result in an exception, handled below
             assert resp.getStatusCode() == 200 : "Unknown response code " + resp.getStatusCode();
             return Optional.ofNullable(resp.evaluate("supported"));
@@ -305,4 +351,17 @@ public Optional<Boolean> clusterHasCapabilities(String method, String path, Stri
             throw new UncheckedIOException(ioException);
         }
     }
+
+    private record SpecificNodeSelector(Node node) implements NodeSelector {
+        @Override
+        public void select(Iterable<Node> nodes) {
+            // between getting the list of nodes, and checking here, the thing that is consistent is the host
+            // which becomes one of the bound addresses
+            for (var it = nodes.iterator(); it.hasNext();) {
+                if (it.next().getBoundHosts().contains(node.getHost()) == false) {
+                    it.remove();
+                }
+            }
+        }
+    }
 }
diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
index 5c9bcdebd6da8..ba0546662bb5d 100644
--- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
+++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
@@ -374,9 +374,10 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx
             // This is really difficult to express just with features, so I will break it down into 2 parts: version check for v7,
             // and feature check for v8. This way the version check can be removed once we move to v9
             @UpdateForV9
-            var fixedInV7 = executionContext.clusterHasFeature("gte_v7.17.2") && executionContext.clusterHasFeature("gte_v8.0.0") == false;
+            var fixedInV7 = executionContext.clusterHasFeature("gte_v7.17.2", false)
+                && executionContext.clusterHasFeature("gte_v8.0.0", false) == false;
             var fixedProductionHeader = fixedInV7
-                || executionContext.clusterHasFeature(RestTestLegacyFeatures.REST_ELASTIC_PRODUCT_HEADER_PRESENT.id());
+                || executionContext.clusterHasFeature(RestTestLegacyFeatures.REST_ELASTIC_PRODUCT_HEADER_PRESENT.id(), false);
             if (fixedProductionHeader) {
                 checkElasticProductHeader(response.getHeaders("X-elastic-product"));
             }
diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/Prerequisites.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/Prerequisites.java
index 96b5aff5d7047..b5e1e85795e36 100644
--- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/Prerequisites.java
+++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/Prerequisites.java
@@ -41,32 +41,32 @@ static Predicate<ClientYamlTestExecutionContext> hasXPack() {
     }
 
     static Predicate<ClientYamlTestExecutionContext> requireClusterFeatures(Set<String> clusterFeatures) {
-        return context -> clusterFeatures.stream().allMatch(context::clusterHasFeature);
+        return context -> clusterFeatures.stream().allMatch(f -> context.clusterHasFeature(f, false));
     }
 
     static Predicate<ClientYamlTestExecutionContext> skipOnClusterFeatures(Set<String> clusterFeatures) {
-        return context -> clusterFeatures.stream().anyMatch(context::clusterHasFeature);
+        return context -> clusterFeatures.stream().anyMatch(f -> context.clusterHasFeature(f, true));
     }
 
     static Predicate<ClientYamlTestExecutionContext> skipOnKnownIssue(List<KnownIssue> knownIssues) {
         return context -> knownIssues.stream()
-            .anyMatch(i -> context.clusterHasFeature(i.clusterFeature()) && context.clusterHasFeature(i.fixedBy()) == false);
+            .anyMatch(i -> context.clusterHasFeature(i.clusterFeature(), true) && context.clusterHasFeature(i.fixedBy(), false) == false);
     }
 
     static CapabilitiesPredicate requireCapabilities(List<CapabilitiesCheck> checks) {
         // requirement not fulfilled if unknown / capabilities API not supported
-        return context -> checks.stream().allMatch(check -> checkCapabilities(context, check).orElse(false));
+        return context -> checks.stream().allMatch(check -> checkCapabilities(context, check, false).orElse(false));
     }
 
     static CapabilitiesPredicate skipCapabilities(List<CapabilitiesCheck> checks) {
         // skip if unknown / capabilities API not supported
-        return context -> checks.stream().anyMatch(check -> checkCapabilities(context, check).orElse(true));
+        return context -> checks.stream().anyMatch(check -> checkCapabilities(context, check, true).orElse(true));
     }
 
     interface CapabilitiesPredicate extends Predicate<ClientYamlTestExecutionContext> {}
 
-    private static Optional<Boolean> checkCapabilities(ClientYamlTestExecutionContext context, CapabilitiesCheck check) {
-        Optional<Boolean> b = context.clusterHasCapabilities(check.method(), check.path(), check.parameters(), check.capabilities());
+    private static Optional<Boolean> checkCapabilities(ClientYamlTestExecutionContext context, CapabilitiesCheck check, boolean any) {
+        Optional<Boolean> b = context.clusterHasCapabilities(check.method(), check.path(), check.parameters(), check.capabilities(), any);
         return b;
     }
 }
diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSectionTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSectionTests.java
index cc27b7fc20b76..1025d2f5ff39f 100644
--- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSectionTests.java
+++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSectionTests.java
@@ -36,6 +36,7 @@
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.oneOf;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -457,8 +458,8 @@ public void testSkipClusterFeaturesAllRequiredMatch() {
         );
 
         var mockContext = mock(ClientYamlTestExecutionContext.class);
-        when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true);
-        when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(true);
+        when(mockContext.clusterHasFeature("required-feature-1", false)).thenReturn(true);
+        when(mockContext.clusterHasFeature("required-feature-2", false)).thenReturn(true);
 
         assertFalse(section.skipCriteriaMet(mockContext));
         assertTrue(section.requiresCriteriaMet(mockContext));
@@ -474,8 +475,8 @@ public void testSkipClusterFeaturesSomeRequiredMatch() {
         );
 
         var mockContext = mock(ClientYamlTestExecutionContext.class);
-        when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true);
-        when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(false);
+        when(mockContext.clusterHasFeature("required-feature-1", false)).thenReturn(true);
+        when(mockContext.clusterHasFeature("required-feature-2", false)).thenReturn(false);
 
         assertFalse(section.skipCriteriaMet(mockContext));
         assertFalse(section.requiresCriteriaMet(mockContext));
@@ -491,7 +492,7 @@ public void testSkipClusterFeaturesSomeToSkipMatch() {
         );
 
         var mockContext = mock(ClientYamlTestExecutionContext.class);
-        when(mockContext.clusterHasFeature("undesired-feature-1")).thenReturn(true);
+        when(mockContext.clusterHasFeature("undesired-feature-1", true)).thenReturn(true);
 
         assertTrue(section.skipCriteriaMet(mockContext));
     }
@@ -521,9 +522,9 @@ public void testSkipClusterFeaturesAllRequiredSomeToSkipMatch() {
         assertFalse(section.hasCapabilitiesCheck());
 
         var mockContext = mock(ClientYamlTestExecutionContext.class);
-        when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true);
-        when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(true);
-        when(mockContext.clusterHasFeature("undesired-feature-1")).thenReturn(true);
+        when(mockContext.clusterHasFeature("required-feature-1", false)).thenReturn(true);
+        when(mockContext.clusterHasFeature("required-feature-2", false)).thenReturn(true);
+        when(mockContext.clusterHasFeature("undesired-feature-1", true)).thenReturn(true);
 
         assertTrue(section.skipCriteriaMet(mockContext));
         assertTrue(section.requiresCriteriaMet(mockContext));
@@ -540,8 +541,8 @@ public void testSkipClusterFeaturesAllRequiredNoneToSkipMatch() {
         assertFalse(section.hasCapabilitiesCheck());
 
         var mockContext = mock(ClientYamlTestExecutionContext.class);
-        when(mockContext.clusterHasFeature("required-feature-1")).thenReturn(true);
-        when(mockContext.clusterHasFeature("required-feature-2")).thenReturn(true);
+        when(mockContext.clusterHasFeature("required-feature-1", false)).thenReturn(true);
+        when(mockContext.clusterHasFeature("required-feature-2", false)).thenReturn(true);
 
         assertFalse(section.skipCriteriaMet(mockContext));
         assertTrue(section.requiresCriteriaMet(mockContext));
@@ -560,16 +561,16 @@ public void testSkipKnownIssue() {
         var mockContext = mock(ClientYamlTestExecutionContext.class);
         assertFalse(section.skipCriteriaMet(mockContext));
 
-        when(mockContext.clusterHasFeature("bug1")).thenReturn(true);
+        when(mockContext.clusterHasFeature("bug1", true)).thenReturn(true);
         assertTrue(section.skipCriteriaMet(mockContext));
 
-        when(mockContext.clusterHasFeature("fix1")).thenReturn(true);
+        when(mockContext.clusterHasFeature("fix1", false)).thenReturn(true);
         assertFalse(section.skipCriteriaMet(mockContext));
 
-        when(mockContext.clusterHasFeature("bug2")).thenReturn(true);
+        when(mockContext.clusterHasFeature("bug2", true)).thenReturn(true);
         assertTrue(section.skipCriteriaMet(mockContext));
 
-        when(mockContext.clusterHasFeature("fix2")).thenReturn(true);
+        when(mockContext.clusterHasFeature("fix2", false)).thenReturn(true);
         assertFalse(section.skipCriteriaMet(mockContext));
     }
 
@@ -596,17 +597,17 @@ public void testEvaluateCapabilities() {
         assertTrue(section.skipCriteriaMet(context)); // always skip if unavailable
         assertFalse(section.requiresCriteriaMet(context)); // always fail requirements / skip if unavailable
 
-        when(context.clusterHasCapabilities(anyString(), anyString(), any(), any())).thenReturn(Optional.of(FALSE));
+        when(context.clusterHasCapabilities(anyString(), anyString(), any(), any(), anyBoolean())).thenReturn(Optional.of(FALSE));
         assertFalse(section.skipCriteriaMet(context));
         assertFalse(section.requiresCriteriaMet(context));
 
-        when(context.clusterHasCapabilities("GET", "/s", null, "c1,c2")).thenReturn(Optional.of(TRUE));
+        when(context.clusterHasCapabilities("GET", "/s", null, "c1,c2", true)).thenReturn(Optional.of(TRUE));
         assertTrue(section.skipCriteriaMet(context));
 
-        when(context.clusterHasCapabilities("GET", "/r", null, null)).thenReturn(Optional.of(TRUE));
+        when(context.clusterHasCapabilities("GET", "/r", null, null, false)).thenReturn(Optional.of(TRUE));
         assertFalse(section.requiresCriteriaMet(context));
 
-        when(context.clusterHasCapabilities("GET", "/r", "p1", null)).thenReturn(Optional.of(TRUE));
+        when(context.clusterHasCapabilities("GET", "/r", "p1", null, false)).thenReturn(Optional.of(TRUE));
         assertTrue(section.requiresCriteriaMet(context));
     }
 
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportAction.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportAction.java
index 53748cc387d7e..4cf6c820d89dc 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportAction.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.analytics.action;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java
index f9c6b3f79f31c..7d22ea0b02983 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsAction.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsAction.java
index beeceec726244..d20ef5abe2388 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsAction.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
index c1f35b9a58d55..e80279934a090 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
@@ -528,7 +528,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
         );
     }
 
-    private class HistogramSyntheticFieldLoader implements CompositeSyntheticFieldLoader.SyntheticFieldLoaderLayer {
+    private class HistogramSyntheticFieldLoader implements CompositeSyntheticFieldLoader.Layer {
         private final InternalHistogramValue value = new InternalHistogramValue();
         private BytesRef binaryValue;
 
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistogramPercentileAggregationTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistogramPercentileAggregationTests.java
index f60466bcf43cc..7c6f85104b5f8 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistogramPercentileAggregationTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistogramPercentileAggregationTests.java
@@ -241,7 +241,6 @@ public void testTDigestHistogram() throws Exception {
         );
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/110406")
     public void testBoxplotHistogram() throws Exception {
         int compression = TestUtil.nextInt(random(), 200, 300);
         setupTDigestHistogram(compression);
diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java
index 04b0257f4180a..6f5d4e13dc56b 100644
--- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java
+++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java
@@ -7,53 +7,24 @@
 
 package org.elasticsearch.xpack.apmdata;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.metadata.ComponentTemplate;
-import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.Nullable;
 import org.elasticsearch.features.FeatureService;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xcontent.XContentParserConfiguration;
-import org.elasticsearch.xcontent.yaml.YamlXContent;
 import org.elasticsearch.xpack.core.ClientHelper;
-import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
-import org.elasticsearch.xpack.core.template.IngestPipelineConfig;
+import org.elasticsearch.xpack.core.template.YamlTemplateRegistry;
 
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.xpack.apmdata.ResourceUtils.APM_TEMPLATE_VERSION_VARIABLE;
-import static org.elasticsearch.xpack.apmdata.ResourceUtils.loadResource;
-import static org.elasticsearch.xpack.apmdata.ResourceUtils.loadVersionedResourceUTF8;
+import static org.elasticsearch.xpack.apmdata.APMPlugin.APM_DATA_REGISTRY_ENABLED;
 
 /**
  * Creates all index templates and ingest pipelines that are required for using Elastic APM.
  */
-public class APMIndexTemplateRegistry extends IndexTemplateRegistry {
-    private static final Logger logger = LogManager.getLogger(APMIndexTemplateRegistry.class);
-    // this node feature is a redefinition of {@link DataStreamFeatures#DATA_STREAM_LIFECYCLE} and it's meant to avoid adding a
-    // dependency to the data-streams module just for this
-    public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle");
-    private final int version;
+public class APMIndexTemplateRegistry extends YamlTemplateRegistry {
 
-    private final Map<String, ComponentTemplate> componentTemplates;
-    private final Map<String, ComposableIndexTemplate> composableIndexTemplates;
-    private final List<IngestPipelineConfig> ingestPipelines;
-    private final FeatureService featureService;
-    private volatile boolean enabled;
+    public static final String APM_TEMPLATE_VERSION_VARIABLE = "xpack.apmdata.template.version";
 
-    @SuppressWarnings("unchecked")
     public APMIndexTemplateRegistry(
         Settings nodeSettings,
         ClusterService clusterService,
@@ -62,133 +33,29 @@ public APMIndexTemplateRegistry(
         NamedXContentRegistry xContentRegistry,
         FeatureService featureService
     ) {
-        super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
-
-        try {
-            final Map<String, Object> apmResources = XContentHelper.convertToMap(
-                YamlXContent.yamlXContent,
-                loadResource("/resources.yaml"),
-                false
-            );
-            version = (((Number) apmResources.get("version")).intValue());
-            final List<Object> componentTemplateNames = (List<Object>) apmResources.get("component-templates");
-            final List<Object> indexTemplateNames = (List<Object>) apmResources.get("index-templates");
-            final List<Object> ingestPipelineConfigs = (List<Object>) apmResources.get("ingest-pipelines");
-
-            componentTemplates = componentTemplateNames.stream()
-                .map(o -> (String) o)
-                .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version)));
-            composableIndexTemplates = indexTemplateNames.stream()
-                .map(o -> (String) o)
-                .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version)));
-            ingestPipelines = ingestPipelineConfigs.stream().map(o -> (Map<String, Map<String, Object>>) o).map(map -> {
-                Map.Entry<String, Map<String, Object>> pipelineConfig = map.entrySet().iterator().next();
-                return loadIngestPipeline(pipelineConfig.getKey(), version, (List<String>) pipelineConfig.getValue().get("dependencies"));
-            }).collect(Collectors.toList());
-            this.featureService = featureService;
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public int getVersion() {
-        return version;
-    }
-
-    void setEnabled(boolean enabled) {
-        logger.info("APM index template registry is {}", enabled ? "enabled" : "disabled");
-        this.enabled = enabled;
-    }
-
-    public boolean isEnabled() {
-        return enabled;
-    }
-
-    public void close() {
-        clusterService.removeListener(this);
-    }
-
-    @Override
-    protected String getOrigin() {
-        return ClientHelper.APM_ORIGIN;
-    }
-
-    @Override
-    protected boolean isClusterReady(ClusterChangedEvent event) {
-        // Ensure current version of the components are installed only after versions that support data stream lifecycle
-        // due to the use of the feature in all the `@lifecycle` component templates
-        return featureService.clusterHasFeature(event.state(), DATA_STREAM_LIFECYCLE);
+        super(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService);
     }
 
     @Override
-    protected boolean requiresMasterNode() {
-        return true;
+    public String getName() {
+        return "apm";
     }
 
     @Override
-    protected Map<String, ComponentTemplate> getComponentTemplateConfigs() {
-        if (enabled) {
-            return componentTemplates;
-        } else {
-            return Map.of();
+    public void initialize() {
+        super.initialize();
+        if (isEnabled()) {
+            clusterService.getClusterSettings().addSettingsUpdateConsumer(APM_DATA_REGISTRY_ENABLED, this::setEnabled);
         }
     }
 
     @Override
-    protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
-        if (enabled) {
-            return composableIndexTemplates;
-        } else {
-            return Map.of();
-        }
-    }
-
-    @Override
-    protected List<IngestPipelineConfig> getIngestPipelines() {
-        if (enabled) {
-            return ingestPipelines;
-        } else {
-            return Collections.emptyList();
-        }
-    }
-
-    private static ComponentTemplate loadComponentTemplate(String name, int version) {
-        try {
-            final byte[] content = loadVersionedResourceUTF8("/component-templates/" + name + ".yaml", version);
-            try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) {
-                return ComponentTemplate.parse(parser);
-            }
-        } catch (Exception e) {
-            throw new RuntimeException("failed to load APM Ingest plugin's component template: " + name, e);
-        }
-    }
-
-    private static ComposableIndexTemplate loadIndexTemplate(String name, int version) {
-        try {
-            final byte[] content = loadVersionedResourceUTF8("/index-templates/" + name + ".yaml", version);
-            try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) {
-                return ComposableIndexTemplate.parse(parser);
-            }
-        } catch (Exception e) {
-            throw new RuntimeException("failed to load APM Ingest plugin's index template: " + name, e);
-        }
-    }
-
-    private static IngestPipelineConfig loadIngestPipeline(String name, int version, @Nullable List<String> dependencies) {
-        if (dependencies == null) {
-            dependencies = Collections.emptyList();
-        }
-        return new YamlIngestPipelineConfig(
-            name,
-            "/ingest-pipelines/" + name + ".yaml",
-            version,
-            APM_TEMPLATE_VERSION_VARIABLE,
-            dependencies
-        );
+    protected String getVersionProperty() {
+        return APM_TEMPLATE_VERSION_VARIABLE;
     }
 
     @Override
-    protected boolean applyRolloverAfterTemplateV2Upgrade() {
-        return true;
+    protected String getOrigin() {
+        return ClientHelper.APM_ORIGIN;
     }
 }
diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java
index 102b0d38461c3..aefb45f6186c1 100644
--- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java
+++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java
@@ -60,7 +60,6 @@ public Collection<?> createComponents(PluginServices services) {
         if (enabled) {
             APMIndexTemplateRegistry registryInstance = registry.get();
             registryInstance.setEnabled(APM_DATA_REGISTRY_ENABLED.get(settings));
-            clusterService.getClusterSettings().addSettingsUpdateConsumer(APM_DATA_REGISTRY_ENABLED, registryInstance::setEnabled);
             registryInstance.initialize();
         }
         return Collections.emptyList();
diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/ResourceUtils.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/ResourceUtils.java
deleted file mode 100644
index 1e6a9a9998a82..0000000000000
--- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/ResourceUtils.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.apmdata;
-
-import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.xpack.core.template.TemplateUtils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.Map;
-
-public class ResourceUtils {
-
-    public static final String APM_TEMPLATE_VERSION_VARIABLE = "xpack.apmdata.template.version";
-
-    static byte[] loadVersionedResourceUTF8(String name, int version) {
-        return loadVersionedResourceUTF8(name, version, Map.of());
-    }
-
-    static byte[] loadVersionedResourceUTF8(String name, int version, Map<String, String> variables) {
-        try {
-            String content = loadResource(name);
-            content = TemplateUtils.replaceVariables(content, String.valueOf(version), APM_TEMPLATE_VERSION_VARIABLE, variables);
-            return content.getBytes(StandardCharsets.UTF_8);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    static String loadResource(String name) throws IOException {
-        InputStream is = APMIndexTemplateRegistry.class.getResourceAsStream(name);
-        if (is == null) {
-            throw new IOException("Resource [" + name + "] not found in classpath.");
-        }
-        return Streams.readFully(is).utf8ToString();
-    }
-}
diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/YamlIngestPipelineConfig.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/YamlIngestPipelineConfig.java
deleted file mode 100644
index de1b715dd138d..0000000000000
--- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/YamlIngestPipelineConfig.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.apmdata;
-
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.xcontent.XContentType;
-import org.elasticsearch.xpack.core.template.IngestPipelineConfig;
-
-import java.util.List;
-
-import static org.elasticsearch.xpack.apmdata.ResourceUtils.loadVersionedResourceUTF8;
-
-/**
- * An APM-plugin-specific implementation that loads ingest pipelines in yaml format from a local resources repository
- */
-public class YamlIngestPipelineConfig extends IngestPipelineConfig {
-    public YamlIngestPipelineConfig(String id, String resource, int version, String versionProperty, List<String> dependencies) {
-        super(id, resource, version, versionProperty, dependencies);
-    }
-
-    @Override
-    public XContentType getXContentType() {
-        return XContentType.YAML;
-    }
-
-    @Override
-    public BytesReference loadConfig() {
-        return new BytesArray(loadVersionedResourceUTF8("/ingest-pipelines/" + id + ".yaml", version, variables));
-    }
-}
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml
index 1e2a6a679dc30..c1d004b4e7bf4 100644
--- a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml
+++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml
@@ -28,9 +28,14 @@ template:
             return;
           }
           def exception = params['_source'].error?.exception;
-          def exceptionMessage = exception != null && exception.length > 0 ? exception[0]?.message : null;
-          if (exceptionMessage != null && exceptionMessage != "") {
-            emit(exception[0].message);
+          if (exception != null && exception.isEmpty() == false) {
+            def exceptionMessage = exception instanceof Map ? exception?.message : exception[0]?.message;
+            if (exceptionMessage instanceof List) {
+              exceptionMessage = exceptionMessage[0]
+            }
+            if (exceptionMessage != null && exceptionMessage != "") {
+              emit(exceptionMessage);
+            }
           }
 
       # http.*
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces@mappings.yaml
deleted file mode 100644
index 51c987df4df60..0000000000000
--- a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces@mappings.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
----
-version: ${xpack.apmdata.template.version}
-_meta:
-  description: Default mappings for traces data streams
-  managed: true
-template:
-  mappings:
-    properties:
-      data_stream.type:
-        type: constant_keyword
-        value: traces
diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml
index efa6ae694c464..3e66769d939ad 100644
--- a/x-pack/plugin/apm-data/src/main/resources/resources.yaml
+++ b/x-pack/plugin/apm-data/src/main/resources/resources.yaml
@@ -1,7 +1,7 @@
 # "version" holds the version of the templates and ingest pipelines installed
 # by xpack-plugin apm-data. This must be increased whenever an existing template or
 # pipeline is changed, in order for it to be updated on Elasticsearch upgrade.
-version: 6
+version: 8
 
 component-templates:
   # Data lifecycle.
@@ -23,7 +23,6 @@ component-templates:
   - metrics-apm.service_summary@mappings
   - metrics-apm.service_transaction@mappings
   - metrics-apm.transaction@mappings
-  - traces@mappings
   - traces-apm@mappings
   - traces-apm.rum@mappings
 
diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java
index e9f0775836c71..1d6faa0f403d4 100644
--- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java
+++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java
@@ -75,7 +75,6 @@
 public class APMIndexTemplateRegistryTests extends ESTestCase {
     private APMIndexTemplateRegistry apmIndexTemplateRegistry;
     private StackTemplateRegistryAccessor stackTemplateRegistryAccessor;
-    private ClusterService clusterService;
     private ThreadPool threadPool;
     private VerifyingClient client;
 
@@ -89,7 +88,7 @@ public void createRegistryAndClient() {
 
         threadPool = new TestThreadPool(this.getClass().getName());
         client = new VerifyingClient(threadPool);
-        clusterService = ClusterServiceUtils.createClusterService(threadPool, clusterSettings);
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool, clusterSettings);
         FeatureService featureService = new FeatureService(List.of(new DataStreamFeatures()));
         stackTemplateRegistryAccessor = new StackTemplateRegistryAccessor(
             new StackTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, NamedXContentRegistry.EMPTY, featureService)
diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_error_grouping.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_error_grouping.yml
index f7cd386227fe8..37a1651da562b 100644
--- a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_error_grouping.yml
+++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_error_grouping.yml
@@ -39,6 +39,10 @@ setup:
           - create: {}
           - '{"@timestamp": "2017-06-22", "error": {"log": {"message": ""}, "exception": [{"message": "exception_used"}]}}'
 
+          # Non-empty error.exception.message used from array
+          - create: {}
+          - '{"@timestamp": "2017-06-22", "error": {"log": {"message": ""}, "exception": [{"message": "first_exception_used"}, {"message": "2_ignored"}]}}'
+
   - is_false: errors
 
   - do:
@@ -46,7 +50,7 @@ setup:
         index: logs-apm.error-testing
         body:
           fields: ["error.grouping_name"]
-  - length: { hits.hits: 7 }
+  - length: { hits.hits: 8 }
   - match: { hits.hits.0.fields: null }
   - match: { hits.hits.1.fields: null }
   - match: { hits.hits.2.fields: null }
@@ -54,3 +58,4 @@ setup:
   - match: { hits.hits.4.fields: null }
   - match: { hits.hits.5.fields: {"error.grouping_name": ["log_used"]} }
   - match: { hits.hits.6.fields: {"error.grouping_name": ["exception_used"]} }
+  - match: { hits.hits.7.fields: {"error.grouping_name": ["first_exception_used"]} }
diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CCSUsageTelemetryAsyncSearchIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CCSUsageTelemetryAsyncSearchIT.java
new file mode 100644
index 0000000000000..4f8fa122047e1
--- /dev/null
+++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CCSUsageTelemetryAsyncSearchIT.java
@@ -0,0 +1,378 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.search;
+
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot;
+import org.elasticsearch.action.search.TransportSearchAction;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.FeatureFlag;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.test.AbstractMultiClustersTestCase;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.usage.UsageService;
+import org.elasticsearch.xpack.async.AsyncResultsIndexPlugin;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.async.GetAsyncResultRequest;
+import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse;
+import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction;
+import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE;
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.MRT_FEATURE;
+import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.Result.CANCELED;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class CCSUsageTelemetryAsyncSearchIT extends AbstractMultiClustersTestCase {
+    private static final String REMOTE1 = "cluster-a";
+    private static final String REMOTE2 = "cluster-b";
+    private static final FeatureFlag CCS_TELEMETRY_FEATURE_FLAG = new FeatureFlag("ccs_telemetry");
+
+    @BeforeClass
+    protected static void skipIfTelemetryDisabled() {
+        assumeTrue("Skipping test as CCS_TELEMETRY_FEATURE_FLAG is disabled", CCS_TELEMETRY_FEATURE_FLAG.isEnabled());
+    }
+
+    @Override
+    protected boolean reuseClusters() {
+        return false;
+    }
+
+    @Override
+    protected Collection<String> remoteClusterAlias() {
+        return List.of(REMOTE1, REMOTE2);
+    }
+
+    @Override
+    protected Map<String, Boolean> skipUnavailableForRemoteClusters() {
+        return Map.of(REMOTE1, true, REMOTE2, true);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
+        List<Class<? extends Plugin>> plugs = Arrays.asList(
+            CrossClusterAsyncSearchIT.SearchListenerPlugin.class,
+            AsyncSearch.class,
+            AsyncResultsIndexPlugin.class,
+            LocalStateCompositeXPackPlugin.class,
+            CrossClusterAsyncSearchIT.TestQueryBuilderPlugin.class
+        );
+        return Stream.concat(super.nodePlugins(clusterAlias).stream(), plugs.stream()).collect(Collectors.toList());
+    }
+
+    @Before
+    public void resetSearchListenerPlugin() {
+        CrossClusterAsyncSearchIT.SearchListenerPlugin.reset();
+    }
+
+    private SubmitAsyncSearchRequest makeSearchRequest(String... indices) {
+        CrossClusterAsyncSearchIT.SearchListenerPlugin.blockQueryPhase();
+
+        SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(indices);
+        request.setCcsMinimizeRoundtrips(randomBoolean());
+        request.setWaitForCompletionTimeout(TimeValue.timeValueMillis(1));
+        request.setKeepOnCompletion(true);
+        request.getSearchRequest().allowPartialSearchResults(false);
+        request.getSearchRequest().source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(10));
+        if (randomBoolean()) {
+            request.setBatchedReduceSize(randomIntBetween(2, 256));
+        }
+
+        return request;
+    }
+
+    /**
+    * Run async search request and get telemetry from it
+    */
+    private CCSTelemetrySnapshot getTelemetryFromSearch(SubmitAsyncSearchRequest searchRequest) throws Exception {
+        // We want to send search to a specific node (we don't care which one) so that we could
+        // collect the CCS telemetry from it later
+        String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName();
+        final AsyncSearchResponse response = cluster(LOCAL_CLUSTER).client(nodeName)
+            .execute(SubmitAsyncSearchAction.INSTANCE, searchRequest)
+            .get();
+        // We don't care here too much about the response, we just want to trigger the telemetry collection.
+        // So we check it's not null and leave the rest to other tests.
+        final String responseId;
+        try {
+            assertNotNull(response.getSearchResponse());
+            responseId = response.getId();
+        } finally {
+            response.decRef();
+        }
+        waitForSearchTasksToFinish();
+        final AsyncSearchResponse finishedResponse = cluster(LOCAL_CLUSTER).client(nodeName)
+            .execute(GetAsyncSearchAction.INSTANCE, new GetAsyncResultRequest(responseId))
+            .get();
+        try {
+            assertNotNull(finishedResponse.getSearchResponse());
+        } finally {
+            finishedResponse.decRef();
+        }
+        return getTelemetrySnapshot(nodeName);
+
+    }
+
+    private void waitForSearchTasksToFinish() throws Exception {
+        assertBusy(() -> {
+            ListTasksResponse listTasksResponse = client(LOCAL_CLUSTER).admin()
+                .cluster()
+                .prepareListTasks()
+                .setActions(TransportSearchAction.TYPE.name())
+                .get();
+            List<TaskInfo> tasks = listTasksResponse.getTasks();
+            assertThat(tasks.size(), equalTo(0));
+
+            for (String clusterAlias : remoteClusterAlias()) {
+                ListTasksResponse remoteTasksResponse = client(clusterAlias).admin()
+                    .cluster()
+                    .prepareListTasks()
+                    .setActions(TransportSearchAction.TYPE.name())
+                    .get();
+                List<TaskInfo> remoteTasks = remoteTasksResponse.getTasks();
+                assertThat(remoteTasks.size(), equalTo(0));
+            }
+        });
+
+        assertBusy(() -> {
+            for (String clusterAlias : remoteClusterAlias()) {
+                final Iterable<TransportService> transportServices = cluster(clusterAlias).getInstances(TransportService.class);
+                for (TransportService transportService : transportServices) {
+                    assertThat(transportService.getTaskManager().getBannedTaskIds(), Matchers.empty());
+                }
+            }
+        });
+    }
+
+    /**
+     * Create search request for indices and get telemetry from it
+     */
+    private CCSTelemetrySnapshot getTelemetryFromSearch(String... indices) throws Exception {
+        return getTelemetryFromSearch(makeSearchRequest(indices));
+    }
+
+    /**
+     * Async search on all remotes
+     */
+    public void testAllRemotesSearch() throws Exception {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        SubmitAsyncSearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex);
+        boolean minimizeRoundtrips = TransportSearchAction.shouldMinimizeRoundtrips(searchRequest.getSearchRequest());
+        CrossClusterAsyncSearchIT.SearchListenerPlugin.negate();
+
+        CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(searchRequest);
+
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(1L));
+        assertThat(telemetry.getFailureReasons().size(), equalTo(0));
+        assertThat(telemetry.getTook().count(), equalTo(1L));
+        assertThat(telemetry.getTookMrtTrue().count(), equalTo(minimizeRoundtrips ? 1L : 0L));
+        assertThat(telemetry.getTookMrtFalse().count(), equalTo(minimizeRoundtrips ? 0L : 1L));
+        assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
+        assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L));
+        assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(1L));
+        if (minimizeRoundtrips) {
+            assertThat(telemetry.getFeatureCounts().get(MRT_FEATURE), equalTo(1L));
+        } else {
+            assertThat(telemetry.getFeatureCounts().get(MRT_FEATURE), equalTo(null));
+        }
+        var perCluster = telemetry.getByRemoteCluster();
+        assertThat(perCluster.size(), equalTo(3));
+        for (String clusterAlias : remoteClusterAlias()) {
+            var clusterTelemetry = perCluster.get(clusterAlias);
+            assertThat(clusterTelemetry.getCount(), equalTo(1L));
+            assertThat(clusterTelemetry.getSkippedCount(), equalTo(0L));
+            assertThat(clusterTelemetry.getTook().count(), equalTo(1L));
+        }
+    }
+
+    /**
+     * Search that is cancelled
+     */
+    public void testCancelledSearch() throws Exception {
+        Map<String, Object> testClusterInfo = setupClusters();
+        String localIndex = (String) testClusterInfo.get("local.index");
+        String remoteIndex = (String) testClusterInfo.get("remote.index");
+
+        SubmitAsyncSearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex);
+        CrossClusterAsyncSearchIT.SearchListenerPlugin.blockQueryPhase();
+
+        String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName();
+        final AsyncSearchResponse response = cluster(LOCAL_CLUSTER).client(nodeName)
+            .execute(SubmitAsyncSearchAction.INSTANCE, searchRequest)
+            .get();
+        try {
+            assertNotNull(response.getSearchResponse());
+        } finally {
+            response.decRef();
+            assertTrue(response.isRunning());
+        }
+        CrossClusterAsyncSearchIT.SearchListenerPlugin.waitSearchStarted();
+
+        ActionFuture<ListTasksResponse> cancelFuture;
+        try {
+            ListTasksResponse listTasksResponse = client(LOCAL_CLUSTER).admin()
+                .cluster()
+                .prepareListTasks()
+                .setActions(TransportSearchAction.TYPE.name())
+                .get();
+            List<TaskInfo> tasks = listTasksResponse.getTasks();
+            assertThat(tasks.size(), equalTo(1));
+            final TaskInfo rootTask = tasks.get(0);
+
+            AtomicReference<List<TaskInfo>> remoteClusterSearchTasks = new AtomicReference<>();
+            assertBusy(() -> {
+                List<TaskInfo> remoteSearchTasks = client(REMOTE1).admin()
+                    .cluster()
+                    .prepareListTasks()
+                    .get()
+                    .getTasks()
+                    .stream()
+                    .filter(t -> t.action().contains(TransportSearchAction.TYPE.name()))
+                    .collect(Collectors.toList());
+                assertThat(remoteSearchTasks.size(), greaterThan(0));
+                remoteClusterSearchTasks.set(remoteSearchTasks);
+            });
+
+            for (TaskInfo taskInfo : remoteClusterSearchTasks.get()) {
+                assertFalse("taskInfo on remote cluster should not be cancelled yet: " + taskInfo, taskInfo.cancelled());
+            }
+
+            final CancelTasksRequest cancelRequest = new CancelTasksRequest().setTargetTaskId(rootTask.taskId());
+            cancelRequest.setWaitForCompletion(randomBoolean());
+            cancelFuture = client().admin().cluster().cancelTasks(cancelRequest);
+            assertBusy(() -> {
+                final Iterable<TransportService> transportServices = cluster(REMOTE1).getInstances(TransportService.class);
+                for (TransportService transportService : transportServices) {
+                    Collection<CancellableTask> cancellableTasks = transportService.getTaskManager().getCancellableTasks().values();
+                    for (CancellableTask cancellableTask : cancellableTasks) {
+                        if (cancellableTask.getAction().contains(TransportSearchAction.TYPE.name())) {
+                            assertTrue(cancellableTask.getDescription(), cancellableTask.isCancelled());
+                        }
+                    }
+                }
+            });
+
+            List<TaskInfo> remoteSearchTasksAfterCancellation = client(REMOTE1).admin()
+                .cluster()
+                .prepareListTasks()
+                .get()
+                .getTasks()
+                .stream()
+                .filter(t -> t.action().contains(TransportSearchAction.TYPE.name()))
+                .toList();
+            for (TaskInfo taskInfo : remoteSearchTasksAfterCancellation) {
+                assertTrue(taskInfo.description(), taskInfo.cancelled());
+            }
+        } finally {
+            CrossClusterAsyncSearchIT.SearchListenerPlugin.allowQueryPhase();
+        }
+
+        assertBusy(() -> assertTrue(cancelFuture.isDone()));
+        waitForSearchTasksToFinish();
+
+        CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName);
+        assertThat(telemetry.getTotalCount(), equalTo(1L));
+        assertThat(telemetry.getSuccessCount(), equalTo(0L));
+        assertThat(telemetry.getFailureReasons().size(), equalTo(1));
+        assertThat(telemetry.getFailureReasons().get(CANCELED.getName()), equalTo(1L));
+        assertThat(telemetry.getTook().count(), equalTo(0L));
+        assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(1.0));
+        assertThat(telemetry.getRemotesPerSearchMax(), equalTo(1L));
+        // Still counts as async search
+        assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(1L));
+    }
+
+    private CCSTelemetrySnapshot getTelemetrySnapshot(String nodeName) {
+        var usage = cluster(LOCAL_CLUSTER).getInstance(UsageService.class, nodeName);
+        return usage.getCcsUsageHolder().getCCSTelemetrySnapshot();
+    }
+
+    private Map<String, Object> setupClusters() {
+        String localIndex = "demo";
+        int numShardsLocal = randomIntBetween(2, 10);
+        Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build();
+        assertAcked(
+            client(LOCAL_CLUSTER).admin()
+                .indices()
+                .prepareCreate(localIndex)
+                .setSettings(localSettings)
+                .setMapping("@timestamp", "type=date", "f", "type=text")
+        );
+        indexDocs(client(LOCAL_CLUSTER), localIndex);
+
+        String remoteIndex = "prod";
+        int numShardsRemote = randomIntBetween(2, 10);
+        for (String clusterAlias : remoteClusterAlias()) {
+            final InternalTestCluster remoteCluster = cluster(clusterAlias);
+            remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(1, 3));
+            assertAcked(
+                client(clusterAlias).admin()
+                    .indices()
+                    .prepareCreate(remoteIndex)
+                    .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1)))
+                    .setMapping("@timestamp", "type=date", "f", "type=text")
+            );
+            assertFalse(
+                client(clusterAlias).admin()
+                    .cluster()
+                    .prepareHealth(remoteIndex)
+                    .setWaitForYellowStatus()
+                    .setTimeout(TimeValue.timeValueSeconds(10))
+                    .get()
+                    .isTimedOut()
+            );
+            indexDocs(client(clusterAlias), remoteIndex);
+        }
+
+        Map<String, Object> clusterInfo = new HashMap<>();
+        clusterInfo.put("local.num_shards", numShardsLocal);
+        clusterInfo.put("local.index", localIndex);
+        clusterInfo.put("remote.num_shards", numShardsRemote);
+        clusterInfo.put("remote.index", remoteIndex);
+        clusterInfo.put("remote.skip_unavailable", true);
+        return clusterInfo;
+    }
+
+    private int indexDocs(Client client, String index) {
+        int numDocs = between(5, 20);
+        for (int i = 0; i < numDocs; i++) {
+            client.prepareIndex(index).setSource("f", "v", "@timestamp", randomNonNegativeLong()).get();
+        }
+        client.admin().indices().prepareRefresh(index).get();
+        return numDocs;
+    }
+}
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
index c0305f873327d..5068ac69e462a 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
@@ -545,4 +545,9 @@ public void onFailure(Exception exc) {
             executeCompletionListeners();
         }
     }
+
+    @Override
+    public boolean isAsync() {
+        return true;
+    }
 }
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java
index d499c4a57ac14..38df763856242 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java
@@ -13,10 +13,10 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncStatusAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncStatusAction.java
index cc27e82a69388..99719fec5dc9e 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncStatusAction.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncStatusAction.java
@@ -15,10 +15,10 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
index c99404c217219..f254fd6171d31 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java
@@ -16,12 +16,12 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageIT.java
index 9fcda7af54f52..a91eb3c38319c 100644
--- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageIT.java
+++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageIT.java
@@ -131,6 +131,9 @@ private static void createDataStreamAndTemplate(String dataStreamName) throws IO
                     .build()
             )
         ).actionGet();
-        client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet();
+        client().execute(
+            CreateDataStreamAction.INSTANCE,
+            new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName)
+        ).actionGet();
     }
 }
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java
index 1084efe09c3af..70a5da97aa32b 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java
@@ -22,10 +22,10 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java
index a2024dec3b8ca..97da6a98bc386 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.snapshots.SnapshotsInfoService;
 import org.elasticsearch.tasks.CancellableTask;
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyAction.java
index 9157643578c7f..1e8a5d36d690b 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyAction.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyAction.java
index 71c3383add0d4..b07eabf2ed0c3 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyAction.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyAction.java
@@ -21,9 +21,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java
index 9c563eef9065e..75106a967a9af 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java
@@ -9,13 +9,13 @@
 
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.Processors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java
index 14364cf189072..7490c7a73aecb 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java
@@ -22,11 +22,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.unit.Processors;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.monitor.os.OsInfo;
 import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata;
 import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicy;
diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java
index 7ca37f376045f..075621e8cdccb 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java
@@ -7,15 +7,43 @@
 
 package org.elasticsearch.blobcache;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.telemetry.TelemetryProvider;
+import org.elasticsearch.telemetry.metric.DoubleHistogram;
 import org.elasticsearch.telemetry.metric.LongCounter;
 import org.elasticsearch.telemetry.metric.LongHistogram;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
 public class BlobCacheMetrics {
+    private static final Logger logger = LogManager.getLogger(BlobCacheMetrics.class);
+
+    private static final double BYTES_PER_NANOSECONDS_TO_MEBIBYTES_PER_SECOND = 1e9D / (1 << 20);
+    public static final String CACHE_POPULATION_REASON_ATTRIBUTE_KEY = "reason";
+    public static final String CACHE_POPULATION_SOURCE_ATTRIBUTE_KEY = "source";
+    public static final String SHARD_ID_ATTRIBUTE_KEY = "shard_id";
+    public static final String INDEX_ATTRIBUTE_KEY = "index_name";
+
     private final LongCounter cacheMissCounter;
     private final LongCounter evictedCountNonZeroFrequency;
     private final LongHistogram cacheMissLoadTimes;
+    private final DoubleHistogram cachePopulationThroughput;
+    private final LongCounter cachePopulationBytes;
+    private final LongCounter cachePopulationTime;
+
+    public enum CachePopulationReason {
+        /**
+         * When warming the cache
+         */
+        Warming,
+        /**
+         * When the data we need is not in the cache
+         */
+        CacheMiss
+    }
 
     public BlobCacheMetrics(MeterRegistry meterRegistry) {
         this(
@@ -31,16 +59,41 @@ public BlobCacheMetrics(MeterRegistry meterRegistry) {
             ),
             meterRegistry.registerLongHistogram(
                 "es.blob_cache.cache_miss_load_times.histogram",
-                "The time in microseconds for populating entries in the blob store resulting from a cache miss, expressed as a histogram.",
-                "micros"
+                "The time in milliseconds for populating entries in the blob store resulting from a cache miss, expressed as a histogram.",
+                "ms"
+            ),
+            meterRegistry.registerDoubleHistogram(
+                "es.blob_cache.population.throughput.histogram",
+                "The throughput observed when populating the the cache",
+                "MiB/second"
+            ),
+            meterRegistry.registerLongCounter(
+                "es.blob_cache.population.bytes.total",
+                "The number of bytes that have been copied into the cache",
+                "bytes"
+            ),
+            meterRegistry.registerLongCounter(
+                "es.blob_cache.population.time.total",
+                "The time spent copying data into the cache",
+                "milliseconds"
             )
         );
     }
 
-    BlobCacheMetrics(LongCounter cacheMissCounter, LongCounter evictedCountNonZeroFrequency, LongHistogram cacheMissLoadTimes) {
+    BlobCacheMetrics(
+        LongCounter cacheMissCounter,
+        LongCounter evictedCountNonZeroFrequency,
+        LongHistogram cacheMissLoadTimes,
+        DoubleHistogram cachePopulationThroughput,
+        LongCounter cachePopulationBytes,
+        LongCounter cachePopulationTime
+    ) {
         this.cacheMissCounter = cacheMissCounter;
         this.evictedCountNonZeroFrequency = evictedCountNonZeroFrequency;
         this.cacheMissLoadTimes = cacheMissLoadTimes;
+        this.cachePopulationThroughput = cachePopulationThroughput;
+        this.cachePopulationBytes = cachePopulationBytes;
+        this.cachePopulationTime = cachePopulationTime;
     }
 
     public static BlobCacheMetrics NOOP = new BlobCacheMetrics(TelemetryProvider.NOOP.getMeterRegistry());
@@ -56,4 +109,55 @@ public LongCounter getEvictedCountNonZeroFrequency() {
     public LongHistogram getCacheMissLoadTimes() {
         return cacheMissLoadTimes;
     }
+
+    /**
+     * Record the various cache population metrics after a chunk is copied to the cache
+     *
+     * @param bytesCopied The number of bytes copied
+     * @param copyTimeNanos The time taken to copy the bytes in nanoseconds
+     * @param index The index being loaded
+     * @param shardId The ID of the shard being loaded
+     * @param cachePopulationReason The reason for the cache being populated
+     * @param cachePopulationSource The source from which the data is being loaded
+     */
+    public void recordCachePopulationMetrics(
+        int bytesCopied,
+        long copyTimeNanos,
+        String index,
+        int shardId,
+        CachePopulationReason cachePopulationReason,
+        CachePopulationSource cachePopulationSource
+    ) {
+        Map<String, Object> metricAttributes = Map.of(
+            INDEX_ATTRIBUTE_KEY,
+            index,
+            SHARD_ID_ATTRIBUTE_KEY,
+            shardId,
+            CACHE_POPULATION_REASON_ATTRIBUTE_KEY,
+            cachePopulationReason.name(),
+            CACHE_POPULATION_SOURCE_ATTRIBUTE_KEY,
+            cachePopulationSource.name()
+        );
+        assert bytesCopied > 0 : "We shouldn't be recording zero-sized copies";
+        cachePopulationBytes.incrementBy(bytesCopied, metricAttributes);
+
+        // This is almost certainly paranoid, but if we had a very fast/small copy with a very coarse nanosecond timer it might happen?
+        if (copyTimeNanos > 0) {
+            cachePopulationThroughput.record(toMebibytesPerSecond(bytesCopied, copyTimeNanos), metricAttributes);
+            cachePopulationTime.incrementBy(TimeUnit.NANOSECONDS.toMillis(copyTimeNanos), metricAttributes);
+        } else {
+            logger.warn("Zero-time copy being reported, ignoring");
+        }
+    }
+
+    /**
+     * Calculate throughput as MiB/second
+     *
+     * @param numberOfBytes The number of bytes transferred
+     * @param timeInNanoseconds The time taken to transfer in nanoseconds
+     * @return The throughput as MiB/second
+     */
+    private double toMebibytesPerSecond(int numberOfBytes, long timeInNanoseconds) {
+        return ((double) numberOfBytes / timeInNanoseconds) * BYTES_PER_NANOSECONDS_TO_MEBIBYTES_PER_SECOND;
+    }
 }
diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/CachePopulationSource.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/CachePopulationSource.java
new file mode 100644
index 0000000000000..8cf4b1b548f7d
--- /dev/null
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/CachePopulationSource.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.blobcache;
+
+/**
+ * The places we populate the cache from
+ */
+public enum CachePopulationSource {
+    /**
+     * When loading data from the blob-store
+     */
+    BlobStore,
+    /**
+     * When fetching data from a peer node
+     */
+    Peer,
+    /**
+     * We cannot determine the source (should not be used except in exceptional cases)
+     */
+    Unknown
+}
diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
index 9cb83e35b63d6..09fcd1cbeba92 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.RefCountingListener;
 import org.elasticsearch.action.support.RefCountingRunnable;
-import org.elasticsearch.action.support.UnsafePlainActionFuture;
 import org.elasticsearch.blobcache.BlobCacheMetrics;
 import org.elasticsearch.blobcache.BlobCacheUtils;
 import org.elasticsearch.blobcache.common.ByteRange;
@@ -39,7 +38,6 @@
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.monitor.fs.FsProbe;
 import org.elasticsearch.node.NodeRoleSettings;
-import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
@@ -313,9 +311,9 @@ private CacheEntry(T chunk) {
     private final int numRegions;
     private final ConcurrentLinkedQueue<SharedBytes.IO> freeRegions = new ConcurrentLinkedQueue<>();
 
-    private final Cache<KeyType, CacheFileRegion> cache;
+    private final Cache<KeyType, CacheFileRegion<KeyType>> cache;
 
-    private final ConcurrentHashMap<SharedBytes.IO, CacheFileRegion> regionOwners; // to assert exclusive access of regions
+    private final ConcurrentHashMap<SharedBytes.IO, CacheFileRegion<KeyType>> regionOwners; // to assert exclusive access of regions
 
     private final LongAdder writeCount = new LongAdder();
     private final LongAdder writeBytes = new LongAdder();
@@ -335,7 +333,7 @@ public SharedBlobCacheService(
         NodeEnvironment environment,
         Settings settings,
         ThreadPool threadPool,
-        String ioExecutor,
+        Executor ioExecutor,
         BlobCacheMetrics blobCacheMetrics
     ) {
         this(environment, settings, threadPool, ioExecutor, blobCacheMetrics, System::nanoTime);
@@ -345,12 +343,12 @@ public SharedBlobCacheService(
         NodeEnvironment environment,
         Settings settings,
         ThreadPool threadPool,
-        String ioExecutor,
+        Executor ioExecutor,
         BlobCacheMetrics blobCacheMetrics,
         LongSupplier relativeTimeInNanosSupplier
     ) {
         this.threadPool = threadPool;
-        this.ioExecutor = threadPool.executor(ioExecutor);
+        this.ioExecutor = ioExecutor;
         long totalFsSize;
         try {
             totalFsSize = FsProbe.getTotal(Environment.getFileStore(environment.nodeDataPaths()[0]));
@@ -398,6 +396,10 @@ public static long calculateCacheSize(Settings settings, long totalFsSize) {
             .getBytes();
     }
 
+    public BlobCacheMetrics getBlobCacheMetrics() {
+        return blobCacheMetrics;
+    }
+
     public int getRangeSize() {
         return rangeSize;
     }
@@ -469,7 +471,7 @@ public int getRegionSize() {
         return regionSize;
     }
 
-    CacheFileRegion get(KeyType cacheKey, long fileLength, int region) {
+    CacheFileRegion<KeyType> get(KeyType cacheKey, long fileLength, int region) {
         return cache.get(cacheKey, fileLength, region).chunk;
     }
 
@@ -514,7 +516,7 @@ public boolean maybeFetchFullEntry(
                     return true;
                 }
                 final ActionListener<Integer> regionListener = refCountingListener.acquire(ignored -> {});
-                final CacheFileRegion entry;
+                final CacheFileRegion<KeyType> entry;
                 try {
                     entry = get(cacheKey, length, region);
                 } catch (AlreadyClosedException e) {
@@ -581,7 +583,7 @@ public void maybeFetchRegion(
                 listener.onResponse(false);
                 return;
             }
-            final CacheFileRegion entry = get(cacheKey, blobLength, region);
+            final CacheFileRegion<KeyType> entry = get(cacheKey, blobLength, region);
             entry.populate(regionRange, writer, fetchExecutor, listener);
         } catch (Exception e) {
             listener.onFailure(e);
@@ -629,7 +631,7 @@ public void maybeFetchRange(
                 listener.onResponse(false);
                 return;
             }
-            final CacheFileRegion entry = get(cacheKey, blobLength, region);
+            final CacheFileRegion<KeyType> entry = get(cacheKey, blobLength, region);
             entry.populate(
                 regionRange,
                 writerWithOffset(writer, Math.toIntExact(range.start() - getRegionStart(region))),
@@ -646,13 +648,14 @@ private RangeMissingHandler writerWithOffset(RangeMissingHandler writer, int wri
             // no need to allocate a new capturing lambda if the offset isn't adjusted
             return writer;
         }
-        return (channel, channelPos, streamFactory, relativePos, len, progressUpdater) -> writer.fillCacheRange(
+        return (channel, channelPos, streamFactory, relativePos, len, progressUpdater, completionListener) -> writer.fillCacheRange(
             channel,
             channelPos,
             streamFactory,
             relativePos - writeOffset,
             len,
-            progressUpdater
+            progressUpdater,
+            completionListener
         );
     }
 
@@ -702,7 +705,7 @@ public int forceEvict(Predicate<KeyType> cacheKeyPredicate) {
     }
 
     // used by tests
-    int getFreq(CacheFileRegion cacheFileRegion) {
+    int getFreq(CacheFileRegion<KeyType> cacheFileRegion) {
         if (cache instanceof LFUCache lfuCache) {
             return lfuCache.getFreq(cacheFileRegion);
         }
@@ -784,25 +787,45 @@ protected boolean assertOffsetsWithinFileLength(long offset, long length, long f
     /**
      * While this class has incRef and tryIncRef methods, incRefEnsureOpen and tryIncrefEnsureOpen should
      * always be used, ensuring the right ordering between incRef/tryIncRef and ensureOpen
-     * (see {@link LFUCache#maybeEvictAndTakeForFrequency(Runnable, int)})
+     * (see {@link SharedBlobCacheService.LFUCache#maybeEvictAndTakeForFrequency(Runnable, int)})
      */
-    class CacheFileRegion extends EvictableRefCounted {
+    static class CacheFileRegion<KeyType> extends EvictableRefCounted {
+
+        private static final VarHandle VH_IO = findIOVarHandle();
+
+        private static VarHandle findIOVarHandle() {
+            try {
+                return MethodHandles.lookup().in(CacheFileRegion.class).findVarHandle(CacheFileRegion.class, "io", SharedBytes.IO.class);
+            } catch (NoSuchFieldException | IllegalAccessException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        final SharedBlobCacheService<KeyType> blobCacheService;
 
         final RegionKey<KeyType> regionKey;
         final SparseFileTracker tracker;
         // io can be null when not init'ed or after evict/take
-        volatile SharedBytes.IO io = null;
-
-        CacheFileRegion(RegionKey<KeyType> regionKey, int regionSize) {
+        // io does not need volatile access on the read path, since it goes from null to a single value (and then possbily back to null).
+        // "cache.get" never returns a `CacheFileRegion` without checking the value is non-null (with a volatile read, ensuring the value is
+        // visible in that thread).
+        // We assume any IndexInput passing among threads is done with proper happens-before semantics (otherwise they'd themselves break).
+        // In general, assertions should use `nonVolatileIO` (when they can) to access this over `volatileIO` to avoid memory visibility
+        // side effects
+        private SharedBytes.IO io = null;
+
+        CacheFileRegion(SharedBlobCacheService<KeyType> blobCacheService, RegionKey<KeyType> regionKey, int regionSize) {
+            this.blobCacheService = blobCacheService;
             this.regionKey = regionKey;
             assert regionSize > 0;
             // NOTE we use a constant string for description to avoid consume extra heap space
             tracker = new SparseFileTracker("file", regionSize);
         }
 
-        public long physicalStartOffset() {
-            var ioRef = io;
-            return ioRef == null ? -1L : (long) regionKey.region * regionSize;
+        // only used for logging
+        private long physicalStartOffset() {
+            var ioRef = nonVolatileIO();
+            return ioRef == null ? -1L : (long) regionKey.region * blobCacheService.regionSize;
         }
 
         public boolean tryIncRefEnsureOpen() {
@@ -829,10 +852,10 @@ private void ensureOpenOrDecRef() {
         // tries to evict this chunk if noone is holding onto its resources anymore
         // visible for tests.
         boolean tryEvict() {
-            assert Thread.holdsLock(SharedBlobCacheService.this) : "must hold lock when evicting";
+            assert Thread.holdsLock(blobCacheService) : "must hold lock when evicting";
             if (refCount() <= 1 && evict()) {
                 logger.trace("evicted {} with channel offset {}", regionKey, physicalStartOffset());
-                evictCount.increment();
+                blobCacheService.evictCount.increment();
                 decRef();
                 return true;
             }
@@ -840,10 +863,10 @@ boolean tryEvict() {
         }
 
         boolean tryEvictNoDecRef() {
-            assert Thread.holdsLock(SharedBlobCacheService.this) : "must hold lock when evicting";
+            assert Thread.holdsLock(blobCacheService) : "must hold lock when evicting";
             if (refCount() <= 1 && evict()) {
                 logger.trace("evicted and take {} with channel offset {}", regionKey, physicalStartOffset());
-                evictCount.increment();
+                blobCacheService.evictCount.increment();
                 return true;
             }
 
@@ -851,10 +874,10 @@ boolean tryEvictNoDecRef() {
         }
 
         public boolean forceEvict() {
-            assert Thread.holdsLock(SharedBlobCacheService.this) : "must hold lock when evicting";
+            assert Thread.holdsLock(blobCacheService) : "must hold lock when evicting";
             if (evict()) {
                 logger.trace("force evicted {} with channel offset {}", regionKey, physicalStartOffset());
-                evictCount.increment();
+                blobCacheService.evictCount.increment();
                 decRef();
                 return true;
             }
@@ -865,9 +888,10 @@ public boolean forceEvict() {
         protected void closeInternal() {
             // now actually free the region associated with this chunk
             // we held the "this" lock when this was evicted, hence if io is not filled in, chunk will never be registered.
+            SharedBytes.IO io = volatileIO();
             if (io != null) {
-                assert regionOwners.remove(io) == this;
-                freeRegions.add(io);
+                assert blobCacheService.regionOwners.remove(io) == this;
+                blobCacheService.freeRegions.add(io);
             }
             logger.trace("closed {} with channel offset {}", regionKey, physicalStartOffset());
         }
@@ -876,14 +900,31 @@ private static void throwAlreadyEvicted() {
             throwAlreadyClosed("File chunk is evicted");
         }
 
+        private SharedBytes.IO volatileIO() {
+            return (SharedBytes.IO) VH_IO.getVolatile(this);
+        }
+
+        private void volatileIO(SharedBytes.IO io) {
+            VH_IO.setVolatile(this, io);
+        }
+
+        private SharedBytes.IO nonVolatileIO() {
+            return io;
+        }
+
+        // for use in tests *only*
+        SharedBytes.IO testOnlyNonVolatileIO() {
+            return io;
+        }
+
         /**
          * Optimistically try to read from the region
          * @return true if successful, i.e., not evicted and data available, false if evicted
          */
         boolean tryRead(ByteBuffer buf, long offset) throws IOException {
-            SharedBytes.IO ioRef = this.io;
+            SharedBytes.IO ioRef = nonVolatileIO();
             if (ioRef != null) {
-                int readBytes = ioRef.read(buf, getRegionRelativePosition(offset));
+                int readBytes = ioRef.read(buf, blobCacheService.getRegionRelativePosition(offset));
                 if (isEvicted()) {
                     buf.position(buf.position() - readBytes);
                     return false;
@@ -919,19 +960,55 @@ void populate(
                         rangeToWrite,
                         rangeToWrite,
                         Assertions.ENABLED ? ActionListener.releaseAfter(ActionListener.running(() -> {
-                            assert regionOwners.get(io) == this;
+                            assert blobCacheService.regionOwners.get(nonVolatileIO()) == this;
                         }), refs.acquire()) : refs.acquireListener()
                     );
                     if (gaps.isEmpty()) {
                         listener.onResponse(false);
                         return;
                     }
-                    try (var gapsListener = new RefCountingListener(listener.map(unused -> true))) {
-                        assert writer.sharedInputStreamFactory(gaps) == null;
-                        for (SparseFileTracker.Gap gap : gaps) {
-                            executor.execute(
-                                fillGapRunnable(gap, writer, null, ActionListener.releaseAfter(gapsListener.acquire(), refs.acquire()))
-                            );
+                    final SourceInputStreamFactory streamFactory = writer.sharedInputStreamFactory(gaps);
+                    logger.trace(
+                        () -> Strings.format(
+                            "fill gaps %s %s shared input stream factory",
+                            gaps,
+                            streamFactory == null ? "without" : "with"
+                        )
+                    );
+                    if (streamFactory == null) {
+                        try (var parallelGapsListener = new RefCountingListener(listener.map(unused -> true))) {
+                            for (SparseFileTracker.Gap gap : gaps) {
+                                executor.execute(
+                                    fillGapRunnable(
+                                        gap,
+                                        writer,
+                                        null,
+                                        ActionListener.releaseAfter(parallelGapsListener.acquire(), refs.acquire())
+                                    )
+                                );
+                            }
+                        }
+                    } else {
+                        try (
+                            var sequentialGapsListener = new RefCountingListener(
+                                ActionListener.runBefore(listener.map(unused -> true), streamFactory::close)
+                            )
+                        ) {
+                            final List<Runnable> gapFillingTasks = gaps.stream()
+                                .map(
+                                    gap -> fillGapRunnable(
+                                        gap,
+                                        writer,
+                                        streamFactory,
+                                        ActionListener.releaseAfter(sequentialGapsListener.acquire(), refs.acquire())
+                                    )
+                                )
+                                .toList();
+                            executor.execute(() -> {
+                                // Fill the gaps in order. If a gap fails to fill for whatever reason, the task for filling the next
+                                // gap will still be executed.
+                                gapFillingTasks.forEach(Runnable::run);
+                            });
                         }
                     }
                 }
@@ -955,8 +1032,8 @@ void populateAndRead(
                         rangeToWrite,
                         rangeToRead,
                         ActionListener.releaseAfter(listener, refs.acquire()).delegateFailureAndWrap((l, success) -> {
-                            var ioRef = io;
-                            assert regionOwners.get(ioRef) == this;
+                            var ioRef = nonVolatileIO();
+                            assert blobCacheService.regionOwners.get(ioRef) == this;
                             final int start = Math.toIntExact(rangeToRead.start());
                             final int read = reader.onRangeAvailable(ioRef, start, start, Math.toIntExact(rangeToRead.length()));
                             assert read == rangeToRead.length()
@@ -967,7 +1044,7 @@ void populateAndRead(
                                     + '-'
                                     + rangeToRead.start()
                                     + ']';
-                            readCount.increment();
+                            blobCacheService.readCount.increment();
                             l.onResponse(read);
                         })
                     );
@@ -978,8 +1055,7 @@ void populateAndRead(
                             () -> Strings.format(
                                 "fill gaps %s %s shared input stream factory",
                                 gaps,
-                                (streamFactory == null ? "without" : "with"),
-                                (streamFactory == null ? "" : " " + streamFactory)
+                                streamFactory == null ? "without" : "with"
                             )
                         );
                         if (streamFactory == null) {
@@ -987,16 +1063,17 @@ void populateAndRead(
                                 executor.execute(fillGapRunnable(gap, writer, null, refs.acquireListener()));
                             }
                         } else {
-                            final List<AbstractRunnable> gapFillingTasks = gaps.stream()
-                                .map(gap -> fillGapRunnable(gap, writer, streamFactory, refs.acquireListener()))
-                                .toList();
-                            executor.execute(() -> {
-                                try (streamFactory) {
+                            var gapFillingListener = refs.acquireListener();
+                            try (var gfRefs = new RefCountingRunnable(ActionRunnable.run(gapFillingListener, streamFactory::close))) {
+                                final List<Runnable> gapFillingTasks = gaps.stream()
+                                    .map(gap -> fillGapRunnable(gap, writer, streamFactory, gfRefs.acquireListener()))
+                                    .toList();
+                                executor.execute(() -> {
                                     // Fill the gaps in order. If a gap fails to fill for whatever reason, the task for filling the next
                                     // gap will still be executed.
                                     gapFillingTasks.forEach(Runnable::run);
-                                }
-                            });
+                                });
+                            }
                         }
                     }
                 }
@@ -1005,15 +1082,15 @@ void populateAndRead(
             }
         }
 
-        private AbstractRunnable fillGapRunnable(
+        private Runnable fillGapRunnable(
             SparseFileTracker.Gap gap,
             RangeMissingHandler writer,
             @Nullable SourceInputStreamFactory streamFactory,
             ActionListener<Void> listener
         ) {
-            return ActionRunnable.run(listener.delegateResponse((l, e) -> failGapAndListener(gap, l, e)), () -> {
-                var ioRef = io;
-                assert regionOwners.get(ioRef) == CacheFileRegion.this;
+            return () -> ActionListener.run(listener, l -> {
+                var ioRef = nonVolatileIO();
+                assert blobCacheService.regionOwners.get(ioRef) == CacheFileRegion.this;
                 assert CacheFileRegion.this.hasReferences() : CacheFileRegion.this;
                 int start = Math.toIntExact(gap.start());
                 writer.fillCacheRange(
@@ -1022,10 +1099,15 @@ private AbstractRunnable fillGapRunnable(
                     streamFactory,
                     start,
                     Math.toIntExact(gap.end() - start),
-                    progress -> gap.onProgress(start + progress)
+                    progress -> gap.onProgress(start + progress),
+                    l.<Void>map(unused -> {
+                        assert blobCacheService.regionOwners.get(ioRef) == CacheFileRegion.this;
+                        assert CacheFileRegion.this.hasReferences() : CacheFileRegion.this;
+                        blobCacheService.writeCount.increment();
+                        gap.onCompletion();
+                        return null;
+                    }).delegateResponse((delegate, e) -> failGapAndListener(gap, delegate, e))
                 );
-                writeCount.increment();
-                gap.onCompletion();
             });
         }
 
@@ -1049,7 +1131,7 @@ public class CacheFile {
         private final KeyType cacheKey;
         private final long length;
 
-        private CacheEntry<CacheFileRegion> lastAccessedRegion;
+        private CacheEntry<CacheFileRegion<KeyType>> lastAccessedRegion;
 
         private CacheFile(KeyType cacheKey, long length) {
             this.cacheKey = cacheKey;
@@ -1113,12 +1195,23 @@ public void fillCacheRange(
                     SourceInputStreamFactory streamFactory,
                     int relativePos,
                     int length,
-                    IntConsumer progressUpdater
+                    IntConsumer progressUpdater,
+                    ActionListener<Void> completionListener
                 ) throws IOException {
-                    writer.fillCacheRange(channel, channelPos, streamFactory, relativePos, length, progressUpdater);
-                    var elapsedTime = TimeUnit.NANOSECONDS.toMicros(relativeTimeInNanosSupplier.getAsLong() - startTime);
-                    SharedBlobCacheService.this.blobCacheMetrics.getCacheMissLoadTimes().record(elapsedTime);
-                    SharedBlobCacheService.this.blobCacheMetrics.getCacheMissCounter().increment();
+                    writer.fillCacheRange(
+                        channel,
+                        channelPos,
+                        streamFactory,
+                        relativePos,
+                        length,
+                        progressUpdater,
+                        completionListener.map(unused -> {
+                            var elapsedTime = TimeUnit.NANOSECONDS.toMillis(relativeTimeInNanosSupplier.getAsLong() - startTime);
+                            blobCacheMetrics.getCacheMissLoadTimes().record(elapsedTime);
+                            blobCacheMetrics.getCacheMissCounter().increment();
+                            return null;
+                        })
+                    );
                 }
             };
             if (rangeToRead.isEmpty()) {
@@ -1140,10 +1233,8 @@ private int readSingleRegion(
             RangeMissingHandler writer,
             int region
         ) throws InterruptedException, ExecutionException {
-            final PlainActionFuture<Integer> readFuture = new UnsafePlainActionFuture<>(
-                BlobStoreRepository.STATELESS_SHARD_PREWARMING_THREAD_NAME
-            );
-            final CacheFileRegion fileRegion = get(cacheKey, length, region);
+            final PlainActionFuture<Integer> readFuture = new PlainActionFuture<>();
+            final CacheFileRegion<KeyType> fileRegion = get(cacheKey, length, region);
             final long regionStart = getRegionStart(region);
             fileRegion.populateAndRead(
                 mapSubRangeToRegion(rangeToWrite, region),
@@ -1164,9 +1255,7 @@ private int readMultiRegions(
             int startRegion,
             int endRegion
         ) throws InterruptedException, ExecutionException {
-            final PlainActionFuture<Void> readsComplete = new UnsafePlainActionFuture<>(
-                BlobStoreRepository.STATELESS_SHARD_PREWARMING_THREAD_NAME
-            );
+            final PlainActionFuture<Void> readsComplete = new PlainActionFuture<>();
             final AtomicInteger bytesRead = new AtomicInteger();
             try (var listeners = new RefCountingListener(1, readsComplete)) {
                 for (int region = startRegion; region <= endRegion; region++) {
@@ -1177,7 +1266,7 @@ private int readMultiRegions(
                     }
                     ActionListener<Integer> listener = listeners.acquire(i -> bytesRead.updateAndGet(j -> Math.addExact(i, j)));
                     try {
-                        final CacheFileRegion fileRegion = get(cacheKey, length, region);
+                        final CacheFileRegion<KeyType> fileRegion = get(cacheKey, length, region);
                         final long regionStart = getRegionStart(region);
                         fileRegion.populateAndRead(
                             mapSubRangeToRegion(rangeToWrite, region),
@@ -1197,7 +1286,7 @@ private int readMultiRegions(
             return bytesRead.get();
         }
 
-        private RangeMissingHandler writerWithOffset(RangeMissingHandler writer, CacheFileRegion fileRegion, int writeOffset) {
+        private RangeMissingHandler writerWithOffset(RangeMissingHandler writer, CacheFileRegion<KeyType> fileRegion, int writeOffset) {
             final RangeMissingHandler adjustedWriter;
             if (writeOffset == 0) {
                 // no need to allocate a new capturing lambda if the offset isn't adjusted
@@ -1211,9 +1300,18 @@ public void fillCacheRange(
                         SourceInputStreamFactory streamFactory,
                         int relativePos,
                         int len,
-                        IntConsumer progressUpdater
+                        IntConsumer progressUpdater,
+                        ActionListener<Void> completionListener
                     ) throws IOException {
-                        delegate.fillCacheRange(channel, channelPos, streamFactory, relativePos - writeOffset, len, progressUpdater);
+                        delegate.fillCacheRange(
+                            channel,
+                            channelPos,
+                            streamFactory,
+                            relativePos - writeOffset,
+                            len,
+                            progressUpdater,
+                            completionListener
+                        );
                     }
                 };
             }
@@ -1226,19 +1324,30 @@ public void fillCacheRange(
                         SourceInputStreamFactory streamFactory,
                         int relativePos,
                         int len,
-                        IntConsumer progressUpdater
+                        IntConsumer progressUpdater,
+                        ActionListener<Void> completionListener
                     ) throws IOException {
                         assert assertValidRegionAndLength(fileRegion, channelPos, len);
-                        delegate.fillCacheRange(channel, channelPos, streamFactory, relativePos, len, progressUpdater);
-                        assert regionOwners.get(fileRegion.io) == fileRegion
-                            : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.io + "]";
+                        delegate.fillCacheRange(
+                            channel,
+                            channelPos,
+                            streamFactory,
+                            relativePos,
+                            len,
+                            progressUpdater,
+                            Assertions.ENABLED ? ActionListener.runBefore(completionListener, () -> {
+                                assert regionOwners.get(fileRegion.nonVolatileIO()) == fileRegion
+                                    : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.nonVolatileIO() + "]";
+                            }) : completionListener
+                        );
                     }
                 };
+
             }
             return adjustedWriter;
         }
 
-        private RangeAvailableHandler readerWithOffset(RangeAvailableHandler reader, CacheFileRegion fileRegion, int readOffset) {
+        private RangeAvailableHandler readerWithOffset(RangeAvailableHandler reader, CacheFileRegion<KeyType> fileRegion, int readOffset) {
             final RangeAvailableHandler adjustedReader = (channel, channelPos, relativePos, len) -> reader.onRangeAvailable(
                 channel,
                 channelPos,
@@ -1249,18 +1358,18 @@ private RangeAvailableHandler readerWithOffset(RangeAvailableHandler reader, Cac
                 return (channel, channelPos, relativePos, len) -> {
                     assert assertValidRegionAndLength(fileRegion, channelPos, len);
                     final int bytesRead = adjustedReader.onRangeAvailable(channel, channelPos, relativePos, len);
-                    assert regionOwners.get(fileRegion.io) == fileRegion
-                        : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.io + "]";
+                    assert regionOwners.get(fileRegion.nonVolatileIO()) == fileRegion
+                        : "File chunk [" + fileRegion.regionKey + "] no longer owns IO [" + fileRegion.nonVolatileIO() + "]";
                     return bytesRead;
                 };
             }
             return adjustedReader;
         }
 
-        private boolean assertValidRegionAndLength(CacheFileRegion fileRegion, int channelPos, int len) {
-            assert fileRegion.io != null;
+        private boolean assertValidRegionAndLength(CacheFileRegion<KeyType> fileRegion, int channelPos, int len) {
+            assert fileRegion.nonVolatileIO() != null;
             assert fileRegion.hasReferences();
-            assert regionOwners.get(fileRegion.io) == fileRegion;
+            assert regionOwners.get(fileRegion.nonVolatileIO()) == fileRegion;
             assert channelPos >= 0 && channelPos + len <= regionSize;
             return true;
         }
@@ -1320,6 +1429,7 @@ default SourceInputStreamFactory sharedInputStreamFactory(List<SparseFileTracker
          * @param length of data to fetch
          * @param progressUpdater consumer to invoke with the number of copied bytes as they are written in cache.
          *                        This is used to notify waiting readers that data become available in cache.
+         * @param completionListener listener that has to be called when the callback method completes
          */
         void fillCacheRange(
             SharedBytes.IO channel,
@@ -1327,7 +1437,8 @@ void fillCacheRange(
             @Nullable SourceInputStreamFactory streamFactory,
             int relativePos,
             int length,
-            IntConsumer progressUpdater
+            IntConsumer progressUpdater,
+            ActionListener<Void> completionListener
         ) throws IOException;
     }
 
@@ -1339,9 +1450,9 @@ public interface SourceInputStreamFactory extends Releasable {
         /**
          * Create the input stream at the specified position.
          * @param relativePos the relative position in the remote storage to read from.
-         * @return the input stream ready to be read from.
+         * @param listener listener for the input stream ready to be read from.
          */
-        InputStream create(int relativePos) throws IOException;
+        void create(int relativePos, ActionListener<InputStream> listener) throws IOException;
     }
 
     private abstract static class DelegatingRangeMissingHandler implements RangeMissingHandler {
@@ -1363,9 +1474,10 @@ public void fillCacheRange(
             SourceInputStreamFactory streamFactory,
             int relativePos,
             int length,
-            IntConsumer progressUpdater
+            IntConsumer progressUpdater,
+            ActionListener<Void> completionListener
         ) throws IOException {
-            delegate.fillCacheRange(channel, channelPos, streamFactory, relativePos, length, progressUpdater);
+            delegate.fillCacheRange(channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener);
         }
     }
 
@@ -1382,15 +1494,15 @@ public record Stats(
         public static final Stats EMPTY = new Stats(0, 0L, 0L, 0L, 0L, 0L, 0L, 0L);
     }
 
-    private class LFUCache implements Cache<KeyType, CacheFileRegion> {
+    private class LFUCache implements Cache<KeyType, CacheFileRegion<KeyType>> {
 
-        class LFUCacheEntry extends CacheEntry<CacheFileRegion> {
+        class LFUCacheEntry extends CacheEntry<CacheFileRegion<KeyType>> {
             LFUCacheEntry prev;
             LFUCacheEntry next;
             int freq;
             volatile long lastAccessedEpoch;
 
-            LFUCacheEntry(CacheFileRegion chunk, long lastAccessed) {
+            LFUCacheEntry(CacheFileRegion<KeyType> chunk, long lastAccessed) {
                 super(chunk);
                 this.lastAccessedEpoch = lastAccessed;
                 // todo: consider whether freq=1 is still right for new entries.
@@ -1428,7 +1540,7 @@ public void close() {
             decayAndNewEpochTask.close();
         }
 
-        int getFreq(CacheFileRegion cacheFileRegion) {
+        int getFreq(CacheFileRegion<KeyType> cacheFileRegion) {
             return keyMapping.get(cacheFileRegion.regionKey).freq;
         }
 
@@ -1441,12 +1553,15 @@ public LFUCacheEntry get(KeyType cacheKey, long fileLength, int region) {
             var entry = keyMapping.get(regionKey);
             if (entry == null) {
                 final int effectiveRegionSize = computeCacheFileRegionSize(fileLength, region);
-                entry = keyMapping.computeIfAbsent(regionKey, key -> new LFUCacheEntry(new CacheFileRegion(key, effectiveRegionSize), now));
+                entry = keyMapping.computeIfAbsent(
+                    regionKey,
+                    key -> new LFUCacheEntry(new CacheFileRegion<KeyType>(SharedBlobCacheService.this, key, effectiveRegionSize), now)
+                );
             }
-            // io is volatile, double locking is fine, as long as we assign it last.
-            if (entry.chunk.io == null) {
+            // checks using volatile, double locking is fine, as long as we assign io last.
+            if (entry.chunk.volatileIO() == null) {
                 synchronized (entry.chunk) {
-                    if (entry.chunk.io == null && entry.chunk.isEvicted() == false) {
+                    if (entry.chunk.volatileIO() == null && entry.chunk.isEvicted() == false) {
                         return initChunk(entry);
                     }
                 }
@@ -1476,7 +1591,7 @@ public int forceEvict(Predicate<KeyType> cacheKeyPredicate) {
                     for (LFUCacheEntry entry : matchingEntries) {
                         int frequency = entry.freq;
                         boolean evicted = entry.chunk.forceEvict();
-                        if (evicted && entry.chunk.io != null) {
+                        if (evicted && entry.chunk.volatileIO() != null) {
                             unlink(entry);
                             keyMapping.remove(entry.chunk.regionKey, entry);
                             evictedCount++;
@@ -1537,7 +1652,7 @@ private void assignToSlot(LFUCacheEntry entry, SharedBytes.IO freeSlot) {
                 }
                 pushEntryToBack(entry);
                 // assign io only when chunk is ready for use. Under lock to avoid concurrent tryEvict.
-                entry.chunk.io = freeSlot;
+                entry.chunk.volatileIO(freeSlot);
             }
         }
 
@@ -1602,7 +1717,7 @@ private boolean assertChunkActiveOrEvicted(LFUCacheEntry entry) {
                 assert entry.prev != null || entry.chunk.isEvicted();
 
             }
-            SharedBytes.IO io = entry.chunk.io;
+            SharedBytes.IO io = entry.chunk.nonVolatileIO();
             assert io != null || entry.chunk.isEvicted();
             assert io == null || regionOwners.get(io) == entry.chunk || entry.chunk.isEvicted();
             return true;
@@ -1725,13 +1840,13 @@ private SharedBytes.IO maybeEvictAndTakeForFrequency(Runnable evictedNotificatio
                 boolean evicted = entry.chunk.tryEvictNoDecRef();
                 if (evicted) {
                     try {
-                        SharedBytes.IO ioRef = entry.chunk.io;
+                        SharedBytes.IO ioRef = entry.chunk.volatileIO();
                         if (ioRef != null) {
                             try {
                                 if (entry.chunk.refCount() == 1) {
                                     // we own that one refcount (since we CAS'ed evicted to 1)
                                     // grab io, rely on incref'ers also checking evicted field.
-                                    entry.chunk.io = null;
+                                    entry.chunk.volatileIO(null);
                                     assert regionOwners.remove(ioRef) == entry.chunk;
                                     return ioRef;
                                 }
@@ -1770,7 +1885,7 @@ public boolean maybeEvictLeastUsed() {
             synchronized (SharedBlobCacheService.this) {
                 for (LFUCacheEntry entry = freqs[0]; entry != null; entry = entry.next) {
                     boolean evicted = entry.chunk.tryEvict();
-                    if (evicted && entry.chunk.io != null) {
+                    if (evicted && entry.chunk.volatileIO() != null) {
                         unlink(entry);
                         keyMapping.remove(entry.chunk.regionKey, entry);
                         return true;
diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheMetricsTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheMetricsTests.java
new file mode 100644
index 0000000000000..ea9d0b7356f0e
--- /dev/null
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheMetricsTests.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.blobcache;
+
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.telemetry.InstrumentType;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.RecordingMeterRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.concurrent.TimeUnit;
+
+public class BlobCacheMetricsTests extends ESTestCase {
+
+    private RecordingMeterRegistry recordingMeterRegistry;
+    private BlobCacheMetrics metrics;
+
+    @Before
+    public void createMetrics() {
+        recordingMeterRegistry = new RecordingMeterRegistry();
+        metrics = new BlobCacheMetrics(recordingMeterRegistry);
+    }
+
+    public void testRecordCachePopulationMetricsRecordsThroughput() {
+        int mebiBytesSent = randomIntBetween(1, 4);
+        int secondsTaken = randomIntBetween(1, 5);
+        String indexName = randomIdentifier();
+        int shardId = randomIntBetween(0, 10);
+        BlobCacheMetrics.CachePopulationReason cachePopulationReason = randomFrom(BlobCacheMetrics.CachePopulationReason.values());
+        CachePopulationSource cachePopulationSource = randomFrom(CachePopulationSource.values());
+        metrics.recordCachePopulationMetrics(
+            Math.toIntExact(ByteSizeValue.ofMb(mebiBytesSent).getBytes()),
+            TimeUnit.SECONDS.toNanos(secondsTaken),
+            indexName,
+            shardId,
+            cachePopulationReason,
+            cachePopulationSource
+        );
+
+        // throughput histogram
+        Measurement throughputMeasurement = recordingMeterRegistry.getRecorder()
+            .getMeasurements(InstrumentType.DOUBLE_HISTOGRAM, "es.blob_cache.population.throughput.histogram")
+            .get(0);
+        assertEquals(throughputMeasurement.getDouble(), (double) mebiBytesSent / secondsTaken, 0.0);
+        assertExpectedAttributesPresent(throughputMeasurement, shardId, indexName, cachePopulationReason, cachePopulationSource);
+
+        // bytes counter
+        Measurement totalBytesMeasurement = recordingMeterRegistry.getRecorder()
+            .getMeasurements(InstrumentType.LONG_COUNTER, "es.blob_cache.population.bytes.total")
+            .get(0);
+        assertEquals(totalBytesMeasurement.getLong(), ByteSizeValue.ofMb(mebiBytesSent).getBytes());
+        assertExpectedAttributesPresent(totalBytesMeasurement, shardId, indexName, cachePopulationReason, cachePopulationSource);
+
+        // time counter
+        Measurement totalTimeMeasurement = recordingMeterRegistry.getRecorder()
+            .getMeasurements(InstrumentType.LONG_COUNTER, "es.blob_cache.population.time.total")
+            .get(0);
+        assertEquals(totalTimeMeasurement.getLong(), TimeUnit.SECONDS.toMillis(secondsTaken));
+        assertExpectedAttributesPresent(totalTimeMeasurement, shardId, indexName, cachePopulationReason, cachePopulationSource);
+    }
+
+    private static void assertExpectedAttributesPresent(
+        Measurement measurement,
+        int shardId,
+        String indexName,
+        BlobCacheMetrics.CachePopulationReason cachePopulationReason,
+        CachePopulationSource cachePopulationSource
+    ) {
+        assertEquals(measurement.attributes().get(BlobCacheMetrics.SHARD_ID_ATTRIBUTE_KEY), shardId);
+        assertEquals(measurement.attributes().get(BlobCacheMetrics.INDEX_ATTRIBUTE_KEY), indexName);
+        assertEquals(measurement.attributes().get(BlobCacheMetrics.CACHE_POPULATION_REASON_ATTRIBUTE_KEY), cachePopulationReason.name());
+        assertEquals(measurement.attributes().get(BlobCacheMetrics.CACHE_POPULATION_SOURCE_ATTRIBUTE_KEY), cachePopulationSource.name());
+    }
+}
diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java
index e477673c90d6d..e405ed578b5a5 100644
--- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.StoppableExecutorServiceWrapper;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.TestEnvironment;
@@ -72,6 +73,13 @@ private static long size(long numPages) {
         return numPages * SharedBytes.PAGE_SIZE;
     }
 
+    private static <E extends Exception> void completeWith(ActionListener<Void> listener, CheckedRunnable<E> runnable) {
+        ActionListener.completeWith(listener, () -> {
+            runnable.run();
+            return null;
+        });
+    }
+
     public void testBasicEviction() throws IOException {
         Settings settings = Settings.builder()
             .put(NODE_NAME_SETTING.getKey(), "node")
@@ -86,7 +94,7 @@ public void testBasicEviction() throws IOException {
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
-                ThreadPool.Names.GENERIC,
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -115,7 +123,10 @@ public void testBasicEviction() throws IOException {
                 ByteRange.of(0L, 1L),
                 ByteRange.of(0L, 1L),
                 (channel, channelPos, relativePos, length) -> 1,
-                (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept(length),
+                (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                    completionListener,
+                    () -> progressUpdater.accept(length)
+                ),
                 taskQueue.getThreadPool().generic(),
                 bytesReadFuture
             );
@@ -138,7 +149,7 @@ public void testBasicEviction() throws IOException {
         }
     }
 
-    private static boolean tryEvict(SharedBlobCacheService<Object>.CacheFileRegion region1) {
+    private static boolean tryEvict(SharedBlobCacheService.CacheFileRegion<Object> region1) {
         if (randomBoolean()) {
             return region1.tryEvict();
         } else {
@@ -164,7 +175,7 @@ public void testAutoEviction() throws IOException {
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
-                ThreadPool.Names.GENERIC,
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -208,7 +219,7 @@ public void testForceEviction() throws IOException {
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
-                ThreadPool.Names.GENERIC,
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -242,7 +253,7 @@ public void testForceEvictResponse() throws IOException {
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
-                ThreadPool.Names.GENERIC,
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -276,7 +287,7 @@ public void testDecay() throws IOException {
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
-                ThreadPool.Names.GENERIC,
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -384,7 +395,7 @@ public void testMassiveDecay() throws IOException {
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
-                ThreadPool.Names.GENERIC,
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -433,6 +444,7 @@ public void testMassiveDecay() throws IOException {
      * Exercise SharedBlobCacheService#get in multiple threads to trigger any assertion errors.
      * @throws IOException
      */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/112305")
     public void testGetMultiThreaded() throws IOException {
         final int threads = between(2, 10);
         final int regionCount = between(1, 20);
@@ -459,7 +471,7 @@ public void testGetMultiThreaded() throws IOException {
                 environment,
                 settings,
                 threadPool,
-                ThreadPool.Names.GENERIC,
+                threadPool.executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -475,7 +487,7 @@ public void testGetMultiThreaded() throws IOException {
                         ready.await();
                         for (int i = 0; i < iterations; ++i) {
                             try {
-                                SharedBlobCacheService<String>.CacheFileRegion cacheFileRegion;
+                                SharedBlobCacheService.CacheFileRegion<String> cacheFileRegion;
                                 try {
                                     cacheFileRegion = cacheService.get(cacheKeys[i], fileLength, regions[i]);
                                 } catch (AlreadyClosedException e) {
@@ -486,6 +498,7 @@ public void testGetMultiThreaded() throws IOException {
                                     if (yield[i] == 0) {
                                         Thread.yield();
                                     }
+                                    assertNotNull(cacheFileRegion.testOnlyNonVolatileIO());
                                     cacheFileRegion.decRef();
                                 }
                                 if (evict[i] == 0) {
@@ -539,7 +552,7 @@ public void execute(Runnable command) {
                 environment,
                 settings,
                 threadPool,
-                ThreadPool.Names.GENERIC,
+                threadPool.executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -552,11 +565,14 @@ public void execute(Runnable command) {
                 cacheService.maybeFetchFullEntry(
                     cacheKey,
                     size,
-                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> {
-                        assert streamFactory == null : streamFactory;
-                        bytesRead.addAndGet(-length);
-                        progressUpdater.accept(length);
-                    },
+                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                        completionListener,
+                        () -> {
+                            assert streamFactory == null : streamFactory;
+                            bytesRead.addAndGet(-length);
+                            progressUpdater.accept(length);
+                        }
+                    ),
                     bulkExecutor,
                     future
                 );
@@ -570,9 +586,15 @@ public void execute(Runnable command) {
                 // a download that would use up all regions should not run
                 final var cacheKey = generateCacheKey();
                 assertEquals(2, cacheService.freeRegionCount());
-                var configured = cacheService.maybeFetchFullEntry(cacheKey, size(500), (ch, chPos, streamFactory, relPos, len, update) -> {
-                    throw new AssertionError("Should never reach here");
-                }, bulkExecutor, ActionListener.noop());
+                var configured = cacheService.maybeFetchFullEntry(
+                    cacheKey,
+                    size(500),
+                    (ch, chPos, streamFactory, relPos, len, update, completionListener) -> completeWith(completionListener, () -> {
+                        throw new AssertionError("Should never reach here");
+                    }),
+                    bulkExecutor,
+                    ActionListener.noop()
+                );
                 assertFalse(configured);
                 assertEquals(2, cacheService.freeRegionCount());
             }
@@ -598,7 +620,7 @@ public void testFetchFullCacheEntryConcurrently() throws Exception {
                 environment,
                 settings,
                 threadPool,
-                ThreadPool.Names.GENERIC,
+                threadPool.executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -613,9 +635,14 @@ public void testFetchFullCacheEntryConcurrently() throws Exception {
                             (ActionListener<Void> listener) -> cacheService.maybeFetchFullEntry(
                                 cacheKey,
                                 size,
-                                (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept(
-                                    length
-                                ),
+                                (
+                                    channel,
+                                    channelPos,
+                                    streamFactory,
+                                    relativePos,
+                                    length,
+                                    progressUpdater,
+                                    completionListener) -> completeWith(completionListener, () -> progressUpdater.accept(length)),
                                 bulkExecutor,
                                 listener
                             )
@@ -801,7 +828,7 @@ public void testCacheSizeChanges() throws IOException {
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
-                ThreadPool.Names.GENERIC,
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -819,7 +846,7 @@ public void testCacheSizeChanges() throws IOException {
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
-                ThreadPool.Names.GENERIC,
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -840,15 +867,15 @@ public void testMaybeEvictLeastUsed() throws Exception {
         final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
         try (
             NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
-            var cacheService = new SharedBlobCacheService<>(
+            var cacheService = new SharedBlobCacheService<Object>(
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
-                ThreadPool.Names.GENERIC,
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
-            final Map<Object, SharedBlobCacheService<Object>.CacheFileRegion> cacheEntries = new HashMap<>();
+            final Map<Object, SharedBlobCacheService.CacheFileRegion<Object>> cacheEntries = new HashMap<>();
 
             assertThat("All regions are free", cacheService.freeRegionCount(), equalTo(numRegions));
             assertThat("Cache has no entries", cacheService.maybeEvictLeastUsed(), is(false));
@@ -859,7 +886,10 @@ public void testMaybeEvictLeastUsed() throws Exception {
                 var entry = cacheService.get(cacheKey, regionSize, 0);
                 entry.populate(
                     ByteRange.of(0L, regionSize),
-                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept(length),
+                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                        completionListener,
+                        () -> progressUpdater.accept(length)
+                    ),
                     taskQueue.getThreadPool().generic(),
                     ActionListener.noop()
                 );
@@ -939,7 +969,7 @@ public void execute(Runnable command) {
                 environment,
                 settings,
                 threadPool,
-                ThreadPool.Names.GENERIC,
+                threadPool.executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -954,11 +984,14 @@ public void execute(Runnable command) {
                     cacheKey,
                     0,
                     blobLength,
-                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> {
-                        assert streamFactory == null : streamFactory;
-                        bytesRead.addAndGet(length);
-                        progressUpdater.accept(length);
-                    },
+                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                        completionListener,
+                        () -> {
+                            assert streamFactory == null : streamFactory;
+                            bytesRead.addAndGet(length);
+                            progressUpdater.accept(length);
+                        }
+                    ),
                     bulkExecutor,
                     future
                 );
@@ -985,11 +1018,14 @@ public void execute(Runnable command) {
                         cacheKey,
                         region,
                         blobLength,
-                        (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> {
-                            assert streamFactory == null : streamFactory;
-                            bytesRead.addAndGet(length);
-                            progressUpdater.accept(length);
-                        },
+                        (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                            completionListener,
+                            () -> {
+                                assert streamFactory == null : streamFactory;
+                                bytesRead.addAndGet(length);
+                                progressUpdater.accept(length);
+                            }
+                        ),
                         bulkExecutor,
                         listener
                     );
@@ -1010,9 +1046,12 @@ public void execute(Runnable command) {
                     cacheKey,
                     randomIntBetween(0, 10),
                     randomLongBetween(1L, regionSize),
-                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> {
-                        throw new AssertionError("should not be executed");
-                    },
+                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                        completionListener,
+                        () -> {
+                            throw new AssertionError("should not be executed");
+                        }
+                    ),
                     bulkExecutor,
                     future
                 );
@@ -1032,11 +1071,14 @@ public void execute(Runnable command) {
                     cacheKey,
                     0,
                     blobLength,
-                    (channel, channelPos, ignore, relativePos, length, progressUpdater) -> {
-                        assert ignore == null : ignore;
-                        bytesRead.addAndGet(length);
-                        progressUpdater.accept(length);
-                    },
+                    (channel, channelPos, ignore, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                        completionListener,
+                        () -> {
+                            assert ignore == null : ignore;
+                            bytesRead.addAndGet(length);
+                            progressUpdater.accept(length);
+                        }
+                    ),
                     bulkExecutor,
                     future
                 );
@@ -1077,7 +1119,7 @@ public void execute(Runnable command) {
                 environment,
                 settings,
                 threadPool,
-                ThreadPool.Names.GENERIC,
+                threadPool.executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -1110,12 +1152,15 @@ public void execute(Runnable command) {
                     region,
                     range,
                     blobLength,
-                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> {
-                        assertThat(range.start() + relativePos, equalTo(cacheService.getRegionStart(region) + regionRange.start()));
-                        assertThat(channelPos, equalTo(Math.toIntExact(regionRange.start())));
-                        assertThat(length, equalTo(Math.toIntExact(regionRange.length())));
-                        bytesCopied.addAndGet(length);
-                    },
+                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                        completionListener,
+                        () -> {
+                            assertThat(range.start() + relativePos, equalTo(cacheService.getRegionStart(region) + regionRange.start()));
+                            assertThat(channelPos, equalTo(Math.toIntExact(regionRange.start())));
+                            assertThat(length, equalTo(Math.toIntExact(regionRange.length())));
+                            bytesCopied.addAndGet(length);
+                        }
+                    ),
                     bulkExecutor,
                     future
                 );
@@ -1150,7 +1195,10 @@ public void execute(Runnable command) {
                         region,
                         ByteRange.of(0L, blobLength),
                         blobLength,
-                        (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> bytesCopied.addAndGet(length),
+                        (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                            completionListener,
+                            () -> bytesCopied.addAndGet(length)
+                        ),
                         bulkExecutor,
                         listener
                     );
@@ -1173,9 +1221,12 @@ public void execute(Runnable command) {
                     randomIntBetween(0, 10),
                     ByteRange.of(0L, blobLength),
                     blobLength,
-                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> {
-                        throw new AssertionError("should not be executed");
-                    },
+                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                        completionListener,
+                        () -> {
+                            throw new AssertionError("should not be executed");
+                        }
+                    ),
                     bulkExecutor,
                     future
                 );
@@ -1196,7 +1247,10 @@ public void execute(Runnable command) {
                     0,
                     ByteRange.of(0L, blobLength),
                     blobLength,
-                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> bytesCopied.addAndGet(length),
+                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                        completionListener,
+                        () -> bytesCopied.addAndGet(length)
+                    ),
                     bulkExecutor,
                     future
                 );
@@ -1226,7 +1280,7 @@ public void testPopulate() throws Exception {
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
-                ThreadPool.Names.GENERIC,
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -1237,10 +1291,18 @@ public void testPopulate() throws Exception {
             var entry = cacheService.get(cacheKey, blobLength, 0);
             AtomicLong bytesWritten = new AtomicLong(0L);
             final PlainActionFuture<Boolean> future1 = new PlainActionFuture<>();
-            entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> {
-                bytesWritten.addAndGet(length);
-                progressUpdater.accept(length);
-            }, taskQueue.getThreadPool().generic(), future1);
+            entry.populate(
+                ByteRange.of(0, regionSize - 1),
+                (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                    completionListener,
+                    () -> {
+                        bytesWritten.addAndGet(length);
+                        progressUpdater.accept(length);
+                    }
+                ),
+                taskQueue.getThreadPool().generic(),
+                future1
+            );
 
             assertThat(future1.isDone(), is(false));
             assertThat(taskQueue.hasRunnableTasks(), is(true));
@@ -1248,18 +1310,34 @@ public void testPopulate() throws Exception {
             // start populating the second region
             entry = cacheService.get(cacheKey, blobLength, 1);
             final PlainActionFuture<Boolean> future2 = new PlainActionFuture<>();
-            entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> {
-                bytesWritten.addAndGet(length);
-                progressUpdater.accept(length);
-            }, taskQueue.getThreadPool().generic(), future2);
+            entry.populate(
+                ByteRange.of(0, regionSize - 1),
+                (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                    completionListener,
+                    () -> {
+                        bytesWritten.addAndGet(length);
+                        progressUpdater.accept(length);
+                    }
+                ),
+                taskQueue.getThreadPool().generic(),
+                future2
+            );
 
             // start populating again the first region, listener should be called immediately
             entry = cacheService.get(cacheKey, blobLength, 0);
             final PlainActionFuture<Boolean> future3 = new PlainActionFuture<>();
-            entry.populate(ByteRange.of(0, regionSize - 1), (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> {
-                bytesWritten.addAndGet(length);
-                progressUpdater.accept(length);
-            }, taskQueue.getThreadPool().generic(), future3);
+            entry.populate(
+                ByteRange.of(0, regionSize - 1),
+                (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                    completionListener,
+                    () -> {
+                        bytesWritten.addAndGet(length);
+                        progressUpdater.accept(length);
+                    }
+                ),
+                taskQueue.getThreadPool().generic(),
+                future3
+            );
 
             assertThat(future3.isDone(), is(true));
             var written = future3.get(10L, TimeUnit.SECONDS);
@@ -1318,7 +1396,7 @@ public void testUseFullRegionSize() throws IOException {
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
-                ThreadPool.Names.GENERIC,
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             ) {
                 @Override
@@ -1332,7 +1410,7 @@ protected int computeCacheFileRegionSize(long fileLength, int region) {
             final var blobLength = randomLongBetween(1L, cacheSize);
 
             int regions = Math.toIntExact(blobLength / regionSize);
-            regions += (blobLength % regionSize == 0L ? 0L : 1L);
+            regions += (blobLength % regionSize == 0 ? 0 : 1);
             assertThat(
                 cacheService.computeCacheFileRegionSize(blobLength, randomFrom(regions)),
                 equalTo(BlobCacheUtils.toIntBytes(regionSize))
@@ -1344,7 +1422,7 @@ protected int computeCacheFileRegionSize(long fileLength, int region) {
         }
     }
 
-    public void testSharedSourceInputStreamFactory() throws Exception {
+    public void testUsageSharedSourceInputStreamFactoryInCachePopulation() throws Exception {
         final long regionSizeInBytes = size(100);
         final Settings settings = Settings.builder()
             .put(NODE_NAME_SETTING.getKey(), "node")
@@ -1359,7 +1437,7 @@ public void testSharedSourceInputStreamFactory() throws Exception {
                 environment,
                 settings,
                 threadPool,
-                ThreadPool.Names.GENERIC,
+                threadPool.executor(ThreadPool.Names.GENERIC),
                 BlobCacheMetrics.NOOP
             )
         ) {
@@ -1377,7 +1455,10 @@ public void testSharedSourceInputStreamFactory() throws Exception {
                     range,
                     range,
                     (channel, channelPos, relativePos, length) -> length,
-                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater) -> progressUpdater.accept(length),
+                    (channel, channelPos, streamFactory, relativePos, length, progressUpdater, completionListener) -> completeWith(
+                        completionListener,
+                        () -> progressUpdater.accept(length)
+                    ),
                     EsExecutors.DIRECT_EXECUTOR_SERVICE,
                     future
                 );
@@ -1394,8 +1475,8 @@ public void testSharedSourceInputStreamFactory() throws Exception {
             final var factoryClosed = new AtomicBoolean(false);
             final var dummyStreamFactory = new SourceInputStreamFactory() {
                 @Override
-                public InputStream create(int relativePos) {
-                    return null;
+                public void create(int relativePos, ActionListener<InputStream> listener) {
+                    listener.onResponse(null);
                 }
 
                 @Override
@@ -1420,31 +1501,40 @@ public void fillCacheRange(
                     SourceInputStreamFactory streamFactory,
                     int relativePos,
                     int length,
-                    IntConsumer progressUpdater
+                    IntConsumer progressUpdater,
+                    ActionListener<Void> completion
                 ) throws IOException {
-                    if (invocationCounter.incrementAndGet() == 1) {
-                        final Thread witness = invocationThread.compareAndExchange(null, Thread.currentThread());
-                        assertThat(witness, nullValue());
-                    } else {
-                        assertThat(invocationThread.get(), sameInstance(Thread.currentThread()));
-                    }
-                    assertThat(streamFactory, sameInstance(dummyStreamFactory));
-                    assertThat(position.getAndSet(relativePos), lessThan(relativePos));
-                    progressUpdater.accept(length);
+                    completeWith(completion, () -> {
+                        if (invocationCounter.incrementAndGet() == 1) {
+                            final Thread witness = invocationThread.compareAndExchange(null, Thread.currentThread());
+                            assertThat(witness, nullValue());
+                        } else {
+                            assertThat(invocationThread.get(), sameInstance(Thread.currentThread()));
+                        }
+                        assertThat(streamFactory, sameInstance(dummyStreamFactory));
+                        assertThat(position.getAndSet(relativePos), lessThan(relativePos));
+                        progressUpdater.accept(length);
+                    });
                 }
             };
 
             final var range = ByteRange.of(0, regionSizeInBytes);
-            final PlainActionFuture<Integer> future = new PlainActionFuture<>();
-            region.populateAndRead(
-                range,
-                range,
-                (channel, channelPos, relativePos, length) -> length,
-                rangeMissingHandler,
-                threadPool.generic(),
-                future
-            );
-            safeGet(future);
+            if (randomBoolean()) {
+                final PlainActionFuture<Integer> future = new PlainActionFuture<>();
+                region.populateAndRead(
+                    range,
+                    range,
+                    (channel, channelPos, relativePos, length) -> length,
+                    rangeMissingHandler,
+                    threadPool.generic(),
+                    future
+                );
+                assertThat(safeGet(future).longValue(), equalTo(regionSizeInBytes));
+            } else {
+                final PlainActionFuture<Boolean> future = new PlainActionFuture<>();
+                region.populate(range, rangeMissingHandler, threadPool.generic(), future);
+                assertThat(safeGet(future), equalTo(true));
+            }
             assertThat(invocationCounter.get(), equalTo(numberGaps));
             assertThat(region.tracker.checkAvailable(regionSizeInBytes), is(true));
             assertBusy(() -> assertThat(factoryClosed.get(), is(true)));
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
index bbd1905374d24..2d77d2c770845 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
@@ -103,10 +103,7 @@ public String getFeatureDescription() {
     }
 
     public void testAutoFollow() throws Exception {
-        Settings leaderIndexSettings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            .build();
+        Settings leaderIndexSettings = indexSettings(1, 0).build();
 
         createLeaderIndex("logs-201812", leaderIndexSettings);
 
@@ -139,10 +136,7 @@ public void testAutoFollowDoNotFollowSystemIndices() throws Exception {
         // Trigger system index creation
         leaderClient().prepareIndex(FakeSystemIndex.SYSTEM_INDEX_NAME).setSource(Map.of("a", "b")).get();
 
-        Settings leaderIndexSettings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            .build();
+        Settings leaderIndexSettings = indexSettings(1, 0).build();
         createLeaderIndex("logs-201901", leaderIndexSettings);
         assertLongBusy(() -> {
             AutoFollowStats autoFollowStats = getAutoFollowStats();
@@ -153,10 +147,7 @@ public void testAutoFollowDoNotFollowSystemIndices() throws Exception {
     }
 
     public void testCleanFollowedLeaderIndexUUIDs() throws Exception {
-        Settings leaderIndexSettings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            .build();
+        Settings leaderIndexSettings = indexSettings(1, 0).build();
 
         putAutoFollowPatterns("my-pattern", new String[] { "logs-*" });
         createLeaderIndex("logs-201901", leaderIndexSettings);
@@ -192,10 +183,7 @@ public void testCleanFollowedLeaderIndexUUIDs() throws Exception {
     }
 
     public void testAutoFollowManyIndices() throws Exception {
-        Settings leaderIndexSettings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            .build();
+        Settings leaderIndexSettings = indexSettings(1, 0).build();
 
         putAutoFollowPatterns("my-pattern", new String[] { "logs-*" });
         long numIndices = randomIntBetween(4, 8);
@@ -267,10 +255,7 @@ public void testAutoFollowManyIndices() throws Exception {
     }
 
     public void testAutoFollowParameterAreDelegated() throws Exception {
-        Settings leaderIndexSettings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            .build();
+        Settings leaderIndexSettings = indexSettings(1, 0).build();
 
         // Enabling auto following:
         PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
@@ -377,10 +362,7 @@ public void testAutoFollowParameterAreDelegated() throws Exception {
     }
 
     public void testConflictingPatterns() throws Exception {
-        Settings leaderIndexSettings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            .build();
+        Settings leaderIndexSettings = indexSettings(1, 0).build();
 
         // Enabling auto following:
         putAutoFollowPatterns("my-pattern1", new String[] { "logs-*" });
@@ -422,10 +404,7 @@ public void testConflictingPatterns() throws Exception {
     }
 
     public void testPauseAndResumeAutoFollowPattern() throws Exception {
-        final Settings leaderIndexSettings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            .build();
+        final Settings leaderIndexSettings = indexSettings(1, 0).build();
 
         // index created in the remote cluster before the auto follow pattern exists won't be auto followed
         createLeaderIndex("test-existing-index-is-ignored", leaderIndexSettings);
@@ -504,10 +483,7 @@ public void testPauseAndResumeAutoFollowPattern() throws Exception {
     }
 
     public void testPauseAndResumeWithMultipleAutoFollowPatterns() throws Exception {
-        final Settings leaderIndexSettings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            .build();
+        final Settings leaderIndexSettings = indexSettings(1, 0).build();
 
         final String[] prefixes = { "logs-", "users-", "docs-", "monitoring-", "data-", "system-", "events-", "files-" };
 
@@ -609,10 +585,7 @@ public void testPauseAndResumeWithMultipleAutoFollowPatterns() throws Exception
     }
 
     public void testAutoFollowExclusion() throws Exception {
-        Settings leaderIndexSettings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-            .build();
+        Settings leaderIndexSettings = indexSettings(1, 0).build();
 
         putAutoFollowPatterns("my-pattern1", new String[] { "logs-*" }, Collections.singletonList("logs-2018*"));
 
@@ -655,7 +628,11 @@ public void testAutoFollowDatastreamWithClosingFollowerIndex() throws Exception
         );
         assertAcked(leaderClient().execute(TransportPutComposableIndexTemplateAction.TYPE, request).get());
 
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(datastream);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            datastream
+        );
         assertAcked(leaderClient().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get());
         leaderClient().prepareIndex(datastream)
             .setCreate(true)
@@ -701,7 +678,11 @@ public void testAutoFollowDatastreamWithClosingFollowerIndex() throws Exception
             .get();
         leaderClient().execute(
             ModifyDataStreamsAction.INSTANCE,
-            new ModifyDataStreamsAction.Request(List.of(DataStreamAction.addBackingIndex(datastream, indexInDatastream)))
+            new ModifyDataStreamsAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                List.of(DataStreamAction.addBackingIndex(datastream, indexInDatastream))
+            )
         ).get();
 
         assertLongBusy(() -> {
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRInfoTransportAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRInfoTransportAction.java
index 84586d6b2fa06..d4e31f25a0a91 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRInfoTransportAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRInfoTransportAction.java
@@ -9,10 +9,10 @@
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.XContentBuilder;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRUsageTransportAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRUsageTransportAction.java
index 283e95b000f3e..9f3b9a3bf16dc 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRUsageTransportAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRUsageTransportAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
index efea78ee4730a..395af34c59b3a 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
@@ -192,7 +192,7 @@ public Collection<?> createComponents(PluginServices services) {
                 client,
                 services.clusterService(),
                 ccrLicenseChecker,
-                services.threadPool()::relativeTimeInMillis,
+                services.threadPool().relativeTimeInMillisSupplier(),
                 services.threadPool()::absoluteTimeInMillis,
                 services.threadPool().executor(Ccr.CCR_THREAD_POOL_NAME)
             )
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
index ca6fb5683e540..167c5f66300a9 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -40,6 +39,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RawIndexingDataTransportRequest;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java
index a4665ae56f08d..eef4ffa9f2499 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java
@@ -18,9 +18,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java
index b089c1a4cb84a..dfc0b0dd37ac1 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java
index 0f41b6ba05dc9..1cc3510b73117 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java
@@ -18,10 +18,10 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java
index a66a79a0f7d76..7bc33ce2a74cb 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Predicates;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java
index 44710372058a3..6d84682815769 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java
index 312adde64a68a..431210264ebbf 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Assertions;
@@ -30,6 +29,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java
index d596eae43abde..532773f857d97 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java
index 8207228f0d21c..17cd10c6c3e46 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java
index ef27988d0416f..b990b738e4bc9 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java
@@ -23,10 +23,10 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
index a74aa4c323426..d8e634a297bfa 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreClusterStateListener;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.action.support.ActionFilters;
@@ -26,12 +27,12 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.snapshots.RestoreInfo;
 import org.elasticsearch.snapshots.RestoreService;
@@ -206,15 +207,17 @@ private void createFollowerIndex(
         ActionListener<RestoreService.RestoreCompletionResponse> delegatelistener = listener.delegateFailure(
             (delegatedListener, response) -> afterRestoreStarted(clientWithHeaders, request, delegatedListener, response)
         );
+
+        final BiConsumer<ClusterState, Metadata.Builder> updater;
         if (remoteDataStream == null) {
             // If the index we're following is not part of a data stream, start the
             // restoration of the index normally.
-            restoreService.restoreSnapshot(restoreRequest, delegatelistener);
+            updater = (clusterState, mdBuilder) -> {};
         } else {
             String followerIndexName = request.getFollowerIndex();
             // This method is used to update the metadata in the same cluster state
             // update as the snapshot is restored.
-            BiConsumer<ClusterState, Metadata.Builder> updater = (currentState, mdBuilder) -> {
+            updater = (currentState, mdBuilder) -> {
                 final String localDataStreamName;
 
                 // If we have been given a data stream name, use that name for the local
@@ -239,8 +242,9 @@ private void createFollowerIndex(
                 );
                 mdBuilder.put(updatedDataStream);
             };
-            restoreService.restoreSnapshot(restoreRequest, delegatelistener, updater);
         }
+        threadPool.executor(ThreadPool.Names.SNAPSHOT_META)
+            .execute(ActionRunnable.wrap(delegatelistener, l -> restoreService.restoreSnapshot(restoreRequest, l, updater)));
     }
 
     private void afterRestoreStarted(
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
index f57e0f0a85a43..01a4076c58bd8 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -42,6 +41,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesRequestCache;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
index 5e7204c7fb972..addf47662276d 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -37,6 +36,7 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.seqno.RetentionLeaseNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java
index 2d0c43315f746..d9e26ae0708c2 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.action.support.replication.TransportWriteAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Releasable;
@@ -29,6 +28,7 @@
 import org.elasticsearch.indices.ExecutorSelector;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -69,7 +69,8 @@ public TransportBulkShardOperationsAction(
             ExecutorSelector.getWriteExecutorForShard(threadPool),
             PrimaryActionExecution.RejectOnOverload,
             indexingPressure,
-            systemIndices
+            systemIndices,
+            ReplicaActionExecution.SubjectToCircuitBreaker
         );
     }
 
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java
index 7e1769e6528ac..66210d43f2f7a 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.action.RemoteClusterActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportActionProxy;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java
index f8e4cda1501b6..61922ee0b8878 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java
@@ -12,7 +12,8 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
@@ -38,7 +39,7 @@ public TransportDeleteInternalRepositoryAction(
             ActionFilters actionFilters,
             TransportService transportService
         ) {
-            super(NAME, actionFilters, transportService.getTaskManager());
+            super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
             this.repositoriesService = repositoriesService;
         }
 
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java
index 5b1382a3ec09a..63d5b1f95b228 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java
@@ -15,12 +15,12 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ByteArray;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportActionProxy;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java
index 3641cda5fc7d9..e090b50480d55 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -26,6 +25,7 @@
 import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java
index 0de1715e17a14..ee44b43037603 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java
@@ -12,7 +12,8 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
@@ -38,7 +39,7 @@ public TransportPutInternalRepositoryAction(
             ActionFilters actionFilters,
             TransportService transportService
         ) {
-            super(NAME, actionFilters, transportService.getTaskManager());
+            super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
             this.repositoriesService = repositoriesService;
         }
 
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
index d5a6e3c7e65c8..97e3a409d590d 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
@@ -82,7 +82,6 @@
 import org.elasticsearch.repositories.SnapshotShardContext;
 import org.elasticsearch.repositories.blobstore.FileRestoreContext;
 import org.elasticsearch.snapshots.Snapshot;
-import org.elasticsearch.snapshots.SnapshotDeleteListener;
 import org.elasticsearch.snapshots.SnapshotException;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
@@ -371,9 +370,10 @@ public void deleteSnapshots(
         Collection<SnapshotId> snapshotIds,
         long repositoryDataGeneration,
         IndexVersion minimumNodeVersion,
-        SnapshotDeleteListener listener
+        ActionListener<RepositoryData> repositoryDataUpdateListener,
+        Runnable onCompletion
     ) {
-        listener.onFailure(new UnsupportedOperationException("Unsupported for repository of type: " + TYPE));
+        repositoryDataUpdateListener.onFailure(new UnsupportedOperationException("Unsupported for repository of type: " + TYPE));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierTelemetryPlugin.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierTelemetryPlugin.java
index b67d4e417f97c..df13cfeb1740c 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierTelemetryPlugin.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierTelemetryPlugin.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse;
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java
index 973b94f05c114..cb790399bd2df 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/DataTiersUsageRestCancellationIT.java
@@ -20,9 +20,9 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.test.ESIntegTestCase;
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/XPackUsageRestCancellationIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/XPackUsageRestCancellationIT.java
index d019851263f6b..f17a89774f71d 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/XPackUsageRestCancellationIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/XPackUsageRestCancellationIT.java
@@ -23,9 +23,9 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java
index 5834f0484df06..10d47a724dc40 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java
@@ -69,7 +69,11 @@ public void testRollover() throws Exception {
         registry.clusterChanged(new ClusterChangedEvent(IndexTemplateRegistryRolloverIT.class.getName(), state, state));
         assertBusy(() -> { assertTrue(clusterService.state().metadata().templatesV2().containsKey(TEST_INDEX_TEMPLATE_ID)); });
         String dsName = TEST_INDEX_PATTERN.replace('*', '1');
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dsName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dsName
+        );
         AcknowledgedResponse acknowledgedResponse = client.execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         assertTrue(acknowledgedResponse.isAcknowledged());
         assertNumberOfBackingIndices(1);
@@ -94,7 +98,10 @@ private void assertNumberOfBackingIndices(final int expected) {
     }
 
     private DataStream getDataStream() {
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { TEST_INDEX_PATTERN });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { TEST_INDEX_PATTERN }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
         List<GetDataStreamAction.Response.DataStreamInfo> dataStreams = getDataStreamResponse.getDataStreams();
         assertThat(dataStreams, hasSize(1));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java
index 0a13aab82aced..3b242ca94ac61 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java
@@ -63,8 +63,6 @@ public final class FrozenEngine extends ReadOnlyEngine {
     );
     private final SegmentsStats segmentsStats;
     private final DocsStats docsStats;
-    private final DenseVectorStats denseVectorStats;
-    private final SparseVectorStats sparseVectorStats;
     private volatile ElasticsearchDirectoryReader lastOpenedReader;
     private final ElasticsearchDirectoryReader canMatchReader;
     private final Object cacheIdentity = new Object();
@@ -95,8 +93,6 @@ public FrozenEngine(
                 fillSegmentStats(segmentReader, true, segmentsStats);
             }
             this.docsStats = docsStats(reader);
-            this.denseVectorStats = denseVectorStats(reader);
-            this.sparseVectorStats = sparseVectorStats(reader, null);
             canMatchReader = ElasticsearchDirectoryReader.wrap(
                 new RewriteCachingDirectoryReader(directory, reader.leaves(), null),
                 config.getShardId()
@@ -334,13 +330,17 @@ public DocsStats docStats() {
     }
 
     @Override
-    public DenseVectorStats denseVectorStats() {
-        return denseVectorStats;
+    public DenseVectorStats denseVectorStats(MappingLookup mappingLookup) {
+        // We could cache the result on first call but dense vectors on frozen tier
+        // are very unlikely, so we just don't count them in the stats.
+        return new DenseVectorStats(0);
     }
 
     @Override
     public SparseVectorStats sparseVectorStats(MappingLookup mappingLookup) {
-        return sparseVectorStats;
+        // We could cache the result on first call but sparse vectors on frozen tier
+        // are very unlikely, so we just don't count them in the stats.
+        return new SparseVectorStats(0);
     }
 
     synchronized boolean isReaderOpen() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java
index b352a9abce886..f5123b9352fe3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java
@@ -289,11 +289,11 @@ public void triggered(SchedulerEngine.Event event) {
         final LicensesMetadata licensesMetadata = getLicensesMetadata();
         if (licensesMetadata != null) {
             final License license = licensesMetadata.getLicense();
-            if (event.getJobName().equals(LICENSE_JOB)) {
+            if (event.jobName().equals(LICENSE_JOB)) {
                 updateXPackLicenseState(license);
-            } else if (event.getJobName().startsWith(ExpirationCallback.EXPIRATION_JOB_PREFIX)) {
+            } else if (event.jobName().startsWith(ExpirationCallback.EXPIRATION_JOB_PREFIX)) {
                 expirationCallbacks.stream()
-                    .filter(expirationCallback -> expirationCallback.getId().equals(event.getJobName()))
+                    .filter(expirationCallback -> expirationCallback.getId().equals(event.jobName()))
                     .forEach(expirationCallback -> expirationCallback.on(license));
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartTrialClusterTask.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartTrialClusterTask.java
index 67731b03d3e65..22f4de105cb2d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartTrialClusterTask.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/StartTrialClusterTask.java
@@ -24,7 +24,7 @@
 
 public class StartTrialClusterTask implements ClusterStateTaskListener {
 
-    private static final String ACKNOWLEDGEMENT_HEADER = "This API initiates a free 30-day trial for all platinum features. "
+    private static final String ACKNOWLEDGEMENT_HEADER = "This API initiates a free 30-day trial for all subscription features. "
         + "By starting this trial, you agree that it is subject to the terms and conditions at"
         + " https://www.elastic.co/legal/trial_license/. To begin your free trial, call /start_trial again and specify "
         + "the \"acknowledge=true\" parameter.";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
index 5739655c88eea..5697b0065f2c9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.internal.MutableLicenseService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java
index ec7cc6526ee53..ea8e8c84940b4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetFeatureUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetFeatureUsageAction.java
index 00023f9c30e70..ff6df38d9d742 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetFeatureUsageAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetFeatureUsageAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java
index 0f6b2b201298d..c2135772194bd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java
index 8f2655b0ae64f..78b00286f0d72 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java
index e3de0a8797e5c..556e3ea80dfb1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.internal.MutableLicenseService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java
index 27f7cbff2e3ec..8a7ead2caf3be 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.internal.MutableLicenseService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java
index 6b9846e636f05..4990dfb2d4ae4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.internal.MutableLicenseService;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
index d27d7a21ddb73..4e7aa37fe1a0b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
@@ -194,6 +194,7 @@ private static String maybeRewriteSingleAuthenticationHeaderForVersion(
     public static final String CONNECTORS_ORIGIN = "connectors";
     public static final String INFERENCE_ORIGIN = "inference";
     public static final String APM_ORIGIN = "apm";
+    public static final String OTEL_ORIGIN = "otel";
 
     private ClientHelper() {}
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/HealthApiUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/HealthApiUsageTransportAction.java
index bd641d1eb8c17..06393dfa3bade 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/HealthApiUsageTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/HealthApiUsageTransportAction.java
@@ -13,10 +13,10 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.health.stats.HealthApiStatsAction;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/RemoteClusterUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/RemoteClusterUsageTransportAction.java
index e78dfdd5eac15..7d8c6830d0def 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/RemoteClusterUsageTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/RemoteClusterUsageTransportAction.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index a2c3e40c76ae4..2e806a24ad469 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -149,7 +149,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             new NamedWriteableRegistry.Entry(ClusterState.Custom.class, TokenMetadata.TYPE, TokenMetadata::new),
             new NamedWriteableRegistry.Entry(NamedDiff.class, TokenMetadata.TYPE, TokenMetadata::readDiffFrom),
             new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SECURITY, SecurityFeatureSetUsage::new),
-            // security : conditional privileges
+            // security : configurable cluster privileges
             new NamedWriteableRegistry.Entry(
                 ConfigurableClusterPrivilege.class,
                 ConfigurableClusterPrivileges.ManageApplicationPrivileges.WRITEABLE_NAME,
@@ -160,6 +160,11 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 ConfigurableClusterPrivileges.WriteProfileDataPrivileges.WRITEABLE_NAME,
                 ConfigurableClusterPrivileges.WriteProfileDataPrivileges::createFrom
             ),
+            new NamedWriteableRegistry.Entry(
+                ConfigurableClusterPrivilege.class,
+                ConfigurableClusterPrivileges.ManageRolesPrivilege.WRITEABLE_NAME,
+                ConfigurableClusterPrivileges.ManageRolesPrivilege::createFrom
+            ),
             // security : role-mappings
             new NamedWriteableRegistry.Entry(Metadata.Custom.class, RoleMappingMetadata.TYPE, RoleMappingMetadata::new),
             new NamedWriteableRegistry.Entry(NamedDiff.class, RoleMappingMetadata.TYPE, RoleMappingMetadata::readDiffFrom),
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
index d33b2aecdab04..f76b0d2bb6d8d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
@@ -93,6 +93,13 @@ public Iterator<Setting<?>> settings() {
     /** Setting for enabling or disabling APM Data. Defaults to true. */
     public static final Setting<Boolean> APM_DATA_ENABLED = Setting.boolSetting("xpack.apm_data.enabled", true, Setting.Property.NodeScope);
 
+    /** Setting for enabling or disabling OTel Data. Defaults to true. */
+    public static final Setting<Boolean> OTEL_DATA_ENABLED = Setting.boolSetting(
+        "xpack.otel_data.enabled",
+        true,
+        Setting.Property.NodeScope
+    );
+
     /** Setting for enabling or disabling enterprise search. Defaults to true. */
     public static final Setting<Boolean> ENTERPRISE_SEARCH_ENABLED = Setting.boolSetting(
         "xpack.ent_search.enabled",
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportSetResetModeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportSetResetModeAction.java
index 6e570d2f6fa17..d3133f071c70c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportSetResetModeAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportSetResetModeAction.java
@@ -21,9 +21,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamInfoTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamInfoTransportAction.java
index d12f3dbb7f650..d05b6665b7c57 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamInfoTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamInfoTransportAction.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.core.action;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportAction.java
index 947adf9f8462f..657a13aa0f00c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportAction.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java
index ba76788d695d4..26f3bdd7654ca 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java
index 46bef2214de99..84e27b08c1d38 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.LicenseUtils;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
index e9be5a62c8eb6..3f26dfdd78ca8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncExecutionId.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncExecutionId.java
index b4cc8bc0d3f30..8316b4cfa605a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncExecutionId.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncExecutionId.java
@@ -21,6 +21,9 @@
  * A class that contains all information related to a submitted async execution.
  */
 public final class AsyncExecutionId {
+    public static final String ASYNC_EXECUTION_ID_HEADER = "X-Elasticsearch-Async-Id";
+    public static final String ASYNC_EXECUTION_IS_RUNNING_HEADER = "X-Elasticsearch-Async-Is-Running";
+
     private final String docId;
     private final TaskId taskId;
     private final String encoded;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/TransportDeleteAsyncResultAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/TransportDeleteAsyncResultAction.java
index 2a80d8fede821..8c67c69f17ef6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/TransportDeleteAsyncResultAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/TransportDeleteAsyncResultAction.java
@@ -15,10 +15,10 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
index a0ecc5dd566c2..1ac164d600115 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
@@ -40,7 +40,7 @@ public static class Request extends MasterNodeRequest<Request> {
 
         public Request(StreamInput in) throws IOException {
             super(in);
-            if (in.getTransportVersion().onOrAfter(TransportVersions.CCR_STATS_API_TIMEOUT_PARAM)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 timeout = in.readOptionalTimeValue();
             }
         }
@@ -57,7 +57,7 @@ public ActionRequestValidationException validate() {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.CCR_STATS_API_TIMEOUT_PARAM)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 out.writeOptionalTimeValue(timeout);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersInfoTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersInfoTransportAction.java
index 3af1945c53d3f..4754021fa6cfd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersInfoTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersInfoTransportAction.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.core.datatiers;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java
index 728309926302a..7308ce028cbd4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java
@@ -22,9 +22,9 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.indices.NodeIndicesStats;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.search.aggregations.metrics.TDigestState;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java
index c1903a2910629..5a91e997ca5fe 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodeDataTiersUsage.java
@@ -108,6 +108,6 @@ public Map<String, UsageStats> getUsageStatsByTier() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeMap(usageStatsByTier, (o, v) -> v.writeTo(o));
+        out.writeMap(usageStatsByTier, StreamOutput::writeWriteable);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java
index b89b73f58c9b2..eb35ba651df20 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.features.NodeFeature;
@@ -32,6 +31,7 @@
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.NodeIndicesStats;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java
index 282f32da28a6b..663978088f22a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.DataStream;
@@ -52,7 +53,10 @@ public void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState cu
                 // This is the last backing index in the data stream, and it's being deleted because the policy doesn't have a rollover
                 // phase. The entire stream needs to be deleted, because we can't have an empty list of data stream backing indices.
                 // We do this even if there are multiple failure store indices because otherwise we would never delete the index.
-                DeleteDataStreamAction.Request deleteReq = new DeleteDataStreamAction.Request(new String[] { dataStream.getName() });
+                DeleteDataStreamAction.Request deleteReq = new DeleteDataStreamAction.Request(
+                    MasterNodeRequest.infiniteMasterNodeTimeout(currentState.getMinTransportVersion()),
+                    dataStream.getName()
+                );
                 getClient().execute(
                     DeleteDataStreamAction.INSTANCE,
                     deleteReq,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java
index 60f9b7b001060..9cbcd6c62dc3b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java
@@ -106,7 +106,7 @@ public ShrinkAction(StreamInput in) throws IOException {
             this.numberOfShards = null;
             this.maxPrimaryShardSize = ByteSizeValue.readFrom(in);
         }
-        this.allowWriteAfterShrink = in.getTransportVersion().onOrAfter(TransportVersions.ILM_SHRINK_ENABLE_WRITE) && in.readBoolean();
+        this.allowWriteAfterShrink = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) && in.readBoolean();
     }
 
     public Integer getNumberOfShards() {
@@ -130,7 +130,7 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             maxPrimaryShardSize.writeTo(out);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ILM_SHRINK_ENABLE_WRITE)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeBoolean(this.allowWriteAfterShrink);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java
index 229285510249c..c38f508db1b6a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java
@@ -14,8 +14,12 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -24,8 +28,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.inference.results.LegacyTextEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
@@ -34,9 +37,11 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.Flow;
 
 import static org.elasticsearch.core.Strings.format;
 
@@ -122,15 +127,11 @@ public Request(StreamInput in) throws IOException {
                 this.inputType = InputType.UNSPECIFIED;
             }
 
-            if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_COHERE_RERANK)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 this.query = in.readOptionalString();
-            } else {
-                this.query = null;
-            }
-
-            if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_TIMEOUT_ADDED)) {
                 this.inferenceTimeout = in.readTimeValue();
             } else {
+                this.query = null;
                 this.inferenceTimeout = DEFAULT_TIMEOUT;
             }
         }
@@ -209,11 +210,8 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeEnum(getInputTypeToWrite(inputType, out.getTransportVersion()));
             }
 
-            if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_COHERE_RERANK)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 out.writeOptionalString(query);
-            }
-
-            if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_TIMEOUT_ADDED)) {
                 out.writeTimeValue(inferenceTimeout);
             }
         }
@@ -325,7 +323,7 @@ public String toString() {
         }
     }
 
-    public static class Response extends ActionResponse implements ToXContentObject {
+    public static class Response extends ActionResponse implements ChunkedToXContentObject {
 
         private final InferenceServiceResults results;
 
@@ -395,6 +393,24 @@ public InferenceServiceResults getResults() {
             return results;
         }
 
+        /**
+         * Returns {@code true} if these results are streamed as chunks, or {@code false} if these results contain the entire payload.
+         * Currently set to false while it is being implemented.
+         */
+        public boolean isStreaming() {
+            return false;
+        }
+
+        /**
+         * When {@link #isStreaming()} is {@code true}, the RestHandler will subscribe to this publisher.
+         * When the RestResponse is finished with the current chunk, it will request the next chunk using the subscription.
+         * If the RestResponse is closed, it will cancel the subscription.
+         */
+        public Flow.Publisher<ChunkedToXContent> publisher() {
+            assert isStreaming() == false : "This must be implemented when isStreaming() == true";
+            throw new UnsupportedOperationException("This must be implemented when isStreaming() == true");
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
@@ -405,11 +421,12 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            results.toXContent(builder, params);
-            builder.endObject();
-            return builder;
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+            return Iterators.concat(
+                ChunkedToXContentHelper.startObject(),
+                results.toXContentChunked(params),
+                ChunkedToXContentHelper.endObject()
+            );
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java
index bbd4d026f0d55..902c69cef558e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java
@@ -10,12 +10,15 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -46,13 +49,8 @@ public ChatCompletionResults(StreamInput in) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(COMPLETION);
-        for (Result result : results) {
-            result.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContentHelper.array(COMPLETION, results.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java
index 376b8763a5eb9..18f88a8ff022a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java
@@ -11,10 +11,11 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.inference.InferenceResults;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -89,9 +90,8 @@ public String toString() {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(NAME, exception.getMessage());
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContentHelper.field(NAME, exception.getMessage());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java
index f1265873ad6dd..187b186fcd91d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.xcontent.ToXContent;
@@ -77,13 +78,8 @@ public List<MlChunkedTextExpansionResults.ChunkedResult> getChunkedResults() {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(FIELD_NAME);
-        for (MlChunkedTextExpansionResults.ChunkedResult chunk : chunkedResults) {
-            chunk.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContentHelper.array(FIELD_NAME, chunkedResults.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java
index b78bce8c5c2cd..cc245c40c51e3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java
@@ -12,8 +12,10 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.inference.InferenceResults;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -61,14 +63,8 @@ public InferenceChunkedTextEmbeddingByteResults(StreamInput in) throws IOExcepti
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        // TODO add isTruncated flag
-        builder.startArray(FIELD_NAME);
-        for (var embedding : chunks) {
-            embedding.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContentHelper.array(FIELD_NAME, chunks.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java
index 9fead334dcbc0..4b4d77cd3f043 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.xcontent.ToXContent;
@@ -74,14 +75,9 @@ public static InferenceChunkedTextEmbeddingFloatResults ofMlResults(MlChunkedTex
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         // TODO add isTruncated flag
-        builder.startArray(FIELD_NAME);
-        for (var embedding : chunks) {
-            embedding.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
+        return ChunkedToXContentHelper.array(FIELD_NAME, chunks.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java
index 8d94083bf3241..16dca7b04d526 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java
@@ -13,8 +13,10 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults;
@@ -22,6 +24,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -58,13 +61,8 @@ public int getFirstEmbeddingSize() {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(TEXT_EMBEDDING_BYTES);
-        for (InferenceByteEmbedding embedding : embeddings) {
-            embedding.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContentHelper.array(TEXT_EMBEDDING_BYTES, embeddings.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java
index 1822e3af28c2d..9f9bdfec7cfae 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java
@@ -14,10 +14,12 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults;
@@ -25,6 +27,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -99,13 +102,8 @@ public int getFirstEmbeddingSize() {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(TEXT_EMBEDDING);
-        for (InferenceFloatEmbedding embedding : embeddings) {
-            embedding.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContentHelper.array(TEXT_EMBEDDING, embeddings.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java
index f82ee8b73c7a2..6ebf15bf34937 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java
@@ -7,27 +7,30 @@
 
 package org.elasticsearch.xpack.core.inference.results;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.TransportVersions.ML_INFERENCE_RERANK_NEW_RESPONSE_FORMAT;
 import static org.elasticsearch.TransportVersions.ML_RERANK_DOC_OPTIONAL;
 
 public class RankedDocsResults implements InferenceServiceResults {
@@ -113,7 +116,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         public static RankedDoc of(StreamInput in) throws IOException {
             if (in.getTransportVersion().onOrAfter(ML_RERANK_DOC_OPTIONAL)) {
                 return new RankedDoc(in.readInt(), in.readFloat(), in.readOptionalString());
-            } else if (in.getTransportVersion().onOrAfter(ML_INFERENCE_RERANK_NEW_RESPONSE_FORMAT)) {
+            } else if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 return new RankedDoc(in.readInt(), in.readFloat(), in.readString());
             } else {
                 return new RankedDoc(Integer.parseInt(in.readString()), Float.parseFloat(in.readString()), in.readString());
@@ -126,7 +129,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeInt(index);
                 out.writeFloat(relevanceScore);
                 out.writeOptionalString(text);
-            } else if (out.getTransportVersion().onOrAfter(ML_INFERENCE_RERANK_NEW_RESPONSE_FORMAT)) {
+            } else if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 out.writeInt(index);
                 out.writeFloat(relevanceScore);
                 out.writeString(text == null ? "" : text);
@@ -172,13 +175,8 @@ public List<RankedDoc> getRankedDocs() {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(RERANK);
-        for (RankedDoc rankedDoc : rankedDocs) {
-            rankedDoc.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContentHelper.array(RERANK, rankedDocs.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
index 1db6dcc802d00..dd8229c604ecb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
@@ -12,10 +12,12 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults;
@@ -23,6 +25,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -68,15 +71,8 @@ public static SparseEmbeddingResults of(List<? extends InferenceResults> results
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(SPARSE_EMBEDDING);
-
-        for (Embedding embedding : embeddings) {
-            embedding.toXContent(builder, params);
-        }
-
-        builder.endArray();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return ChunkedToXContentHelper.array(SPARSE_EMBEDDING, embeddings.iterator());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
index 59eaf4affa9a8..7ae9a4a4b2d58 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
@@ -357,7 +357,7 @@ public ActionRequestValidationException validate() {
                 if (numberOfAllocations < 1) {
                     validationException.addValidationError("[" + NUMBER_OF_ALLOCATIONS + "] must be a positive integer");
                 }
-                if (adaptiveAllocationsSettings != null && adaptiveAllocationsSettings.getEnabled()) {
+                if (adaptiveAllocationsSettings != null && adaptiveAllocationsSettings.getEnabled() == Boolean.TRUE) {
                     validationException.addValidationError(
                         "[" + NUMBER_OF_ALLOCATIONS + "] cannot be set if adaptive allocations is enabled"
                     );
@@ -620,6 +620,9 @@ public String getDeploymentId() {
             return deploymentId;
         }
 
+        /**
+         * @return the estimated memory (in bytes) required for the model deployment to run
+         */
         public long estimateMemoryUsageBytes() {
             // We already take into account 2x the model bytes. If the cache size is larger than the model bytes, then
             // we need to take it into account when returning the estimate.
@@ -729,10 +732,16 @@ public long getModelBytes() {
             return modelBytes;
         }
 
+        /**
+         * @return the number of threads per allocation used by the model during inference. each thread requires one processor.
+         */
         public int getThreadsPerAllocation() {
             return threadsPerAllocation;
         }
 
+        /**
+         * @return the number of allocations requested by the user
+         */
         public int getNumberOfAllocations() {
             return numberOfAllocations;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java
index 28152bc0d5556..61601c402e8e7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java
@@ -167,7 +167,9 @@ public ActionRequestValidationException validate() {
                 if (numberOfAllocations < 1) {
                     validationException.addValidationError("[" + NUMBER_OF_ALLOCATIONS + "] must be a positive integer");
                 }
-                if (isInternal == false && adaptiveAllocationsSettings != null && adaptiveAllocationsSettings.getEnabled()) {
+                if (isInternal == false
+                    && adaptiveAllocationsSettings != null
+                    && adaptiveAllocationsSettings.getEnabled() == Boolean.TRUE) {
                     validationException.addValidationError(
                         "[" + NUMBER_OF_ALLOCATIONS + "] cannot be set if adaptive allocations is enabled"
                     );
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/autoscaling/MlAutoscalingStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/autoscaling/MlAutoscalingStats.java
index 645d80525cf6b..ffadf4cafaf12 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/autoscaling/MlAutoscalingStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/autoscaling/MlAutoscalingStats.java
@@ -3,6 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
+ *
+ * this file was contributed to by a Generative AI
  */
 
 package org.elasticsearch.xpack.core.ml.autoscaling;
@@ -13,48 +15,79 @@
 
 import java.io.IOException;
 
+/**
+ * MlAutoscalingStats is the record which is transmitted to the elasticsearch-autoscaler to decide which nodes to deliver.
+ * <p>
+ * The "existing" attributes exist only so that the autoscaler can confirm that elasticsearch has the same view of the available hardware.
+ * <p>
+ * The "extra" attributes are used to communicate the additional resources that are required.
+ * <p>
+ * The "perNode" attributes define the minimum amount of resources that must be available on every node.
+ * <p>
+ * unwantedNodeMemoryBytesToRemove is used to communicate the amount of memory that should be removed from the node.
+ * No attribute exists to remove processors.
+ * <p>
+ * The word "total" in an attribute name indicates that the attribute is a sum across all nodes.
+ *
+ * @param currentTotalNodes                 the count of nodes that are currently in the cluster
+ * @param currentPerNodeMemoryBytes         the minimum size (memory) of all nodes in the cluster
+ * @param currentTotalModelMemoryBytes      the sum of model memory over every assignment/deployment
+ * @param currentTotalProcessorsInUse       the sum of processors used over every assignment/deployment
+ * @param currentPerNodeMemoryOverheadBytes always equal to MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD
+ * @param wantedMinNodes                    the minimum number of nodes that must be provided by the autoscaler
+ * @param wantedExtraPerNodeMemoryBytes     the amount of additional memory that must be provided on every node
+ *                                          (this value must be >0 to trigger a scale up based on memory)
+ * @param wantedExtraPerNodeNodeProcessors  the number of additional processors that must be provided on every node
+ *                                          (this value must be >0 to trigger a scale up based on processors)
+ * @param wantedExtraModelMemoryBytes       the amount of additional model memory that is newly required
+ *                                          (due to a new assignment/deployment)
+ * @param wantedExtraProcessors             the number of additional processors that are required to be added to the cluster
+ * @param unwantedNodeMemoryBytesToRemove   the amount of memory that should be removed from the cluster. If this is equal to the amount of
+ *                                          memory provided by a node, a node will be removed.
+ */
+
 public record MlAutoscalingStats(
-    int nodes,
-    long perNodeMemoryInBytes,
-    long modelMemoryInBytesSum,
-    int processorsSum,
-    int minNodes,
-    long extraSingleNodeModelMemoryInBytes,
-    int extraSingleNodeProcessors,
-    long extraModelMemoryInBytes,
-    int extraProcessors,
-    long removeNodeMemoryInBytes,
-    long perNodeMemoryOverheadInBytes
+    int currentTotalNodes,
+    long currentPerNodeMemoryBytes,
+    long currentTotalModelMemoryBytes,
+    int currentTotalProcessorsInUse,
+    int wantedMinNodes,
+    long wantedExtraPerNodeMemoryBytes,
+    int wantedExtraPerNodeNodeProcessors,
+    long wantedExtraModelMemoryBytes,
+    int wantedExtraProcessors,
+    long unwantedNodeMemoryBytesToRemove,
+    long currentPerNodeMemoryOverheadBytes
 ) implements Writeable {
 
     public MlAutoscalingStats(StreamInput in) throws IOException {
         this(
-            in.readVInt(), // nodes
-            in.readVLong(),  // perNodeMemoryInBytes
+            in.readVInt(), // currentTotalNodes
+            in.readVLong(),  // currentPerNodeMemoryBytes
             in.readVLong(), // modelMemoryInBytes
-            in.readVInt(), // processorsSum
-            in.readVInt(), // minNodes
-            in.readVLong(), // extraSingleNodeModelMemoryInBytes
-            in.readVInt(), // extraSingleNodeProcessors
-            in.readVLong(), // extraModelMemoryInBytes
-            in.readVInt(), // extraProcessors
-            in.readVLong(), // removeNodeMemoryInBytes
-            in.readVLong() // perNodeMemoryOverheadInBytes
+            in.readVInt(), // currentTotalProcessorsInUse
+            in.readVInt(), // wantedMinNodes
+            in.readVLong(), // wantedExtraPerNodeMemoryBytes
+            in.readVInt(), // wantedExtraPerNodeNodeProcessors
+            in.readVLong(), // wantedExtraModelMemoryBytes
+            in.readVInt(), // wantedExtraProcessors
+            in.readVLong(), // unwantedNodeMemoryBytesToRemove
+            in.readVLong() // currentPerNodeMemoryOverheadBytes
         );
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(nodes);
-        out.writeVLong(perNodeMemoryInBytes);
-        out.writeVLong(modelMemoryInBytesSum);
-        out.writeVLong(processorsSum);
-        out.writeVInt(minNodes);
-        out.writeVLong(extraSingleNodeModelMemoryInBytes);
-        out.writeVInt(extraSingleNodeProcessors);
-        out.writeVLong(extraModelMemoryInBytes);
-        out.writeVInt(extraProcessors);
-        out.writeVLong(removeNodeMemoryInBytes);
-        out.writeVLong(perNodeMemoryOverheadInBytes);
+        out.writeVInt(currentTotalNodes);
+        out.writeVLong(currentPerNodeMemoryBytes);
+        out.writeVLong(currentTotalModelMemoryBytes);
+        out.writeVLong(currentTotalProcessorsInUse);
+        out.writeVInt(wantedMinNodes);
+        out.writeVLong(wantedExtraPerNodeMemoryBytes);
+        out.writeVInt(wantedExtraPerNodeNodeProcessors);
+        out.writeVLong(wantedExtraModelMemoryBytes);
+        out.writeVInt(wantedExtraProcessors);
+        out.writeVLong(unwantedNodeMemoryBytesToRemove);
+        out.writeVLong(currentPerNodeMemoryOverheadBytes);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationsSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationsSettings.java
index 0b5a62ccb588c..19af6a3a4ef4c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationsSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationsSettings.java
@@ -104,9 +104,7 @@ public Integer getMaxNumberOfAllocations() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        if (enabled != null) {
-            builder.field(ENABLED.getPreferredName(), enabled);
-        }
+        builder.field(ENABLED.getPreferredName(), enabled != null ? enabled : false);
         if (minNumberOfAllocations != null) {
             builder.field(MIN_NUMBER_OF_ALLOCATIONS.getPreferredName(), minNumberOfAllocations);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfo.java
index 826b0785aa563..6e7cdf6e9ab03 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfo.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfo.java
@@ -59,6 +59,14 @@ private RoutingInfo(
         this(currentAllocations == null ? 0 : currentAllocations, targetAllocations == null ? 0 : targetAllocations, state, reason);
     }
 
+    /**
+     * RoutingInfo defines the state of a particular trained model assignment on a particular node.
+     * @param currentAllocations The number of allocations currently running on a node.
+     * @param targetAllocations The number of allocations that have been assigned to a node, and will run on the node. Should never be
+     *                          higher than the number of available processors on the node.
+     * @param state Indicates the availability of the allocations on the node.
+     * @param reason Will contain the reason that currentAllocations != targetAllocations, if applicable, otherwise empty string.
+     */
     public RoutingInfo(int currentAllocations, int targetAllocations, RoutingState state, String reason) {
         this.currentAllocations = currentAllocations;
         this.targetAllocations = targetAllocations;
@@ -78,10 +86,17 @@ public RoutingInfo(StreamInput in) throws IOException {
         this.reason = in.readOptionalString();
     }
 
+    /**
+     * @return The number of allocations currently running on a node.
+     */
     public int getCurrentAllocations() {
         return currentAllocations;
     }
 
+    /**
+     * @return The number of allocations that have been assigned to a node, and will run on the node. Should never be
+     * higher than the number of available processors on the node.
+     */
     public int getTargetAllocations() {
         return targetAllocations;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
index 60e0c0e86a828..4a87b8e24f481 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
@@ -136,6 +136,17 @@ private TrainedModelAssignment(
         );
     }
 
+    /**
+     * A long-lived object which defines a trained model deployment/assignment.
+     *
+     * @param taskParams the parameters provided by the StartTrainedModelDeploymentAction during the creation of the deployment/assignment
+     * @param nodeRoutingTable shows where allocations for this assignment/deployment are located (on which nodes)
+     * @param assignmentState used to track the state of the assignment for rebalancing, autoscaling, and more
+     * @param reason may contain a human-readable explanation for the current state
+     * @param startTime the time when the assignment was created
+     * @param maxAssignedAllocations used for adaptive allocations
+     * @param adaptiveAllocationsSettings how the assignment should scale based on usage
+     */
     TrainedModelAssignment(
         StartTrainedModelDeploymentAction.TaskParams taskParams,
         Map<String, RoutingInfo> nodeRoutingTable,
@@ -178,6 +189,9 @@ public boolean isRoutedToNode(String nodeId) {
         return nodeRoutingTable.containsKey(nodeId);
     }
 
+    /**
+     * @return shows where allocations for this assignment/deployment are located (on which nodes)
+     */
     public Map<String, RoutingInfo> getNodeRoutingTable() {
         return Collections.unmodifiableMap(nodeRoutingTable);
     }
@@ -310,6 +324,10 @@ public int totalTargetAllocations() {
         return nodeRoutingTable.values().stream().mapToInt(RoutingInfo::getTargetAllocations).sum();
     }
 
+    public int totalTargetProcessors() {
+        return nodeRoutingTable.values().stream().mapToInt(r -> r.getTargetAllocations() * getTaskParams().getThreadsPerAllocation()).sum();
+    }
+
     public int totalFailedAllocations() {
         return nodeRoutingTable.values().stream().mapToInt(RoutingInfo::getFailedAllocations).sum();
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceConfig.java
index 92e833c250873..d50db1bd171ee 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceConfig.java
@@ -12,6 +12,8 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.license.License;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xpack.core.ml.MlConfigVersion;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput;
@@ -22,6 +24,7 @@
 import java.util.Arrays;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.xpack.core.ml.MachineLearningField.ML_API_FEATURE;
 
 public interface InferenceConfig extends NamedXContentObject, VersionedNamedWriteable {
 
@@ -114,4 +117,12 @@ default ElasticsearchStatusException incompatibleUpdateException(String updateNa
             updateName
         );
     }
+
+    default License.OperationMode getMinLicenseSupported() {
+        return ML_API_FEATURE.getMinimumOperationMode();
+    }
+
+    default License.OperationMode getMinLicenseSupportedForAction(RestRequest.Method method) {
+        return getMinLicenseSupported();
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java
index 7d515c9509c41..9929e59a9c803 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfig.java
@@ -7,12 +7,15 @@
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.Rewriteable;
+import org.elasticsearch.license.License;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -38,7 +41,7 @@
 public class LearningToRankConfig extends RegressionConfig implements Rewriteable<LearningToRankConfig> {
 
     public static final ParseField NAME = new ParseField("learning_to_rank");
-    static final TransportVersion MIN_SUPPORTED_TRANSPORT_VERSION = TransportVersion.current();
+    static final TransportVersion MIN_SUPPORTED_TRANSPORT_VERSION = TransportVersions.LTR_SERVERLESS_RELEASE;
     public static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values");
     public static final ParseField FEATURE_EXTRACTORS = new ParseField("feature_extractors");
     public static final ParseField DEFAULT_PARAMS = new ParseField("default_params");
@@ -226,6 +229,14 @@ public TransportVersion getMinimalSupportedTransportVersion() {
         return MIN_SUPPORTED_TRANSPORT_VERSION;
     }
 
+    @Override
+    public License.OperationMode getMinLicenseSupportedForAction(RestRequest.Method method) {
+        if (method == RestRequest.Method.PUT) {
+            return License.OperationMode.ENTERPRISE;
+        }
+        return super.getMinLicenseSupportedForAction(method);
+    }
+
     @Override
     public LearningToRankConfig rewrite(QueryRewriteContext ctx) throws IOException {
         if (this.featureExtractorBuilders.isEmpty()) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java
index ff2921d755b40..eb952a7dc7e5c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.job.config;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -30,6 +31,7 @@ public class DetectionRule implements ToXContentObject, Writeable {
     public static final ParseField ACTIONS_FIELD = new ParseField("actions");
     public static final ParseField SCOPE_FIELD = new ParseField("scope");
     public static final ParseField CONDITIONS_FIELD = new ParseField("conditions");
+    public static final ParseField PARAMS_FIELD = new ParseField("params");
 
     // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
     public static final ObjectParser<Builder, Void> LENIENT_PARSER = createParser(true);
@@ -45,6 +47,7 @@ private static ObjectParser<Builder, Void> createParser(boolean ignoreUnknownFie
             ignoreUnknownFields ? RuleCondition.LENIENT_PARSER : RuleCondition.STRICT_PARSER,
             CONDITIONS_FIELD
         );
+        parser.declareObject(Builder::setParams, ignoreUnknownFields ? RuleParams.LENIENT_PARSER : RuleParams.STRICT_PARSER, PARAMS_FIELD);
 
         return parser;
     }
@@ -52,17 +55,24 @@ private static ObjectParser<Builder, Void> createParser(boolean ignoreUnknownFie
     private final EnumSet<RuleAction> actions;
     private final RuleScope scope;
     private final List<RuleCondition> conditions;
+    private final RuleParams params;
 
-    private DetectionRule(EnumSet<RuleAction> actions, RuleScope scope, List<RuleCondition> conditions) {
+    private DetectionRule(EnumSet<RuleAction> actions, RuleScope scope, List<RuleCondition> conditions, RuleParams params) {
         this.actions = Objects.requireNonNull(actions);
         this.scope = Objects.requireNonNull(scope);
         this.conditions = Collections.unmodifiableList(conditions);
+        this.params = params;
     }
 
     public DetectionRule(StreamInput in) throws IOException {
         actions = in.readEnumSet(RuleAction.class);
         scope = new RuleScope(in);
         conditions = in.readCollectionAsList(RuleCondition::new);
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ML_ADD_DETECTION_RULE_PARAMS)) {
+            params = new RuleParams(in);
+        } else {
+            params = new RuleParams();
+        }
     }
 
     @Override
@@ -70,6 +80,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeEnumSet(actions);
         scope.writeTo(out);
         out.writeCollection(conditions);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ML_ADD_DETECTION_RULE_PARAMS)) {
+            params.writeTo(out);
+        }
     }
 
     @Override
@@ -82,6 +95,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (conditions.isEmpty() == false) {
             builder.field(CONDITIONS_FIELD.getPreferredName(), conditions);
         }
+        if (this.params.isEmpty() == false) {
+            builder.field(PARAMS_FIELD.getPreferredName(), this.params);
+        }
         builder.endObject();
         return builder;
     }
@@ -98,6 +114,10 @@ public List<RuleCondition> getConditions() {
         return conditions;
     }
 
+    public RuleParams getParams() {
+        return params;
+    }
+
     public Set<String> extractReferencedFilters() {
         return scope.getReferencedFilters();
     }
@@ -113,18 +133,22 @@ public boolean equals(Object obj) {
         }
 
         DetectionRule other = (DetectionRule) obj;
-        return Objects.equals(actions, other.actions) && Objects.equals(scope, other.scope) && Objects.equals(conditions, other.conditions);
+        return Objects.equals(actions, other.actions)
+            && Objects.equals(scope, other.scope)
+            && Objects.equals(conditions, other.conditions)
+            && Objects.equals(params, other.params);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(actions, scope, conditions);
+        return Objects.hash(actions, scope, conditions, params);
     }
 
     public static class Builder {
         private EnumSet<RuleAction> actions = EnumSet.of(RuleAction.SKIP_RESULT);
         private RuleScope scope = new RuleScope();
         private List<RuleCondition> conditions = Collections.emptyList();
+        private RuleParams params = new RuleParams();
 
         public Builder(RuleScope.Builder scope) {
             this.scope = scope.build();
@@ -163,12 +187,27 @@ public Builder setConditions(List<RuleCondition> conditions) {
             return this;
         }
 
+        public Builder setParams(RuleParams params) {
+            this.params = params;
+            return this;
+        }
+
         public DetectionRule build() {
             if (scope.isEmpty() && conditions.isEmpty()) {
                 String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_REQUIRES_SCOPE_OR_CONDITION);
                 throw ExceptionsHelper.badRequestException(msg);
             }
-            return new DetectionRule(actions, scope, conditions);
+            // if actions contain FORCE_TIME_SHIFT, then params must contain RuleParamsForForceTimeShift
+            if (actions.contains(RuleAction.FORCE_TIME_SHIFT) && params.getForceTimeShift() == null) {
+                String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_REQUIRES_FORCE_TIME_SHIFT_PARAMS);
+                throw ExceptionsHelper.badRequestException(msg);
+            }
+            // Return error if params must contain RuleParamsForForceTimeShift, but actions do not contain FORCE_TIME_SHIFT
+            if (actions.contains(RuleAction.FORCE_TIME_SHIFT) == false && params.getForceTimeShift() != null) {
+                String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_PARAMS_FORCE_TIME_SHIFT_NOT_REQUIRED);
+                throw ExceptionsHelper.badRequestException(msg);
+            }
+            return new DetectionRule(actions, scope, conditions, params);
         }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleAction.java
index cce1d8a2f87ee..933a68ca287a2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleAction.java
@@ -15,7 +15,8 @@
 
 public enum RuleAction implements Writeable {
     SKIP_RESULT,
-    SKIP_MODEL_UPDATE;
+    SKIP_MODEL_UPDATE,
+    FORCE_TIME_SHIFT;
 
     /**
      * Case-insensitive from string method.
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleParams.java
new file mode 100644
index 0000000000000..fc9126de9944f
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleParams.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ml.job.config;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class RuleParams implements ToXContentObject, Writeable {
+
+    public static final ParseField RULE_PARAMS_FIELD = new ParseField("params");
+    public static final ParseField FORCE_TIME_SHIFT_FIELD = new ParseField("force_time_shift");
+
+    // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
+    public static final ConstructingObjectParser<RuleParams, Void> LENIENT_PARSER = createParser(true);
+    public static final ConstructingObjectParser<RuleParams, Void> STRICT_PARSER = createParser(false);
+
+    public static ConstructingObjectParser<RuleParams, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<RuleParams, Void> parser = new ConstructingObjectParser<>(
+            RULE_PARAMS_FIELD.getPreferredName(),
+            ignoreUnknownFields,
+            a -> new RuleParams((RuleParamsForForceTimeShift) a[0])
+        );
+
+        parser.declareObject(
+            ConstructingObjectParser.optionalConstructorArg(),
+            RuleParamsForForceTimeShift.LENIENT_PARSER,
+            FORCE_TIME_SHIFT_FIELD
+        );
+        return parser;
+    }
+
+    private final RuleParamsForForceTimeShift forceTimeShift;
+
+    public RuleParams() {
+        this.forceTimeShift = null;
+    }
+
+    public RuleParams(RuleParamsForForceTimeShift forceTimeShift) {
+        this.forceTimeShift = forceTimeShift;
+    }
+
+    public RuleParams(StreamInput in) throws IOException {
+        // initialize optional forceTimeShift from in
+        forceTimeShift = in.readOptionalWriteable(RuleParamsForForceTimeShift::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        // write optional forceTimeShift to out
+        out.writeOptionalWriteable(forceTimeShift);
+    }
+
+    boolean isEmpty() {
+        return forceTimeShift == null;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (forceTimeShift != null) {
+            builder.field(FORCE_TIME_SHIFT_FIELD.getPreferredName(), forceTimeShift);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj instanceof RuleParams == false) {
+            return false;
+        }
+
+        RuleParams other = (RuleParams) obj;
+        return Objects.equals(forceTimeShift, other.forceTimeShift);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(forceTimeShift);
+    }
+
+    public RuleParamsForForceTimeShift getForceTimeShift() {
+        return forceTimeShift;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleParamsForForceTimeShift.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleParamsForForceTimeShift.java
new file mode 100644
index 0000000000000..02c762196d8ce
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleParamsForForceTimeShift.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ml.job.config;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+public class RuleParamsForForceTimeShift implements ToXContentObject, Writeable {
+    public static final ParseField TYPE_FIELD = new ParseField("force_time_shift_params");
+    public static final ParseField TIME_SHIFT_AMOUNT_FIELD = new ParseField("time_shift_amount");
+
+    public static final ConstructingObjectParser<RuleParamsForForceTimeShift, Void> LENIENT_PARSER = createParser(true);
+    public static final ConstructingObjectParser<RuleParamsForForceTimeShift, Void> STRICT_PARSER = createParser(false);
+
+    private static ConstructingObjectParser<RuleParamsForForceTimeShift, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<RuleParamsForForceTimeShift, Void> parser = new ConstructingObjectParser<>(
+            TYPE_FIELD.getPreferredName(),
+            ignoreUnknownFields,
+            a -> new RuleParamsForForceTimeShift((Long) a[0])
+        );
+        parser.declareLong(ConstructingObjectParser.constructorArg(), TIME_SHIFT_AMOUNT_FIELD);
+        return parser;
+    }
+
+    private final long timeShiftAmount;
+
+    public RuleParamsForForceTimeShift(long timeShiftAmount) {
+        this.timeShiftAmount = timeShiftAmount;
+    }
+
+    public RuleParamsForForceTimeShift(StreamInput in) throws IOException {
+        timeShiftAmount = in.readLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeLong(timeShiftAmount);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(TIME_SHIFT_AMOUNT_FIELD.getPreferredName(), timeShiftAmount);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj instanceof RuleParamsForForceTimeShift == false) {
+            return false;
+        }
+
+        RuleParamsForForceTimeShift other = (RuleParamsForForceTimeShift) obj;
+        return timeShiftAmount == other.timeShiftAmount;
+    }
+
+    @Override
+    public int hashCode() {
+        return Long.hashCode(timeShiftAmount);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
index 52c97ece1b017..6ebed55451ae7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
@@ -204,6 +204,10 @@ public final class Messages {
         "Invalid detector rule: function {0} only supports conditions that apply to time";
     public static final String JOB_CONFIG_DETECTION_RULE_REQUIRES_SCOPE_OR_CONDITION =
         "Invalid detector rule: at least scope or a condition is required";
+    public static final String JOB_CONFIG_DETECTION_RULE_REQUIRES_FORCE_TIME_SHIFT_PARAMS =
+        "Invalid detector rule: actions contain force_time_shift, but corresponding parameters are missing";
+    public static final String JOB_CONFIG_DETECTION_RULE_PARAMS_FORCE_TIME_SHIFT_NOT_REQUIRED =
+        "Invalid detector rule: actions do not contain force_time_shift, but corresponding parameters are present";
     public static final String JOB_CONFIG_DETECTION_RULE_SCOPE_NO_AVAILABLE_FIELDS =
         "Invalid detector rule: scope field ''{0}'' is invalid; detector has no available fields for scoping";
     public static final String JOB_CONFIG_DETECTION_RULE_SCOPE_HAS_INVALID_FIELD =
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java
index 1ba625a507a46..f7ad1f65628b2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java
@@ -74,7 +74,7 @@ public RollupJobStatus(IndexerState state, @Nullable Map<String, Object> positio
 
     public RollupJobStatus(StreamInput in) throws IOException {
         state = IndexerState.fromStream(in);
-        currentPosition = in.readBoolean() ? new TreeMap<>(in.readGenericMap()) : null;
+        currentPosition = in.readOptional(CURRENT_POSITION_READER);
         if (in.getTransportVersion().before(TransportVersions.V_8_0_0)) {
             // 7.x nodes serialize `upgradedDocumentID` flag. We don't need it anymore, but
             // we need to pull it off the stream
@@ -83,6 +83,8 @@ public RollupJobStatus(StreamInput in) throws IOException {
         }
     }
 
+    private static final Reader<TreeMap<String, Object>> CURRENT_POSITION_READER = in -> new TreeMap<>(in.readGenericMap());
+
     public IndexerState getIndexerState() {
         return state;
     }
@@ -118,10 +120,7 @@ public String getWriteableName() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         state.writeTo(out);
-        out.writeBoolean(currentPosition != null);
-        if (currentPosition != null) {
-            out.writeGenericMap(currentPosition);
-        }
+        out.writeOptional(StreamOutput::writeGenericMap, currentPosition);
         if (out.getTransportVersion().before(TransportVersions.V_8_0_0)) {
             // 7.x nodes expect a boolean `upgradedDocumentID` flag. We don't have it anymore,
             // but we need to tell them we are upgraded in case there is a mixed cluster
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ActionTypes.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ActionTypes.java
index 52f8c7cf456d9..bf9484c52800a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ActionTypes.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ActionTypes.java
@@ -20,8 +20,13 @@
 public final class ActionTypes {
     private ActionTypes() {};
 
+    // Note: this action is *not* prefixed with `cluster:admin/xpack/security` since it would otherwise be excluded from the `manage`
+    // privilege -- instead it matches its prefix to `TransportNodesReloadSecureSettingsAction` which is the "parent" transport action
+    // that invokes the overall reload flow.
+    // This allows us to maintain the invariant that the parent reload secure settings action can be executed with the `manage` privilege
+    // without trappy system-context switches.
     public static final ActionType<ActionResponse.Empty> RELOAD_REMOTE_CLUSTER_CREDENTIALS_ACTION = new ActionType<>(
-        "cluster:admin/xpack/security/remote_cluster_credentials/reload"
+        "cluster:admin/nodes/reload_secure_settings/security/remote_cluster_credentials"
     );
 
     public static final ActionType<QueryUserResponse> QUERY_USER_ACTION = new ActionType<>("cluster:admin/xpack/security/user/query");
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/UpdateIndexMigrationVersionAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/UpdateIndexMigrationVersionAction.java
index f28014afde295..6f37abcdc0b0c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/UpdateIndexMigrationVersionAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/UpdateIndexMigrationVersionAction.java
@@ -25,11 +25,11 @@
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderFactory.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderFactory.java
index ff5592e339634..cfdf8147d8439 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderFactory.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderFactory.java
@@ -10,12 +10,11 @@
 import org.elasticsearch.client.internal.Client;
 
 public interface CreateApiKeyRequestBuilderFactory {
-    CreateApiKeyRequestBuilder create(Client client, boolean restrictRequest);
+    CreateApiKeyRequestBuilder create(Client client);
 
     class Default implements CreateApiKeyRequestBuilderFactory {
         @Override
-        public CreateApiKeyRequestBuilder create(Client client, boolean restrictRequest) {
-            assert false == restrictRequest;
+        public CreateApiKeyRequestBuilder create(Client client) {
             return new CreateApiKeyRequestBuilder(client);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTranslator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTranslator.java
index 2d018ae2f1b2f..0a1f49ffa04ef 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTranslator.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTranslator.java
@@ -9,11 +9,10 @@
 
 public interface GetBuiltinPrivilegesResponseTranslator {
 
-    GetBuiltinPrivilegesResponse translate(GetBuiltinPrivilegesResponse response, boolean restrictResponse);
+    GetBuiltinPrivilegesResponse translate(GetBuiltinPrivilegesResponse response);
 
     class Default implements GetBuiltinPrivilegesResponseTranslator {
-        public GetBuiltinPrivilegesResponse translate(GetBuiltinPrivilegesResponse response, boolean restrictResponse) {
-            assert false == restrictResponse;
+        public GetBuiltinPrivilegesResponse translate(GetBuiltinPrivilegesResponse response) {
             return response;
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java
index 27f7c42d74018..46ba00a4f2768 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java
@@ -45,7 +45,6 @@ public class PutRoleRequest extends ActionRequest {
     private Map<String, Object> metadata;
     private List<RoleDescriptor.RemoteIndicesPrivileges> remoteIndicesPrivileges = new ArrayList<>();
     private RemoteClusterPermissions remoteClusterPermissions = RemoteClusterPermissions.NONE;
-    private boolean restrictRequest = false;
     private String description;
 
     public PutRoleRequest() {}
@@ -84,14 +83,6 @@ public void addRemoteIndex(RoleDescriptor.RemoteIndicesPrivileges... privileges)
         remoteIndicesPrivileges.addAll(Arrays.asList(privileges));
     }
 
-    public void restrictRequest(boolean restrictRequest) {
-        this.restrictRequest = restrictRequest;
-    }
-
-    public boolean restrictRequest() {
-        return restrictRequest;
-    }
-
     public void putRemoteCluster(RemoteClusterPermissions remoteClusterPermissions) {
         this.remoteClusterPermissions = remoteClusterPermissions;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilderFactory.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilderFactory.java
index 169bd5d4cc1f7..e8c5b465ace36 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilderFactory.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilderFactory.java
@@ -10,13 +10,11 @@
 import org.elasticsearch.client.internal.Client;
 
 public interface PutRoleRequestBuilderFactory {
-    PutRoleRequestBuilder create(Client client, boolean restrictRequest);
+    PutRoleRequestBuilder create(Client client);
 
     class Default implements PutRoleRequestBuilderFactory {
         @Override
-        public PutRoleRequestBuilder create(Client client, boolean restrictRequest) {
-            // by default, we don't apply extra restrictions to Put Role requests and don't require checks against file-based roles
-            // these dependencies are only used by our stateless implementation
+        public PutRoleRequestBuilder create(Client client) {
             return new PutRoleRequestBuilder(client);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsAction.java
index 2d59911ec7ecb..c2bf1466fd41f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsAction.java
@@ -14,9 +14,9 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -28,6 +28,8 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.BiFunction;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
@@ -42,10 +44,35 @@ public class UpdateSecuritySettingsAction {
     public static final String TOKENS_INDEX_NAME = "security-tokens";
     public static final String PROFILES_INDEX_NAME = "security-profile";
 
-    public static final Set<String> ALLOWED_SETTING_KEYS = Set.of(
-        IndexMetadata.SETTING_NUMBER_OF_REPLICAS,
-        IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS
-    );
+    /**
+     * A map of allowed settings to validators for those settings. Values should take the value which is being assigned to the setting
+     * and an existing {@link ActionRequestValidationException}, to which they should add if the value is disallowed.
+     */
+    public static final Map<
+        String,
+        BiFunction<Object, ActionRequestValidationException, ActionRequestValidationException>> ALLOWED_SETTING_VALIDATORS = Map.of(
+            IndexMetadata.SETTING_NUMBER_OF_REPLICAS,
+            (it, ex) -> ex, // no additional validation
+            IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS,
+            (it, ex) -> ex, // no additional validation
+            DataTier.TIER_PREFERENCE,
+            (it, ex) -> {
+                Set<String> allowedTiers = Set.of(DataTier.DATA_CONTENT, DataTier.DATA_HOT, DataTier.DATA_WARM, DataTier.DATA_COLD);
+                if (it instanceof String preference) {
+                    String disallowedTiers = DataTier.parseTierList(preference)
+                        .stream()
+                        .filter(tier -> allowedTiers.contains(tier) == false)
+                        .collect(Collectors.joining(","));
+                    if (disallowedTiers.isEmpty() == false) {
+                        return ValidateActions.addValidationError(
+                            "disallowed data tiers [" + disallowedTiers + "] found, allowed tiers are [" + String.join(",", allowedTiers),
+                            ex
+                        );
+                    }
+                }
+                return ex;
+            }
+        );
 
     private UpdateSecuritySettingsAction() {/* no instances */}
 
@@ -154,19 +181,26 @@ private static ActionRequestValidationException validateIndexSettings(
             String indexName,
             ActionRequestValidationException existingExceptions
         ) {
-            Set<String> forbiddenSettings = Sets.difference(indexSettings.keySet(), ALLOWED_SETTING_KEYS);
-            if (forbiddenSettings.size() > 0) {
-                return ValidateActions.addValidationError(
-                    "illegal settings for index ["
-                        + indexName
-                        + "]: "
-                        + forbiddenSettings
-                        + ", these settings may not be configured. Only the following settings may be configured for that index: "
-                        + ALLOWED_SETTING_KEYS,
-                    existingExceptions
-                );
+            ActionRequestValidationException errors = existingExceptions;
+
+            for (Map.Entry<String, Object> entry : indexSettings.entrySet()) {
+                String setting = entry.getKey();
+                if (ALLOWED_SETTING_VALIDATORS.containsKey(setting)) {
+                    errors = ALLOWED_SETTING_VALIDATORS.get(setting).apply(entry.getValue(), errors);
+                } else {
+                    errors = ValidateActions.addValidationError(
+                        "illegal setting for index ["
+                            + indexName
+                            + "]: ["
+                            + setting
+                            + "], this setting may not be configured. Only the following settings may be configured for that index: "
+                            + ALLOWED_SETTING_VALIDATORS.keySet(),
+                        existingExceptions
+                    );
+                }
             }
-            return existingExceptions;
+
+            return errors;
         }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
index 6ef2441011e6a..3e3b1178b372c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
@@ -54,6 +54,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.elasticsearch.TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.elasticsearch.transport.RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
@@ -1319,6 +1320,24 @@ private static Map<String, Object> maybeRewriteMetadataForApiKeyRoleDescriptors(
                     )
                 );
             }
+
+            if (authentication.getEffectiveSubject().getTransportVersion().onOrAfter(ROLE_REMOTE_CLUSTER_PRIVS)
+                && streamVersion.before(ROLE_REMOTE_CLUSTER_PRIVS)) {
+                metadata = new HashMap<>(metadata);
+                metadata.put(
+                    AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY,
+                    maybeRemoveRemoteClusterFromRoleDescriptors(
+                        (BytesReference) metadata.get(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY)
+                    )
+                );
+                metadata.put(
+                    AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY,
+                    maybeRemoveRemoteClusterFromRoleDescriptors(
+                        (BytesReference) metadata.get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)
+                    )
+                );
+            }
+
             if (authentication.getEffectiveSubject().getTransportVersion().onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES)
                 && streamVersion.before(VERSION_API_KEY_ROLES_AS_BYTES)) {
                 metadata = new HashMap<>(metadata);
@@ -1397,7 +1416,15 @@ private static BytesReference convertRoleDescriptorsMapToBytes(Map<String, Objec
         }
     }
 
+    static BytesReference maybeRemoveRemoteClusterFromRoleDescriptors(BytesReference roleDescriptorsBytes) {
+        return maybeRemoveTopLevelFromRoleDescriptors(roleDescriptorsBytes, RoleDescriptor.Fields.REMOTE_CLUSTER.getPreferredName());
+    }
+
     static BytesReference maybeRemoveRemoteIndicesFromRoleDescriptors(BytesReference roleDescriptorsBytes) {
+        return maybeRemoveTopLevelFromRoleDescriptors(roleDescriptorsBytes, RoleDescriptor.Fields.REMOTE_INDICES.getPreferredName());
+    }
+
+    static BytesReference maybeRemoveTopLevelFromRoleDescriptors(BytesReference roleDescriptorsBytes, String topLevelField) {
         if (roleDescriptorsBytes == null || roleDescriptorsBytes.length() == 0) {
             return roleDescriptorsBytes;
         }
@@ -1408,7 +1435,7 @@ static BytesReference maybeRemoveRemoteIndicesFromRoleDescriptors(BytesReference
             if (value instanceof Map) {
                 @SuppressWarnings("unchecked")
                 Map<String, Object> roleDescriptor = (Map<String, Object>) value;
-                boolean removed = roleDescriptor.remove(RoleDescriptor.Fields.REMOTE_INDICES.getPreferredName()) != null;
+                boolean removed = roleDescriptor.remove(topLevelField) != null;
                 if (removed) {
                     removedAtLeastOne.set(true);
                 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
index c70f2a05bfe93..9c41786f39eeb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java
@@ -10,6 +10,7 @@
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.RestrictedIndices;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.xpack.core.security.support.Automatons;
 
@@ -17,6 +18,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.function.Predicate;
 
 /**
@@ -84,6 +86,16 @@ public static class Builder {
         private final List<Automaton> actionAutomatons = new ArrayList<>();
         private final List<PermissionCheck> permissionChecks = new ArrayList<>();
 
+        private final RestrictedIndices restrictedIndices;
+
+        public Builder(RestrictedIndices restrictedIndices) {
+            this.restrictedIndices = restrictedIndices;
+        }
+
+        public Builder() {
+            this.restrictedIndices = null;
+        }
+
         public Builder add(
             final ClusterPrivilege clusterPrivilege,
             final Set<String> allowedActionPatterns,
@@ -110,6 +122,16 @@ public Builder add(final ClusterPrivilege clusterPrivilege, final PermissionChec
             return this;
         }
 
+        public Builder addWithPredicateSupplier(
+            final ClusterPrivilege clusterPrivilege,
+            final Set<String> allowedActionPatterns,
+            final Function<RestrictedIndices, Predicate<TransportRequest>> requestPredicateSupplier
+        ) {
+            final Automaton actionAutomaton = createAutomaton(allowedActionPatterns, Set.of());
+            Predicate<TransportRequest> requestPredicate = requestPredicateSupplier.apply(restrictedIndices);
+            return add(clusterPrivilege, new ActionRequestBasedPermissionCheck(clusterPrivilege, actionAutomaton, requestPredicate));
+        }
+
         public ClusterPermission build() {
             if (clusterPrivileges.isEmpty()) {
                 return NONE;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
index d29b1dd67757a..e1b72cc43b38e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xpack.core.security.authz.RestrictedIndices;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
@@ -86,6 +87,7 @@ public Builder addGroup(
         public IndicesPermission build() {
             return new IndicesPermission(restrictedIndices, groups.toArray(Group.EMPTY_ARRAY));
         }
+
     }
 
     private IndicesPermission(RestrictedIndices restrictedIndices, Group[] groups) {
@@ -238,6 +240,21 @@ public boolean check(String action) {
         return false;
     }
 
+    public boolean checkResourcePrivileges(
+        Set<String> checkForIndexPatterns,
+        boolean allowRestrictedIndices,
+        Set<String> checkForPrivileges,
+        @Nullable ResourcePrivilegesMap.Builder resourcePrivilegesMapBuilder
+    ) {
+        return checkResourcePrivileges(
+            checkForIndexPatterns,
+            allowRestrictedIndices,
+            checkForPrivileges,
+            false,
+            resourcePrivilegesMapBuilder
+        );
+    }
+
     /**
      * For given index patterns and index privileges determines allowed privileges and creates an instance of {@link ResourcePrivilegesMap}
      * holding a map of resource to {@link ResourcePrivileges} where resource is index pattern and the map of index privilege to whether it
@@ -246,6 +263,7 @@ public boolean check(String action) {
      * @param checkForIndexPatterns check permission grants for the set of index patterns
      * @param allowRestrictedIndices if {@code true} then checks permission grants even for restricted indices by index matching
      * @param checkForPrivileges check permission grants for the set of index privileges
+     * @param combineIndexGroups combine index groups to enable checking against regular expressions
      * @param resourcePrivilegesMapBuilder out-parameter for returning the details on which privilege over which resource is granted or not.
      *                                     Can be {@code null} when no such details are needed so the method can return early, after
      *                                     encountering the first privilege that is not granted over some resource.
@@ -255,10 +273,13 @@ public boolean checkResourcePrivileges(
         Set<String> checkForIndexPatterns,
         boolean allowRestrictedIndices,
         Set<String> checkForPrivileges,
+        boolean combineIndexGroups,
         @Nullable ResourcePrivilegesMap.Builder resourcePrivilegesMapBuilder
     ) {
-        final Map<IndicesPermission.Group, Automaton> predicateCache = new HashMap<>();
         boolean allMatch = true;
+        Map<Automaton, Automaton> indexGroupAutomatons = indexGroupAutomatons(
+            combineIndexGroups && checkForIndexPatterns.stream().anyMatch(Automatons::isLuceneRegex)
+        );
         for (String forIndexPattern : checkForIndexPatterns) {
             Automaton checkIndexAutomaton = Automatons.patterns(forIndexPattern);
             if (false == allowRestrictedIndices && false == isConcreteRestrictedIndex(forIndexPattern)) {
@@ -266,15 +287,14 @@ public boolean checkResourcePrivileges(
             }
             if (false == Operations.isEmpty(checkIndexAutomaton)) {
                 Automaton allowedIndexPrivilegesAutomaton = null;
-                for (Group group : groups) {
-                    final Automaton groupIndexAutomaton = predicateCache.computeIfAbsent(group, Group::getIndexMatcherAutomaton);
-                    if (Operations.subsetOf(checkIndexAutomaton, groupIndexAutomaton)) {
+                for (var indexAndPrivilegeAutomaton : indexGroupAutomatons.entrySet()) {
+                    if (Operations.subsetOf(checkIndexAutomaton, indexAndPrivilegeAutomaton.getValue())) {
                         if (allowedIndexPrivilegesAutomaton != null) {
                             allowedIndexPrivilegesAutomaton = Automatons.unionAndMinimize(
-                                Arrays.asList(allowedIndexPrivilegesAutomaton, group.privilege().getAutomaton())
+                                Arrays.asList(allowedIndexPrivilegesAutomaton, indexAndPrivilegeAutomaton.getKey())
                             );
                         } else {
-                            allowedIndexPrivilegesAutomaton = group.privilege().getAutomaton();
+                            allowedIndexPrivilegesAutomaton = indexAndPrivilegeAutomaton.getKey();
                         }
                     }
                 }
@@ -656,6 +676,61 @@ private static boolean containsPrivilegeThatGrantsMappingUpdatesForBwc(Group gro
         return group.privilege().name().stream().anyMatch(PRIVILEGE_NAME_SET_BWC_ALLOW_MAPPING_UPDATE::contains);
     }
 
+    /**
+     * Get all automatons for the index groups in this permission and optionally combine the index groups to enable checking if a set of
+     * index patterns specified using a regular expression grants a set of index privileges.
+     *
+     * <p>An index group is defined as a set of index patterns and a set of privileges (excluding field permissions and DLS queries).
+     * {@link IndicesPermission} consist of a set of index groups. For non-regular expression privilege checks, an index pattern is checked
+     * against each index group, to see if it's a sub-pattern of the index pattern for the group and then if that group grants some or all
+     * of the privileges requested. For regular expressions it's not sufficient to check per group since the index patterns covered by a
+     * group can be distinct sets and a regular expression can cover several distinct sets.
+     *
+     * <p>For example the two index groups: {"names": ["a"], "privileges": ["read", "create"]} and {"names": ["b"],
+     * "privileges": ["read","delete"]} will not match on ["\[ab]\"], while a single index group:
+     * {"names": ["a", "b"], "privileges": ["read"]} will. This happens because the index groups are evaluated against a request index
+     * pattern without first being combined. In the example above, the two index patterns should be combined to:
+     * {"names": ["a", "b"], "privileges": ["read"]} before being checked.
+     *
+     *
+     * @param combine combine index groups to allow for checking against regular expressions
+     *
+     * @return a map of all index and privilege pattern automatons
+     */
+    private Map<Automaton, Automaton> indexGroupAutomatons(boolean combine) {
+        // Map of privilege automaton object references (cached by IndexPrivilege::CACHE)
+        Map<Automaton, Automaton> allAutomatons = new HashMap<>();
+        for (Group group : groups) {
+            Automaton indexAutomaton = group.getIndexMatcherAutomaton();
+            allAutomatons.compute(
+                group.privilege().getAutomaton(),
+                (key, value) -> value == null ? indexAutomaton : Automatons.unionAndMinimize(List.of(value, indexAutomaton))
+            );
+            if (combine) {
+                List<Tuple<Automaton, Automaton>> combinedAutomatons = new ArrayList<>();
+                for (var indexAndPrivilegeAutomatons : allAutomatons.entrySet()) {
+                    Automaton intersectingPrivileges = Operations.intersection(
+                        indexAndPrivilegeAutomatons.getKey(),
+                        group.privilege().getAutomaton()
+                    );
+                    if (Operations.isEmpty(intersectingPrivileges) == false) {
+                        Automaton indexPatternAutomaton = Automatons.unionAndMinimize(
+                            List.of(indexAndPrivilegeAutomatons.getValue(), indexAutomaton)
+                        );
+                        combinedAutomatons.add(new Tuple<>(intersectingPrivileges, indexPatternAutomaton));
+                    }
+                }
+                combinedAutomatons.forEach(
+                    automatons -> allAutomatons.compute(
+                        automatons.v1(),
+                        (key, value) -> value == null ? automatons.v2() : Automatons.unionAndMinimize(List.of(value, automatons.v2()))
+                    )
+                );
+            }
+        }
+        return allAutomatons;
+    }
+
     public static class Group {
         public static final Group[] EMPTY_ARRAY = new Group[0];
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java
index 0fc04e8cc9a52..d8d56a4fbb247 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java
@@ -233,7 +233,7 @@ private Builder(RestrictedIndices restrictedIndices, String[] names) {
         }
 
         public Builder cluster(Set<String> privilegeNames, Iterable<ConfigurableClusterPrivilege> configurableClusterPrivileges) {
-            ClusterPermission.Builder builder = ClusterPermission.builder();
+            ClusterPermission.Builder builder = new ClusterPermission.Builder(restrictedIndices);
             if (privilegeNames.isEmpty() == false) {
                 for (String name : privilegeNames) {
                     builder = ClusterPrivilegeResolver.resolve(name).buildPermission(builder);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
index a435f7e877250..3d1b378f4f51e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
@@ -399,11 +399,11 @@ public class ClusterPrivilegeResolver {
     );
     public static final NamedClusterPrivilege MONITOR_GLOBAL_RETENTION = new ActionClusterPrivilege(
         "monitor_data_stream_global_retention",
-        Set.of("cluster:monitor/data_stream/global_retention/*")
+        Set.of()
     );
     public static final NamedClusterPrivilege MANAGE_GLOBAL_RETENTION = new ActionClusterPrivilege(
         "manage_data_stream_global_retention",
-        Set.of("cluster:admin/data_stream/global_retention/*", "cluster:monitor/data_stream/global_retention/*")
+        Set.of()
     );
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivilege.java
index f9722ca42f20d..edb0cb8f9e79d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivilege.java
@@ -41,7 +41,8 @@ public interface ConfigurableClusterPrivilege extends NamedWriteable, ToXContent
      */
     enum Category {
         APPLICATION(new ParseField("application")),
-        PROFILE(new ParseField("profile"));
+        PROFILE(new ParseField("profile")),
+        ROLE(new ParseField("role"));
 
         public final ParseField field;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivileges.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivileges.java
index fed8b7e0d7a1c..b93aa079a28d2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivileges.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivileges.java
@@ -7,6 +7,9 @@
 
 package org.elasticsearch.xpack.core.security.authz.privilege;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -17,10 +20,21 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParseException;
 import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.security.action.ActionTypes;
 import org.elasticsearch.xpack.core.security.action.privilege.ApplicationPrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.profile.UpdateProfileDataAction;
 import org.elasticsearch.xpack.core.security.action.profile.UpdateProfileDataRequest;
+import org.elasticsearch.xpack.core.security.action.role.BulkDeleteRolesRequest;
+import org.elasticsearch.xpack.core.security.action.role.BulkPutRolesRequest;
+import org.elasticsearch.xpack.core.security.action.role.DeleteRoleAction;
+import org.elasticsearch.xpack.core.security.action.role.DeleteRoleRequest;
+import org.elasticsearch.xpack.core.security.action.role.PutRoleAction;
+import org.elasticsearch.xpack.core.security.action.role.PutRoleRequest;
+import org.elasticsearch.xpack.core.security.authz.RestrictedIndices;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.permission.ClusterPermission;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
+import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege.Category;
 import org.elasticsearch.xpack.core.security.support.StringMatcher;
 import org.elasticsearch.xpack.core.security.xcontent.XContentUtils;
@@ -30,12 +44,18 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.Function;
 import java.util.function.Predicate;
 
+import static org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege.DELETE_INDEX;
+
 /**
  * Static utility class for working with {@link ConfigurableClusterPrivilege} instances
  */
@@ -43,6 +63,7 @@ public final class ConfigurableClusterPrivileges {
 
     public static final ConfigurableClusterPrivilege[] EMPTY_ARRAY = new ConfigurableClusterPrivilege[0];
 
+    private static final Logger logger = LogManager.getLogger(ConfigurableClusterPrivileges.class);
     public static final Writeable.Reader<ConfigurableClusterPrivilege> READER = in1 -> in1.readNamedWriteable(
         ConfigurableClusterPrivilege.class
     );
@@ -61,7 +82,16 @@ public static ConfigurableClusterPrivilege[] readArray(StreamInput in) throws IO
      * Utility method to write an array of {@link ConfigurableClusterPrivilege} objects to a {@link StreamOutput}
      */
     public static void writeArray(StreamOutput out, ConfigurableClusterPrivilege[] privileges) throws IOException {
-        out.writeArray(WRITER, privileges);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE)) {
+            out.writeArray(WRITER, privileges);
+        } else {
+            out.writeArray(
+                WRITER,
+                Arrays.stream(privileges)
+                    .filter(privilege -> privilege instanceof ManageRolesPrivilege == false)
+                    .toArray(ConfigurableClusterPrivilege[]::new)
+            );
+        }
     }
 
     /**
@@ -97,7 +127,7 @@ public static List<ConfigurableClusterPrivilege> parse(XContentParser parser) th
         while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
             expectedToken(parser.currentToken(), parser, XContentParser.Token.FIELD_NAME);
 
-            expectFieldName(parser, Category.APPLICATION.field, Category.PROFILE.field);
+            expectFieldName(parser, Category.APPLICATION.field, Category.PROFILE.field, Category.ROLE.field);
             if (Category.APPLICATION.field.match(parser.currentName(), parser.getDeprecationHandler())) {
                 expectedToken(parser.nextToken(), parser, XContentParser.Token.START_OBJECT);
                 while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
@@ -106,8 +136,7 @@ public static List<ConfigurableClusterPrivilege> parse(XContentParser parser) th
                     expectFieldName(parser, ManageApplicationPrivileges.Fields.MANAGE);
                     privileges.add(ManageApplicationPrivileges.parse(parser));
                 }
-            } else {
-                assert Category.PROFILE.field.match(parser.currentName(), parser.getDeprecationHandler());
+            } else if (Category.PROFILE.field.match(parser.currentName(), parser.getDeprecationHandler())) {
                 expectedToken(parser.nextToken(), parser, XContentParser.Token.START_OBJECT);
                 while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
                     expectedToken(parser.currentToken(), parser, XContentParser.Token.FIELD_NAME);
@@ -115,9 +144,16 @@ public static List<ConfigurableClusterPrivilege> parse(XContentParser parser) th
                     expectFieldName(parser, WriteProfileDataPrivileges.Fields.WRITE);
                     privileges.add(WriteProfileDataPrivileges.parse(parser));
                 }
+            } else if (Category.ROLE.field.match(parser.currentName(), parser.getDeprecationHandler())) {
+                expectedToken(parser.nextToken(), parser, XContentParser.Token.START_OBJECT);
+                while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                    expectedToken(parser.currentToken(), parser, XContentParser.Token.FIELD_NAME);
+
+                    expectFieldName(parser, ManageRolesPrivilege.Fields.MANAGE);
+                    privileges.add(ManageRolesPrivilege.parse(parser));
+                }
             }
         }
-
         return privileges;
     }
 
@@ -362,4 +398,277 @@ private interface Fields {
             ParseField APPLICATIONS = new ParseField("applications");
         }
     }
+
+    public static class ManageRolesPrivilege implements ConfigurableClusterPrivilege {
+        public static final String WRITEABLE_NAME = "manage-roles-privilege";
+        private final List<ManageRolesIndexPermissionGroup> indexPermissionGroups;
+        private final Function<RestrictedIndices, Predicate<TransportRequest>> requestPredicateSupplier;
+
+        private static final Set<String> EXPECTED_INDEX_GROUP_FIELDS = Set.of(
+            Fields.NAMES.getPreferredName(),
+            Fields.PRIVILEGES.getPreferredName()
+        );
+
+        public ManageRolesPrivilege(List<ManageRolesIndexPermissionGroup> manageRolesIndexPermissionGroups) {
+            this.indexPermissionGroups = manageRolesIndexPermissionGroups;
+            this.requestPredicateSupplier = (restrictedIndices) -> {
+                IndicesPermission.Builder indicesPermissionBuilder = new IndicesPermission.Builder(restrictedIndices);
+                for (ManageRolesIndexPermissionGroup indexPatternPrivilege : manageRolesIndexPermissionGroups) {
+                    indicesPermissionBuilder.addGroup(
+                        IndexPrivilege.get(Set.of(indexPatternPrivilege.privileges())),
+                        FieldPermissions.DEFAULT,
+                        null,
+                        false,
+                        indexPatternPrivilege.indexPatterns()
+                    );
+                }
+                final IndicesPermission indicesPermission = indicesPermissionBuilder.build();
+
+                return (TransportRequest request) -> {
+                    if (request instanceof final PutRoleRequest putRoleRequest) {
+                        return hasNonIndexPrivileges(putRoleRequest.roleDescriptor()) == false
+                            && Arrays.stream(putRoleRequest.indices())
+                                .noneMatch(
+                                    indexPrivilege -> requestIndexPatternsAllowed(
+                                        indicesPermission,
+                                        indexPrivilege.getIndices(),
+                                        indexPrivilege.getPrivileges()
+                                    ) == false
+                                );
+                    } else if (request instanceof final BulkPutRolesRequest bulkPutRoleRequest) {
+                        return bulkPutRoleRequest.getRoles().stream().noneMatch(ManageRolesPrivilege::hasNonIndexPrivileges)
+                            && bulkPutRoleRequest.getRoles()
+                                .stream()
+                                .allMatch(
+                                    roleDescriptor -> Arrays.stream(roleDescriptor.getIndicesPrivileges())
+                                        .noneMatch(
+                                            indexPrivilege -> requestIndexPatternsAllowed(
+                                                indicesPermission,
+                                                indexPrivilege.getIndices(),
+                                                indexPrivilege.getPrivileges()
+                                            ) == false
+                                        )
+                                );
+                    } else if (request instanceof final DeleteRoleRequest deleteRoleRequest) {
+                        return requestIndexPatternsAllowed(
+                            indicesPermission,
+                            new String[] { deleteRoleRequest.name() },
+                            DELETE_INDEX.name().toArray(String[]::new)
+                        );
+                    } else if (request instanceof final BulkDeleteRolesRequest bulkDeleteRoleRequest) {
+                        return requestIndexPatternsAllowed(
+                            indicesPermission,
+                            bulkDeleteRoleRequest.getRoleNames().toArray(String[]::new),
+                            DELETE_INDEX.name().toArray(String[]::new)
+                        );
+                    }
+                    throw new IllegalArgumentException("Unsupported request type [" + request.getClass() + "]");
+                };
+            };
+        }
+
+        @Override
+        public Category getCategory() {
+            return Category.ROLE;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return WRITEABLE_NAME;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeCollection(indexPermissionGroups);
+        }
+
+        public static ManageRolesPrivilege createFrom(StreamInput in) throws IOException {
+            final List<ManageRolesIndexPermissionGroup> indexPatternPrivileges = in.readCollectionAsList(
+                ManageRolesIndexPermissionGroup::createFrom
+            );
+            return new ManageRolesPrivilege(indexPatternPrivileges);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.field(
+                Fields.MANAGE.getPreferredName(),
+                Map.of(Fields.INDICES.getPreferredName(), indexPermissionGroups.stream().map(indexPatternPrivilege -> {
+                    Map<String, String[]> sortedMap = new TreeMap<>();
+                    sortedMap.put(Fields.NAMES.getPreferredName(), indexPatternPrivilege.indexPatterns());
+                    sortedMap.put(Fields.PRIVILEGES.getPreferredName(), indexPatternPrivilege.privileges());
+                    return sortedMap;
+                }).toList())
+            );
+        }
+
+        private static void expectedIndexGroupFields(String fieldName, XContentParser parser) {
+            if (EXPECTED_INDEX_GROUP_FIELDS.contains(fieldName) == false) {
+                throw new XContentParseException(
+                    parser.getTokenLocation(),
+                    "failed to parse privilege. expected one of "
+                        + Arrays.toString(EXPECTED_INDEX_GROUP_FIELDS.toArray(String[]::new))
+                        + " but found ["
+                        + fieldName
+                        + "] instead"
+                );
+            }
+        }
+
+        public static ManageRolesPrivilege parse(XContentParser parser) throws IOException {
+            expectedToken(parser.currentToken(), parser, XContentParser.Token.FIELD_NAME);
+            expectFieldName(parser, Fields.MANAGE);
+            expectedToken(parser.nextToken(), parser, XContentParser.Token.START_OBJECT);
+            expectedToken(parser.nextToken(), parser, XContentParser.Token.FIELD_NAME);
+            expectFieldName(parser, Fields.INDICES);
+            expectedToken(parser.nextToken(), parser, XContentParser.Token.START_ARRAY);
+            List<ManageRolesIndexPermissionGroup> indexPrivileges = new ArrayList<>();
+            Map<String, String[]> parsedArraysByFieldName = new HashMap<>();
+
+            XContentParser.Token token;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                expectedToken(token, parser, XContentParser.Token.START_OBJECT);
+                expectedToken(parser.nextToken(), parser, XContentParser.Token.FIELD_NAME);
+                String currentFieldName = parser.currentName();
+                expectedIndexGroupFields(currentFieldName, parser);
+                expectedToken(parser.nextToken(), parser, XContentParser.Token.START_ARRAY);
+                parsedArraysByFieldName.put(currentFieldName, XContentUtils.readStringArray(parser, false));
+                expectedToken(parser.nextToken(), parser, XContentParser.Token.FIELD_NAME);
+                currentFieldName = parser.currentName();
+                expectedIndexGroupFields(currentFieldName, parser);
+                expectedToken(parser.nextToken(), parser, XContentParser.Token.START_ARRAY);
+                parsedArraysByFieldName.put(currentFieldName, XContentUtils.readStringArray(parser, false));
+                expectedToken(parser.nextToken(), parser, XContentParser.Token.END_OBJECT);
+                indexPrivileges.add(
+                    new ManageRolesIndexPermissionGroup(
+                        parsedArraysByFieldName.get(Fields.NAMES.getPreferredName()),
+                        parsedArraysByFieldName.get(Fields.PRIVILEGES.getPreferredName())
+                    )
+                );
+            }
+            expectedToken(parser.nextToken(), parser, XContentParser.Token.END_OBJECT);
+
+            for (var indexPrivilege : indexPrivileges) {
+                if (indexPrivilege.indexPatterns == null || indexPrivilege.indexPatterns.length == 0) {
+                    throw new IllegalArgumentException("Indices privileges must refer to at least one index name or index name pattern");
+                }
+                if (indexPrivilege.privileges == null || indexPrivilege.privileges.length == 0) {
+                    throw new IllegalArgumentException("Indices privileges must define at least one privilege");
+                }
+            }
+            return new ManageRolesPrivilege(indexPrivileges);
+        }
+
+        public record ManageRolesIndexPermissionGroup(String[] indexPatterns, String[] privileges) implements Writeable {
+            public static ManageRolesIndexPermissionGroup createFrom(StreamInput in) throws IOException {
+                return new ManageRolesIndexPermissionGroup(in.readStringArray(), in.readStringArray());
+            }
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+                out.writeStringArray(indexPatterns);
+                out.writeStringArray(privileges);
+            }
+
+            @Override
+            public String toString() {
+                return "{"
+                    + Fields.NAMES
+                    + ":"
+                    + Arrays.toString(indexPatterns())
+                    + ":"
+                    + Fields.PRIVILEGES
+                    + ":"
+                    + Arrays.toString(privileges())
+                    + "}";
+            }
+
+            @Override
+            public boolean equals(Object o) {
+                if (this == o) return true;
+                if (o == null || getClass() != o.getClass()) return false;
+                ManageRolesIndexPermissionGroup that = (ManageRolesIndexPermissionGroup) o;
+                return Arrays.equals(indexPatterns, that.indexPatterns) && Arrays.equals(privileges, that.privileges);
+            }
+
+            @Override
+            public int hashCode() {
+                return Objects.hash(Arrays.hashCode(indexPatterns), Arrays.hashCode(privileges));
+            }
+        }
+
+        @Override
+        public String toString() {
+            return "{"
+                + getCategory()
+                + ":"
+                + Fields.MANAGE.getPreferredName()
+                + ":"
+                + Fields.INDICES.getPreferredName()
+                + "=["
+                + Strings.collectionToDelimitedString(indexPermissionGroups, ",")
+                + "]}";
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            final ManageRolesPrivilege that = (ManageRolesPrivilege) o;
+
+            if (this.indexPermissionGroups.size() != that.indexPermissionGroups.size()) {
+                return false;
+            }
+
+            for (int i = 0; i < this.indexPermissionGroups.size(); i++) {
+                if (Objects.equals(this.indexPermissionGroups.get(i), that.indexPermissionGroups.get(i)) == false) {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(indexPermissionGroups.hashCode());
+        }
+
+        @Override
+        public ClusterPermission.Builder buildPermission(final ClusterPermission.Builder builder) {
+            return builder.addWithPredicateSupplier(
+                this,
+                Set.of(PutRoleAction.NAME, ActionTypes.BULK_PUT_ROLES.name(), ActionTypes.BULK_DELETE_ROLES.name(), DeleteRoleAction.NAME),
+                requestPredicateSupplier
+            );
+        }
+
+        private static boolean requestIndexPatternsAllowed(
+            IndicesPermission indicesPermission,
+            String[] requestIndexPatterns,
+            String[] privileges
+        ) {
+            return indicesPermission.checkResourcePrivileges(Set.of(requestIndexPatterns), false, Set.of(privileges), true, null);
+        }
+
+        private static boolean hasNonIndexPrivileges(RoleDescriptor roleDescriptor) {
+            return roleDescriptor.hasApplicationPrivileges()
+                || roleDescriptor.hasClusterPrivileges()
+                || roleDescriptor.hasConfigurableClusterPrivileges()
+                || roleDescriptor.hasRemoteIndicesPrivileges()
+                || roleDescriptor.hasRemoteClusterPermissions()
+                || roleDescriptor.hasRunAs()
+                || roleDescriptor.hasWorkflowsRestriction();
+        }
+
+        private interface Fields {
+            ParseField MANAGE = new ParseField("manage");
+            ParseField INDICES = new ParseField("indices");
+            ParseField PRIVILEGES = new ParseField("privileges");
+            ParseField NAMES = new ParseField("names");
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
index df13bcb9342bf..7174b2f616c2a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
@@ -86,7 +86,6 @@ public final class IndexPrivilege extends Privilege {
         TransportClusterSearchShardsAction.TYPE.name(),
         TransportSearchShardsAction.TYPE.name(),
         TransportResolveClusterAction.NAME,
-        "indices:data/read/esql/resolve_fields",
         "indices:data/read/esql",
         "indices:data/read/esql/compute"
     );
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
index 4f3d7a245fc8f..74434adf61fbb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
@@ -868,6 +868,11 @@ private static RoleDescriptor buildViewerRoleDescriptor() {
                     .indices("/~(([.]|ilm-history-).*)/")
                     .privileges("read", "view_index_metadata")
                     .build(),
+                // Observability
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".slo-observability.*")
+                    .privileges("read", "view_index_metadata")
+                    .build(),
                 // Security
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(ReservedRolesStore.ALERTS_LEGACY_INDEX, ReservedRolesStore.LISTS_INDEX, ReservedRolesStore.LISTS_ITEMS_INDEX)
@@ -915,6 +920,10 @@ private static RoleDescriptor buildEditorRoleDescriptor() {
                     .indices("observability-annotations")
                     .privileges("read", "view_index_metadata", "write")
                     .build(),
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".slo-observability.*")
+                    .privileges("read", "view_index_metadata", "write", "manage")
+                    .build(),
                 // Security
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(ReservedRolesStore.ALERTS_LEGACY_INDEX, ReservedRolesStore.LISTS_INDEX, ReservedRolesStore.LISTS_ITEMS_INDEX)
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java
index fb892a318f07c..23bf21004040a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicy.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.core.slm;
 
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.cluster.SimpleDiffable;
@@ -15,6 +14,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.scheduler.SchedulerEngine;
+import org.elasticsearch.common.scheduler.TimeValueSchedule;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.snapshots.SnapshotsService;
@@ -24,9 +25,11 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.scheduler.Cron;
+import org.elasticsearch.xpack.core.scheduler.CronSchedule;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.time.Clock;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -48,6 +51,7 @@ public class SnapshotLifecyclePolicy implements SimpleDiffable<SnapshotLifecycle
     private final String repository;
     private final Map<String, Object> configuration;
     private final SnapshotRetentionConfiguration retentionPolicy;
+    private final boolean isCronSchedule;
 
     private static final ParseField NAME = new ParseField("name");
     private static final ParseField SCHEDULE = new ParseField("schedule");
@@ -92,6 +96,7 @@ public SnapshotLifecyclePolicy(
         this.repository = Objects.requireNonNull(repository, "policy snapshot repository is required");
         this.configuration = configuration;
         this.retentionPolicy = retentionPolicy;
+        this.isCronSchedule = isCronSchedule(schedule);
     }
 
     public SnapshotLifecyclePolicy(StreamInput in) throws IOException {
@@ -101,6 +106,7 @@ public SnapshotLifecyclePolicy(StreamInput in) throws IOException {
         this.repository = in.readString();
         this.configuration = in.readGenericMap();
         this.retentionPolicy = in.readOptionalWriteable(SnapshotRetentionConfiguration::new);
+        this.isCronSchedule = isCronSchedule(schedule);
     }
 
     public String getId() {
@@ -129,9 +135,43 @@ public SnapshotRetentionConfiguration getRetentionPolicy() {
         return this.retentionPolicy;
     }
 
-    public long calculateNextExecution() {
-        final Cron scheduleEvaluator = new Cron(this.schedule);
-        return scheduleEvaluator.getNextValidTimeAfter(System.currentTimeMillis());
+    boolean isCronSchedule() {
+        return this.isCronSchedule;
+    }
+
+    /**
+     * @return whether `schedule` is a cron expression
+     */
+    static boolean isCronSchedule(String schedule) {
+        try {
+            new Cron(schedule);
+            return true;
+        } catch (IllegalArgumentException e) {
+            return false;
+        }
+    }
+
+    /**
+     * @return whether `schedule` is an interval time unit expression
+     */
+    public static boolean isIntervalSchedule(String schedule) {
+        try {
+            TimeValue.parseTimeValue(schedule, "schedule");
+            return true;
+        } catch (IllegalArgumentException e) {
+            return false;
+        }
+    }
+
+    public long calculateNextExecution(long modifiedDate, Clock clock) {
+        if (isCronSchedule()) {
+            final Cron scheduleEvaluator = new Cron(this.schedule);
+            return scheduleEvaluator.getNextValidTimeAfter(clock.millis());
+        } else {
+            final TimeValue interval = TimeValue.parseTimeValue(this.schedule, SCHEDULE.getPreferredName());
+            final TimeValueSchedule timeValueSchedule = new TimeValueSchedule(interval);
+            return timeValueSchedule.nextScheduledTimeAfter(modifiedDate, clock.millis());
+        }
     }
 
     /**
@@ -139,13 +179,17 @@ public long calculateNextExecution() {
      * <p>
      * In ordinary cases, this can be treated as the interval between executions of the schedule (for schedules like 'twice an hour' or
      * 'every five minutes').
-     *
+     * @param clock a clock to provide current time
      * @return a {@link TimeValue} representing the difference between the next two valid times after now, or {@link TimeValue#MINUS_ONE}
      *         if either of the next two times after now is unsupported according to @{@link Cron#getNextValidTimeAfter(long)}
      */
-    public TimeValue calculateNextInterval() {
+    public TimeValue calculateNextInterval(Clock clock) {
+        if (isCronSchedule() == false) {
+            return TimeValue.parseTimeValue(schedule, SCHEDULE.getPreferredName());
+        }
+
         final Cron scheduleEvaluator = new Cron(this.schedule);
-        long next1 = scheduleEvaluator.getNextValidTimeAfter(System.currentTimeMillis());
+        long next1 = scheduleEvaluator.getNextValidTimeAfter(clock.millis());
         long next2 = scheduleEvaluator.getNextValidTimeAfter(next1);
         if (next1 > 0 && next2 > 0) {
             return TimeValue.timeValueMillis(next2 - next1);
@@ -154,6 +198,15 @@ public TimeValue calculateNextInterval() {
         }
     }
 
+    public SchedulerEngine.Job buildSchedulerJob(String jobId, long modifiedDate) {
+        if (isCronSchedule()) {
+            return new SchedulerEngine.Job(jobId, new CronSchedule(schedule));
+        } else {
+            TimeValue timeValue = TimeValue.parseTimeValue(schedule, "schedule");
+            return new SchedulerEngine.Job(jobId, new TimeValueSchedule(timeValue), modifiedDate);
+        }
+    }
+
     public ActionRequestValidationException validate() {
         ActionRequestValidationException err = new ActionRequestValidationException();
 
@@ -182,13 +235,19 @@ public ActionRequestValidationException validate() {
         }
 
         // Schedule validation
+        // n.b. there's more validation beyond this in SnapshotLifecycleService#validateMinimumInterval
         if (Strings.hasText(schedule) == false) {
             err.addValidationError("invalid schedule [" + schedule + "]: must not be empty");
         } else {
             try {
-                new Cron(schedule);
-            } catch (IllegalArgumentException e) {
-                err.addValidationError("invalid schedule: " + ExceptionsHelper.unwrapCause(e).getMessage());
+                var intervalTimeValue = TimeValue.parseTimeValue(schedule, SCHEDULE.getPreferredName());
+                if (intervalTimeValue.millis() == 0) {
+                    err.addValidationError("invalid schedule [" + schedule + "]: time unit must be at least 1 millisecond");
+                }
+            } catch (IllegalArgumentException e1) {
+                if (isCronSchedule(schedule) == false) {
+                    err.addValidationError("invalid schedule [" + schedule + "]: must be a valid cron expression or time unit");
+                }
             }
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java
index 6a352461c2e1e..c3c70e595eb75 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItem.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.time.Clock;
 import java.util.Objects;
 
 /**
@@ -171,7 +172,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.timeField(
             SnapshotLifecyclePolicyMetadata.NEXT_EXECUTION_MILLIS.getPreferredName(),
             SnapshotLifecyclePolicyMetadata.NEXT_EXECUTION.getPreferredName(),
-            policy.calculateNextExecution()
+            policy.calculateNextExecution(modifiedDate, Clock.systemUTC())
         );
         if (snapshotInProgress != null) {
             builder.field(SNAPSHOT_IN_PROGRESS.getPreferredName(), snapshotInProgress);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java
index daec14df094c4..672578787762e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.scheduler.SchedulerEngine;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -181,6 +182,10 @@ public long getInvocationsSinceLastSuccess() {
         return invocationsSinceLastSuccess;
     }
 
+    public SchedulerEngine.Job buildSchedulerJob(String jobId) {
+        return policy.buildSchedulerJob(jobId, modifiedDate);
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -277,6 +282,11 @@ public Builder setInvocationsSinceLastSuccess(long invocationsSinceLastSuccess)
             return this;
         }
 
+        public Builder incrementInvocationsSinceLastSuccess() {
+            this.invocationsSinceLastSuccess++;
+            return this;
+        }
+
         public SnapshotLifecyclePolicyMetadata build() {
             return new SnapshotLifecyclePolicyMetadata(
                 Objects.requireNonNull(policy),
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStats.java
index d1b71a92c061d..8dd0510d2b199 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStats.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -27,7 +26,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -38,12 +36,11 @@
  */
 public class SnapshotLifecycleStats implements Writeable, ToXContentObject {
 
-    private final CounterMetric retentionRunCount = new CounterMetric();
-    private final CounterMetric retentionFailedCount = new CounterMetric();
-    private final CounterMetric retentionTimedOut = new CounterMetric();
-    private final CounterMetric retentionTimeMs = new CounterMetric();
+    private final long retentionRun;
+    private final long retentionFailed;
+    private final long retentionTimedOut;
+    private final long retentionTimeMs;
     private final Map<String, SnapshotPolicyStats> policyStats;
-
     public static final ParseField RETENTION_RUNS = new ParseField("retention_runs");
     public static final ParseField RETENTION_FAILED = new ParseField("retention_failed");
     public static final ParseField RETENTION_TIMED_OUT = new ParseField("retention_timed_out");
@@ -66,7 +63,7 @@ public class SnapshotLifecycleStats implements Writeable, ToXContentObject {
             long timeMs = (long) a[3];
             Map<String, SnapshotPolicyStats> policyStatsMap = ((List<SnapshotPolicyStats>) a[4]).stream()
                 .collect(Collectors.toMap(m -> m.policyId, Function.identity()));
-            return new SnapshotLifecycleStats(runs, failed, timedOut, timeMs, policyStatsMap);
+            return new SnapshotLifecycleStats(runs, failed, timedOut, timeMs, Collections.unmodifiableMap(policyStatsMap));
         }
     );
 
@@ -79,7 +76,7 @@ public class SnapshotLifecycleStats implements Writeable, ToXContentObject {
     }
 
     public SnapshotLifecycleStats() {
-        this.policyStats = new ConcurrentHashMap<>();
+        this(0, 0, 0, 0, Map.of());
     }
 
     // public for testing
@@ -90,19 +87,23 @@ public SnapshotLifecycleStats(
         long retentionTimeMs,
         Map<String, SnapshotPolicyStats> policyStats
     ) {
-        this.retentionRunCount.inc(retentionRuns);
-        this.retentionFailedCount.inc(retentionFailed);
-        this.retentionTimedOut.inc(retentionTimedOut);
-        this.retentionTimeMs.inc(retentionTimeMs);
-        this.policyStats = policyStats;
+        this.retentionRun = retentionRuns;
+        this.retentionFailed = retentionFailed;
+        this.retentionTimedOut = retentionTimedOut;
+        this.retentionTimeMs = retentionTimeMs;
+        this.policyStats = Collections.unmodifiableMap(policyStats);
+    }
+
+    private SnapshotLifecycleStats(Map<String, SnapshotPolicyStats> policyStats) {
+        this(0, 0, 0, 0, policyStats);
     }
 
     public SnapshotLifecycleStats(StreamInput in) throws IOException {
-        this.policyStats = new ConcurrentHashMap<>(in.readMap(SnapshotPolicyStats::new));
-        this.retentionRunCount.inc(in.readVLong());
-        this.retentionFailedCount.inc(in.readVLong());
-        this.retentionTimedOut.inc(in.readVLong());
-        this.retentionTimeMs.inc(in.readVLong());
+        this.policyStats = in.readImmutableMap(SnapshotPolicyStats::new);
+        this.retentionRun = in.readVLong();
+        this.retentionFailed = in.readVLong();
+        this.retentionTimedOut = in.readVLong();
+        this.retentionTimeMs = in.readVLong();
     }
 
     public static SnapshotLifecycleStats parse(XContentParser parser) {
@@ -110,7 +111,6 @@ public static SnapshotLifecycleStats parse(XContentParser parser) {
     }
 
     public SnapshotLifecycleStats merge(SnapshotLifecycleStats other) {
-
         HashMap<String, SnapshotPolicyStats> newPolicyStats = new HashMap<>(this.policyStats);
         // Merges the per-run stats (the stats in "other") with the stats already present
         other.policyStats.forEach((policyId, perRunPolicyStats) -> {
@@ -124,11 +124,11 @@ public SnapshotLifecycleStats merge(SnapshotLifecycleStats other) {
         });
 
         return new SnapshotLifecycleStats(
-            this.retentionRunCount.count() + other.retentionRunCount.count(),
-            this.retentionFailedCount.count() + other.retentionFailedCount.count(),
-            this.retentionTimedOut.count() + other.retentionTimedOut.count(),
-            this.retentionTimeMs.count() + other.retentionTimeMs.count(),
-            newPolicyStats
+            this.retentionRun + other.retentionRun,
+            this.retentionFailed + other.retentionFailed,
+            this.retentionTimedOut + other.retentionTimedOut,
+            this.retentionTimeMs + other.retentionTimeMs,
+            Collections.unmodifiableMap(newPolicyStats)
         );
     }
 
@@ -136,11 +136,11 @@ public SnapshotLifecycleStats removePolicy(String policyId) {
         Map<String, SnapshotPolicyStats> policyStatsCopy = new HashMap<>(this.policyStats);
         policyStatsCopy.remove(policyId);
         return new SnapshotLifecycleStats(
-            this.retentionRunCount.count(),
-            this.retentionFailedCount.count(),
-            this.retentionTimedOut.count(),
-            this.retentionTimeMs.count(),
-            policyStatsCopy
+            this.retentionRun,
+            this.retentionFailed,
+            this.retentionTimedOut,
+            this.retentionTimeMs,
+            Collections.unmodifiableMap(policyStatsCopy)
         );
     }
 
@@ -148,82 +148,83 @@ public SnapshotLifecycleStats removePolicy(String policyId) {
      * @return a map of per-policy stats for each SLM policy
      */
     public Map<String, SnapshotPolicyStats> getMetrics() {
-        return Collections.unmodifiableMap(this.policyStats);
+        return this.policyStats;
     }
 
     /**
-     * Increment the number of times SLM retention has been run
+     * Return new stats with number of times SLM retention has been run incremented
      */
-    public void retentionRun() {
-        this.retentionRunCount.inc();
+    public SnapshotLifecycleStats withRetentionRunIncremented() {
+        return new SnapshotLifecycleStats(retentionRun + 1, retentionFailed, retentionTimedOut, retentionTimeMs, policyStats);
     }
 
     /**
-     * Increment the number of times SLM retention has failed
+     * Return new stats with number of times SLM retention has failed incremented
      */
-    public void retentionFailed() {
-        this.retentionFailedCount.inc();
+    public SnapshotLifecycleStats withRetentionFailedIncremented() {
+        return new SnapshotLifecycleStats(retentionRun, retentionFailed + 1, retentionTimedOut, retentionTimeMs, policyStats);
     }
 
     /**
-     * Increment the number of times that SLM retention timed out due to the max delete time
-     * window being exceeded.
+     * Return new stats the number of times that SLM retention timed out due to the max delete time
+     * window being exceeded incremented
      */
-    public void retentionTimedOut() {
-        this.retentionTimedOut.inc();
+    public SnapshotLifecycleStats withRetentionTimedOutIncremented() {
+        return new SnapshotLifecycleStats(retentionRun, retentionFailed, retentionTimedOut + 1, retentionTimeMs, policyStats);
     }
 
     /**
-     * Register the amount of time taken for deleting snapshots during SLM retention
+     * Return new stats with the amount of time taken for deleting snapshots during SLM retention updated
      */
-    public void deletionTime(TimeValue elapsedTime) {
-        this.retentionTimeMs.inc(elapsedTime.millis());
+    public SnapshotLifecycleStats withDeletionTimeUpdated(TimeValue elapsedTime) {
+        final long newRetentionTimeMs = retentionTimeMs + elapsedTime.millis();
+        return new SnapshotLifecycleStats(retentionRun, retentionFailed, retentionTimedOut, newRetentionTimeMs, policyStats);
     }
 
     /**
-     * Increment the per-policy snapshot taken count for the given policy id
+     * Return new stats with the per-policy snapshot taken count for the given policy id incremented
      */
-    public void snapshotTaken(String slmPolicy) {
-        this.policyStats.computeIfAbsent(slmPolicy, SnapshotPolicyStats::new).snapshotTaken();
+    public SnapshotLifecycleStats withTakenIncremented(String slmPolicy) {
+        return merge(new SnapshotLifecycleStats(Map.of(slmPolicy, SnapshotPolicyStats.taken(slmPolicy))));
     }
 
     /**
-     * Increment the per-policy snapshot failure count for the given policy id
+     * Return new stats with the per-policy snapshot failure count for the given policy id incremented
      */
-    public void snapshotFailed(String slmPolicy) {
-        this.policyStats.computeIfAbsent(slmPolicy, SnapshotPolicyStats::new).snapshotFailed();
+    public SnapshotLifecycleStats withFailedIncremented(String slmPolicy) {
+        return merge(new SnapshotLifecycleStats(Map.of(slmPolicy, SnapshotPolicyStats.failed(slmPolicy))));
     }
 
     /**
-     * Increment the per-policy snapshot deleted count for the given policy id
+     * Return new stats with the per-policy snapshot deleted count for the given policy id incremented
      */
-    public void snapshotDeleted(String slmPolicy) {
-        this.policyStats.computeIfAbsent(slmPolicy, SnapshotPolicyStats::new).snapshotDeleted();
+    public SnapshotLifecycleStats withDeletedIncremented(String slmPolicy) {
+        return merge(new SnapshotLifecycleStats(Map.of(slmPolicy, SnapshotPolicyStats.deleted(slmPolicy))));
     }
 
     /**
-     * Increment the per-policy snapshot deletion failure count for the given policy id
+     * Return new stats with the per-policy snapshot deletion failure count for the given policy id incremented
      */
-    public void snapshotDeleteFailure(String slmPolicy) {
-        this.policyStats.computeIfAbsent(slmPolicy, SnapshotPolicyStats::new).snapshotDeleteFailure();
+    public SnapshotLifecycleStats withDeleteFailureIncremented(String slmPolicy) {
+        return merge(new SnapshotLifecycleStats(Map.of(slmPolicy, SnapshotPolicyStats.deleteFailure(slmPolicy))));
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeMap(policyStats, StreamOutput::writeWriteable);
-        out.writeVLong(retentionRunCount.count());
-        out.writeVLong(retentionFailedCount.count());
-        out.writeVLong(retentionTimedOut.count());
-        out.writeVLong(retentionTimeMs.count());
+        out.writeVLong(retentionRun);
+        out.writeVLong(retentionFailed);
+        out.writeVLong(retentionTimedOut);
+        out.writeVLong(retentionTimeMs);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(RETENTION_RUNS.getPreferredName(), this.retentionRunCount.count());
-        builder.field(RETENTION_FAILED.getPreferredName(), this.retentionFailedCount.count());
-        builder.field(RETENTION_TIMED_OUT.getPreferredName(), this.retentionTimedOut.count());
-        TimeValue retentionTime = TimeValue.timeValueMillis(this.retentionTimeMs.count());
+        builder.field(RETENTION_RUNS.getPreferredName(), this.retentionRun);
+        builder.field(RETENTION_FAILED.getPreferredName(), this.retentionFailed);
+        builder.field(RETENTION_TIMED_OUT.getPreferredName(), this.retentionTimedOut);
+        TimeValue retentionTime = TimeValue.timeValueMillis(this.retentionTimeMs);
         builder.field(RETENTION_TIME.getPreferredName(), retentionTime);
         builder.field(RETENTION_TIME_MILLIS.getPreferredName(), retentionTime.millis());
 
@@ -231,10 +232,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             .stream()
             .sorted(Comparator.comparing(SnapshotPolicyStats::getPolicyId)) // maintain a consistent order when serializing
             .toList();
-        long totalTaken = metrics.stream().mapToLong(s -> s.snapshotsTaken.count()).sum();
-        long totalFailed = metrics.stream().mapToLong(s -> s.snapshotsFailed.count()).sum();
-        long totalDeleted = metrics.stream().mapToLong(s -> s.snapshotsDeleted.count()).sum();
-        long totalDeleteFailures = metrics.stream().mapToLong(s -> s.snapshotDeleteFailures.count()).sum();
+        long totalTaken = metrics.stream().mapToLong(s -> s.snapshotsTaken).sum();
+        long totalFailed = metrics.stream().mapToLong(s -> s.snapshotsFailed).sum();
+        long totalDeleted = metrics.stream().mapToLong(s -> s.snapshotsDeleted).sum();
+        long totalDeleteFailures = metrics.stream().mapToLong(s -> s.snapshotDeleteFailures).sum();
         builder.field(TOTAL_TAKEN.getPreferredName(), totalTaken);
         builder.field(TOTAL_FAILED.getPreferredName(), totalFailed);
         builder.field(TOTAL_DELETIONS.getPreferredName(), totalDeleted);
@@ -253,13 +254,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public int hashCode() {
-        return Objects.hash(
-            retentionRunCount.count(),
-            retentionFailedCount.count(),
-            retentionTimedOut.count(),
-            retentionTimeMs.count(),
-            policyStats
-        );
+        return Objects.hash(retentionRun, retentionFailed, retentionTimedOut, retentionTimeMs, policyStats);
     }
 
     @Override
@@ -271,10 +266,10 @@ public boolean equals(Object obj) {
             return false;
         }
         SnapshotLifecycleStats other = (SnapshotLifecycleStats) obj;
-        return Objects.equals(retentionRunCount.count(), other.retentionRunCount.count())
-            && Objects.equals(retentionFailedCount.count(), other.retentionFailedCount.count())
-            && Objects.equals(retentionTimedOut.count(), other.retentionTimedOut.count())
-            && Objects.equals(retentionTimeMs.count(), other.retentionTimeMs.count())
+        return Objects.equals(retentionRun, other.retentionRun)
+            && Objects.equals(retentionFailed, other.retentionFailed)
+            && Objects.equals(retentionTimedOut, other.retentionTimedOut)
+            && Objects.equals(retentionTimeMs, other.retentionTimeMs)
             && Objects.equals(policyStats, other.policyStats);
     }
 
@@ -285,10 +280,10 @@ public String toString() {
 
     public static class SnapshotPolicyStats implements Writeable, ToXContentFragment {
         private final String policyId;
-        private final CounterMetric snapshotsTaken = new CounterMetric();
-        private final CounterMetric snapshotsFailed = new CounterMetric();
-        private final CounterMetric snapshotsDeleted = new CounterMetric();
-        private final CounterMetric snapshotDeleteFailures = new CounterMetric();
+        private final long snapshotsTaken;
+        private final long snapshotsFailed;
+        private final long snapshotsDeleted;
+        private final long snapshotDeleteFailures;
 
         public static final ParseField POLICY_ID = new ParseField("policy");
         public static final ParseField SNAPSHOTS_TAKEN = new ParseField("snapshots_taken");
@@ -318,73 +313,71 @@ public static class SnapshotPolicyStats implements Writeable, ToXContentFragment
         }
 
         public SnapshotPolicyStats(String slmPolicy) {
-            this.policyId = slmPolicy;
+            this(slmPolicy, 0, 0, 0, 0);
         }
 
         public SnapshotPolicyStats(String policyId, long snapshotsTaken, long snapshotsFailed, long deleted, long failedDeletes) {
             this.policyId = policyId;
-            this.snapshotsTaken.inc(snapshotsTaken);
-            this.snapshotsFailed.inc(snapshotsFailed);
-            this.snapshotsDeleted.inc(deleted);
-            this.snapshotDeleteFailures.inc(failedDeletes);
+            this.snapshotsTaken = snapshotsTaken;
+            this.snapshotsFailed = snapshotsFailed;
+            this.snapshotsDeleted = deleted;
+            this.snapshotDeleteFailures = failedDeletes;
         }
 
         public SnapshotPolicyStats(StreamInput in) throws IOException {
-            this.policyId = in.readString();
-            this.snapshotsTaken.inc(in.readVLong());
-            this.snapshotsFailed.inc(in.readVLong());
-            this.snapshotsDeleted.inc(in.readVLong());
-            this.snapshotDeleteFailures.inc(in.readVLong());
+            this(in.readString(), in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong());
         }
 
         public SnapshotPolicyStats merge(SnapshotPolicyStats other) {
             return new SnapshotPolicyStats(
                 this.policyId,
-                this.snapshotsTaken.count() + other.snapshotsTaken.count(),
-                this.snapshotsFailed.count() + other.snapshotsFailed.count(),
-                this.snapshotsDeleted.count() + other.snapshotsDeleted.count(),
-                this.snapshotDeleteFailures.count() + other.snapshotDeleteFailures.count()
+                this.snapshotsTaken + other.snapshotsTaken,
+                this.snapshotsFailed + other.snapshotsFailed,
+                this.snapshotsDeleted + other.snapshotsDeleted,
+                this.snapshotDeleteFailures + other.snapshotDeleteFailures
             );
         }
 
-        void snapshotTaken() {
-            snapshotsTaken.inc();
+        private static SnapshotPolicyStats taken(String policyId) {
+            return new SnapshotPolicyStats(policyId, 1, 0, 0, 0);
         }
 
-        void snapshotFailed() {
-            snapshotsFailed.inc();
+        private static SnapshotPolicyStats failed(String policyId) {
+            return new SnapshotPolicyStats(policyId, 0, 1, 0, 0);
         }
 
-        void snapshotDeleted() {
-            snapshotsDeleted.inc();
+        private static SnapshotPolicyStats deleted(String policyId) {
+            return new SnapshotPolicyStats(policyId, 0, 0, 1, 0);
         }
 
-        void snapshotDeleteFailure() {
-            snapshotDeleteFailures.inc();
+        private static SnapshotPolicyStats deleteFailure(String policyId) {
+            return new SnapshotPolicyStats(policyId, 0, 0, 0, 1);
         }
 
         public String getPolicyId() {
             return policyId;
         }
 
+        public long getSnapshotTakenCount() {
+            return snapshotsTaken;
+        }
+
+        public long getSnapshotFailedCount() {
+            return snapshotsFailed;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(policyId);
-            out.writeVLong(snapshotsTaken.count());
-            out.writeVLong(snapshotsFailed.count());
-            out.writeVLong(snapshotsDeleted.count());
-            out.writeVLong(snapshotDeleteFailures.count());
+            out.writeVLong(snapshotsTaken);
+            out.writeVLong(snapshotsFailed);
+            out.writeVLong(snapshotsDeleted);
+            out.writeVLong(snapshotDeleteFailures);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(
-                policyId,
-                snapshotsTaken.count(),
-                snapshotsFailed.count(),
-                snapshotsDeleted.count(),
-                snapshotDeleteFailures.count()
-            );
+            return Objects.hash(policyId, snapshotsTaken, snapshotsFailed, snapshotsDeleted, snapshotDeleteFailures);
         }
 
         @Override
@@ -397,19 +390,19 @@ public boolean equals(Object obj) {
             }
             SnapshotPolicyStats other = (SnapshotPolicyStats) obj;
             return Objects.equals(policyId, other.policyId)
-                && Objects.equals(snapshotsTaken.count(), other.snapshotsTaken.count())
-                && Objects.equals(snapshotsFailed.count(), other.snapshotsFailed.count())
-                && Objects.equals(snapshotsDeleted.count(), other.snapshotsDeleted.count())
-                && Objects.equals(snapshotDeleteFailures.count(), other.snapshotDeleteFailures.count());
+                && Objects.equals(snapshotsTaken, other.snapshotsTaken)
+                && Objects.equals(snapshotsFailed, other.snapshotsFailed)
+                && Objects.equals(snapshotsDeleted, other.snapshotsDeleted)
+                && Objects.equals(snapshotDeleteFailures, other.snapshotDeleteFailures);
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field(SnapshotPolicyStats.POLICY_ID.getPreferredName(), policyId);
-            builder.field(SnapshotPolicyStats.SNAPSHOTS_TAKEN.getPreferredName(), snapshotsTaken.count());
-            builder.field(SnapshotPolicyStats.SNAPSHOTS_FAILED.getPreferredName(), snapshotsFailed.count());
-            builder.field(SnapshotPolicyStats.SNAPSHOTS_DELETED.getPreferredName(), snapshotsDeleted.count());
-            builder.field(SnapshotPolicyStats.SNAPSHOT_DELETION_FAILURES.getPreferredName(), snapshotDeleteFailures.count());
+            builder.field(SnapshotPolicyStats.SNAPSHOTS_TAKEN.getPreferredName(), snapshotsTaken);
+            builder.field(SnapshotPolicyStats.SNAPSHOTS_FAILED.getPreferredName(), snapshotsFailed);
+            builder.field(SnapshotPolicyStats.SNAPSHOTS_DELETED.getPreferredName(), snapshotsDeleted);
+            builder.field(SnapshotPolicyStats.SNAPSHOT_DELETION_FAILURES.getPreferredName(), snapshotDeleteFailures);
             return builder;
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleStatsAction.java
index 6279c4208b878..478f2bd17cb8d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleStatsAction.java
@@ -46,6 +46,10 @@ public Response(StreamInput in) throws IOException {
             this.slmStats = new SnapshotLifecycleStats(in);
         }
 
+        public SnapshotLifecycleStats getSlmStats() {
+            return slmStats;
+        }
+
         @Override
         public String toString() {
             return Strings.toString(this);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/action/TransportGetCertificateInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/action/TransportGetCertificateInfoAction.java
index 90a6397e90b54..22eecb0c789d7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/action/TransportGetCertificateInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/action/TransportGetCertificateInfoAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ssl.SSLService;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java
new file mode 100644
index 0000000000000..9840535989a7c
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.template;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+public class ResourceUtils {
+    static byte[] loadVersionedResourceUTF8(Class<?> clazz, String name, int version, String versionProperty) {
+        return loadVersionedResourceUTF8(clazz, name, version, versionProperty, Map.of());
+    }
+
+    static byte[] loadVersionedResourceUTF8(
+        Class<?> clazz,
+        String name,
+        int version,
+        String versionProperty,
+        Map<String, String> variables
+    ) {
+        try {
+            String content = loadResource(clazz, name);
+            content = TemplateUtils.replaceVariables(content, String.valueOf(version), versionProperty, variables);
+            return content.getBytes(StandardCharsets.UTF_8);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static String loadResource(Class<?> clazz, String name) throws IOException {
+        InputStream is = clazz.getResourceAsStream(name);
+        if (is == null) {
+            throw new IOException("Resource [" + name + "] not found in classpath.");
+        }
+        return new String(is.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java
new file mode 100644
index 0000000000000..0cb69b490a73a
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlIngestPipelineConfig.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.template;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.util.List;
+
+import static org.elasticsearch.xpack.core.template.ResourceUtils.loadVersionedResourceUTF8;
+
+public class YamlIngestPipelineConfig extends IngestPipelineConfig {
+    private final Class<?> clazz;
+
+    public YamlIngestPipelineConfig(
+        String id,
+        String resource,
+        int version,
+        String versionProperty,
+        List<String> dependencies,
+        Class<?> clazz
+    ) {
+        super(id, resource, version, versionProperty, dependencies);
+        this.clazz = clazz;
+    }
+
+    @Override
+    public XContentType getXContentType() {
+        return XContentType.YAML;
+    }
+
+    @Override
+    public BytesReference loadConfig() {
+        return new BytesArray(loadVersionedResourceUTF8(clazz, "/ingest-pipelines/" + id + ".yaml", version, versionProperty, variables));
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java
new file mode 100644
index 0000000000000..a4e5d7cdc4b44
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java
@@ -0,0 +1,219 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.template;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.yaml.YamlXContent;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.core.template.ResourceUtils.loadResource;
+import static org.elasticsearch.xpack.core.template.ResourceUtils.loadVersionedResourceUTF8;
+
+/**
+ * Creates index templates and ingest pipelines based on YAML files from resources.
+ */
+public abstract class YamlTemplateRegistry extends IndexTemplateRegistry {
+    private static final Logger logger = LogManager.getLogger(YamlTemplateRegistry.class);
+    // this node feature is a redefinition of {@link DataStreamFeatures#DATA_STREAM_LIFECYCLE} and it's meant to avoid adding a
+    // dependency to the data-streams module just for this
+    public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle");
+    private final int version;
+
+    private final Map<String, ComponentTemplate> componentTemplates;
+    private final Map<String, ComposableIndexTemplate> composableIndexTemplates;
+    private final List<IngestPipelineConfig> ingestPipelines;
+    private final FeatureService featureService;
+    private volatile boolean enabled;
+
+    @SuppressWarnings({ "unchecked", "this-escape" })
+    public YamlTemplateRegistry(
+        Settings nodeSettings,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        Client client,
+        NamedXContentRegistry xContentRegistry,
+        FeatureService featureService
+    ) {
+        super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
+
+        try {
+            final Map<String, Object> resources = XContentHelper.convertToMap(
+                YamlXContent.yamlXContent,
+                loadResource(this.getClass(), "/resources.yaml"),
+                false
+            );
+            version = (((Number) resources.get("version")).intValue());
+
+            final List<Object> componentTemplateNames = (List<Object>) resources.get("component-templates");
+            final List<Object> indexTemplateNames = (List<Object>) resources.get("index-templates");
+            final List<Object> ingestPipelineConfigs = (List<Object>) resources.get("ingest-pipelines");
+
+            componentTemplates = Optional.ofNullable(componentTemplateNames)
+                .orElse(Collections.emptyList())
+                .stream()
+                .map(o -> (String) o)
+                .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version)));
+            composableIndexTemplates = Optional.ofNullable(indexTemplateNames)
+                .orElse(Collections.emptyList())
+                .stream()
+                .map(o -> (String) o)
+                .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version)));
+            ingestPipelines = Optional.ofNullable(ingestPipelineConfigs)
+                .orElse(Collections.emptyList())
+                .stream()
+                .map(o -> (Map<String, Map<String, Object>>) o)
+                .map(map -> {
+                    Map.Entry<String, Map<String, Object>> pipelineConfig = map.entrySet().iterator().next();
+                    return loadIngestPipeline(
+                        pipelineConfig.getKey(),
+                        version,
+                        (List<String>) pipelineConfig.getValue().get("dependencies")
+                    );
+                })
+                .collect(Collectors.toList());
+            this.featureService = featureService;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public int getVersion() {
+        return version;
+    }
+
+    /***
+     *
+     * @return A friendly, human readable name of the index template regisry
+     */
+    public abstract String getName();
+
+    public void setEnabled(boolean enabled) {
+        logger.info("{} index template registry is {}", getName(), enabled ? "enabled" : "disabled");
+        this.enabled = enabled;
+    }
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    public void close() {
+        clusterService.removeListener(this);
+    }
+
+    @Override
+    protected boolean isClusterReady(ClusterChangedEvent event) {
+        // Ensure current version of the components are installed only after versions that support data stream lifecycle
+        // due to the use of the feature in all the `@lifecycle` component templates
+        return featureService.clusterHasFeature(event.state(), DATA_STREAM_LIFECYCLE);
+    }
+
+    @Override
+    protected boolean requiresMasterNode() {
+        return true;
+    }
+
+    @Override
+    public Map<String, ComponentTemplate> getComponentTemplateConfigs() {
+        if (enabled) {
+            return componentTemplates;
+        } else {
+            return Map.of();
+        }
+    }
+
+    @Override
+    public Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
+        if (enabled) {
+            return composableIndexTemplates;
+        } else {
+            return Map.of();
+        }
+    }
+
+    @Override
+    public List<IngestPipelineConfig> getIngestPipelines() {
+        if (enabled) {
+            return ingestPipelines;
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    protected abstract String getVersionProperty();
+
+    private ComponentTemplate loadComponentTemplate(String name, int version) {
+        try {
+            final byte[] content = loadVersionedResourceUTF8(
+                this.getClass(),
+                "/component-templates/" + name + ".yaml",
+                version,
+                getVersionProperty()
+            );
+            try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) {
+                return ComponentTemplate.parse(parser);
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("failed to load " + getName() + " Ingest plugin's component template: " + name, e);
+        }
+    }
+
+    private ComposableIndexTemplate loadIndexTemplate(String name, int version) {
+        try {
+            final byte[] content = loadVersionedResourceUTF8(
+                this.getClass(),
+                "/index-templates/" + name + ".yaml",
+                version,
+                getVersionProperty()
+            );
+            try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)) {
+                return ComposableIndexTemplate.parse(parser);
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("failed to load " + getName() + " Ingest plugin's index template: " + name, e);
+        }
+    }
+
+    private IngestPipelineConfig loadIngestPipeline(String name, int version, @Nullable List<String> dependencies) {
+        if (dependencies == null) {
+            dependencies = Collections.emptyList();
+        }
+        return new YamlIngestPipelineConfig(
+            name,
+            "/ingest-pipelines/" + name + ".yaml",
+            version,
+            getVersionProperty(),
+            dependencies,
+            this.getClass()
+        );
+    }
+
+    @Override
+    protected boolean applyRolloverAfterTemplateV2Upgrade() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java
index 734f674336809..9164fd88b6395 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
@@ -46,6 +45,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchService;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/history/WatchRecord.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/history/WatchRecord.java
index 64215c0b1acf1..f6e8a80884d66 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/history/WatchRecord.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/history/WatchRecord.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -21,8 +22,10 @@
 import org.elasticsearch.xpack.core.watcher.execution.Wid;
 import org.elasticsearch.xpack.core.watcher.input.ExecutableInput;
 import org.elasticsearch.xpack.core.watcher.input.Input;
+import org.elasticsearch.xpack.core.watcher.support.WatcherDateTimeUtils;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
+import org.elasticsearch.xpack.core.watcher.watch.Payload;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.core.watcher.watch.WatchField;
 
@@ -45,13 +48,16 @@ public abstract class WatchRecord implements ToXContentObject {
     private static final ParseField EXECUTION_RESULT = new ParseField("result");
     private static final ParseField EXCEPTION = new ParseField("exception");
     private static final ParseField USER = new ParseField("user");
+    public static final String TRUNCATED_RECORD_KEY = "truncated";
+    public static final String TRUNCATED_RECORD_VALUE = "Watch history record exceeded the value of the "
+        + "`xpack.watcher.max.history.record.size' setting";
 
     protected final Wid id;
     protected final Watch watch;
     private final String nodeId;
     protected final TriggerEvent triggerEvent;
     protected final ExecutionState state;
-    private final String user;
+    protected final String user;
 
     // only emitted to xcontent in "debug" mode
     protected final Map<String, Object> vars;
@@ -254,6 +260,8 @@ public String toString() {
         return id.toString();
     }
 
+    public abstract WatchRecord dropLargeFields() throws Exception;
+
     public static class MessageWatchRecord extends WatchRecord {
         @Nullable
         private final String[] messages;
@@ -299,6 +307,24 @@ public MessageWatchRecord(WatchRecord record, ExecutionState state, String messa
             }
         }
 
+        private MessageWatchRecord(
+            Wid id,
+            TriggerEvent triggerEvent,
+            ExecutionState state,
+            Map<String, Object> vars,
+            ExecutableInput<? extends Input, ? extends Input.Result> redactedInput,
+            ExecutableCondition condition,
+            Map<String, Object> metadata,
+            Watch watch,
+            WatchExecutionResult redactedResult,
+            String nodeId,
+            String user,
+            String[] messages
+        ) {
+            super(id, triggerEvent, state, vars, redactedInput, condition, metadata, watch, redactedResult, nodeId, user);
+            this.messages = messages;
+        }
+
         public String[] messages() {
             return messages;
         }
@@ -309,10 +335,46 @@ void innerToXContent(XContentBuilder builder, Params params) throws IOException
                 builder.array(MESSAGES.getPreferredName(), messages);
             }
         }
+
+        @Override
+        public WatchRecord dropLargeFields() throws Exception {
+            return new MessageWatchRecord(
+                this.id,
+                this.triggerEvent,
+                this.state,
+                this.vars,
+                this.input == null ? null : getTruncatedInput(),
+                this.condition,
+                this.metadata,
+                this.watch,
+                this.executionResult == null ? null : getTruncatedWatchExecutionResult(this),
+                this.getNodeId(),
+                this.user,
+                this.messages
+            );
+        }
     }
 
     public static class ExceptionWatchRecord extends WatchRecord {
 
+        private ExceptionWatchRecord(
+            Wid id,
+            TriggerEvent triggerEvent,
+            ExecutionState state,
+            Map<String, Object> vars,
+            ExecutableInput<? extends Input, ? extends Input.Result> redactedInput,
+            ExecutableCondition condition,
+            Map<String, Object> metadata,
+            Watch watch,
+            WatchExecutionResult redactedResult,
+            String nodeId,
+            String user,
+            Exception exception
+        ) {
+            super(id, triggerEvent, state, vars, redactedInput, condition, metadata, watch, redactedResult, nodeId, user);
+            this.exception = exception;
+        }
+
         private static final Map<String, String> STACK_TRACE_ENABLED_PARAMS = Map.of(
             ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE,
             "false"
@@ -356,5 +418,101 @@ void innerToXContent(XContentBuilder builder, Params params) throws IOException
                 }
             }
         }
+
+        @Override
+        public WatchRecord dropLargeFields() throws Exception {
+            return new ExceptionWatchRecord(
+                this.id,
+                triggerEvent,
+                this.state,
+                this.vars,
+                this.input == null ? null : getTruncatedInput(),
+                this.condition,
+                this.metadata,
+                this.watch,
+                this.executionResult == null ? null : getTruncatedWatchExecutionResult(this),
+                this.getNodeId(),
+                this.user,
+                this.exception
+            );
+        }
+    }
+
+    /*
+     * This returns a ExecutableInput whose toXContent() returns no information other than a new TRUNCATED_MESSAGE field. It
+     * drops other information to avoid having a document that is too large to index into Elasticsearch.
+     */
+    private static ExecutableInput<? extends Input, ? extends Input.Result> getTruncatedInput() {
+        return new ExecutableInput<>(new Input() {
+            @Override
+            public String type() {
+                return TRUNCATED_RECORD_KEY;
+            }
+
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                builder.value(TRUNCATED_RECORD_VALUE);
+                return builder;
+            }
+        }) {
+            @Override
+            public Input.Result execute(WatchExecutionContext ctx, Payload payload) {
+                throw new UnsupportedOperationException("Redacted input cannot be executed");
+            }
+        };
+    }
+
+    /*
+     * This returns a WatchExecutionResult whose toXContent() returns minimal information, including a new TRUNCATED_MESSAGE field. It
+     * drops most other information to avoid having a document that is too large to index into Elasticsearch.
+     */
+    private static WatchExecutionResult getTruncatedWatchExecutionResult(WatchRecord watchRecord) {
+        WatchExecutionContext watchExecutionContext = new WatchExecutionContext(
+            watchRecord.id.watchId(),
+            watchRecord.executionResult.executionTime(),
+            null,
+            TimeValue.ZERO
+        ) {
+            @Override
+            public boolean knownWatch() {
+                return false;
+            }
+
+            @Override
+            public boolean simulateAction(String actionId) {
+                return false;
+            }
+
+            @Override
+            public boolean skipThrottling(String actionId) {
+                return false;
+            }
+
+            @Override
+            public boolean shouldBeExecuted() {
+                return false;
+            }
+
+            @Override
+            public boolean recordExecution() {
+                return false;
+            }
+        };
+
+        return new WatchExecutionResult(watchExecutionContext, watchRecord.executionResult.executionDurationMs()) {
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                builder.startObject();
+                WatcherDateTimeUtils.writeDate(
+                    Field.EXECUTION_TIME.getPreferredName(),
+                    builder,
+                    watchRecord.executionResult.executionTime()
+                );
+                builder.field(Field.EXECUTION_DURATION.getPreferredName(), watchRecord.executionResult.executionDurationMs());
+                builder.field(TRUNCATED_RECORD_KEY, TRUNCATED_RECORD_VALUE);
+                builder.endObject();
+                return builder;
+            }
+        };
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java
index 681b004dd1d28..2f2617f956ed9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java
@@ -59,12 +59,8 @@ public ExecuteWatchRequest(StreamInput in) throws IOException {
         id = in.readOptionalString();
         ignoreCondition = in.readBoolean();
         recordExecution = in.readBoolean();
-        if (in.readBoolean()) {
-            alternativeInput = in.readGenericMap();
-        }
-        if (in.readBoolean()) {
-            triggerData = in.readGenericMap();
-        }
+        alternativeInput = in.readOptional(StreamInput::readGenericMap);
+        triggerData = in.readOptional(StreamInput::readGenericMap);
         long actionModesCount = in.readLong();
         actionModes = new HashMap<>();
         for (int i = 0; i < actionModesCount; i++) {
@@ -83,14 +79,8 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(id);
         out.writeBoolean(ignoreCondition);
         out.writeBoolean(recordExecution);
-        out.writeBoolean(alternativeInput != null);
-        if (alternativeInput != null) {
-            out.writeGenericMap(alternativeInput);
-        }
-        out.writeBoolean(triggerData != null);
-        if (triggerData != null) {
-            out.writeGenericMap(triggerData);
-        }
+        out.writeOptional(StreamOutput::writeGenericMap, alternativeInput);
+        out.writeOptional(StreamOutput::writeGenericMap, triggerData);
         out.writeLong(actionModes.size());
         for (Map.Entry<String, ActionExecutionMode> entry : actionModes.entrySet()) {
             out.writeString(entry.getKey());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java
index be43705984435..e2218dfab1f1c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java
@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.license;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
@@ -121,13 +120,12 @@ public void testLicenseTombstoneFromXContext() throws Exception {
         assertThat(metadataFromXContent.getLicense(), equalTo(LicensesMetadata.LICENSE_TOMBSTONE));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103093")
     public void testLicenseTombstoneWithUsedTrialFromXContext() throws Exception {
         final XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.startObject();
         builder.startObject("licenses");
         builder.nullField("license");
-        builder.field("trial_license", Version.CURRENT.toString());
+        builder.field("trial_license", TrialLicenseVersion.CURRENT);
         builder.endObject();
         builder.endObject();
         LicensesMetadata metadataFromXContent = getLicensesMetadataFromXContent(createParser(builder));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java
index 7c1e71139cb0c..b4389377dff34 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java
@@ -55,7 +55,7 @@ public void testSerializationBwc() throws IOException {
         if (randomBoolean()) {
             request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(20, 25, 30)));
         }
-        assertSerialization(request, TransportVersionUtils.getPreviousVersion(TransportVersions.CCR_STATS_API_TIMEOUT_PARAM));
+        assertSerialization(request, TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_14_0));
         assertSerialization(request, TransportVersions.MINIMUM_CCS_VERSION);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java
index fae437c1d2fc9..50ed7ddcc3f33 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexVersion;
@@ -124,13 +123,7 @@ public void onFailure(Exception e) {
         {
             // regular indices (non-ts) meet the step condition
             IndexMetadata indexMeta = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30))
-                .settings(
-                    Settings.builder()
-                        .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                        .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-                        .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current())
-                        .build()
-                )
+                .settings(indexSettings(1, 1).put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()).build())
                 .build();
 
             Metadata newMetadata = Metadata.builder(clusterState.metadata()).put(indexMeta, true).build();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java
index d4d4146c6a5ba..f41e117e75b9f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/InferenceActionRequestTests.java
@@ -315,7 +315,7 @@ protected InferenceAction.Request mutateInstanceForVersion(InferenceAction.Reque
                             InputType.UNSPECIFIED,
                             InferenceAction.Request.DEFAULT_TIMEOUT
                         );
-                    } else if (version.before(TransportVersions.ML_INFERENCE_COHERE_RERANK)) {
+                    } else if (version.before(TransportVersions.V_8_14_0)) {
                         return new InferenceAction.Request(
                             instance.getTaskType(),
                             instance.getInferenceEntityId(),
@@ -325,16 +325,6 @@ protected InferenceAction.Request mutateInstanceForVersion(InferenceAction.Reque
                             instance.getInputType(),
                             InferenceAction.Request.DEFAULT_TIMEOUT
                         );
-                    } else if (version.before(TransportVersions.ML_INFERENCE_TIMEOUT_ADDED)) {
-                        return new InferenceAction.Request(
-                            instance.getTaskType(),
-                            instance.getInferenceEntityId(),
-                            instance.getQuery(),
-                            instance.getInput(),
-                            instance.getTaskSettings(),
-                            instance.getInputType(),
-                            InferenceAction.Request.DEFAULT_TIMEOUT
-                        );
                     }
 
         return instance;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java
index 603531f0aedf9..b84aaa2bcfc1b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+import org.elasticsearch.xpack.core.ml.AbstractChunkedBWCSerializationTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -18,7 +18,7 @@
 
 import static org.elasticsearch.TransportVersions.ML_RERANK_DOC_OPTIONAL;
 
-public class RankedDocsResultsTests extends AbstractBWCSerializationTestCase<RankedDocsResults> {
+public class RankedDocsResultsTests extends AbstractChunkedBWCSerializationTestCase<RankedDocsResults> {
 
     @Override
     protected Writeable.Reader<RankedDocsResults> instanceReader() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractChunkedBWCSerializationTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractChunkedBWCSerializationTestCase.java
new file mode 100644
index 0000000000000..a23ce2c107fe3
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractChunkedBWCSerializationTestCase.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ml;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase.DEFAULT_BWC_VERSIONS;
+
+public abstract class AbstractChunkedBWCSerializationTestCase<T extends ChunkedToXContent & Writeable> extends
+    AbstractChunkedSerializingTestCase<T> {
+
+    /**
+     * Returns the expected instance if serialized from the given version.
+     */
+    protected abstract T mutateInstanceForVersion(T instance, TransportVersion version);
+
+    /**
+     * The bwc versions to test serialization against
+     */
+    protected List<TransportVersion> bwcVersions() {
+        return DEFAULT_BWC_VERSIONS;
+    }
+
+    /**
+     * Test serialization and deserialization of the test instance across versions
+     */
+    public final void testBwcSerialization() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T testInstance = createTestInstance();
+            for (TransportVersion bwcVersion : bwcVersions()) {
+                assertBwcSerialization(testInstance, bwcVersion);
+            }
+        }
+    }
+
+    /**
+     * Assert that instances copied at a particular version are equal. The version is useful
+     * for sanity checking the backwards compatibility of the wire. It isn't a substitute for
+     * real backwards compatibility tests but it is *so* much faster.
+     */
+    protected final void assertBwcSerialization(T testInstance, TransportVersion version) throws IOException {
+        T deserializedInstance = copyWriteable(testInstance, getNamedWriteableRegistry(), instanceReader(), version);
+        assertOnBWCObject(deserializedInstance, mutateInstanceForVersion(testInstance, version), version);
+    }
+
+    /**
+     * @param bwcSerializedObject The object deserialized from the previous version
+     * @param testInstance The original test instance
+     * @param version The version which serialized
+     */
+    protected void assertOnBWCObject(T bwcSerializedObject, T testInstance, TransportVersion version) {
+        assertNotSame(version.toString(), bwcSerializedObject, testInstance);
+        assertEquals(version.toString(), bwcSerializedObject, testInstance);
+        assertEquals(version.toString(), bwcSerializedObject.hashCode(), testInstance.hashCode());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/autoscaling/MlAutoscalingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/autoscaling/MlAutoscalingStatsTests.java
index 007f3a657dd8f..af6e16eabed8d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/autoscaling/MlAutoscalingStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/autoscaling/MlAutoscalingStatsTests.java
@@ -17,16 +17,16 @@ public class MlAutoscalingStatsTests extends AbstractWireSerializingTestCase<MlA
     public static MlAutoscalingStats randomAutoscalingResources() {
         return new MlAutoscalingStats(
             randomIntBetween(0, 100), // nodes
-            randomNonNegativeLong(), // perNodeMemoryInBytes
+            randomNonNegativeLong(), // currentPerNodeMemoryBytes
             randomNonNegativeLong(), // modelMemoryInBytes
-            randomIntBetween(0, 100), // processorsSum
-            randomIntBetween(0, 100), // minNodes
-            randomNonNegativeLong(), // extraSingleNodeModelMemoryInBytes
-            randomIntBetween(0, 100), // extraSingleNodeProcessors
-            randomNonNegativeLong(), // extraModelMemoryInBytes
-            randomIntBetween(0, 100), // extraProcessors
-            randomNonNegativeLong(), // removeNodeMemoryInBytes
-            randomNonNegativeLong() // perNodeMemoryOverheadInBytes
+            randomIntBetween(0, 100), // currentTotalProcessorsInUse
+            randomIntBetween(0, 100), // wantedMinNodes
+            randomNonNegativeLong(), // extraPerNodeMemoryBytes
+            randomIntBetween(0, 100), // wantedExtraPerNodeNodeProcessors
+            randomNonNegativeLong(), // wantedExtraModelMemoryBytes
+            randomIntBetween(0, 100), // wantedExtraProcessors
+            randomNonNegativeLong(), // unwantedNodeMemoryBytesToRemove
+            randomNonNegativeLong() // currentPerNodeMemoryOverheadBytes
         );
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
index 6d70105dfedba..6a213d6f5e379 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
@@ -51,6 +52,29 @@ public static TrainedModelAssignment randomInstance() {
         return builder.build();
     }
 
+    public static TrainedModelAssignment.Builder randomInstanceBuilder(
+        @Nullable StartTrainedModelDeploymentAction.TaskParams taskParams,
+        AssignmentState assignmentState
+    ) {
+        TrainedModelAssignment.Builder builder = TrainedModelAssignment.Builder.empty(
+            taskParams != null ? taskParams : randomParams(),
+            null
+        );
+        List<String> nodes = Stream.generate(() -> randomAlphaOfLength(10)).limit(randomInt(5)).toList();
+        for (String node : nodes) {
+            builder.addRoutingEntry(node, RoutingInfoTests.randomInstance());
+        }
+        if (assignmentState == null) {
+            builder.setAssignmentState(randomFrom(AssignmentState.values()));
+        } else {
+            builder.setAssignmentState(assignmentState);
+        }
+        if (randomBoolean()) {
+            builder.setReason(randomAlphaOfLength(10));
+        }
+        return builder;
+    }
+
     @Override
     protected TrainedModelAssignment doParseInstance(XContentParser parser) throws IOException {
         return TrainedModelAssignment.fromXContent(parser);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfigTests.java
index 09d2366984383..5adf62a3b67ef 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearningToRankConfigTests.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.license.License;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -36,6 +38,7 @@
 import java.util.stream.Stream;
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.hamcrest.Matchers.is;
 
 public class LearningToRankConfigTests extends InferenceConfigItemTestCase<LearningToRankConfig> {
     private boolean lenient;
@@ -140,6 +143,16 @@ public void testDuplicateFeatureNames() {
         expectThrows(IllegalArgumentException.class, () -> builder.build());
     }
 
+    public void testLicenseSupport_ForPutAction_RequiresEnterprise() {
+        var config = randomLearningToRankConfig();
+        assertThat(config.getMinLicenseSupportedForAction(RestRequest.Method.PUT), is(License.OperationMode.ENTERPRISE));
+    }
+
+    public void testLicenseSupport_ForGetAction_RequiresPlatinum() {
+        var config = randomLearningToRankConfig();
+        assertThat(config.getMinLicenseSupportedForAction(RestRequest.Method.GET), is(License.OperationMode.PLATINUM));
+    }
+
     @Override
     protected NamedXContentRegistry xContentRegistry() {
         List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java
index d78d929fd580f..127088d82bade 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java
@@ -38,7 +38,7 @@ public void testExtractReferencedLists() {
     @Override
     protected DetectionRule createTestInstance() {
         DetectionRule.Builder builder = new DetectionRule.Builder();
-
+        boolean hasForceTimeShiftAction = false;
         if (randomBoolean()) {
             EnumSet<RuleAction> actions = EnumSet.noneOf(RuleAction.class);
             int actionsCount = randomIntBetween(1, RuleAction.values().length);
@@ -46,10 +46,12 @@ protected DetectionRule createTestInstance() {
                 actions.add(randomFrom(RuleAction.values()));
             }
             builder.setActions(actions);
+            hasForceTimeShiftAction = actions.contains(RuleAction.FORCE_TIME_SHIFT);
         }
 
         boolean hasScope = randomBoolean();
         boolean hasConditions = randomBoolean();
+        boolean hasParams = randomBoolean() || hasForceTimeShiftAction;
 
         if (hasScope == false && hasConditions == false) {
             // at least one of the two should be present
@@ -79,6 +81,15 @@ protected DetectionRule createTestInstance() {
             builder.setConditions(ruleConditions);
         }
 
+        if (hasParams) {
+            if (hasForceTimeShiftAction) {
+                long timeShiftAmount = randomLong();
+                builder.setParams(new RuleParams(new RuleParamsForForceTimeShift(timeShiftAmount)));
+            } else {
+                builder.setParams(new RuleParams());
+            }
+        }
+
         return builder.build();
     }
 
@@ -97,8 +108,9 @@ protected DetectionRule mutateInstance(DetectionRule instance) {
         List<RuleCondition> conditions = instance.getConditions();
         RuleScope scope = instance.getScope();
         EnumSet<RuleAction> actions = instance.getActions();
+        RuleParams params = instance.getParams();
 
-        switch (between(0, 2)) {
+        switch (between(0, 3)) {
             case 0:
                 if (actions.size() == RuleAction.values().length) {
                     actions = EnumSet.of(randomFrom(RuleAction.values()));
@@ -113,11 +125,25 @@ protected DetectionRule mutateInstance(DetectionRule instance) {
             case 2:
                 scope = new RuleScope.Builder(scope).include("another_field", "another_filter").build();
                 break;
+            case 3:
+                if (params.getForceTimeShift() != null) {
+                    params = new RuleParams(new RuleParamsForForceTimeShift(randomLong()));
+                } else {
+                    params = new RuleParams(new RuleParamsForForceTimeShift(randomLong()));
+                    actions.add(RuleAction.FORCE_TIME_SHIFT);
+                }
+                break;
             default:
                 throw new AssertionError("Illegal randomisation branch");
         }
 
-        return new DetectionRule.Builder(conditions).setActions(actions).setScope(scope).build();
+        if (actions.contains(RuleAction.FORCE_TIME_SHIFT) && params.getForceTimeShift() == null) {
+            params = new RuleParams(new RuleParamsForForceTimeShift(randomLong()));
+        } else if (actions.contains(RuleAction.FORCE_TIME_SHIFT) == false && params.getForceTimeShift() != null) {
+            params = new RuleParams();
+        }
+
+        return new DetectionRule.Builder(conditions).setActions(actions).setScope(scope).setParams(params).build();
     }
 
     private static List<RuleCondition> createCondition(double value) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsActionTests.java
index 893f7474c3e6e..50ab034a34ef3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/settings/UpdateSecuritySettingsActionTests.java
@@ -7,15 +7,19 @@
 
 package org.elasticsearch.xpack.core.security.action.settings;
 
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.xpack.core.security.action.settings.UpdateSecuritySettingsAction.ALLOWED_SETTING_KEYS;
+import static org.elasticsearch.xpack.core.security.action.settings.UpdateSecuritySettingsAction.ALLOWED_SETTING_VALIDATORS;
 import static org.elasticsearch.xpack.core.security.action.settings.UpdateSecuritySettingsAction.MAIN_INDEX_NAME;
 import static org.elasticsearch.xpack.core.security.action.settings.UpdateSecuritySettingsAction.PROFILES_INDEX_NAME;
 import static org.elasticsearch.xpack.core.security.action.settings.UpdateSecuritySettingsAction.TOKENS_INDEX_NAME;
@@ -27,6 +31,15 @@
 
 public class UpdateSecuritySettingsActionTests extends ESTestCase {
 
+    static final Map<String, Supplier<String>> ALLOWED_SETTING_GENERATORS = Map.of(
+        IndexMetadata.SETTING_NUMBER_OF_REPLICAS,
+        () -> randomAlphaOfLength(5), // no additional validation
+        IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS,
+        () -> randomAlphaOfLength(5), // no additional validation
+        DataTier.TIER_PREFERENCE,
+        () -> randomFrom(DataTier.DATA_CONTENT, DataTier.DATA_HOT, DataTier.DATA_WARM, DataTier.DATA_COLD)
+    );
+
     public void testValidateSettingsEmpty() {
         var req = new UpdateSecuritySettingsAction.Request(
             TEST_REQUEST_TIMEOUT,
@@ -43,9 +56,10 @@ public void testValidateSettingsEmpty() {
 
     public void testAllowedSettingsOk() {
         Map<String, Object> allAllowedSettingsMap = new HashMap<>();
-        for (String allowedSetting : ALLOWED_SETTING_KEYS) {
-            Map<String, Object> allowedSettingMap = Map.of(allowedSetting, randomAlphaOfLength(5));
-            allAllowedSettingsMap.put(allowedSetting, randomAlphaOfLength(5));
+        for (String allowedSetting : ALLOWED_SETTING_VALIDATORS.keySet()) {
+            String settingValue = ALLOWED_SETTING_GENERATORS.get(allowedSetting).get();
+            Map<String, Object> allowedSettingMap = Map.of(allowedSetting, settingValue);
+            allAllowedSettingsMap.put(allowedSetting, settingValue);
             var req = new UpdateSecuritySettingsAction.Request(
                 TEST_REQUEST_TIMEOUT,
                 TEST_REQUEST_TIMEOUT,
@@ -86,11 +100,12 @@ public void testAllowedSettingsOk() {
 
     public void testDisallowedSettingsFailsValidation() {
         String disallowedSetting = "index."
-            + randomValueOtherThanMany((value) -> ALLOWED_SETTING_KEYS.contains("index." + value), () -> randomAlphaOfLength(5));
+            + randomValueOtherThanMany((value) -> ALLOWED_SETTING_VALIDATORS.containsKey("index." + value), () -> randomAlphaOfLength(5));
         Map<String, Object> disallowedSettingMap = Map.of(disallowedSetting, randomAlphaOfLength(5));
+        String validSetting = randomFrom(ALLOWED_SETTING_VALIDATORS.keySet());
         Map<String, Object> validOrEmptySettingMap = randomFrom(
             Collections.emptyMap(),
-            Map.of(randomFrom(ALLOWED_SETTING_KEYS), randomAlphaOfLength(5))
+            Map.of(validSetting, ALLOWED_SETTING_GENERATORS.get(validSetting).get())
         );
         {
             var req = new UpdateSecuritySettingsAction.Request(
@@ -106,11 +121,11 @@ public void testDisallowedSettingsFailsValidation() {
                 assertThat(
                     errorMsg,
                     matchesRegex(
-                        "illegal settings for index \\["
+                        "illegal setting for index \\["
                             + Pattern.quote(TOKENS_INDEX_NAME)
                             + "\\]: \\["
                             + disallowedSetting
-                            + "\\], these settings may not be configured. Only the following settings may be configured for that index.*"
+                            + "\\], this setting may not be configured. Only the following settings may be configured for that index.*"
                     )
                 );
             }
@@ -130,13 +145,13 @@ public void testDisallowedSettingsFailsValidation() {
                 assertThat(
                     errorMsg,
                     matchesRegex(
-                        "illegal settings for index \\[("
+                        "illegal setting for index \\[("
                             + Pattern.quote(MAIN_INDEX_NAME)
                             + "|"
                             + Pattern.quote(PROFILES_INDEX_NAME)
                             + ")\\]: \\["
                             + disallowedSetting
-                            + "\\], these settings may not be configured. Only the following settings may be configured for that index.*"
+                            + "\\], this setting may not be configured. Only the following settings may be configured for that index.*"
                     )
                 );
             }
@@ -156,7 +171,7 @@ public void testDisallowedSettingsFailsValidation() {
                 assertThat(
                     errorMsg,
                     matchesRegex(
-                        "illegal settings for index \\[("
+                        "illegal setting for index \\[("
                             + Pattern.quote(MAIN_INDEX_NAME)
                             + "|"
                             + Pattern.quote(TOKENS_INDEX_NAME)
@@ -164,11 +179,64 @@ public void testDisallowedSettingsFailsValidation() {
                             + Pattern.quote(PROFILES_INDEX_NAME)
                             + ")\\]: \\["
                             + disallowedSetting
-                            + "\\], these settings may not be configured. Only the following settings may be configured for that index.*"
+                            + "\\], this setting may not be configured. Only the following settings may be configured for that index.*"
                     )
                 );
             }
         }
     }
 
+    public void testSettingValuesAreValidated() {
+        Map<String, Object> forbiddenSettingsMap = Map.of(DataTier.TIER_PREFERENCE, DataTier.DATA_FROZEN);
+        String badTier = randomAlphaOfLength(5);
+        Map<String, Object> badSettingsMap = Map.of(DataTier.TIER_PREFERENCE, badTier);
+        Map<String, Object> allowedSettingMap = Map.of(
+            DataTier.TIER_PREFERENCE,
+            randomFrom(DataTier.DATA_HOT, DataTier.DATA_WARM, DataTier.DATA_CONTENT, DataTier.DATA_COLD)
+        );
+        {
+            var req = new UpdateSecuritySettingsAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                allowedSettingMap,
+                Collections.emptyMap(),
+                Collections.emptyMap()
+            );
+            assertThat(req.validate(), nullValue());
+        }
+
+        {
+            var req = new UpdateSecuritySettingsAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                forbiddenSettingsMap,
+                Collections.emptyMap(),
+                Collections.emptyMap()
+            );
+            ActionRequestValidationException exception = req.validate();
+            assertThat(exception, notNullValue());
+            assertThat(exception.validationErrors(), hasSize(1));
+            assertThat(
+                exception.validationErrors().get(0),
+                containsString("disallowed data tiers [" + DataTier.DATA_FROZEN + "] found, allowed tiers are ")
+            );
+        }
+
+        {
+            var req = new UpdateSecuritySettingsAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                badSettingsMap,
+                Collections.emptyMap(),
+                Collections.emptyMap()
+            );
+            var exception = req.validate();
+            assertThat(exception, notNullValue());
+            assertThat(exception.validationErrors(), hasSize(1));
+            assertThat(
+                exception.validationErrors().get(0),
+                containsString("disallowed data tiers [" + badTier + "] found, allowed tiers are ")
+            );
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
index 1e33a7f54394b..3d4d6106a7eaf 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
@@ -1089,6 +1089,51 @@ public void testMaybeRewriteMetadataForApiKeyRoleDescriptorsWithRemoteIndices()
         );
     }
 
+    public void testMaybeRewriteMetadataForApiKeyRoleDescriptorsWithRemoteCluster() {
+        final String apiKeyId = randomAlphaOfLengthBetween(1, 10);
+        final String apiKeyName = randomAlphaOfLengthBetween(1, 10);
+        final Map<String, Object> metadata = Map.ofEntries(
+            entry(AuthenticationField.API_KEY_ID_KEY, apiKeyId),
+            entry(AuthenticationField.API_KEY_NAME_KEY, apiKeyName),
+            entry(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY, new BytesArray("""
+                {"base_role":{"cluster":["all"],
+                 "remote_cluster":[{"privileges":["monitor_enrich"],"clusters":["*"]}]
+                }}""")),
+            entry(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, new BytesArray("""
+                {"limited_by_role":{"cluster":["*"],
+                 "remote_cluster":[{"privileges":["monitor_enrich"],"clusters":["*"]}]
+                }}"""))
+        );
+
+        final Authentication original = AuthenticationTestHelper.builder()
+            .apiKey()
+            .metadata(metadata)
+            .transportVersion(TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS)
+            .build();
+
+        // pick a version before that of the authentication instance to force a rewrite
+        final TransportVersion olderVersion = TransportVersionUtils.randomVersionBetween(
+            random(),
+            Authentication.VERSION_API_KEY_ROLES_AS_BYTES,
+            TransportVersionUtils.getPreviousVersion(original.getEffectiveSubject().getTransportVersion())
+        );
+
+        final Map<String, Object> rewrittenMetadata = original.maybeRewriteForOlderVersion(olderVersion)
+            .getEffectiveSubject()
+            .getMetadata();
+        assertThat(rewrittenMetadata.keySet(), equalTo(original.getAuthenticatingSubject().getMetadata().keySet()));
+        assertThat(
+            ((BytesReference) rewrittenMetadata.get(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY)).toBytesRef(),
+            equalTo(new BytesArray("""
+                {"base_role":{"cluster":["all"]}}""").toBytesRef())
+        );
+        assertThat(
+            ((BytesReference) rewrittenMetadata.get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)).toBytesRef(),
+            equalTo(new BytesArray("""
+                {"limited_by_role":{"cluster":["*"]}}""").toBytesRef())
+        );
+    }
+
     public void testMaybeRemoveRemoteIndicesFromRoleDescriptors() {
         final boolean includeClusterPrivileges = randomBoolean();
         final BytesReference roleWithoutRemoteIndices = new BytesArray(Strings.format("""
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTestHelper.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTestHelper.java
index 2d8b62335f4ef..77a37cec45b25 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTestHelper.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTestHelper.java
@@ -26,6 +26,7 @@
 
 import static org.elasticsearch.test.ESTestCase.generateRandomStringArray;
 import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween;
+import static org.elasticsearch.test.ESTestCase.randomArray;
 import static org.elasticsearch.test.ESTestCase.randomBoolean;
 import static org.elasticsearch.test.ESTestCase.randomInt;
 import static org.elasticsearch.test.ESTestCase.randomIntBetween;
@@ -52,6 +53,7 @@ public static RoleDescriptor randomRoleDescriptor() {
             .allowRestriction(randomBoolean())
             .allowDescription(randomBoolean())
             .allowRemoteClusters(randomBoolean())
+            .allowConfigurableClusterPrivileges(randomBoolean())
             .build();
     }
 
@@ -69,7 +71,7 @@ public static Map<String, Object> randomRoleDescriptorMetadata(boolean allowRese
     }
 
     public static ConfigurableClusterPrivilege[] randomClusterPrivileges() {
-        final ConfigurableClusterPrivilege[] configurableClusterPrivileges = switch (randomIntBetween(0, 4)) {
+        return switch (randomIntBetween(0, 5)) {
             case 0 -> new ConfigurableClusterPrivilege[0];
             case 1 -> new ConfigurableClusterPrivilege[] {
                 new ConfigurableClusterPrivileges.ManageApplicationPrivileges(
@@ -93,9 +95,9 @@ public static ConfigurableClusterPrivilege[] randomClusterPrivileges() {
                 new ConfigurableClusterPrivileges.WriteProfileDataPrivileges(
                     Sets.newHashSet(generateRandomStringArray(3, randomIntBetween(4, 12), false, false))
                 ) };
+            case 5 -> randomManageRolesPrivileges();
             default -> throw new IllegalStateException("Unexpected value");
         };
-        return configurableClusterPrivileges;
     }
 
     public static RoleDescriptor.ApplicationResourcePrivileges[] randomApplicationPrivileges() {
@@ -119,6 +121,27 @@ public static RoleDescriptor.ApplicationResourcePrivileges[] randomApplicationPr
         return applicationPrivileges;
     }
 
+    public static ConfigurableClusterPrivilege[] randomManageRolesPrivileges() {
+        List<ConfigurableClusterPrivileges.ManageRolesPrivilege.ManageRolesIndexPermissionGroup> indexPatternPrivileges = randomList(
+            1,
+            10,
+            () -> {
+                String[] indexPatterns = randomArray(1, 5, String[]::new, () -> randomAlphaOfLengthBetween(5, 100));
+
+                int startIndex = randomIntBetween(0, IndexPrivilege.names().size() - 2);
+                int endIndex = randomIntBetween(startIndex + 1, IndexPrivilege.names().size());
+
+                String[] indexPrivileges = IndexPrivilege.names().stream().toList().subList(startIndex, endIndex).toArray(String[]::new);
+                return new ConfigurableClusterPrivileges.ManageRolesPrivilege.ManageRolesIndexPermissionGroup(
+                    indexPatterns,
+                    indexPrivileges
+                );
+            }
+        );
+
+        return new ConfigurableClusterPrivilege[] { new ConfigurableClusterPrivileges.ManageRolesPrivilege(indexPatternPrivileges) };
+    }
+
     public static RoleDescriptor.RemoteIndicesPrivileges[] randomRemoteIndicesPrivileges(int min, int max) {
         return randomRemoteIndicesPrivileges(min, max, Set.of());
     }
@@ -251,6 +274,7 @@ public static class Builder {
         private boolean allowRestriction = false;
         private boolean allowDescription = false;
         private boolean allowRemoteClusters = false;
+        private boolean allowConfigurableClusterPrivileges = false;
 
         public Builder() {}
 
@@ -259,6 +283,11 @@ public Builder allowReservedMetadata(boolean allowReservedMetadata) {
             return this;
         }
 
+        public Builder allowConfigurableClusterPrivileges(boolean allowConfigurableClusterPrivileges) {
+            this.allowConfigurableClusterPrivileges = allowConfigurableClusterPrivileges;
+            return this;
+        }
+
         public Builder alwaysIncludeRemoteIndices() {
             this.alwaysIncludeRemoteIndices = true;
             return this;
@@ -302,7 +331,7 @@ public RoleDescriptor build() {
                 randomSubsetOf(ClusterPrivilegeResolver.names()).toArray(String[]::new),
                 randomIndicesPrivileges(0, 3),
                 randomApplicationPrivileges(),
-                randomClusterPrivileges(),
+                allowConfigurableClusterPrivileges ? randomClusterPrivileges() : null,
                 generateRandomStringArray(5, randomIntBetween(2, 8), false, true),
                 randomRoleDescriptorMetadata(allowReservedMetadata),
                 Map.of(),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersectionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersectionTests.java
index a892e8b864e6e..b67292e76961f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersectionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersectionTests.java
@@ -48,6 +48,11 @@ public void testSerialization() throws IOException {
                     ConfigurableClusterPrivilege.class,
                     ConfigurableClusterPrivileges.WriteProfileDataPrivileges.WRITEABLE_NAME,
                     ConfigurableClusterPrivileges.WriteProfileDataPrivileges::createFrom
+                ),
+                new NamedWriteableRegistry.Entry(
+                    ConfigurableClusterPrivilege.class,
+                    ConfigurableClusterPrivileges.ManageRolesPrivilege.WRITEABLE_NAME,
+                    ConfigurableClusterPrivileges.ManageRolesPrivilege::createFrom
                 )
             )
         );
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivilegesTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivilegesTests.java
index c6fac77ea26e6..5599b33fbcfe7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivilegesTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ConfigurableClusterPrivilegesTests.java
@@ -61,13 +61,15 @@ public void testGenerateAndParseXContent() throws Exception {
     }
 
     private ConfigurableClusterPrivilege[] buildSecurityPrivileges() {
-        return switch (randomIntBetween(0, 3)) {
+        return switch (randomIntBetween(0, 4)) {
             case 0 -> new ConfigurableClusterPrivilege[0];
             case 1 -> new ConfigurableClusterPrivilege[] { ManageApplicationPrivilegesTests.buildPrivileges() };
             case 2 -> new ConfigurableClusterPrivilege[] { WriteProfileDataPrivilegesTests.buildPrivileges() };
-            case 3 -> new ConfigurableClusterPrivilege[] {
+            case 3 -> new ConfigurableClusterPrivilege[] { ManageRolesPrivilegesTests.buildPrivileges() };
+            case 4 -> new ConfigurableClusterPrivilege[] {
                 ManageApplicationPrivilegesTests.buildPrivileges(),
-                WriteProfileDataPrivilegesTests.buildPrivileges() };
+                WriteProfileDataPrivilegesTests.buildPrivileges(),
+                ManageRolesPrivilegesTests.buildPrivileges() };
             default -> throw new IllegalStateException("Unexpected value");
         };
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageRolesPrivilegesTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageRolesPrivilegesTests.java
new file mode 100644
index 0000000000000..2d47752063d9d
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageRolesPrivilegesTests.java
@@ -0,0 +1,351 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.privilege;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.AbstractNamedWriteableTestCase;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.XPackClientPlugin;
+import org.elasticsearch.xpack.core.security.action.role.BulkDeleteRolesRequest;
+import org.elasticsearch.xpack.core.security.action.role.BulkPutRolesRequest;
+import org.elasticsearch.xpack.core.security.action.role.DeleteRoleRequest;
+import org.elasticsearch.xpack.core.security.action.role.PutRoleRequest;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper;
+import org.elasticsearch.xpack.core.security.authz.RestrictedIndices;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.permission.ClusterPermission;
+import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissionGroup;
+import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges.ManageRolesPrivilege;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsEqual.equalTo;
+
+public class ManageRolesPrivilegesTests extends AbstractNamedWriteableTestCase<ConfigurableClusterPrivilege> {
+
+    private static final int MIN_INDEX_NAME_LENGTH = 4;
+
+    public void testSimplePutRoleRequest() {
+        new ReservedRolesStore();
+        final ManageRolesPrivilege privilege = new ManageRolesPrivilege(
+            List.of(new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "allowed*" }, new String[] { "all" }))
+        );
+        final ClusterPermission permission = privilege.buildPermission(
+            new ClusterPermission.Builder(new RestrictedIndices(TestRestrictedIndices.RESTRICTED_INDICES.getAutomaton()))
+        ).build();
+
+        assertAllowedIndexPatterns(permission, randomArray(1, 10, String[]::new, () -> "allowed-" + randomAlphaOfLength(5)), true);
+        assertAllowedIndexPatterns(permission, randomArray(1, 10, String[]::new, () -> "not-allowed-" + randomAlphaOfLength(5)), false);
+        assertAllowedIndexPatterns(
+            permission,
+            new String[] { "allowed-" + randomAlphaOfLength(5), "not-allowed-" + randomAlphaOfLength(5) },
+            false
+        );
+    }
+
+    public void testDeleteRoleRequest() {
+        new ReservedRolesStore();
+        {
+            final ManageRolesPrivilege privilege = new ManageRolesPrivilege(
+                List.of(new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "allowed*" }, new String[] { "manage" }))
+            );
+            final ClusterPermission permission = privilege.buildPermission(
+                new ClusterPermission.Builder(new RestrictedIndices(TestRestrictedIndices.RESTRICTED_INDICES.getAutomaton()))
+            ).build();
+
+            assertAllowedDeleteIndex(permission, randomArray(1, 10, String[]::new, () -> "allowed-" + randomAlphaOfLength(5)), true);
+            assertAllowedDeleteIndex(permission, randomArray(1, 10, String[]::new, () -> "not-allowed-" + randomAlphaOfLength(5)), false);
+            assertAllowedDeleteIndex(
+                permission,
+                new String[] { "allowed-" + randomAlphaOfLength(5), "not-allowed-" + randomAlphaOfLength(5) },
+                false
+            );
+        }
+        {
+            final ManageRolesPrivilege privilege = new ManageRolesPrivilege(
+                List.of(new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "allowed*" }, new String[] { "read" }))
+            );
+            final ClusterPermission permission = privilege.buildPermission(
+                new ClusterPermission.Builder(new RestrictedIndices(TestRestrictedIndices.RESTRICTED_INDICES.getAutomaton()))
+            ).build();
+            assertAllowedDeleteIndex(permission, randomArray(1, 10, String[]::new, () -> "allowed-" + randomAlphaOfLength(5)), false);
+        }
+    }
+
+    public void testSeveralIndexGroupsPutRoleRequest() {
+        new ReservedRolesStore();
+
+        final ManageRolesPrivilege privilege = new ManageRolesPrivilege(
+            List.of(
+                new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "a", "b" }, new String[] { "read" }),
+                new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "c" }, new String[] { "read" }),
+                new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "d" }, new String[] { "read" })
+            )
+        );
+
+        final ClusterPermission permission = privilege.buildPermission(
+            new ClusterPermission.Builder(new RestrictedIndices(TestRestrictedIndices.RESTRICTED_INDICES.getAutomaton()))
+        ).build();
+
+        assertAllowedIndexPatterns(permission, new String[] { "/[ab]/" }, new String[] { "read" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[cd]/" }, new String[] { "read" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[acd]/" }, new String[] { "read" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[ef]/" }, new String[] { "read" }, false);
+    }
+
+    public void testPrivilegeIntersectionPutRoleRequest() {
+        new ReservedRolesStore();
+
+        final ManageRolesPrivilege privilege = new ManageRolesPrivilege(
+            List.of(
+                new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "a", "b" }, new String[] { "all" }),
+                new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "c" }, new String[] { "create" }),
+                new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "d" }, new String[] { "delete" }),
+                new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "e" }, new String[] { "create_doc" }),
+                new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "f" }, new String[] { "read", "manage" })
+            )
+        );
+
+        final ClusterPermission permission = privilege.buildPermission(
+            new ClusterPermission.Builder(new RestrictedIndices(TestRestrictedIndices.RESTRICTED_INDICES.getAutomaton()))
+        ).build();
+
+        assertAllowedIndexPatterns(permission, new String[] { "/[ab]/" }, new String[] { "all" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[abc]/" }, new String[] { "all" }, false);
+        assertAllowedIndexPatterns(permission, new String[] { "/[ab]/" }, new String[] { "read", "manage" }, true);
+
+        assertAllowedIndexPatterns(permission, new String[] { "/[ac]/" }, new String[] { "create" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[ac]/" }, new String[] { "create", "create_doc" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[ce]/" }, new String[] { "create_doc" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[abce]/" }, new String[] { "create_doc" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[abcde]/" }, new String[] { "create_doc" }, false);
+        assertAllowedIndexPatterns(permission, new String[] { "/[ce]/" }, new String[] { "create_doc" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[eb]/" }, new String[] { "create_doc" }, true);
+
+        assertAllowedIndexPatterns(permission, new String[] { "/[d]/" }, new String[] { "delete" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[ad]/" }, new String[] { "delete" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[de]/" }, new String[] { "delete" }, false);
+
+        assertAllowedIndexPatterns(permission, new String[] { "/[f]/" }, new String[] { "read", "manage" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { "/[f]/" }, new String[] { "read", "write" }, false);
+        assertAllowedIndexPatterns(permission, new String[] { "/[f]/" }, new String[] { "read", "manage" }, true);
+    }
+
+    public void testEmptyPrivileges() {
+        new ReservedRolesStore();
+
+        final ManageRolesPrivilege privilege = new ManageRolesPrivilege(List.of());
+
+        final ClusterPermission permission = privilege.buildPermission(
+            new ClusterPermission.Builder(new RestrictedIndices(TestRestrictedIndices.RESTRICTED_INDICES.getAutomaton()))
+        ).build();
+
+        assertAllowedIndexPatterns(permission, new String[] { "test" }, new String[] { "all" }, false);
+    }
+
+    public void testRestrictedIndexPutRoleRequest() {
+        new ReservedRolesStore();
+
+        final ManageRolesPrivilege privilege = new ManageRolesPrivilege(
+            List.of(new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "*" }, new String[] { "all" }))
+        );
+        final ClusterPermission permission = privilege.buildPermission(
+            new ClusterPermission.Builder(new RestrictedIndices(TestRestrictedIndices.RESTRICTED_INDICES.getAutomaton()))
+        ).build();
+
+        assertAllowedIndexPatterns(permission, new String[] { "security" }, true);
+        assertAllowedIndexPatterns(permission, new String[] { ".security" }, false);
+        assertAllowedIndexPatterns(permission, new String[] { "security", ".security-7" }, false);
+    }
+
+    public void testGenerateAndParseXContent() throws Exception {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            final XContentBuilder builder = new XContentBuilder(xContent, out);
+
+            final ManageRolesPrivilege original = buildPrivileges();
+            builder.startObject();
+            original.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            builder.flush();
+
+            final byte[] bytes = out.toByteArray();
+            try (XContentParser parser = xContent.createParser(XContentParserConfiguration.EMPTY, bytes)) {
+                assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
+                assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
+                final ManageRolesPrivilege clone = ManageRolesPrivilege.parse(parser);
+                assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
+
+                assertThat(clone, equalTo(original));
+                assertThat(original, equalTo(clone));
+            }
+        }
+    }
+
+    public void testPutRoleRequestContainsNonIndexPrivileges() {
+        new ReservedRolesStore();
+        final ManageRolesPrivilege privilege = new ManageRolesPrivilege(
+            List.of(new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "allowed*" }, new String[] { "all" }))
+        );
+        final ClusterPermission permission = privilege.buildPermission(
+            new ClusterPermission.Builder(new RestrictedIndices(TestRestrictedIndices.RESTRICTED_INDICES.getAutomaton()))
+        ).build();
+
+        final PutRoleRequest putRoleRequest = new PutRoleRequest();
+
+        switch (randomIntBetween(0, 5)) {
+            case 0:
+                putRoleRequest.cluster("all");
+                break;
+            case 1:
+                putRoleRequest.runAs("test");
+                break;
+            case 2:
+                putRoleRequest.addApplicationPrivileges(
+                    RoleDescriptor.ApplicationResourcePrivileges.builder()
+                        .privileges("all")
+                        .application("test-app")
+                        .resources("test-resource")
+                        .build()
+                );
+                break;
+            case 3:
+                putRoleRequest.addRemoteIndex(
+                    new RoleDescriptor.RemoteIndicesPrivileges.Builder("test-cluster").privileges("all").indices("test*").build()
+                );
+                break;
+            case 4:
+                putRoleRequest.putRemoteCluster(
+                    new RemoteClusterPermissions().addGroup(
+                        new RemoteClusterPermissionGroup(new String[] { "monitor_enrich" }, new String[] { "test" })
+                    )
+                );
+                break;
+            case 5:
+                putRoleRequest.conditionalCluster(
+                    new ConfigurableClusterPrivileges.ManageRolesPrivilege(
+                        List.of(
+                            new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(new String[] { "test-*" }, new String[] { "read" })
+                        )
+                    )
+                );
+                break;
+        }
+
+        putRoleRequest.name(randomAlphaOfLength(4));
+        assertThat(permissionCheck(permission, "cluster:admin/xpack/security/role/put", putRoleRequest), is(false));
+    }
+
+    private static boolean permissionCheck(ClusterPermission permission, String action, ActionRequest request) {
+        final Authentication authentication = AuthenticationTestHelper.builder().build();
+        assertThat(request.validate(), nullValue());
+        return permission.check(action, request, authentication);
+    }
+
+    private static void assertAllowedIndexPatterns(ClusterPermission permission, String[] indexPatterns, boolean expected) {
+        assertAllowedIndexPatterns(permission, indexPatterns, new String[] { "index", "write", "indices:data/read" }, expected);
+    }
+
+    private static void assertAllowedIndexPatterns(
+        ClusterPermission permission,
+        String[] indexPatterns,
+        String[] privileges,
+        boolean expected
+    ) {
+        {
+            final PutRoleRequest putRoleRequest = new PutRoleRequest();
+            putRoleRequest.name(randomAlphaOfLength(3));
+            putRoleRequest.addIndex(indexPatterns, privileges, null, null, null, false);
+            assertThat(permissionCheck(permission, "cluster:admin/xpack/security/role/put", putRoleRequest), is(expected));
+        }
+        {
+            final BulkPutRolesRequest bulkPutRolesRequest = new BulkPutRolesRequest(
+                List.of(
+                    new RoleDescriptor(
+                        randomAlphaOfLength(3),
+                        new String[] {},
+                        new RoleDescriptor.IndicesPrivileges[] {
+                            RoleDescriptor.IndicesPrivileges.builder().indices(indexPatterns).privileges(privileges).build() },
+                        new String[] {}
+                    )
+                )
+            );
+            assertThat(permissionCheck(permission, "cluster:admin/xpack/security/role/bulk_put", bulkPutRolesRequest), is(expected));
+        }
+    }
+
+    private static void assertAllowedDeleteIndex(ClusterPermission permission, String[] indices, boolean expected) {
+        {
+            final BulkDeleteRolesRequest bulkDeleteRolesRequest = new BulkDeleteRolesRequest(List.of(indices));
+            assertThat(permissionCheck(permission, "cluster:admin/xpack/security/role/bulk_delete", bulkDeleteRolesRequest), is(expected));
+        }
+        {
+            assertThat(Arrays.stream(indices).allMatch(pattern -> {
+                final DeleteRoleRequest deleteRolesRequest = new DeleteRoleRequest();
+                deleteRolesRequest.name(pattern);
+                return permissionCheck(permission, "cluster:admin/xpack/security/role/delete", deleteRolesRequest);
+            }), is(expected));
+        }
+    }
+
+    public static ManageRolesPrivilege buildPrivileges() {
+        return buildPrivileges(randomIntBetween(MIN_INDEX_NAME_LENGTH, 7));
+    }
+
+    private static ManageRolesPrivilege buildPrivileges(int indexNameLength) {
+        String[] indexNames = Objects.requireNonNull(generateRandomStringArray(5, indexNameLength, false, false));
+
+        return new ManageRolesPrivilege(
+            List.of(new ManageRolesPrivilege.ManageRolesIndexPermissionGroup(indexNames, IndexPrivilege.READ.name().toArray(String[]::new)))
+        );
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        try (var xClientPlugin = new XPackClientPlugin()) {
+            return new NamedWriteableRegistry(xClientPlugin.getNamedWriteables());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    protected Class<ConfigurableClusterPrivilege> categoryClass() {
+        return ConfigurableClusterPrivilege.class;
+    }
+
+    @Override
+    protected ConfigurableClusterPrivilege createTestInstance() {
+        return buildPrivileges();
+    }
+
+    @Override
+    protected ConfigurableClusterPrivilege mutateInstance(ConfigurableClusterPrivilege instance) throws IOException {
+        if (instance instanceof ManageRolesPrivilege) {
+            return buildPrivileges(MIN_INDEX_NAME_LENGTH - 1);
+        }
+        fail();
+        return null;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
index f0676f35ae316..0cdf7de63ca99 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.action.admin.indices.get.GetIndexAction;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsAction;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
+import org.elasticsearch.action.admin.indices.mapping.put.TransportAutoPutMappingAction;
 import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryAction;
 import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction;
@@ -3662,6 +3663,9 @@ public void testPredefinedViewerRole() {
         assertOnlyReadAllowed(role, ".profiling-" + randomIntBetween(0, 5));
         assertOnlyReadAllowed(role, randomAlphaOfLength(5));
 
+        assertOnlyReadAllowed(role, ".slo-observability." + randomIntBetween(0, 5));
+        assertViewIndexMetadata(role, ".slo-observability." + randomIntBetween(0, 5));
+
         assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES);
         assertNoAccessAllowed(role, "." + randomAlphaOfLengthBetween(6, 10));
         assertNoAccessAllowed(role, "ilm-history-" + randomIntBetween(0, 5));
@@ -3740,6 +3744,9 @@ public void testPredefinedEditorRole() {
         assertReadWriteDocsAndMaintenanceButNotDeleteIndexAllowed(role, ".preview.alerts-" + randomIntBetween(0, 5));
         assertReadWriteDocsAndMaintenanceButNotDeleteIndexAllowed(role, ".internal.preview.alerts-" + randomIntBetween(0, 5));
 
+        assertViewIndexMetadata(role, ".slo-observability." + randomIntBetween(0, 5));
+        assertReadWriteAndManage(role, ".slo-observability." + randomIntBetween(0, 5));
+
         assertNoAccessAllowed(role, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES);
         assertNoAccessAllowed(role, "." + randomAlphaOfLengthBetween(6, 10));
         assertNoAccessAllowed(role, "ilm-history-" + randomIntBetween(0, 5));
@@ -3865,6 +3872,41 @@ private void assertReadWriteDocsButNotDeleteIndexAllowed(Role role, String index
             role.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(mockIndexAbstraction(index)),
             is(false)
         );
+
+        assertThat(role.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(mockIndexAbstraction(index)), is(true));
+        assertThat(role.indices().allowedIndicesMatcher(TransportGetAction.TYPE.name()).test(mockIndexAbstraction(index)), is(true));
+        assertThat(role.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(mockIndexAbstraction(index)), is(true));
+        assertThat(role.indices().allowedIndicesMatcher(TransportUpdateAction.NAME).test(mockIndexAbstraction(index)), is(true));
+        assertThat(role.indices().allowedIndicesMatcher(TransportDeleteAction.NAME).test(mockIndexAbstraction(index)), is(true));
+        assertThat(role.indices().allowedIndicesMatcher(TransportBulkAction.NAME).test(mockIndexAbstraction(index)), is(true));
+    }
+
+    private void assertReadWriteAndManage(Role role, String index) {
+        assertThat(
+            role.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(mockIndexAbstraction(index)),
+            is(true)
+        );
+        assertThat(
+            role.indices().allowedIndicesMatcher(TransportFieldCapabilitiesAction.NAME + "*").test(mockIndexAbstraction(index)),
+            is(true)
+        );
+        assertThat(
+            role.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(mockIndexAbstraction(index)),
+            is(true)
+        );
+        assertThat(
+            role.indices().allowedIndicesMatcher(TransportUpdateSettingsAction.TYPE.name()).test(mockIndexAbstraction(index)),
+            is(true)
+        );
+        assertThat(role.indices().allowedIndicesMatcher(GetRollupIndexCapsAction.NAME + "*").test(mockIndexAbstraction(index)), is(true));
+        assertThat(role.indices().allowedIndicesMatcher("indices:admin/*").test(mockIndexAbstraction(index)), is(true));
+        assertThat(role.indices().allowedIndicesMatcher("indices:monitor/*").test(mockIndexAbstraction(index)), is(true));
+        assertThat(
+            role.indices().allowedIndicesMatcher(TransportAutoPutMappingAction.TYPE.name()).test(mockIndexAbstraction(index)),
+            is(true)
+        );
+        assertThat(role.indices().allowedIndicesMatcher(AutoCreateAction.NAME).test(mockIndexAbstraction(index)), is(true));
+
         assertThat(role.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(mockIndexAbstraction(index)), is(true));
         assertThat(role.indices().allowedIndicesMatcher(TransportGetAction.TYPE.name()).test(mockIndexAbstraction(index)), is(true));
         assertThat(role.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(mockIndexAbstraction(index)), is(true));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItemTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItemTests.java
index 3eeaa18f0a81e..2dd1d8d4ec13a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItemTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyItemTests.java
@@ -67,7 +67,7 @@ protected SnapshotLifecyclePolicyItem mutateInstance(SnapshotLifecyclePolicyItem
                 return new SnapshotLifecyclePolicyItem(
                     instance.getPolicy(),
                     instance.getVersion(),
-                    randomValueOtherThan(instance.getModifiedDate(), ESTestCase::randomNonNegativeLong),
+                    randomValueOtherThan(instance.getModifiedDate(), SnapshotLifecyclePolicyMetadataTests::randomModifiedTime),
                     instance.getLastSuccess(),
                     instance.getLastFailure(),
                     instance.getSnapshotInProgress(),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java
index 090b4fe78253d..66e25c3b91db2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java
@@ -13,6 +13,8 @@
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.time.Clock;
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -79,7 +81,7 @@ public static SnapshotLifecyclePolicyMetadata createRandomPolicyMetadata(String
         SnapshotLifecyclePolicyMetadata.Builder builder = SnapshotLifecyclePolicyMetadata.builder()
             .setPolicy(randomSnapshotLifecyclePolicy(policyId))
             .setVersion(randomNonNegativeLong())
-            .setModifiedDate(randomNonNegativeLong());
+            .setModifiedDate(randomModifiedTime());
         if (randomBoolean()) {
             builder.setHeaders(randomHeaders());
         }
@@ -102,6 +104,7 @@ public static SnapshotLifecyclePolicy randomSnapshotLifecyclePolicy(String polic
         for (int i = 0; i < randomIntBetween(2, 5); i++) {
             config.put(randomAlphaOfLength(4), randomAlphaOfLength(4));
         }
+
         return new SnapshotLifecyclePolicy(
             policyId,
             randomAlphaOfLength(4),
@@ -122,7 +125,41 @@ public static SnapshotRetentionConfiguration randomRetention() {
             );
     }
 
-    public static String randomSchedule() {
+    public static String randomCronSchedule() {
         return randomIntBetween(0, 59) + " " + randomIntBetween(0, 59) + " " + randomIntBetween(0, 12) + " * * ?";
     }
+
+    public static String randomTimeValueString() {
+        // restrict to intervals greater than slm.minimum_interval value of 15 minutes
+        Duration minInterval = Duration.ofMinutes(15);
+        Map<String, Long> unitMinVal = Map.of(
+            "nanos",
+            minInterval.toNanos(),
+            "micros",
+            minInterval.toNanos() * 1000,
+            "ms",
+            minInterval.toMillis(),
+            "s",
+            minInterval.toSeconds(),
+            "m",
+            minInterval.toMinutes(),
+            "h",
+            minInterval.toHours(),
+            "d",
+            minInterval.toDays()
+        );
+        var unit = randomFrom(unitMinVal.keySet());
+        long minVal = Math.max(1, unitMinVal.get(unit));
+        long value = randomLongBetween(minVal, 1000 * minVal);
+        return value + unit;
+    }
+
+    public static String randomSchedule() {
+        return randomBoolean() ? randomCronSchedule() : randomTimeValueString();
+    }
+
+    public static long randomModifiedTime() {
+        // if modified time is after the current time, validation will fail
+        return randomLongBetween(0, Clock.systemUTC().millis());
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStatsTests.java
index 156df48e79829..0332fcf6e6f71 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStatsTests.java
@@ -7,15 +7,75 @@
 
 package org.elasticsearch.xpack.core.slm;
 
+import org.elasticsearch.common.io.stream.InputStreamStreamInput;
+import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 public class SnapshotLifecycleStatsTests extends AbstractXContentSerializingTestCase<SnapshotLifecycleStats> {
+
+    public void testPolicyStatsMapImmutable() throws IOException {
+        {
+            SnapshotLifecycleStats stats = new SnapshotLifecycleStats();
+            assertThrows(UnsupportedOperationException.class, () -> stats.getMetrics().put("new_policy", null));
+        }
+
+        {
+            SnapshotLifecycleStats stats = new SnapshotLifecycleStats(0, 0, 0, 0, new HashMap<>());
+            assertThrows(UnsupportedOperationException.class, () -> stats.getMetrics().put("new_policy", null));
+        }
+
+        {
+            SnapshotLifecycleStats stats1 = new SnapshotLifecycleStats(1, 0, 0, 0, new HashMap<>());
+            SnapshotLifecycleStats stats2 = new SnapshotLifecycleStats(0, 1, 0, 0, new HashMap<>());
+            SnapshotLifecycleStats stats = stats1.merge(stats2);
+            assertThrows(UnsupportedOperationException.class, () -> stats.getMetrics().put("new_policy", null));
+        }
+
+        {
+            // write
+            ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
+            OutputStreamStreamOutput out = new OutputStreamStreamOutput(outBuffer);
+            SnapshotLifecycleStats stats1 = new SnapshotLifecycleStats(0, 0, 0, 0, new HashMap<>());
+            stats1.writeTo(out);
+
+            // read
+            ByteArrayInputStream inBuffer = new ByteArrayInputStream(outBuffer.toByteArray());
+            InputStreamStreamInput in = new InputStreamStreamInput(inBuffer);
+            SnapshotLifecycleStats stats = new SnapshotLifecycleStats(in);
+            assertThrows(UnsupportedOperationException.class, () -> stats.getMetrics().put("new_policy", null));
+        }
+    }
+
+    public void testIncrementAllFields() {
+        final String policy = "policy";
+        SnapshotLifecycleStats stats = new SnapshotLifecycleStats();
+        SnapshotLifecycleStats updated = stats.withRetentionRunIncremented()
+            .withRetentionFailedIncremented()
+            .withRetentionTimedOutIncremented()
+            .withDeletionTimeUpdated(TimeValue.ONE_MINUTE)
+            .withTakenIncremented(policy)
+            .withDeletedIncremented(policy)
+            .withFailedIncremented(policy)
+            .withDeleteFailureIncremented(policy);
+
+        var policyStats = Map.of(policy, new SnapshotLifecycleStats.SnapshotPolicyStats(policy, 1, 1, 1, 1));
+        SnapshotLifecycleStats expected = new SnapshotLifecycleStats(1, 1, 1, TimeValue.ONE_MINUTE.millis(), policyStats);
+        assertEquals(expected, updated);
+    }
+
     @Override
     protected SnapshotLifecycleStats doParseInstance(XContentParser parser) throws IOException {
         return SnapshotLifecycleStats.parse(parser);
@@ -54,7 +114,19 @@ protected SnapshotLifecycleStats createTestInstance() {
 
     @Override
     protected SnapshotLifecycleStats mutateInstance(SnapshotLifecycleStats instance) {
-        return randomValueOtherThan(instance, () -> instance.merge(createTestInstance()));
+        List<String> policies = new ArrayList<>(instance.getMetrics().keySet());
+        String policy = policies.isEmpty() ? "policy-" + randomAlphaOfLength(4) : randomFrom(policies);
+
+        return switch (between(0, 7)) {
+            case 0 -> instance.withRetentionRunIncremented();
+            case 1 -> instance.withRetentionFailedIncremented();
+            case 2 -> instance.withRetentionTimedOutIncremented();
+            case 3 -> instance.withDeletionTimeUpdated(randomTimeValue(1, 1_000_000, TimeUnit.MILLISECONDS));
+            case 4 -> instance.withTakenIncremented(policy);
+            case 5 -> instance.withFailedIncremented(policy);
+            case 6 -> instance.withDeletedIncremented(policy);
+            default -> instance.withDeleteFailureIncremented(policy);
+        };
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
index 7b19d53663a08..4d1ebf6cbaabc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
@@ -68,6 +68,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
@@ -201,14 +202,19 @@ public void testPEMKeyConfigReloading() throws Exception {
                     throw new RuntimeException("Exception starting or connecting to the mock server", e);
                 }
             };
-            final Runnable modifier = () -> {
+            final List<Runnable> modifierFunctions = List.of(() -> {
                 try {
                     atomicMoveIfPossible(updatedKeyPath, keyPath);
+                } catch (Exception e) {
+                    throw new RuntimeException("failed to modify file", e);
+                }
+            }, () -> {
+                try {
                     atomicMoveIfPossible(updatedCertPath, certPath);
                 } catch (Exception e) {
                     throw new RuntimeException("failed to modify file", e);
                 }
-            };
+            });
 
             // The new server certificate is not in the client's truststore so SSLHandshake should fail
             final Consumer<SSLContext> keyMaterialPostChecks = (updatedContext) -> {
@@ -224,7 +230,7 @@ public void testPEMKeyConfigReloading() throws Exception {
                     throw new RuntimeException("Exception starting or connecting to the mock server", e);
                 }
             };
-            validateSSLConfigurationIsReloaded(env, keyMaterialPreChecks, modifier, keyMaterialPostChecks);
+            validateSSLConfigurationIsReloaded(env, keyMaterialPreChecks, modifierFunctions, keyMaterialPostChecks);
         }
     }
 
@@ -559,28 +565,45 @@ private Settings.Builder baseKeystoreSettings(Path tempDir, MockSecureSettings s
     private void validateSSLConfigurationIsReloaded(
         Environment env,
         Consumer<SSLContext> preChecks,
-        Runnable modificationFunction,
+        Runnable modifierFunction,
+        Consumer<SSLContext> postChecks
+    ) throws Exception {
+        validateSSLConfigurationIsReloaded(env, preChecks, List.of(modifierFunction), postChecks);
+    }
+
+    private void validateSSLConfigurationIsReloaded(
+        Environment env,
+        Consumer<SSLContext> preChecks,
+        List<Runnable> modifierFunctions,
         Consumer<SSLContext> postChecks
     ) throws Exception {
-        final CountDownLatch reloadLatch = new CountDownLatch(1);
+        final CyclicBarrier reloadBarrier = new CyclicBarrier(2);
         final SSLService sslService = new SSLService(env);
         final SslConfiguration config = sslService.getSSLConfiguration("xpack.security.transport.ssl");
         final Consumer<SslConfiguration> reloadConsumer = sslConfiguration -> {
             try {
                 sslService.reloadSSLContext(sslConfiguration);
             } finally {
-                reloadLatch.countDown();
+                try {
+                    reloadBarrier.await();
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
             }
         };
         new SSLConfigurationReloader(reloadConsumer, resourceWatcherService, SSLService.getSSLConfigurations(env).values());
         // Baseline checks
         preChecks.accept(sslService.sslContextHolder(config).sslContext());
 
-        assertEquals("nothing should have called reload", 1, reloadLatch.getCount());
+        assertEquals("nothing should have called reload", 0, reloadBarrier.getNumberWaiting());
 
         // modify
-        modificationFunction.run();
-        reloadLatch.await();
+        for (var modifierFunction : modifierFunctions) {
+            modifierFunction.run();
+            reloadBarrier.await();
+            reloadBarrier.reset();
+        }
+
         // checks after reload
         postChecks.accept(sslService.sslContextHolder(config).sslContext());
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/action/RestTermsEnumActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/action/RestTermsEnumActionTests.java
index 2ea372a84b66c..9df49c72baee0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/action/RestTermsEnumActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/action/RestTermsEnumActionTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -71,7 +72,8 @@ public static void stubTermEnumAction() {
         final TransportAction<? extends ActionRequest, ? extends ActionResponse> transportAction = new TransportAction<>(
             TermsEnumAction.NAME,
             new ActionFilters(Collections.emptySet()),
-            taskManager
+            taskManager,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
         ) {
             @Override
             protected void doExecute(Task task, ActionRequest request, ActionListener<ActionResponse> listener) {}
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json
index 1951431859ffe..138a8d763bae6 100644
--- a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json
+++ b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json
@@ -31,7 +31,7 @@
               "doc_values": false
             },
             "path_match": [
-              "event.original"
+              "*event.original"
             ]
           }
         },
@@ -128,9 +128,9 @@
               "*file.path",
               "*file.target_path",
               "*os.full",
-              "email.subject",
-              "vulnerability.description",
-              "user_agent.original"
+              "*email.subject",
+              "*vulnerability.description",
+              "*user_agent.original"
             ],
             "unmatch_mapping_type": "object"
           }
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json
index ad66ad8796862..8b1c13f3152e8 100644
--- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json
+++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json
@@ -319,6 +319,12 @@
         },
         "namespaces": {
           "type": "keyword"
+        },
+        "audit_unenrolled_time": {
+          "type": "date"
+        },
+        "audit_unenrolled_reason": {
+          "type": "keyword"
         }
       }
     }
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json
index b4aa999697632..9c58322f12d03 100644
--- a/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json
+++ b/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json
@@ -43,6 +43,50 @@
               "default_metric": "value_count"
             }
           }
+        },
+        {
+          "histogram": {
+            "mapping": {
+              "type": "histogram",
+              "ignore_malformed": true
+            }
+          }
+        },
+        {
+          "counter_long": {
+            "mapping": {
+              "type": "long",
+              "time_series_metric": "counter",
+              "ignore_malformed": true
+            }
+          }
+        },
+        {
+          "gauge_long": {
+            "mapping": {
+              "type": "long",
+              "time_series_metric": "gauge",
+              "ignore_malformed": true
+            }
+          }
+        },
+        {
+          "counter_double": {
+            "mapping": {
+              "type": "double",
+              "time_series_metric": "counter",
+              "ignore_malformed": true
+            }
+          }
+        },
+        {
+          "gauge_double": {
+            "mapping": {
+              "type": "double",
+              "time_series_metric": "gauge",
+              "ignore_malformed": true
+            }
+          }
         }
       ],
       "properties": {
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json
index 811810f5d0f2f..80585bea8c2b8 100644
--- a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json
+++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json
@@ -225,6 +225,21 @@
                   "scope" : {
                     "type" : "object",
                     "enabled" : false
+                  },
+                  "params": {
+                    "type": "object",
+                    "enabled": false,
+                    "properties": {
+                      "force_time_shift": {
+                        "type": "object",
+                        "enabled": false,
+                        "properties": {
+                          "time_shift_amount": {
+                            "type": "long"
+                          }
+                        }
+                      }
+                    }
                   }
                 }
               },
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json
index 9b90f97682306..8f50ebd334f16 100644
--- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json
+++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json
@@ -77,6 +77,9 @@
         },
         "service.name": {
           "type": "keyword"
+        },
+        "container.id": {
+          "type": "keyword"
         }
       }
     }
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/traces@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/traces@mappings.json
new file mode 100644
index 0000000000000..e3990a250f0c2
--- /dev/null
+++ b/x-pack/plugin/core/template-resources/src/main/resources/traces@mappings.json
@@ -0,0 +1,28 @@
+{
+  "template": {
+    "mappings": {
+      "date_detection": false,
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "data_stream.type": {
+          "type": "constant_keyword",
+          "value": "traces"
+        },
+        "data_stream.dataset": {
+          "type": "constant_keyword"
+        },
+        "data_stream.namespace": {
+          "type": "constant_keyword"
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default mappings for the traces index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}
diff --git a/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json
new file mode 100644
index 0000000000000..3f4fdba6f4f46
--- /dev/null
+++ b/x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json
@@ -0,0 +1,18 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "codec": "best_compression",
+        "mapping": {
+          "ignore_malformed": true
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default settings for the traces index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java
index 3c16830c2ba97..91e77762870bf 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java
@@ -21,8 +21,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java
index ba72be655a7ff..92b16b6a3430e 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java
@@ -19,12 +19,12 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/TransportDeprecationCacheResetAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/TransportDeprecationCacheResetAction.java
index 252fc04a1aac5..01d9089a153fd 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/TransportDeprecationCacheResetAction.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/TransportDeprecationCacheResetAction.java
@@ -14,9 +14,9 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.logging.RateLimitingFilter;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java
index d8dd030903e0e..68c1d7ab178c2 100644
--- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java
+++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java
@@ -258,7 +258,12 @@ public void testUpdateDownsampleRound() throws Exception {
 
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
-            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, updatedLifecycle)
+            new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName },
+                updatedLifecycle
+            )
         );
 
         String thirtySecondsDownsampleIndex = "downsample-20m-" + firstGenerationBackingIndex;
diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java
index 8f1c32e56d057..aea4a06411e4e 100644
--- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java
+++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java
@@ -44,6 +44,7 @@
 import java.util.Map;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.test.ESTestCase.TEST_REQUEST_TIMEOUT;
 import static org.elasticsearch.test.ESTestCase.indexSettings;
 import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
@@ -78,7 +79,10 @@ public static int setupTSDBDataStreamAndIngestDocs(
     }
 
     public static List<String> getBackingIndices(Client client, String dataStreamName) {
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStreamName }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
         assertThat(getDataStreamResponse.getDataStreams().isEmpty(), is(false));
         assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), is(dataStreamName));
diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java
index 8a0d9edae4993..5c2a05da1e8d2 100644
--- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java
+++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java
@@ -10,6 +10,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
+import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
@@ -324,12 +326,11 @@ private Disruptor(
         public void run() {
             listener.disruptionStart();
             try {
-                final String candidateNode = cluster.client(clientNode)
-                    .admin()
-                    .cluster()
-                    .prepareSearchShards(sourceIndex)
-                    .get()
-                    .getNodes()[0].getName();
+                final String candidateNode = safeExecute(
+                    cluster.client(clientNode),
+                    TransportClusterSearchShardsAction.TYPE,
+                    new ClusterSearchShardsRequest(TEST_REQUEST_TIMEOUT, sourceIndex)
+                ).getNodes()[0].getName();
                 logger.info("Candidate node [" + candidateNode + "]");
                 disruption.accept(candidateNode);
                 ensureGreen(sourceIndex);
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java
index a6ba4346b1a25..7dcda9c2b0032 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java
@@ -137,6 +137,6 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
 
     @Override
     public Collection<?> createComponents(PluginServices services) {
-        return List.of(new DownsampleMetrics(services.telemetryProvider().getMeterRegistry()));
+        return List.of(DownsampleMetrics.class);
     }
 }
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java
index c950658b411ed..b5ac4b0ae37a3 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.downsample;
 
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 
 import java.io.IOException;
@@ -36,8 +37,8 @@ public class DownsampleMetrics extends AbstractLifecycleComponent {
 
     private final MeterRegistry meterRegistry;
 
-    public DownsampleMetrics(MeterRegistry meterRegistry) {
-        this.meterRegistry = meterRegistry;
+    public DownsampleMetrics(TelemetryProvider telemetryProvider) {
+        this.meterRegistry = telemetryProvider.getMeterRegistry();
     }
 
     @Override
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java
index 5e6f8b6b5b18e..e5b9ca32808a7 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -34,6 +33,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
 import org.elasticsearch.persistent.PersistentTaskState;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
@@ -315,7 +315,8 @@ public TA(
                 IndicesService indicesService,
                 DownsampleMetrics downsampleMetrics
             ) {
-                super(NAME, actionFilters, transportService.getTaskManager());
+                // TODO: consider moving to Downsample.DOWSAMPLE_TASK_THREAD_POOL_NAME and simplify realNodeOperation
+                super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
                 this.client = client;
                 this.indicesService = indicesService;
                 this.downsampleMetrics = downsampleMetrics;
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java
index abf629dc9c1fa..d8c9acff156ce 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java
@@ -45,7 +45,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -64,6 +63,7 @@
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTaskParams;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java
index d316ca8a9e675..3d3fe3650ebd9 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java
@@ -21,10 +21,10 @@
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
index e4db5a253f996..3a696a196175f 100644
--- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
+++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
@@ -640,7 +640,10 @@ public void testDownsampleDatastream() throws Exception {
         downsample(sourceIndex, downsampleIndex, config);
         assertDownsampleIndex(sourceIndex, downsampleIndex, config);
 
-        var r = client().execute(GetDataStreamAction.INSTANCE, new GetDataStreamAction.Request(new String[] { dataStreamName })).get();
+        var r = client().execute(
+            GetDataStreamAction.INSTANCE,
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName })
+        ).get();
         assertEquals(1, r.getDataStreams().size());
         List<Index> indices = r.getDataStreams().get(0).getDataStream().getIndices();
         // Assert that the downsample index has not been added to the data stream
@@ -1584,7 +1587,12 @@ private String createDataStream() throws Exception {
             dataStreamName + "_template"
         ).indexTemplate(template);
         assertAcked(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet());
-        assertAcked(client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).get());
+        assertAcked(
+            client().execute(
+                CreateDataStreamAction.INSTANCE,
+                new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName)
+            ).get()
+        );
         return dataStreamName;
     }
 
diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java
index 0a754cbb542b2..834a1e887caa7 100644
--- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java
+++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleDataStreamTests.java
@@ -77,7 +77,10 @@ public void testDataStreamDownsample() throws ExecutionException, InterruptedExc
         // GIVEN
         final String dataStreamName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
         putComposableIndexTemplate("1", List.of(dataStreamName));
-        client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet();
+        client().execute(
+            CreateDataStreamAction.INSTANCE,
+            new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName)
+        ).actionGet();
         indexDocs(dataStreamName, 10, Instant.now().toEpochMilli());
         final RolloverResponse rolloverResponse = indicesAdmin().rolloverIndex(new RolloverRequest(dataStreamName, null)).get();
         // NOTE: here we calculate a delay to index documents because the next data stream write index is created with a start time of
@@ -113,6 +116,8 @@ public void testDataStreamDownsample() throws ExecutionException, InterruptedExc
         ).actionGet();
 
         final ModifyDataStreamsAction.Request modifyDataStreamRequest = new ModifyDataStreamsAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
             List.of(
                 DataStreamAction.removeBackingIndex(dataStreamName, rolloverResponse.getOldIndex()),
                 DataStreamAction.addBackingIndex(dataStreamName, downsampleTargetIndex)
@@ -124,7 +129,7 @@ public void testDataStreamDownsample() throws ExecutionException, InterruptedExc
         assertThat(downsampleResponse.isAcknowledged(), equalTo(true));
         final GetDataStreamAction.Response getDataStreamActionResponse = indicesAdmin().execute(
             GetDataStreamAction.INSTANCE,
-            new GetDataStreamAction.Request(new String[] { dataStreamName })
+            new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName })
         ).actionGet();
         assertThat(getDataStreamActionResponse.getDataStreams().get(0).getDataStream().getIndices().size(), equalTo(2));
         final List<String> backingIndices = getDataStreamActionResponse.getDataStreams()
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java
index 35c2071188864..0130bd5537a11 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.cache.Cache;
 import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -29,6 +30,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.BiConsumer;
 import java.util.function.LongSupplier;
+import java.util.function.ToLongBiFunction;
 
 /**
  * A simple cache for enrich that uses {@link Cache}. There is one instance of this cache and
@@ -61,12 +63,29 @@ public final class EnrichCache {
         this(maxSize, System::nanoTime);
     }
 
+    EnrichCache(ByteSizeValue maxByteSize) {
+        this(maxByteSize, System::nanoTime);
+    }
+
     // non-private for unit testing only
     EnrichCache(long maxSize, LongSupplier relativeNanoTimeProvider) {
         this.relativeNanoTimeProvider = relativeNanoTimeProvider;
-        this.cache = CacheBuilder.<CacheKey, CacheValue>builder().setMaximumWeight(maxSize).removalListener(notification -> {
+        this.cache = createCache(maxSize, null);
+    }
+
+    EnrichCache(ByteSizeValue maxByteSize, LongSupplier relativeNanoTimeProvider) {
+        this.relativeNanoTimeProvider = relativeNanoTimeProvider;
+        this.cache = createCache(maxByteSize.getBytes(), (key, value) -> value.sizeInBytes);
+    }
+
+    private Cache<CacheKey, CacheValue> createCache(long maxWeight, ToLongBiFunction<CacheKey, CacheValue> weigher) {
+        var builder = CacheBuilder.<CacheKey, CacheValue>builder().setMaximumWeight(maxWeight).removalListener(notification -> {
             sizeInBytes.getAndAdd(-1 * notification.getValue().sizeInBytes);
-        }).build();
+        });
+        if (weigher != null) {
+            builder.weigher(weigher);
+        }
+        return builder.build();
     }
 
     /**
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
index 868ec49ff1d97..1a68ada60b6f1 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
@@ -12,17 +12,22 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.MemorySizeValue;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
@@ -121,14 +126,29 @@ public class EnrichPlugin extends Plugin implements SystemIndexPlugin, IngestPlu
         return String.valueOf(maxConcurrentRequests * maxLookupsPerRequest);
     }, val -> Setting.parseInt(val, 1, Integer.MAX_VALUE, QUEUE_CAPACITY_SETTING_NAME), Setting.Property.NodeScope);
 
-    public static final Setting<Long> CACHE_SIZE = Setting.longSetting("enrich.cache_size", 1000, 0, Setting.Property.NodeScope);
+    public static final String CACHE_SIZE_SETTING_NAME = "enrich.cache.size";
+    public static final Setting<FlatNumberOrByteSizeValue> CACHE_SIZE = new Setting<>(
+        "enrich.cache.size",
+        (String) null,
+        (String s) -> FlatNumberOrByteSizeValue.parse(
+            s,
+            CACHE_SIZE_SETTING_NAME,
+            new FlatNumberOrByteSizeValue(ByteSizeValue.ofBytes((long) (0.01 * JvmInfo.jvmInfo().getConfiguredMaxHeapSize())))
+        ),
+        Setting.Property.NodeScope
+    );
 
     private final Settings settings;
     private final EnrichCache enrichCache;
 
     public EnrichPlugin(final Settings settings) {
         this.settings = settings;
-        this.enrichCache = new EnrichCache(CACHE_SIZE.get(settings));
+        FlatNumberOrByteSizeValue maxSize = CACHE_SIZE.get(settings);
+        if (maxSize.byteSizeValue() != null) {
+            this.enrichCache = new EnrichCache(maxSize.byteSizeValue());
+        } else {
+            this.enrichCache = new EnrichCache(maxSize.flatNumber());
+        }
     }
 
     @Override
@@ -265,4 +285,45 @@ public String getFeatureName() {
     public String getFeatureDescription() {
         return "Manages data related to Enrich policies";
     }
+
+    /**
+     * A class that specifies either a flat (unit-less) number or a byte size value.
+     */
+    public static class FlatNumberOrByteSizeValue {
+
+        @Nullable
+        private final Long flatNumber;
+        @Nullable
+        private final ByteSizeValue byteSizeValue;
+
+        public FlatNumberOrByteSizeValue(ByteSizeValue byteSizeValue) {
+            this.byteSizeValue = byteSizeValue;
+            this.flatNumber = null;
+        }
+
+        public FlatNumberOrByteSizeValue(Long flatNumber) {
+            this.flatNumber = flatNumber;
+            this.byteSizeValue = null;
+        }
+
+        public static FlatNumberOrByteSizeValue parse(String value, String settingName, FlatNumberOrByteSizeValue defaultValue) {
+            if (Strings.hasText(value) == false) {
+                return defaultValue;
+            }
+            if (Character.isDigit(value.charAt(value.length() - 1)) == false) {
+                return new FlatNumberOrByteSizeValue(MemorySizeValue.parseBytesSizeValueOrHeapRatio(value, settingName));
+            }
+            return new FlatNumberOrByteSizeValue(Long.parseLong(value));
+        }
+
+        @Nullable
+        public ByteSizeValue byteSizeValue() {
+            return byteSizeValue;
+        }
+
+        @Nullable
+        public Long flatNumber() {
+            return flatNumber;
+        }
+    }
 }
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java
index 2ebe268cc788d..746ae2f4eee2b 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
 import org.elasticsearch.client.internal.Client;
@@ -126,8 +127,9 @@ public void runPolicyLocally(
             }
 
             task.setStatus(new ExecuteEnrichPolicyStatus(ExecuteEnrichPolicyStatus.PolicyPhases.SCHEDULED));
-            Runnable runnable = createPolicyRunner(policyName, policy, enrichIndexName, task, listener);
-            threadPool.executor(ThreadPool.Names.GENERIC).execute(runnable);
+            var policyRunner = createPolicyRunner(policyName, policy, enrichIndexName, task);
+            threadPool.executor(ThreadPool.Names.GENERIC)
+                .execute(ActionRunnable.wrap(ActionListener.assertOnce(listener), policyRunner::run));
         } catch (Exception e) {
             task.setStatus(new ExecuteEnrichPolicyStatus(ExecuteEnrichPolicyStatus.PolicyPhases.FAILED));
             throw e;
@@ -206,18 +208,16 @@ public void onFailure(Exception exception) {
         });
     }
 
-    private Runnable createPolicyRunner(
+    private EnrichPolicyRunner createPolicyRunner(
         String policyName,
         EnrichPolicy policy,
         String enrichIndexName,
-        ExecuteEnrichPolicyTask task,
-        ActionListener<ExecuteEnrichPolicyStatus> listener
+        ExecuteEnrichPolicyTask task
     ) {
         return new EnrichPolicyRunner(
             policyName,
             policy,
             task,
-            listener,
             clusterService,
             indicesService,
             client,
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java
index ca00f49100279..cab115ddc4964 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java
@@ -16,9 +16,12 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.DelegatingActionListener;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
@@ -30,6 +33,10 @@
 import org.elasticsearch.action.admin.indices.segments.ShardSegments;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.action.support.broadcast.BroadcastResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.FilterClient;
 import org.elasticsearch.client.internal.OriginSettingClient;
@@ -76,7 +83,7 @@
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.core.ClientHelper.ENRICH_ORIGIN;
 
-public class EnrichPolicyRunner implements Runnable {
+public class EnrichPolicyRunner {
 
     private static final Logger logger = LogManager.getLogger(EnrichPolicyRunner.class);
 
@@ -92,7 +99,6 @@ public class EnrichPolicyRunner implements Runnable {
     private final String policyName;
     private final EnrichPolicy policy;
     private final ExecuteEnrichPolicyTask task;
-    private final ActionListener<ExecuteEnrichPolicyStatus> listener;
     private final ClusterService clusterService;
     private final IndicesService indicesService;
     private final Client client;
@@ -105,7 +111,6 @@ public class EnrichPolicyRunner implements Runnable {
         String policyName,
         EnrichPolicy policy,
         ExecuteEnrichPolicyTask task,
-        ActionListener<ExecuteEnrichPolicyStatus> listener,
         ClusterService clusterService,
         IndicesService indicesService,
         Client client,
@@ -117,7 +122,6 @@ public class EnrichPolicyRunner implements Runnable {
         this.policyName = Objects.requireNonNull(policyName);
         this.policy = Objects.requireNonNull(policy);
         this.task = Objects.requireNonNull(task);
-        this.listener = Objects.requireNonNull(listener);
         this.clusterService = Objects.requireNonNull(clusterService);
         this.indicesService = indicesService;
         this.client = wrapClient(client, policyName, task, clusterService);
@@ -127,23 +131,37 @@ public class EnrichPolicyRunner implements Runnable {
         this.maxForceMergeAttempts = maxForceMergeAttempts;
     }
 
-    @Override
-    public void run() {
-        try {
-            logger.info("Policy [{}]: Running enrich policy", policyName);
-            task.setStatus(new ExecuteEnrichPolicyStatus(ExecuteEnrichPolicyStatus.PolicyPhases.RUNNING));
-            // Collect the source index information
-            final String[] sourceIndices = policy.getIndices().toArray(new String[0]);
-            logger.debug("Policy [{}]: Checking source indices [{}]", policyName, sourceIndices);
-            GetIndexRequest getIndexRequest = new GetIndexRequest().indices(sourceIndices);
-            // This call does not set the origin to ensure that the user executing the policy has permission to access the source index
-            client.admin().indices().getIndex(getIndexRequest, listener.delegateFailureAndWrap((l, getIndexResponse) -> {
+    public void run(ActionListener<ExecuteEnrichPolicyStatus> listener) {
+        logger.info("Policy [{}]: Running enrich policy", policyName);
+        task.setStatus(new ExecuteEnrichPolicyStatus(ExecuteEnrichPolicyStatus.PolicyPhases.RUNNING));
+
+        SubscribableListener
+
+            .<GetIndexResponse>newForked(l -> {
+                // Collect the source index information
+                final String[] sourceIndices = policy.getIndices().toArray(new String[0]);
+                logger.debug("Policy [{}]: Checking source indices [{}]", policyName, sourceIndices);
+                GetIndexRequest getIndexRequest = new GetIndexRequest().indices(sourceIndices);
+                // This call does not set the origin to ensure that the user executing the policy has permission to access the source index
+                client.admin().indices().getIndex(getIndexRequest, l);
+            })
+            .<CreateIndexResponse>andThen((l, getIndexResponse) -> {
                 validateMappings(getIndexResponse);
-                prepareAndCreateEnrichIndex(toMappings(getIndexResponse), clusterService.getSettings());
-            }));
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
+                prepareAndCreateEnrichIndex(toMappings(getIndexResponse), clusterService.getSettings(), l);
+            })
+            .andThen(this::prepareReindexOperation)
+            .andThen(this::transferDataToEnrichIndex)
+            .andThen(this::forceMergeEnrichIndex)
+            .andThen(this::setIndexReadOnly)
+            .andThen(this::waitForIndexGreen)
+            .andThen(this::updateEnrichPolicyAlias)
+            .andThenApply(r -> {
+                logger.info("Policy [{}]: Policy execution complete", policyName);
+                ExecuteEnrichPolicyStatus completeStatus = new ExecuteEnrichPolicyStatus(ExecuteEnrichPolicyStatus.PolicyPhases.COMPLETE);
+                task.setStatus(completeStatus);
+                return completeStatus;
+            })
+            .addListener(listener);
     }
 
     private static List<Map<String, Object>> toMappings(GetIndexResponse response) {
@@ -204,9 +222,7 @@ static void validateMappings(
         }
     }
 
-    private record MappingTypeAndFormat(String type, String format) {
-
-    }
+    private record MappingTypeAndFormat(String type, String format) {}
 
     private static MappingTypeAndFormat validateAndGetMappingTypeAndFormat(
         String fieldName,
@@ -436,7 +452,11 @@ static boolean isIndexableField(MapperService mapperService, String field, Strin
         }
     }
 
-    private void prepareAndCreateEnrichIndex(List<Map<String, Object>> mappings, Settings settings) {
+    private void prepareAndCreateEnrichIndex(
+        List<Map<String, Object>> mappings,
+        Settings settings,
+        ActionListener<CreateIndexResponse> listener
+    ) {
         int numberOfReplicas = settings.getAsInt(ENRICH_MIN_NUMBER_OF_REPLICAS_NAME, 0);
         Settings enrichIndexSettings = Settings.builder()
             .put("index.number_of_shards", 1)
@@ -449,28 +469,20 @@ private void prepareAndCreateEnrichIndex(List<Map<String, Object>> mappings, Set
         CreateIndexRequest createEnrichIndexRequest = new CreateIndexRequest(enrichIndexName, enrichIndexSettings);
         createEnrichIndexRequest.mapping(createEnrichMapping(mappings));
         logger.debug("Policy [{}]: Creating new enrich index [{}]", policyName, enrichIndexName);
-        enrichOriginClient().admin()
-            .indices()
-            .create(
-                createEnrichIndexRequest,
-                listener.delegateFailure((l, createIndexResponse) -> prepareReindexOperation(enrichIndexName))
-            );
+        enrichOriginClient().admin().indices().create(createEnrichIndexRequest, listener);
     }
 
-    private void prepareReindexOperation(final String destinationIndexName) {
+    private void prepareReindexOperation(ActionListener<AcknowledgedResponse> listener) {
         // Check to make sure that the enrich pipeline exists, and create it if it is missing.
-        if (EnrichPolicyReindexPipeline.exists(clusterService.state()) == false) {
-            EnrichPolicyReindexPipeline.create(
-                enrichOriginClient(),
-                listener.delegateFailure((l, r) -> transferDataToEnrichIndex(destinationIndexName))
-            );
+        if (EnrichPolicyReindexPipeline.exists(clusterService.state())) {
+            listener.onResponse(null);
         } else {
-            transferDataToEnrichIndex(destinationIndexName);
+            EnrichPolicyReindexPipeline.create(enrichOriginClient(), listener);
         }
     }
 
-    private void transferDataToEnrichIndex(final String destinationIndexName) {
-        logger.debug("Policy [{}]: Transferring source data to new enrich index [{}]", policyName, destinationIndexName);
+    private void transferDataToEnrichIndex(ActionListener<Void> listener) {
+        logger.debug("Policy [{}]: Transferring source data to new enrich index [{}]", policyName, enrichIndexName);
         // Filter down the source fields to just the ones required by the policy
         final Set<String> retainFields = new HashSet<>();
         retainFields.add(policy.getMatchField());
@@ -481,7 +493,7 @@ private void transferDataToEnrichIndex(final String destinationIndexName) {
         if (policy.getQuery() != null) {
             searchSourceBuilder.query(QueryBuilders.wrapperQuery(policy.getQuery().getQuery()));
         }
-        ReindexRequest reindexRequest = new ReindexRequest().setDestIndex(destinationIndexName)
+        ReindexRequest reindexRequest = new ReindexRequest().setDestIndex(enrichIndexName)
             .setSourceIndices(policy.getIndices().toArray(new String[0]));
         reindexRequest.getSearchRequest().source(searchSourceBuilder);
         reindexRequest.getDestination().source(new BytesArray(new byte[0]), XContentType.SMILE);
@@ -538,101 +550,142 @@ public void onResponse(BulkByScrollResponse bulkByScrollResponse) {
                         "Policy [{}]: Transferred [{}] documents to enrich index [{}]",
                         policyName,
                         bulkByScrollResponse.getCreated(),
-                        destinationIndexName
+                        enrichIndexName
                     );
-                    forceMergeEnrichIndex(destinationIndexName, 1);
+                    delegate.onResponse(null);
                 }
             }
         });
     }
 
-    private void forceMergeEnrichIndex(final String destinationIndexName, final int attempt) {
+    private void forceMergeEnrichIndex(ActionListener<Void> listener) {
+        forceMergeEnrichIndexOrRetry(1, listener);
+    }
+
+    private void forceMergeEnrichIndexOrRetry(final int attempt, ActionListener<Void> listener) {
         logger.debug(
             "Policy [{}]: Force merging newly created enrich index [{}] (Attempt {}/{})",
             policyName,
-            destinationIndexName,
+            enrichIndexName,
             attempt,
             maxForceMergeAttempts
         );
-        enrichOriginClient().admin()
-            .indices()
-            .forceMerge(
-                new ForceMergeRequest(destinationIndexName).maxNumSegments(1),
-                listener.delegateFailure((l, r) -> refreshEnrichIndex(destinationIndexName, attempt))
-            );
-    }
-
-    private void refreshEnrichIndex(final String destinationIndexName, final int attempt) {
-        logger.debug("Policy [{}]: Refreshing enrich index [{}]", policyName, destinationIndexName);
-        enrichOriginClient().admin()
-            .indices()
-            .refresh(
-                new RefreshRequest(destinationIndexName),
-                listener.delegateFailure((l, r) -> ensureSingleSegment(destinationIndexName, attempt))
-            );
-    }
 
-    protected void ensureSingleSegment(final String destinationIndexName, final int attempt) {
-        enrichOriginClient().admin()
-            .indices()
-            .segments(new IndicesSegmentsRequest(destinationIndexName), new DelegatingActionListener<>(listener) {
-                @Override
-                public void onResponse(IndicesSegmentResponse indicesSegmentResponse) {
-                    IndexSegments indexSegments = indicesSegmentResponse.getIndices().get(destinationIndexName);
-                    if (indexSegments == null) {
-                        throw new ElasticsearchException(
-                            "Could not locate segment information for newly created index [{}]",
-                            destinationIndexName
-                        );
-                    }
-                    Map<Integer, IndexShardSegments> indexShards = indexSegments.getShards();
-                    assert indexShards.size() == 1 : "Expected enrich index to contain only one shard";
-                    ShardSegments[] shardSegments = indexShards.get(0).shards();
-                    assert shardSegments.length == 1 : "Expected enrich index to contain no replicas at this point";
-                    ShardSegments primarySegments = shardSegments[0];
-                    if (primarySegments.getSegments().size() > 1) {
+        SubscribableListener
+
+            .<BroadcastResponse>newForked(
+                l -> enrichOriginClient().admin().indices().forceMerge(new ForceMergeRequest(enrichIndexName).maxNumSegments(1), l)
+            )
+            .andThen(this::refreshEnrichIndex)
+            .andThen(this::afterRefreshEnrichIndex)
+            .andThen(this::getSegments)
+            .andThenApply(this::getSegmentCount)
+            .addListener(
+                // delegateFailureAndWrap() rather than andThen().addListener() to avoid building unnecessary O(#retries) listener chain
+                listener.delegateFailureAndWrap((l, segmentCount) -> {
+                    if (segmentCount > 1) {
                         int nextAttempt = attempt + 1;
                         if (nextAttempt > maxForceMergeAttempts) {
-                            delegate.onFailure(
-                                new ElasticsearchException(
-                                    "Force merging index [{}] attempted [{}] times but did not result in one segment.",
-                                    destinationIndexName,
-                                    attempt,
-                                    maxForceMergeAttempts
-                                )
+                            throw new ElasticsearchException(
+                                "Force merging index [{}] attempted [{}] times but did not result in one segment.",
+                                enrichIndexName,
+                                attempt,
+                                maxForceMergeAttempts
                             );
                         } else {
                             logger.debug(
                                 "Policy [{}]: Force merge result contains more than one segment [{}], retrying (attempt {}/{})",
                                 policyName,
-                                primarySegments.getSegments().size(),
+                                segmentCount,
                                 nextAttempt,
                                 maxForceMergeAttempts
                             );
-                            forceMergeEnrichIndex(destinationIndexName, nextAttempt);
+                            // TransportForceMergeAction always forks so no risk of stack overflow from this recursion
+                            forceMergeEnrichIndexOrRetry(nextAttempt, l);
                         }
                     } else {
-                        // Force merge down to one segment successful
-                        setIndexReadOnly(destinationIndexName);
+                        l.onResponse(null);
                     }
+                })
+            );
+    }
+
+    private void refreshEnrichIndex(ActionListener<BroadcastResponse> listener) {
+        logger.debug("Policy [{}]: Refreshing enrich index [{}]", policyName, enrichIndexName);
+        enrichOriginClient().admin().indices().refresh(new RefreshRequest(enrichIndexName), listener);
+    }
+
+    // hook to allow testing force-merge retries
+    protected void afterRefreshEnrichIndex(ActionListener<Void> listener) {
+        listener.onResponse(null);
+    }
+
+    private void getSegments(ActionListener<IndicesSegmentResponse> listener) {
+        enrichOriginClient().admin().indices().segments(new IndicesSegmentsRequest(enrichIndexName), listener);
+    }
+
+    private int getSegmentCount(IndicesSegmentResponse indicesSegmentResponse) {
+        int failedShards = indicesSegmentResponse.getFailedShards();
+        if (failedShards > 0) {
+            // Encountered a problem while querying the segments for the enrich index. Try and surface the problem in the log.
+            logger.warn(
+                "Policy [{}]: Encountered [{}] shard level failures while querying the segments for enrich index [{}]. "
+                    + "Turn on DEBUG logging for details.",
+                policyName,
+                failedShards,
+                enrichIndexName
+            );
+            if (logger.isDebugEnabled()) {
+                DefaultShardOperationFailedException[] shardFailures = indicesSegmentResponse.getShardFailures();
+                int failureNumber = 1;
+                String logPrefix = "Policy [" + policyName + "]: Encountered shard failure [";
+                String logSuffix = " of "
+                    + shardFailures.length
+                    + "] while querying segments for enrich index ["
+                    + enrichIndexName
+                    + "]. Shard [";
+                for (DefaultShardOperationFailedException shardFailure : shardFailures) {
+                    logger.debug(
+                        logPrefix + failureNumber + logSuffix + shardFailure.index() + "][" + shardFailure.shardId() + "]",
+                        shardFailure.getCause()
+                    );
+                    failureNumber++;
                 }
-            });
+            }
+        }
+        IndexSegments indexSegments = indicesSegmentResponse.getIndices().get(enrichIndexName);
+        if (indexSegments == null) {
+            if (indicesSegmentResponse.getShardFailures().length == 0) {
+                throw new ElasticsearchException("Could not locate segment information for newly created index [{}]", enrichIndexName);
+            } else {
+                DefaultShardOperationFailedException shardFailure = indicesSegmentResponse.getShardFailures()[0];
+                throw new ElasticsearchException(
+                    "Could not obtain segment information for newly created index [{}]; shard info [{}][{}]",
+                    shardFailure.getCause(),
+                    enrichIndexName,
+                    shardFailure.index(),
+                    shardFailure.shardId()
+                );
+            }
+        }
+        Map<Integer, IndexShardSegments> indexShards = indexSegments.getShards();
+        assert indexShards.size() == 1 : "Expected enrich index to contain only one shard";
+        ShardSegments[] shardSegments = indexShards.get(0).shards();
+        assert shardSegments.length == 1 : "Expected enrich index to contain no replicas at this point";
+        ShardSegments primarySegments = shardSegments[0];
+        return primarySegments.getSegments().size();
     }
 
-    private void setIndexReadOnly(final String destinationIndexName) {
-        logger.debug("Policy [{}]: Setting new enrich index [{}] to be read only", policyName, destinationIndexName);
-        UpdateSettingsRequest request = new UpdateSettingsRequest(destinationIndexName).setPreserveExisting(true)
+    private void setIndexReadOnly(ActionListener<AcknowledgedResponse> listener) {
+        logger.debug("Policy [{}]: Setting new enrich index [{}] to be read only", policyName, enrichIndexName);
+        UpdateSettingsRequest request = new UpdateSettingsRequest(enrichIndexName).setPreserveExisting(true)
             .settings(Settings.builder().put("index.auto_expand_replicas", "0-all").put("index.blocks.write", "true"));
-        enrichOriginClient().admin()
-            .indices()
-            .updateSettings(request, listener.delegateFailure((l, r) -> waitForIndexGreen(destinationIndexName)));
+        enrichOriginClient().admin().indices().updateSettings(request, listener);
     }
 
-    private void waitForIndexGreen(final String destinationIndexName) {
-        ClusterHealthRequest request = new ClusterHealthRequest(destinationIndexName).waitForGreenStatus();
-        enrichOriginClient().admin()
-            .cluster()
-            .health(request, listener.delegateFailureAndWrap((l, r) -> updateEnrichPolicyAlias(destinationIndexName)));
+    private void waitForIndexGreen(ActionListener<ClusterHealthResponse> listener) {
+        ClusterHealthRequest request = new ClusterHealthRequest(enrichIndexName).waitForGreenStatus();
+        enrichOriginClient().admin().cluster().health(request, listener);
     }
 
     /**
@@ -686,12 +739,12 @@ private void validateIndexBeforePromotion(String destinationIndexName, ClusterSt
         }
     }
 
-    private void updateEnrichPolicyAlias(final String destinationIndexName) {
+    private void updateEnrichPolicyAlias(ActionListener<IndicesAliasesResponse> listener) {
         String enrichIndexBase = EnrichPolicy.getBaseName(policyName);
-        logger.debug("Policy [{}]: Promoting new enrich index [{}] to alias [{}]", policyName, destinationIndexName, enrichIndexBase);
+        logger.debug("Policy [{}]: Promoting new enrich index [{}] to alias [{}]", policyName, enrichIndexName, enrichIndexBase);
         GetAliasesRequest aliasRequest = new GetAliasesRequest(enrichIndexBase);
         ClusterState clusterState = clusterService.state();
-        validateIndexBeforePromotion(destinationIndexName, clusterState);
+        validateIndexBeforePromotion(enrichIndexName, clusterState);
         String[] concreteIndices = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(clusterState, aliasRequest);
         String[] aliases = aliasRequest.aliases();
         IndicesAliasesRequest aliasToggleRequest = new IndicesAliasesRequest();
@@ -699,13 +752,8 @@ private void updateEnrichPolicyAlias(final String destinationIndexName) {
         if (indices.length > 0) {
             aliasToggleRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove().indices(indices).alias(enrichIndexBase));
         }
-        aliasToggleRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(destinationIndexName).alias(enrichIndexBase));
-        enrichOriginClient().admin().indices().aliases(aliasToggleRequest, listener.safeMap(r -> {
-            logger.info("Policy [{}]: Policy execution complete", policyName);
-            ExecuteEnrichPolicyStatus completeStatus = new ExecuteEnrichPolicyStatus(ExecuteEnrichPolicyStatus.PolicyPhases.COMPLETE);
-            task.setStatus(completeStatus);
-            return completeStatus;
-        }));
+        aliasToggleRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(enrichIndexName).alias(enrichIndexBase));
+        enrichOriginClient().admin().indices().aliases(aliasToggleRequest, listener);
     }
 
     /**
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorProxyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorProxyAction.java
index af5791ac6efd1..40356f2824494 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorProxyAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorProxyAction.java
@@ -17,11 +17,11 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.ElasticsearchClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java
index d540cdb83361d..1213c439c628c 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichInfoTransportAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichInfoTransportAction.java
index f32e3d4da627a..d4850e3945f14 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichInfoTransportAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichInfoTransportAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.enrich.action;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java
index 2aa30614b58f3..c43dc99f147b0 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -50,6 +49,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchModule;
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichUsageTransportAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichUsageTransportAction.java
index bb31690074d2e..d0c7965657c73 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichUsageTransportAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichUsageTransportAction.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java
index ed28599da9fbb..fd07dce89d370 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java
@@ -18,11 +18,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskAwareRequest;
 import org.elasticsearch.tasks.TaskCancelledException;
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyAction.java
index 5dec35149dc52..e88d830111dea 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyAction.java
@@ -22,10 +22,10 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.ingest.PipelineConfiguration;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichReindexAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichReindexAction.java
index b3ccb3b4b4355..cc42199ab1019 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichReindexAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichReindexAction.java
@@ -13,11 +13,11 @@
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.reindex.ReindexRequest;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.reindex.ReindexSslConfig;
 import org.elasticsearch.reindex.TransportReindexAction;
 import org.elasticsearch.script.ScriptService;
@@ -61,7 +61,8 @@ public TransportEnrichReindexAction(
             autoCreateIndex,
             client,
             transportService,
-            new ReindexSslConfig(settings, environment, watcherService)
+            new ReindexSslConfig(settings, environment, watcherService),
+            null
         );
         this.bulkClient = new OriginSettingClient(client, ENRICH_ORIGIN);
     }
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichStatsAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichStatsAction.java
index 7bd2c43f54eaf..b6128491d5916 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichStatsAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportEnrichStatsAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportExecuteEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportExecuteEnrichPolicyAction.java
index c0d447385f228..1f1960bff1bbf 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportExecuteEnrichPolicyAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportExecuteEnrichPolicyAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyAction.java
index e1f0240b8e8ed..43ff0762fc9e3 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportPutEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportPutEnrichPolicyAction.java
index 7433863fcbd5d..b2735a7e64851 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportPutEnrichPolicyAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/TransportPutEnrichPolicyAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java
index 8ce1e7f350ccb..d3dcd7ae36f59 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.DocWriteResponse;
-import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
@@ -33,7 +32,7 @@
 import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.FilterClient;
@@ -78,18 +77,15 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
+import static org.hamcrest.CoreMatchers.allOf;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.notNullValue;
 
@@ -146,17 +142,10 @@ public void testRunner() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -226,17 +215,10 @@ public void testRunnerGeoMatchType() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -317,17 +299,10 @@ private void testNumberRangeMatchType(String rangeType) throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -414,17 +389,10 @@ public void testRunnerRangeTypeWithIpRange() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -513,17 +481,10 @@ public void testRunnerMultiSource() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -632,17 +593,10 @@ public void testRunnerMultiSourceDocIdCollisions() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -755,17 +709,10 @@ public void testRunnerMultiSourceEnrichKeyCollisions() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -819,7 +766,7 @@ public void testRunnerMultiSourceEnrichKeyCollisions() throws Exception {
         ensureEnrichIndexIsReadOnly(createdEnrichIndex);
     }
 
-    public void testRunnerNoSourceIndex() throws Exception {
+    public void testRunnerNoSourceIndex() {
         final String sourceIndex = "source-index";
 
         List<String> enrichFields = List.of("field2", "field5");
@@ -828,24 +775,16 @@ public void testRunnerNoSourceIndex() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            Exception thrown = exception.get();
-            assertThat(thrown, instanceOf(IndexNotFoundException.class));
-            assertThat(thrown.getMessage(), containsString("no such index [" + sourceIndex + "]"));
-        } else {
-            fail("Expected exception but nothing was thrown");
-        }
+        assertThat(
+            asInstanceOf(IndexNotFoundException.class, safeExecuteExpectFailure(enrichPolicyRunner)).getMessage(),
+            containsString("no such index [" + sourceIndex + "]")
+        );
     }
 
-    public void testRunnerNoSourceMapping() throws Exception {
+    public void testRunnerNoSourceMapping() {
         final String sourceIndex = "source-index";
         CreateIndexResponse createResponse = indicesAdmin().create(new CreateIndexRequest(sourceIndex)).actionGet();
         assertTrue(createResponse.isAcknowledged());
@@ -856,32 +795,21 @@ public void testRunnerNoSourceMapping() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            Exception thrown = exception.get();
-            assertThat(thrown, instanceOf(ElasticsearchException.class));
-            assertThat(
-                thrown.getMessage(),
-                containsString(
-                    "Enrich policy execution for ["
-                        + policyName
-                        + "] failed. No mapping available on source ["
-                        + sourceIndex
-                        + "] included in [["
-                        + sourceIndex
-                        + "]]"
-                )
-            );
-        } else {
-            fail("Expected exception but nothing was thrown");
-        }
+        assertThat(
+            asInstanceOf(ElasticsearchException.class, safeExecuteExpectFailure(enrichPolicyRunner)).getMessage(),
+            containsString(
+                "Enrich policy execution for ["
+                    + policyName
+                    + "] failed. No mapping available on source ["
+                    + sourceIndex
+                    + "] included in [["
+                    + sourceIndex
+                    + "]]"
+            )
+        );
     }
 
     public void testRunnerKeyNestedSourceMapping() throws Exception {
@@ -913,36 +841,22 @@ public void testRunnerKeyNestedSourceMapping() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            Exception thrown = exception.get();
-            assertThat(thrown, instanceOf(ElasticsearchException.class));
-            assertThat(
-                thrown.getMessage(),
-                containsString(
-                    "Enrich policy execution for ["
-                        + policyName
-                        + "] failed while validating field mappings for index ["
-                        + sourceIndex
-                        + "]"
-                )
-            );
-            assertThat(
-                thrown.getCause().getMessage(),
-                containsString(
-                    "Could not traverse mapping to field [nesting.key]. The [nesting" + "] field must be regular object but was [nested]."
-                )
-            );
-        } else {
-            fail("Expected exception but nothing was thrown");
-        }
+        final var thrown = asInstanceOf(ElasticsearchException.class, safeExecuteExpectFailure(enrichPolicyRunner));
+        assertThat(
+            thrown.getMessage(),
+            containsString(
+                "Enrich policy execution for [" + policyName + "] failed while validating field mappings for index [" + sourceIndex + "]"
+            )
+        );
+        assertThat(
+            thrown.getCause().getMessage(),
+            containsString(
+                "Could not traverse mapping to field [nesting.key]. The [nesting" + "] field must be regular object but was [nested]."
+            )
+        );
     }
 
     public void testRunnerValueNestedSourceMapping() throws Exception {
@@ -974,37 +888,22 @@ public void testRunnerValueNestedSourceMapping() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            Exception thrown = exception.get();
-            assertThat(thrown, instanceOf(ElasticsearchException.class));
-            assertThat(
-                thrown.getMessage(),
-                containsString(
-                    "Enrich policy execution for ["
-                        + policyName
-                        + "] failed while validating field mappings for index ["
-                        + sourceIndex
-                        + "]"
-                )
-            );
-            assertThat(
-                thrown.getCause().getMessage(),
-                containsString(
-                    "Could not traverse mapping to field [nesting.field2]. "
-                        + "The [nesting] field must be regular object but was [nested]."
-                )
-            );
-        } else {
-            fail("Expected exception but nothing was thrown");
-        }
+        final var thrown = asInstanceOf(ElasticsearchException.class, safeExecuteExpectFailure(enrichPolicyRunner));
+        assertThat(
+            thrown.getMessage(),
+            containsString(
+                "Enrich policy execution for [" + policyName + "] failed while validating field mappings for index [" + sourceIndex + "]"
+            )
+        );
+        assertThat(
+            thrown.getCause().getMessage(),
+            containsString(
+                "Could not traverse mapping to field [nesting.field2]. " + "The [nesting] field must be regular object but was [nested]."
+            )
+        );
     }
 
     public void testRunnerObjectSourceMapping() throws Exception {
@@ -1061,17 +960,10 @@ public void testRunnerObjectSourceMapping() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -1175,17 +1067,10 @@ public void testRunnerExplicitObjectSourceMapping() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -1289,17 +1174,10 @@ public void testRunnerExplicitObjectSourceMappingRangePolicy() throws Exception
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -1413,17 +1291,10 @@ public void testRunnerTwoObjectLevelsSourceMapping() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -1543,17 +1414,10 @@ public void testRunnerTwoObjectLevelsSourceMappingRangePolicy() throws Exception
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -1677,17 +1541,10 @@ public void testRunnerTwoObjectLevelsSourceMappingDateRangeWithFormat() throws E
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -1814,17 +1671,10 @@ public void testRunnerDottedKeyNameSourceMapping() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -1902,9 +1752,6 @@ public void testRunnerWithForceMergeRetry() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
         ClusterService clusterService = getInstanceFromNode(ClusterService.class);
         IndexNameExpressionResolver resolver = getInstanceFromNode(IndexNameExpressionResolver.class);
         Task asyncTask = testTaskManager.register("enrich", "policy_execution", new TaskAwareRequest() {
@@ -1930,22 +1777,6 @@ public String getDescription() {
             }
         });
         ExecuteEnrichPolicyTask task = ((ExecuteEnrichPolicyTask) asyncTask);
-        // The executor would wrap the listener in order to clean up the task in the
-        // task manager, but we're just testing the runner, so we make sure to clean
-        // up after ourselves.
-        ActionListener<ExecuteEnrichPolicyStatus> wrappedListener = new ActionListener<>() {
-            @Override
-            public void onResponse(ExecuteEnrichPolicyStatus policyExecutionResult) {
-                testTaskManager.unregister(task);
-                listener.onResponse(policyExecutionResult);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                testTaskManager.unregister(task);
-                listener.onFailure(e);
-            }
-        };
         AtomicInteger forceMergeAttempts = new AtomicInteger(0);
         final XContentBuilder unmergedDocument = SmileXContent.contentBuilder()
             .startObject()
@@ -1957,7 +1788,6 @@ public void onFailure(Exception e) {
             policyName,
             policy,
             task,
-            wrappedListener,
             clusterService,
             getInstanceFromNode(IndicesService.class),
             client(),
@@ -1967,28 +1797,36 @@ public void onFailure(Exception e) {
             randomIntBetween(3, 10)
         ) {
             @Override
-            protected void ensureSingleSegment(String destinationIndexName, int attempt) {
-                forceMergeAttempts.incrementAndGet();
+            public void run(ActionListener<ExecuteEnrichPolicyStatus> listener) {
+                // The executor would wrap the listener in order to clean up the task in the
+                // task manager, but we're just testing the runner, so we make sure to clean
+                // up after ourselves.
+                super.run(ActionListener.runBefore(listener, () -> testTaskManager.unregister(task)));
+            }
+
+            @Override
+            protected void afterRefreshEnrichIndex(ActionListener<Void> listener) {
+                final var attempt = forceMergeAttempts.incrementAndGet();
                 if (attempt == 1) {
                     // Put and flush a document to increase the number of segments, simulating not
                     // all segments were merged on the first try.
-                    DocWriteResponse indexRequest = client().index(
+                    client().index(
                         new IndexRequest().index(createdEnrichIndex)
                             .source(unmergedDocument)
-                            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
-                    ).actionGet();
-                    assertEquals(RestStatus.CREATED, indexRequest.status());
+                            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE),
+                        listener.delegateFailureAndWrap((l, response) -> {
+                            assertEquals(RestStatus.CREATED, response.status());
+                            super.afterRefreshEnrichIndex(l);
+                        })
+                    );
+                } else {
+                    super.afterRefreshEnrichIndex(listener);
                 }
-                super.ensureSingleSegment(destinationIndexName, attempt);
             }
         };
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        if (exception.get() != null) {
-            throw exception.get();
-        }
+        safeExecute(enrichPolicyRunner);
 
         // Validate number of force merges
         assertThat(forceMergeAttempts.get(), equalTo(2));
@@ -2048,7 +1886,233 @@ protected void ensureSingleSegment(String destinationIndexName, int attempt) {
         ensureEnrichIndexIsReadOnly(createdEnrichIndex);
     }
 
-    public void testRunnerCancel() throws Exception {
+    public void testRunnerWithEmptySegmentsResponse() throws Exception {
+        final String sourceIndex = "source-index";
+        DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source("""
+            {
+              "field1": "value1",
+              "field2": 2,
+              "field3": "ignored",
+              "field4": "ignored",
+              "field5": "value5"
+            }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet();
+        assertEquals(RestStatus.CREATED, indexRequest.status());
+
+        assertResponse(
+            client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
+            sourceSearchResponse -> {
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
+                assertNotNull(sourceDocMap);
+                assertThat(sourceDocMap.get("field1"), is(equalTo("value1")));
+                assertThat(sourceDocMap.get("field2"), is(equalTo(2)));
+                assertThat(sourceDocMap.get("field3"), is(equalTo("ignored")));
+                assertThat(sourceDocMap.get("field4"), is(equalTo("ignored")));
+                assertThat(sourceDocMap.get("field5"), is(equalTo("value5")));
+            }
+        );
+        List<String> enrichFields = List.of("field2", "field5");
+        EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of(sourceIndex), "field1", enrichFields);
+        String policyName = "test1";
+
+        final long createTime = randomNonNegativeLong();
+        String createdEnrichIndex = ".enrich-test1-" + createTime;
+        ClusterService clusterService = getInstanceFromNode(ClusterService.class);
+        IndexNameExpressionResolver resolver = getInstanceFromNode(IndexNameExpressionResolver.class);
+        Task asyncTask = testTaskManager.register("enrich", "policy_execution", new TaskAwareRequest() {
+            @Override
+            public void setParentTask(TaskId taskId) {}
+
+            @Override
+            public void setRequestId(long requestId) {}
+
+            @Override
+            public TaskId getParentTask() {
+                return TaskId.EMPTY_TASK_ID;
+            }
+
+            @Override
+            public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+                return new ExecuteEnrichPolicyTask(id, type, action, getDescription(), parentTaskId, headers);
+            }
+
+            @Override
+            public String getDescription() {
+                return policyName;
+            }
+        });
+        ExecuteEnrichPolicyTask task = ((ExecuteEnrichPolicyTask) asyncTask);
+
+        // Wrap the client so that when we receive the indices segments action, we intercept the request and complete it on another thread
+        // with an empty segments response.
+        Client client = new FilterClient(client()) {
+            @Override
+            protected <Request extends ActionRequest, Response extends ActionResponse> void doExecute(
+                ActionType<Response> action,
+                Request request,
+                ActionListener<Response> listener
+            ) {
+                if (action.equals(IndicesSegmentsAction.INSTANCE)) {
+                    testThreadPool.generic().execute(() -> {
+                        @SuppressWarnings("unchecked")
+                        ActionListener<IndicesSegmentResponse> castListener = ((ActionListener<IndicesSegmentResponse>) listener);
+                        castListener.onResponse(new IndicesSegmentResponse(new ShardSegments[0], 0, 0, 0, List.of()));
+                    });
+                } else {
+                    super.doExecute(action, request, listener);
+                }
+            }
+        };
+
+        EnrichPolicyRunner enrichPolicyRunner = new EnrichPolicyRunner(
+            policyName,
+            policy,
+            task,
+            clusterService,
+            getInstanceFromNode(IndicesService.class),
+            client,
+            resolver,
+            createdEnrichIndex,
+            randomIntBetween(1, 10000),
+            randomIntBetween(3, 10)
+        ) {
+            @Override
+            public void run(ActionListener<ExecuteEnrichPolicyStatus> listener) {
+                // The executor would wrap the listener in order to clean up the task in the
+                // task manager, but we're just testing the runner, so we make sure to clean
+                // up after ourselves.
+                super.run(ActionListener.runBefore(listener, () -> testTaskManager.unregister(task)));
+            }
+        };
+
+        logger.info("Starting policy run");
+        assertThat(
+            asInstanceOf(ElasticsearchException.class, safeExecuteExpectFailure(enrichPolicyRunner)).getMessage(),
+            containsString("Could not locate segment information for newly created index")
+        );
+    }
+
+    public void testRunnerWithShardFailuresInSegmentResponse() throws Exception {
+        final String sourceIndex = "source-index";
+        DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source("""
+            {
+              "field1": "value1",
+              "field2": 2,
+              "field3": "ignored",
+              "field4": "ignored",
+              "field5": "value5"
+            }""", XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)).actionGet();
+        assertEquals(RestStatus.CREATED, indexRequest.status());
+
+        assertResponse(
+            client().search(new SearchRequest(sourceIndex).source(SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()))),
+            sourceSearchResponse -> {
+                assertThat(sourceSearchResponse.getHits().getTotalHits().value, equalTo(1L));
+                Map<String, Object> sourceDocMap = sourceSearchResponse.getHits().getAt(0).getSourceAsMap();
+                assertNotNull(sourceDocMap);
+                assertThat(sourceDocMap.get("field1"), is(equalTo("value1")));
+                assertThat(sourceDocMap.get("field2"), is(equalTo(2)));
+                assertThat(sourceDocMap.get("field3"), is(equalTo("ignored")));
+                assertThat(sourceDocMap.get("field4"), is(equalTo("ignored")));
+                assertThat(sourceDocMap.get("field5"), is(equalTo("value5")));
+            }
+        );
+        List<String> enrichFields = List.of("field2", "field5");
+        EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of(sourceIndex), "field1", enrichFields);
+        String policyName = "test1";
+
+        final long createTime = randomNonNegativeLong();
+        String createdEnrichIndex = ".enrich-test1-" + createTime;
+        ClusterService clusterService = getInstanceFromNode(ClusterService.class);
+        IndexNameExpressionResolver resolver = getInstanceFromNode(IndexNameExpressionResolver.class);
+        Task asyncTask = testTaskManager.register("enrich", "policy_execution", new TaskAwareRequest() {
+            @Override
+            public void setParentTask(TaskId taskId) {}
+
+            @Override
+            public void setRequestId(long requestId) {}
+
+            @Override
+            public TaskId getParentTask() {
+                return TaskId.EMPTY_TASK_ID;
+            }
+
+            @Override
+            public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+                return new ExecuteEnrichPolicyTask(id, type, action, getDescription(), parentTaskId, headers);
+            }
+
+            @Override
+            public String getDescription() {
+                return policyName;
+            }
+        });
+        ExecuteEnrichPolicyTask task = ((ExecuteEnrichPolicyTask) asyncTask);
+        // The executor would wrap the listener in order to clean up the task in the
+        // task manager, but we're just testing the runner, so we make sure to clean
+        // up after ourselves.
+
+        // Wrap the client so that when we receive the indices segments action, we intercept the request and complete it on another thread
+        // with an failed segments response.
+        Client client = new FilterClient(client()) {
+            @Override
+            protected <Request extends ActionRequest, Response extends ActionResponse> void doExecute(
+                ActionType<Response> action,
+                Request request,
+                ActionListener<Response> listener
+            ) {
+                if (action.equals(IndicesSegmentsAction.INSTANCE)) {
+                    testThreadPool.generic().execute(() -> {
+                        @SuppressWarnings("unchecked")
+                        ActionListener<IndicesSegmentResponse> castListener = ((ActionListener<IndicesSegmentResponse>) listener);
+                        castListener.onResponse(
+                            new IndicesSegmentResponse(
+                                new ShardSegments[0],
+                                0,
+                                0,
+                                3,
+                                List.of(
+                                    new DefaultShardOperationFailedException(createdEnrichIndex, 1, new ElasticsearchException("failure1")),
+                                    new DefaultShardOperationFailedException(createdEnrichIndex, 2, new ElasticsearchException("failure2")),
+                                    new DefaultShardOperationFailedException(createdEnrichIndex, 3, new ElasticsearchException("failure3"))
+                                )
+                            )
+                        );
+                    });
+                } else {
+                    super.doExecute(action, request, listener);
+                }
+            }
+        };
+
+        EnrichPolicyRunner enrichPolicyRunner = new EnrichPolicyRunner(
+            policyName,
+            policy,
+            task,
+            clusterService,
+            getInstanceFromNode(IndicesService.class),
+            client,
+            resolver,
+            createdEnrichIndex,
+            randomIntBetween(1, 10000),
+            randomIntBetween(3, 10)
+        ) {
+            @Override
+            public void run(ActionListener<ExecuteEnrichPolicyStatus> listener) {
+                // The executor would wrap the listener in order to clean up the task in the
+                // task manager, but we're just testing the runner, so we make sure to clean
+                // up after ourselves.
+                super.run(ActionListener.runBefore(listener, () -> testTaskManager.unregister(task)));
+            }
+        };
+
+        logger.info("Starting policy run");
+        final var exceptionThrown = asInstanceOf(ElasticsearchException.class, safeExecuteExpectFailure(enrichPolicyRunner));
+        assertThat(exceptionThrown.getMessage(), containsString("Could not obtain segment information for newly created index"));
+        assertThat(asInstanceOf(ElasticsearchException.class, exceptionThrown.getCause()).getMessage(), containsString("failure1"));
+    }
+
+    public void testRunnerCancel() {
         final String sourceIndex = "source-index";
         DocWriteResponse indexRequest = client().index(new IndexRequest().index(sourceIndex).id("id").source("""
             {
@@ -2066,9 +2130,6 @@ public void testRunnerCancel() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
 
         ActionType<?> randomActionType = randomFrom(
             EnrichReindexAction.INSTANCE,
@@ -2100,12 +2161,12 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
             }
         };
 
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(client, policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(client, policyName, policy, createdEnrichIndex);
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        assertThat(exception.get(), notNullValue());
-        assertThat(exception.get().getMessage(), containsString("cancelled policy execution [test1], status ["));
+        assertThat(
+            safeExecuteExpectFailure(enrichPolicyRunner).getMessage(),
+            containsString("cancelled policy execution [test1], status [")
+        );
     }
 
     public void testRunRangePolicyWithObjectFieldAsMatchField() throws Exception {
@@ -2137,17 +2198,13 @@ public void testRunRangePolicyWithObjectFieldAsMatchField() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        Exception e = exception.get();
-        assertThat(e, notNullValue());
-        assertThat(e.getMessage(), equalTo("Field 'field1' has type [object] which doesn't appear to be a range type"));
+        assertThat(
+            safeExecuteExpectFailure(enrichPolicyRunner).getMessage(),
+            equalTo("Field 'field1' has type [object] which doesn't appear to be a range type")
+        );
     }
 
     public void testEnrichFieldsConflictMappingTypes() throws Exception {
@@ -2178,10 +2235,7 @@ public void testEnrichFieldsConflictMappingTypes() throws Exception {
         String policyName = "test1";
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        PlainActionFuture<ExecuteEnrichPolicyStatus> future = new PlainActionFuture<>();
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, future, createdEnrichIndex);
-        enrichPolicyRunner.run();
-        future.actionGet();
+        safeExecute(createPolicyRunner(policyName, policy, createdEnrichIndex));
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -2224,10 +2278,7 @@ public void testEnrichMappingConflictFormats() throws ExecutionException, Interr
         String policyName = "test1";
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        PlainActionFuture<ExecuteEnrichPolicyStatus> future = new PlainActionFuture<>();
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, future, createdEnrichIndex);
-        enrichPolicyRunner.run();
-        future.actionGet();
+        safeExecute(createPolicyRunner(policyName, policy, createdEnrichIndex));
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -2259,10 +2310,7 @@ public void testEnrichObjectField() throws ExecutionException, InterruptedExcept
         String policyName = "test1";
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        PlainActionFuture<ExecuteEnrichPolicyStatus> future = new PlainActionFuture<>();
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, future, createdEnrichIndex);
-        enrichPolicyRunner.run();
-        future.actionGet();
+        safeExecute(createPolicyRunner(policyName, policy, createdEnrichIndex));
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -2317,12 +2365,10 @@ public void testEnrichNestedField() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        PlainActionFuture<ExecuteEnrichPolicyStatus> future = new PlainActionFuture<>();
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, future, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        future.actionGet();
+        safeExecute(enrichPolicyRunner);
 
         // Validate Index definition
         GetIndexResponse enrichIndex = getGetIndexResponseAndCheck(createdEnrichIndex);
@@ -2376,9 +2422,6 @@ public void testRunnerValidatesIndexIntegrity() throws Exception {
 
         final long createTime = randomNonNegativeLong();
         String createdEnrichIndex = ".enrich-test1-" + createTime;
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ActionListener<ExecuteEnrichPolicyStatus> listener = createTestListener(latch, exception::set);
 
         // Wrap the client so that when we receive the reindex action, we delete the index then resume operation. This mimics an invalid
         // state for the resulting index.
@@ -2405,36 +2448,20 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
                 }
             }
         };
-        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(client, policyName, policy, listener, createdEnrichIndex);
+        EnrichPolicyRunner enrichPolicyRunner = createPolicyRunner(client, policyName, policy, createdEnrichIndex);
 
         logger.info("Starting policy run");
-        enrichPolicyRunner.run();
-        latch.await();
-        Exception runnerException = exception.get();
-        if (runnerException == null) {
-            fail("Expected the runner to fail when the underlying index was deleted during policy execution!");
-        }
-        assertThat(runnerException, is(instanceOf(ElasticsearchException.class)));
-        assertThat(runnerException.getMessage(), containsString("Could not verify enrich index"));
-        assertThat(runnerException.getMessage(), containsString("mapping meta field missing"));
+        assertThat(
+            asInstanceOf(ElasticsearchException.class, safeExecuteExpectFailure(enrichPolicyRunner)).getMessage(),
+            allOf(containsString("Could not verify enrich index"), containsString("mapping meta field missing"))
+        );
     }
 
-    private EnrichPolicyRunner createPolicyRunner(
-        String policyName,
-        EnrichPolicy policy,
-        ActionListener<ExecuteEnrichPolicyStatus> listener,
-        String targetIndex
-    ) {
-        return createPolicyRunner(client(), policyName, policy, listener, targetIndex);
+    private EnrichPolicyRunner createPolicyRunner(String policyName, EnrichPolicy policy, String targetIndex) {
+        return createPolicyRunner(client(), policyName, policy, targetIndex);
     }
 
-    private EnrichPolicyRunner createPolicyRunner(
-        Client client,
-        String policyName,
-        EnrichPolicy policy,
-        ActionListener<ExecuteEnrichPolicyStatus> listener,
-        String targetIndex
-    ) {
+    private EnrichPolicyRunner createPolicyRunner(Client client, String policyName, EnrichPolicy policy, String targetIndex) {
         ClusterService clusterService = getInstanceFromNode(ClusterService.class);
         IndexNameExpressionResolver resolver = getInstanceFromNode(IndexNameExpressionResolver.class);
         Task asyncTask = testTaskManager.register("enrich", "policy_execution", new TaskAwareRequest() {
@@ -2460,27 +2487,10 @@ public String getDescription() {
             }
         });
         ExecuteEnrichPolicyTask task = ((ExecuteEnrichPolicyTask) asyncTask);
-        // The executor would wrap the listener in order to clean up the task in the
-        // task manager, but we're just testing the runner, so we make sure to clean
-        // up after ourselves.
-        ActionListener<ExecuteEnrichPolicyStatus> wrappedListener = new ActionListener<>() {
-            @Override
-            public void onResponse(ExecuteEnrichPolicyStatus policyExecutionResult) {
-                testTaskManager.unregister(task);
-                listener.onResponse(policyExecutionResult);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                testTaskManager.unregister(task);
-                listener.onFailure(e);
-            }
-        };
         return new EnrichPolicyRunner(
             policyName,
             policy,
             task,
-            wrappedListener,
             clusterService,
             getInstanceFromNode(IndicesService.class),
             client,
@@ -2488,14 +2498,24 @@ public void onFailure(Exception e) {
             targetIndex,
             randomIntBetween(1, 10000),
             randomIntBetween(1, 10)
-        );
+        ) {
+            @Override
+            public void run(ActionListener<ExecuteEnrichPolicyStatus> listener) {
+                // The executor would wrap the listener in order to clean up the task in the
+                // task manager, but we're just testing the runner, so we make sure to clean
+                // up after ourselves.
+                super.run(ActionListener.runBefore(listener, () -> testTaskManager.unregister(task)));
+            }
+        };
+    }
+
+    private void safeExecute(EnrichPolicyRunner enrichPolicyRunner) {
+        safeAwait(enrichPolicyRunner::run);
+        logger.debug("Run complete");
     }
 
-    private ActionListener<ExecuteEnrichPolicyStatus> createTestListener(
-        final CountDownLatch latch,
-        final Consumer<Exception> exceptionConsumer
-    ) {
-        return new LatchedActionListener<>(ActionListener.wrap((r) -> logger.info("Run complete"), exceptionConsumer), latch);
+    private Exception safeExecuteExpectFailure(EnrichPolicyRunner enrichPolicyRunner) {
+        return safeAwaitFailure(enrichPolicyRunner::run);
     }
 
     private void validateMappingMetadata(Map<?, ?> mapping, String policyName, EnrichPolicy policy) {
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/FlatNumberOrByteSizeValueTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/FlatNumberOrByteSizeValueTests.java
new file mode 100644
index 0000000000000..809b78c50b35a
--- /dev/null
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/FlatNumberOrByteSizeValueTests.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.enrich;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.xpack.enrich.EnrichPlugin.FlatNumberOrByteSizeValue;
+
+public class FlatNumberOrByteSizeValueTests extends ESTestCase {
+
+    private static final String SETTING_NAME = "test.setting";
+
+    public void testParse() {
+        int number = randomIntBetween(1, Integer.MAX_VALUE);
+        assertEquals(
+            new FlatNumberOrByteSizeValue((long) number),
+            FlatNumberOrByteSizeValue.parse(Integer.toString(number), SETTING_NAME, null)
+        );
+        assertEquals(
+            new FlatNumberOrByteSizeValue(ByteSizeValue.ofGb(number)),
+            FlatNumberOrByteSizeValue.parse(number + "GB", SETTING_NAME, null)
+        );
+        assertEquals(
+            new FlatNumberOrByteSizeValue(ByteSizeValue.ofGb(number)),
+            FlatNumberOrByteSizeValue.parse(number + "g", SETTING_NAME, null)
+        );
+        int percentage = randomIntBetween(0, 100);
+        assertEquals(
+            new FlatNumberOrByteSizeValue(
+                ByteSizeValue.ofBytes((long) ((double) percentage / 100 * JvmInfo.jvmInfo().getConfiguredMaxHeapSize()))
+            ),
+            FlatNumberOrByteSizeValue.parse(percentage + "%", SETTING_NAME, null)
+        );
+        assertEquals(new FlatNumberOrByteSizeValue(0L), FlatNumberOrByteSizeValue.parse("0", SETTING_NAME, null));
+        assertEquals(new FlatNumberOrByteSizeValue(ByteSizeValue.ZERO), FlatNumberOrByteSizeValue.parse("0GB", SETTING_NAME, null));
+        assertEquals(new FlatNumberOrByteSizeValue(ByteSizeValue.ZERO), FlatNumberOrByteSizeValue.parse("0%", SETTING_NAME, null));
+        // Assert default value.
+        assertEquals(
+            new FlatNumberOrByteSizeValue((long) number),
+            FlatNumberOrByteSizeValue.parse(null, SETTING_NAME, new FlatNumberOrByteSizeValue((long) number))
+        );
+        assertThrows(ElasticsearchParseException.class, () -> FlatNumberOrByteSizeValue.parse("5GB%", SETTING_NAME, null));
+        assertThrows(ElasticsearchParseException.class, () -> FlatNumberOrByteSizeValue.parse("5%GB", SETTING_NAME, null));
+        assertThrows(ElasticsearchParseException.class, () -> FlatNumberOrByteSizeValue.parse("5GBX", SETTING_NAME, null));
+    }
+
+    private void assertEquals(FlatNumberOrByteSizeValue expected, FlatNumberOrByteSizeValue actual) {
+        assertEquals(expected.byteSizeValue(), actual.byteSizeValue());
+        assertEquals(expected.flatNumber(), actual.flatNumber());
+    }
+}
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/LocalStateEnrich.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/LocalStateEnrich.java
index 8eb30f2688143..cb5bd6f5eb60c 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/LocalStateEnrich.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/LocalStateEnrich.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/10_query_ruleset_put.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/10_query_ruleset_put.yml
index a1f9eeccf2002..ee94cff46d1fb 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/10_query_ruleset_put.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/10_query_ruleset_put.yml
@@ -44,6 +44,18 @@ teardown:
                     '_id': 'id3'
                   - '_index': 'test-index2'
                     '_id': 'id4'
+            - rule_id: query-rule-id3
+              type: exclude
+              criteria:
+                - type: exact
+                  metadata: query_string
+                  values: [ logstash ]
+              actions:
+                docs:
+                  - '_index': 'test-index1'
+                    '_id': 'id4'
+                  - '_index': 'test-index2'
+                    '_id': 'id5'
 
   - match: { result: 'created' }
 
@@ -75,6 +87,18 @@ teardown:
                 '_id': 'id3'
               - '_index': 'test-index2'
                 '_id': 'id4'
+        - rule_id: query-rule-id3
+          type: exclude
+          criteria:
+            - type: exact
+              metadata: query_string
+              values: [ logstash ]
+          actions:
+            docs:
+              - '_index': 'test-index1'
+                '_id': 'id4'
+              - '_index': 'test-index2'
+                '_id': 'id5'
 
 ---
 'Create Query Ruleset - Resource already exists':
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml
index f2ced956b5369..172d38cce5384 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/20_query_ruleset_list.yml
@@ -109,6 +109,16 @@ setup:
                 ids:
                   - 'id7'
                   - 'id8'
+            - rule_id: query-rule-id5
+              type: exclude
+              criteria:
+                - type: fuzzy
+                  metadata: query_string
+                  values: [ inference ]
+              actions:
+                ids:
+                  - 'id9'
+                  - 'id10'
 ---
 teardown:
   - do:
@@ -144,8 +154,8 @@ teardown:
   - match: { results.0.rule_criteria_types_counts: { exact: 3 } }
 
   - match: { results.1.ruleset_id: "test-query-ruleset-2" }
-  - match: { results.1.rule_total_count: 4 }
-  - match: { results.1.rule_criteria_types_counts: { exact: 4 } }
+  - match: { results.1.rule_total_count: 5 }
+  - match: { results.1.rule_criteria_types_counts: { exact: 4, fuzzy: 1 } }
 
   - match: { results.2.ruleset_id: "test-query-ruleset-3" }
   - match: { results.2.rule_total_count: 2 }
@@ -161,8 +171,8 @@ teardown:
 
   # Alphabetical order by ruleset_id for results
   - match: { results.0.ruleset_id: "test-query-ruleset-2" }
-  - match: { results.0.rule_total_count: 4 }
-  - match: { results.0.rule_criteria_types_counts: { exact: 4 } }
+  - match: { results.0.rule_total_count: 5 }
+  - match: { results.0.rule_criteria_types_counts: { exact: 4, fuzzy: 1 } }
 
   - match: { results.1.ruleset_id: "test-query-ruleset-3" }
   - match: { results.1.rule_total_count: 2 }
@@ -182,8 +192,8 @@ teardown:
   - match: { results.0.rule_criteria_types_counts: { exact: 3 } }
 
   - match: { results.1.ruleset_id: "test-query-ruleset-2" }
-  - match: { results.1.rule_total_count: 4 }
-  - match: { results.1.rule_criteria_types_counts: { exact: 4 } }
+  - match: { results.1.rule_total_count: 5 }
+  - match: { results.1.rule_criteria_types_counts: { exact: 4, fuzzy: 1 } }
 
 ---
 "List Query Rulesets - empty":
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/40_rule_query_search.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/40_rule_query_search.yml
index 078e24d86f1c8..845cdb7f9ac19 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/40_rule_query_search.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/40_rule_query_search.yml
@@ -11,6 +11,19 @@ setup:
             index:
               number_of_shards: 1
               number_of_replicas: 0
+          aliases:
+            test-alias1: { }
+
+  - do:
+      indices.create:
+        index: test-index2
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+          aliases:
+            test-alias1: { }
 
   - do:
       bulk:
@@ -38,6 +51,22 @@ setup:
           - index:
               _id: doc7
           - { "text": "observability" }
+          - index:
+              _id: doc8
+          - { "text": "elasticsearch" }
+
+  - do:
+      bulk:
+        refresh: true
+        index: test-index2
+        body:
+          - index:
+              _id: another-doc
+          - { "text": "you know, for search" }
+          - index:
+              _id: doc8
+          - { "text": "elasticsearch" }
+
 
   - do:
       query_rules.put_ruleset:
@@ -82,6 +111,15 @@ setup:
               actions:
                 ids:
                   - 'doc7'
+            - rule_id: rule5
+              type: exclude
+              criteria:
+                - type: exact
+                  metadata: query_string
+                  values: [ search ]
+              actions:
+                ids:
+                  - 'doc8'
 
   - do:
       query_rules.put_ruleset:
@@ -115,6 +153,21 @@ teardown:
         ruleset_id: combined-ruleset
         ignore: 404
 
+  - do:
+      query_rules.delete_ruleset:
+        ruleset_id: alias-ruleset
+        ignore: 404
+
+  - do:
+      query_rules.delete_ruleset:
+        ruleset_id: double-jeopardy-ruleset
+        ignore: 404
+
+  - do:
+      query_rules.delete_ruleset:
+        ruleset_id: multiple-exclude-ruleset
+        ignore: 404
+
 ---
 "Perform a rule query specifying a ruleset that does not exist":
   - do:
@@ -165,7 +218,7 @@ teardown:
                 foo: bar
 
 ---
-"Perform a rule query with malformed rule":
+"Perform a search with malformed rule query":
   - do:
       catch: bad_request
       search:
@@ -184,6 +237,7 @@ teardown:
 
   - do:
       search:
+        index: test-index1
         body:
           query:
             rule:
@@ -208,6 +262,7 @@ teardown:
   - do:
       headers: { Authorization: "Basic ZW50c2VhcmNoLXVzZXI6ZW50c2VhcmNoLXVzZXItcGFzc3dvcmQ=" }  # user
       search:
+        index: test-index1
         body:
           query:
             rule:
@@ -294,6 +349,7 @@ teardown:
 
   - do:
       search:
+        index: test-index1
         body:
           query:
             rule:
@@ -310,6 +366,190 @@ teardown:
   - match: { hits.hits.0._id: 'doc2' }
   - match: { hits.hits.1._id: 'doc3' }
 
+---
+"Perform a query over an alias, where one document is pinned specifying the index":
+  - do:
+      query_rules.put_ruleset:
+        ruleset_id: alias-ruleset
+        body:
+          rules:
+            - rule_id: rule1
+              type: pinned
+              criteria:
+                - type: exact
+                  metadata: foo
+                  values: [ bar ]
+              actions:
+                docs:
+                  - '_index': 'test-index1'
+                    '_id': 'doc8'
+            - rule_id: rule2
+              type: exclude
+              criteria:
+                - type: exact
+                  metadata: foo
+                  values: [ baz ]
+              actions:
+                docs:
+                  - '_index': 'test-index1'
+                    '_id': 'doc8'
+
+  - do:
+      search:
+        body:
+          query:
+            rule:
+              organic:
+                match_none: { }
+              match_criteria:
+                foo: bar
+              ruleset_ids:
+                - alias-ruleset
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: 'doc8' }
+  - match: { hits.hits.0._index: 'test-index1' }
+
+  - do:
+      search:
+        body:
+          query:
+            rule:
+              organic:
+                query_string:
+                  query: elasticsearch
+              match_criteria:
+                foo: baz
+              ruleset_ids:
+                - alias-ruleset
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: 'doc8' }
+  - match: { hits.hits.0._index: 'test-index2' }
+
+  - do:
+      search:
+        body:
+          query:
+            rule:
+              organic:
+                query_string:
+                  query: elasticsearch
+              match_criteria:
+                foo: not-a-match
+              ruleset_ids:
+                - alias-ruleset
+
+  - match: { hits.total.value: 2 }
+
+---
+"Perform a query where the same ID is both pinned and excluded, leading it to be excluded":
+  - do:
+      query_rules.put_ruleset:
+        ruleset_id: double-jeopardy-ruleset
+        body:
+          rules:
+            - rule_id: rule1
+              type: pinned
+              criteria:
+                - type: exact
+                  metadata: foo
+                  values: [ bar ]
+              actions:
+                ids:
+                  - 'doc8'
+            - rule_id: rule2
+              type: exclude
+              criteria:
+                - type: exact
+                  metadata: foo
+                  values: [ bar ]
+              actions:
+                ids:
+                  - 'doc8'
+
+  - do:
+      search:
+        index: test-index2
+        body:
+          query:
+            query_string:
+              query: elasticsearch
+
+  - match: { hits.total.value: 1 }
+
+  - do:
+      search:
+        index: test-index2
+        body:
+          query:
+            rule:
+              organic:
+                query_string:
+                  query: elasticsearch
+              match_criteria:
+                foo: bar
+              ruleset_ids:
+                - double-jeopardy-ruleset
+
+  - match: { hits.total.value: 0 }
+
+---
+"Perform a query that matches multiple exclude rules":
+  - do:
+      query_rules.put_ruleset:
+        ruleset_id: multiple-exclude-ruleset
+        body:
+          rules:
+            - rule_id: rule1
+              type: exclude
+              criteria:
+                - type: exact
+                  metadata: foo
+                  values: [ bar ]
+              actions:
+                ids:
+                  - 'doc1'
+            - rule_id: rule2
+              type: exclude
+              criteria:
+                - type: exact
+                  metadata: foo
+                  values: [ bar ]
+              actions:
+                ids:
+                  - 'doc8'
+
+  - do:
+      search:
+        index: test-index1
+        body:
+          query:
+            query_string:
+              query: elasticsearch is elastic search
+
+  - match: { hits.total.value: 3 }
+  - match: { hits.hits.0._id: 'doc1' }
+  - match: { hits.hits.1._id: 'doc8' }
+  - match: { hits.hits.2._id: 'doc4' }
+
+  - do:
+      search:
+        index: test-index1
+        body:
+          query:
+            rule:
+              organic:
+                query_string:
+                  query: elasticsearch is elastic search
+              match_criteria:
+                foo: bar
+              ruleset_ids:
+                - multiple-exclude-ruleset
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: 'doc4' }
+
 ---
 "Perform a rule query over a ruleset with combined numeric and text rule matching":
 
@@ -509,6 +749,7 @@ teardown:
 
   - do:
       search:
+        index: test-index1
         body:
           query:
             rule:
@@ -634,6 +875,7 @@ teardown:
 
   - do:
       search:
+        index: test-index1
         body:
           query:
             rule_query:
@@ -651,4 +893,3 @@ teardown:
   - match: { hits.total.value: 2 }
   - match: { hits.hits.0._id: 'doc1' }
   - match: { hits.hits.1._id: 'doc4' }
-
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/50_query_rule_put.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/50_query_rule_put.yml
index fb3d7be9d2367..285ae77e1f47f 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/50_query_rule_put.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/rules/50_query_rule_put.yml
@@ -101,7 +101,7 @@ teardown:
                   - 'id1'
                   - 'id2'
             - rule_id: query-rule-id2
-              type: pinned
+              type: exclude
               criteria:
                 - type: exact
                   metadata: query_string
@@ -122,7 +122,7 @@ teardown:
   - match:
       rules:
         - rule_id: query-rule-id2
-          type: pinned
+          type: exclude
           criteria:
             - type: exact
               metadata: query_string
@@ -169,7 +169,7 @@ teardown:
   - match:
       rules:
         - rule_id: query-rule-id2
-          type: pinned
+          type: exclude
           criteria:
             - type: exact
               metadata: query_string
@@ -225,7 +225,7 @@ teardown:
   - match:
       rules:
         - rule_id: query-rule-id2
-          type: pinned
+          type: exclude
           criteria:
             - type: exact
               metadata: query_string
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/52_search_application_render_query.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/52_search_application_render_query.yml
index 7b4d0f16551d7..7a83902b5107d 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/52_search_application_render_query.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/search/52_search_application_render_query.yml
@@ -220,6 +220,36 @@ teardown:
     }
   }
 
+---
+"Render query for search application with floats":
+
+  - do:
+      search_application.render_query:
+        name: test-search-application-with-list
+        body:
+          params:
+            query_string: value3
+            text_fields:
+              - name: field1
+                boost: 1.2
+              - name: field2
+                boost: 2.3
+              - name: field3
+                boost: 3
+
+  - match: {
+    query: {
+      multi_match: {
+        query: "value3",
+        fields: [
+          "field1^1.2",
+          "field2^2.3",
+          "field3^3.0"
+        ]
+      }
+    }
+  }
+
 ---
 "Render query for search application - not found":
 
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchInfoTransportAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchInfoTransportAction.java
index 4523a04c9a8c1..e60df4ed4a0a3 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchInfoTransportAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchInfoTransportAction.java
@@ -8,8 +8,8 @@
 package org.elasticsearch.xpack.application;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.utils.LicenseUtils;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java
index 4a6a2a3590b3d..c079892ccb2b6 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java
@@ -19,8 +19,8 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java
index 72ffc1b978304..f68feebfb5034 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionResolver.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.injection.guice.Inject;
 
 import java.util.ArrayList;
 import java.util.Arrays;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionService.java
index e1afb60bcaec1..fc43783cfd8ad 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionService.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsCollectionService.java
@@ -18,7 +18,8 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.xpack.application.analytics.action.DeleteAnalyticsCollectionAction;
@@ -81,7 +82,11 @@ public void putAnalyticsCollection(
         assert (state.nodes().isLocalNodeElectedMaster());
 
         AnalyticsCollection collection = new AnalyticsCollection(request.getName());
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(collection.getEventDataStream());
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TimeValue.THIRTY_SECONDS /* TODO should we wait longer? */,
+            TimeValue.THIRTY_SECONDS /* TODO should we wait longer? */,
+            collection.getEventDataStream()
+        );
 
         ActionListener<AcknowledgedResponse> createDataStreamListener = ActionListener.wrap(
             r -> listener.onResponse(new PutAnalyticsCollectionAction.Response(r.isAcknowledged(), request.getName())),
@@ -124,7 +129,10 @@ public void deleteAnalyticsCollection(
         assert (state.nodes().isLocalNodeElectedMaster());
 
         AnalyticsCollection collection = new AnalyticsCollection(request.getCollectionName());
-        DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(collection.getEventDataStream());
+        DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(
+            TimeValue.THIRTY_SECONDS /* TODO should we wait longer? */,
+            collection.getEventDataStream()
+        );
         ActionListener<AcknowledgedResponse> deleteDataStreamListener = ActionListener.wrap(listener::onResponse, (Exception e) -> {
             if (e instanceof ResourceNotFoundException) {
                 listener.onFailure(new ResourceNotFoundException("analytics collection [{}] does not exists", request.getCollectionName()));
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsEventIngestService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsEventIngestService.java
index 0f46fbcca45d4..2166646fc4bfd 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsEventIngestService.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsEventIngestService.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.xpack.application.analytics.action.PostAnalyticsEventAction;
 import org.elasticsearch.xpack.application.analytics.ingest.AnalyticsEventEmitter;
 
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java
index 03bf1c2d9adfa..5158e0a2ed415 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportDeleteAnalyticsCollectionAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java
index 41d30017a185b..2d821b7684457 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportGetAnalyticsCollectionAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPostAnalyticsEventAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPostAnalyticsEventAction.java
index 1e57ab91c116a..bf81ba75400b1 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPostAnalyticsEventAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPostAnalyticsEventAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.analytics.AnalyticsEventIngestService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java
index 2f10532b504d4..847032a023fc8 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/TransportPutAnalyticsCollectionAction.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java
index aa22dfbe2e3cb..a49e29654b0d6 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventEmitter.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.rest.RestStatus;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventIngestConfig.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventIngestConfig.java
index 7823b6b9d6ce6..ac78a981c2d0c 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventIngestConfig.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/AnalyticsEventIngestConfig.java
@@ -7,12 +7,12 @@
 
 package org.elasticsearch.xpack.application.analytics.ingest;
 
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 
 /**
  *  Bulk processor configuration.
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/BulkProcessorFactory.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/BulkProcessorFactory.java
index d00fa191b49d0..343d050ffbfde 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/BulkProcessorFactory.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/ingest/BulkProcessorFactory.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportDeleteConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportDeleteConnectorAction.java
index 9c71adbda78f0..e534d969fdaaa 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportDeleteConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportDeleteConnectorAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java
index 633063c37f61d..e0beeb9b19515 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java
index 7c099345fa8a9..8ff180aa6189b 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportPostConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportPostConnectorAction.java
index bf0f9ebd02309..4762f125fe9df 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportPostConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportPostConnectorAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportPutConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportPutConnectorAction.java
index 825b53a163993..b409670e44164 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportPutConnectorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportPutConnectorAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorActiveFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorActiveFilteringAction.java
index e0e4d50598a28..f9c1e429e390d 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorActiveFilteringAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorActiveFilteringAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorApiKeyIdAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorApiKeyIdAction.java
index 62d22784e2896..1ba4ce037fb09 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorApiKeyIdAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorApiKeyIdAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java
index 3f85ff534eb35..782dba32395e7 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorConfigurationAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java
index d71344e74aee9..ba4e38cc590f4 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorErrorAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java
index c86ddf902519f..aabfd15f604a8 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java
index 6704493f6b689..c16bf63b493aa 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorFiltering;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringValidationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringValidationAction.java
index 591f28577480f..3fda3774c8dc6 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringValidationAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFilteringValidationAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorIndexNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorIndexNameAction.java
index 63f76d2f17860..01e785f88d775 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorIndexNameAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorIndexNameAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java
index d23cbea3128c6..6a0cc3e3baf35 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSeenAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java
index 90b5448d484ec..309c9bf5296df 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorLastSyncStatsAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java
index a21919d782665..3f8085cbfae61 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNameAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNativeAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNativeAction.java
index 17bfca44e8e29..5f4a60abd4890 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNativeAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorNativeAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java
index 179507f346aaf..6ff6f0cfe4bc0 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorPipelineAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java
index 5172c3b43311b..4d533f7f70c67 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorSchedulingAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorServiceTypeAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorServiceTypeAction.java
index 291f1e35da9c1..d6aafb74121f0 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorServiceTypeAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorServiceTypeAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorStatusAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorStatusAction.java
index 49359318697e0..012e7ffc83ca5 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorStatusAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorStatusAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportDeleteConnectorSecretAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportDeleteConnectorSecretAction.java
index 7c87598440cfd..8eaa88f195c8e 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportDeleteConnectorSecretAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportDeleteConnectorSecretAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportGetConnectorSecretAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportGetConnectorSecretAction.java
index aaa03fa13298f..ad29185567e74 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportGetConnectorSecretAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportGetConnectorSecretAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportPostConnectorSecretAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportPostConnectorSecretAction.java
index 7cc3195ccbbf2..c84c9ec4cfb17 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportPostConnectorSecretAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportPostConnectorSecretAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportPutConnectorSecretAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportPutConnectorSecretAction.java
index bd20d93db1dc3..b1492ee58cd54 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportPutConnectorSecretAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/secrets/action/TransportPutConnectorSecretAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java
index 179689a148934..6374401ded162 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCancelConnectorSyncJobAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java
index d72d1cd0a04bb..fc0da5b1c206a 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportCheckInConnectorSyncJobAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportClaimConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportClaimConnectorSyncJobAction.java
index 8b43e153a06c9..4c7cb0e9ffec9 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportClaimConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportClaimConnectorSyncJobAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportDeleteConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportDeleteConnectorSyncJobAction.java
index 60a91d53aea5a..3a4cb61105ca4 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportDeleteConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportDeleteConnectorSyncJobAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportGetConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportGetConnectorSyncJobAction.java
index 993606050e5e6..ad45cccef0046 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportGetConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportGetConnectorSyncJobAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportListConnectorSyncJobsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportListConnectorSyncJobsAction.java
index ab8ebf2b87caf..822efa287dbc9 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportListConnectorSyncJobsAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportListConnectorSyncJobsAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportPostConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportPostConnectorSyncJobAction.java
index 79dfb28439292..85538f8681aaf 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportPostConnectorSyncJobAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportPostConnectorSyncJobAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java
index d67c006b61bb2..f1608f4245385 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobErrorAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java
index 4893c22f39a99..118e2aacebaef 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportUpdateConnectorSyncJobIngestionStatsAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/AppliedQueryRules.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/AppliedQueryRules.java
index 96ad782a06f17..885b84b200e04 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/AppliedQueryRules.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/AppliedQueryRules.java
@@ -7,31 +7,31 @@
 
 package org.elasticsearch.xpack.application.rules;
 
+import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument;
+
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder.Item;
-
 public class AppliedQueryRules {
 
-    private final List<String> pinnedIds;
-    private final List<Item> pinnedDocs;
+    private final List<SpecifiedDocument> pinnedDocs;
+    private final List<SpecifiedDocument> excludedDocs;
 
     public AppliedQueryRules() {
         this(new ArrayList<>(0), new ArrayList<>(0));
     }
 
-    public AppliedQueryRules(List<String> pinnedIds, List<Item> pinnedDocs) {
-        this.pinnedIds = pinnedIds;
+    public AppliedQueryRules(List<SpecifiedDocument> pinnedDocs, List<SpecifiedDocument> excludedDocs) {
         this.pinnedDocs = pinnedDocs;
+        this.excludedDocs = excludedDocs;
     }
 
-    public List<String> pinnedIds() {
-        return pinnedIds;
+    public List<SpecifiedDocument> pinnedDocs() {
+        return pinnedDocs;
     }
 
-    public List<Item> pinnedDocs() {
-        return pinnedDocs;
+    public List<SpecifiedDocument> excludedDocs() {
+        return excludedDocs;
     }
 
 }
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java
index 33fa74e5178cf..0ecb35531ac09 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRule.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
-import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder;
+import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -31,14 +31,11 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xpack.application.rules.QueryRuleCriteriaType.ALWAYS;
-import static org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder.DOCS_FIELD;
-import static org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder.IDS_FIELD;
-import static org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder.Item.INDEX_FIELD;
-import static org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder.MAX_NUM_PINNED_HITS;
 
 /**
  * A query rule consists of:
@@ -51,6 +48,11 @@
  */
 public class QueryRule implements Writeable, ToXContentObject {
 
+    public static final int MAX_NUM_DOCS_IN_RULE = 100;
+    public static final ParseField IDS_FIELD = new ParseField("ids");
+    public static final ParseField DOCS_FIELD = new ParseField("docs");
+    public static final ParseField INDEX_FIELD = new ParseField("_index");
+
     private final String id;
     private final QueryRuleType type;
     private final List<QueryRuleCriteria> criteria;
@@ -61,6 +63,7 @@ public class QueryRule implements Writeable, ToXContentObject {
     public static final int MAX_PRIORITY = 1000000;
 
     public enum QueryRuleType {
+        EXCLUDE,
         PINNED;
 
         public static QueryRuleType queryRuleType(String type) {
@@ -137,32 +140,31 @@ public QueryRule(StreamInput in) throws IOException {
     }
 
     private void validate() {
-        if (type == QueryRuleType.PINNED) {
-            boolean ruleContainsPinnedIds = actions.containsKey(IDS_FIELD.getPreferredName());
-            boolean ruleContainsPinnedDocs = actions.containsKey(DOCS_FIELD.getPreferredName());
-            if (ruleContainsPinnedIds ^ ruleContainsPinnedDocs) {
-                validatePinnedAction(actions.get(IDS_FIELD.getPreferredName()));
-                validatePinnedAction(actions.get(DOCS_FIELD.getPreferredName()));
-            } else {
-                throw new ElasticsearchParseException("pinned query rule actions must contain only one of either ids or docs");
-            }
-        } else {
-            throw new IllegalArgumentException("Unsupported QueryRuleType: " + type);
-        }
 
         if (priority != null && (priority < MIN_PRIORITY || priority > MAX_PRIORITY)) {
             throw new IllegalArgumentException("Priority was " + priority + ", must be between " + MIN_PRIORITY + " and " + MAX_PRIORITY);
         }
+
+        if (Set.of(QueryRuleType.PINNED, QueryRuleType.EXCLUDE).contains(type)) {
+            boolean ruleContainsIds = actions.containsKey(IDS_FIELD.getPreferredName());
+            boolean ruleContainsDocs = actions.containsKey(DOCS_FIELD.getPreferredName());
+            if (ruleContainsIds ^ ruleContainsDocs) {
+                validateIdOrDocAction(actions.get(IDS_FIELD.getPreferredName()));
+                validateIdOrDocAction(actions.get(DOCS_FIELD.getPreferredName()));
+            } else {
+                throw new ElasticsearchParseException(type.toString() + " query rule actions must contain only one of either ids or docs");
+            }
+        }
     }
 
-    private void validatePinnedAction(Object action) {
+    private void validateIdOrDocAction(Object action) {
         if (action != null) {
             if (action instanceof List == false) {
-                throw new ElasticsearchParseException("pinned query rule actions must be a list");
+                throw new ElasticsearchParseException(type + " query rule actions must be a list");
             } else if (((List<?>) action).isEmpty()) {
-                throw new ElasticsearchParseException("pinned query rule actions cannot be empty");
-            } else if (((List<?>) action).size() > MAX_NUM_PINNED_HITS) {
-                throw new ElasticsearchParseException("pinned hits cannot exceed " + MAX_NUM_PINNED_HITS);
+                throw new ElasticsearchParseException(type + " query rule actions cannot be empty");
+            } else if (((List<?>) action).size() > MAX_NUM_DOCS_IN_RULE) {
+                throw new ElasticsearchParseException(type + " documents cannot exceed " + MAX_NUM_DOCS_IN_RULE);
             }
         }
     }
@@ -316,14 +318,22 @@ public String toString() {
         return Strings.toString(this);
     }
 
-    @SuppressWarnings("unchecked")
     public AppliedQueryRules applyRule(AppliedQueryRules appliedRules, Map<String, Object> matchCriteria) {
-        if (type != QueryRule.QueryRuleType.PINNED) {
-            throw new UnsupportedOperationException("Only pinned query rules are supported");
+        List<SpecifiedDocument> pinnedDocs = appliedRules.pinnedDocs();
+        List<SpecifiedDocument> excludedDocs = appliedRules.excludedDocs();
+        List<SpecifiedDocument> matchingDocs = identifyMatchingDocs(matchCriteria);
+
+        switch (type) {
+            case PINNED -> pinnedDocs.addAll(matchingDocs);
+            case EXCLUDE -> excludedDocs.addAll(matchingDocs);
+            default -> throw new IllegalStateException("Unsupported query rule type: " + type);
         }
+        return new AppliedQueryRules(pinnedDocs, excludedDocs);
+    }
 
-        List<String> matchingPinnedIds = new ArrayList<>();
-        List<PinnedQueryBuilder.Item> matchingPinnedDocs = new ArrayList<>();
+    @SuppressWarnings("unchecked")
+    private List<SpecifiedDocument> identifyMatchingDocs(Map<String, Object> matchCriteria) {
+        List<SpecifiedDocument> matchingDocs = new ArrayList<>();
         Boolean isRuleMatch = null;
 
         // All specified criteria in a rule must match for the rule to be applied
@@ -342,25 +352,23 @@ public AppliedQueryRules applyRule(AppliedQueryRules appliedRules, Map<String, O
 
         if (isRuleMatch != null && isRuleMatch) {
             if (actions.containsKey(IDS_FIELD.getPreferredName())) {
-                matchingPinnedIds.addAll((List<String>) actions.get(IDS_FIELD.getPreferredName()));
+                matchingDocs.addAll(
+                    ((List<String>) actions.get(IDS_FIELD.getPreferredName())).stream().map(id -> new SpecifiedDocument(null, id)).toList()
+                );
             } else if (actions.containsKey(DOCS_FIELD.getPreferredName())) {
                 List<Map<String, String>> docsToPin = (List<Map<String, String>>) actions.get(DOCS_FIELD.getPreferredName());
-                List<PinnedQueryBuilder.Item> items = docsToPin.stream()
+                List<SpecifiedDocument> specifiedDocuments = docsToPin.stream()
                     .map(
-                        map -> new PinnedQueryBuilder.Item(
+                        map -> new SpecifiedDocument(
                             map.get(INDEX_FIELD.getPreferredName()),
-                            map.get(PinnedQueryBuilder.Item.ID_FIELD.getPreferredName())
+                            map.get(SpecifiedDocument.ID_FIELD.getPreferredName())
                         )
                     )
                     .toList();
-                matchingPinnedDocs.addAll(items);
+                matchingDocs.addAll(specifiedDocuments);
             }
         }
-
-        List<String> pinnedIds = appliedRules.pinnedIds();
-        List<PinnedQueryBuilder.Item> pinnedDocs = appliedRules.pinnedDocs();
-        pinnedIds.addAll(matchingPinnedIds);
-        pinnedDocs.addAll(matchingPinnedDocs);
-        return new AppliedQueryRules(pinnedIds, pinnedDocs);
+        return matchingDocs;
     }
+
 }
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java
index 80fbedc2aa7af..a3703a5005979 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/RuleQueryBuilder.java
@@ -20,8 +20,12 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.HeaderWarning;
+import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.IndexFieldMapper;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.license.LicenseUtils;
@@ -32,7 +36,7 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder;
-import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder.Item;
+import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument;
 
 import java.io.IOException;
 import java.util.List;
@@ -68,8 +72,8 @@ public class RuleQueryBuilder extends AbstractQueryBuilder<RuleQueryBuilder> {
     private final Map<String, Object> matchCriteria;
     private final QueryBuilder organicQuery;
 
-    private final Supplier<List<String>> pinnedIdsSupplier;
-    private final Supplier<List<Item>> pinnedDocsSupplier;
+    private final Supplier<List<SpecifiedDocument>> pinnedDocsSupplier;
+    private final Supplier<List<SpecifiedDocument>> excludedDocsSupplier;
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
@@ -89,18 +93,18 @@ public RuleQueryBuilder(StreamInput in) throws IOException {
         } else {
             rulesetIds = List.of(in.readString());
             in.readOptionalStringCollectionAsList();
-            in.readOptionalCollectionAsList(Item::new);
+            in.readOptionalCollectionAsList(SpecifiedDocument::new);
         }
-        pinnedIdsSupplier = null;
         pinnedDocsSupplier = null;
+        excludedDocsSupplier = null;
     }
 
     private RuleQueryBuilder(
         QueryBuilder organicQuery,
         Map<String, Object> matchCriteria,
         List<String> rulesetIds,
-        Supplier<List<String>> pinnedIdsSupplier,
-        Supplier<List<Item>> pinnedDocsSupplier
+        Supplier<List<SpecifiedDocument>> pinnedDocsSupplier,
+        Supplier<List<SpecifiedDocument>> excludedDocsSupplier
 
     ) {
         if (organicQuery == null) {
@@ -124,18 +128,18 @@ private RuleQueryBuilder(
         this.organicQuery = organicQuery;
         this.matchCriteria = matchCriteria;
         this.rulesetIds = rulesetIds;
-        this.pinnedIdsSupplier = pinnedIdsSupplier;
         this.pinnedDocsSupplier = pinnedDocsSupplier;
+        this.excludedDocsSupplier = excludedDocsSupplier;
     }
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
-        if (pinnedIdsSupplier != null) {
-            throw new IllegalStateException("pinnedIdsSupplier must be null, can't serialize suppliers, missing a rewriteAndFetch?");
-        }
         if (pinnedDocsSupplier != null) {
             throw new IllegalStateException("pinnedDocsSupplier must be null, can't serialize suppliers, missing a rewriteAndFetch?");
         }
+        if (excludedDocsSupplier != null) {
+            throw new IllegalStateException("excludedDocsSupplier must be null, can't serialize suppliers, missing a rewriteAndFetch?");
+        }
 
         out.writeNamedWriteable(organicQuery);
         out.writeGenericMap(matchCriteria);
@@ -176,18 +180,11 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
     @Override
     protected Query doToQuery(SearchExecutionContext context) throws IOException {
         // NOTE: this is old query logic, as in 8.12.2+ and 8.13.0+ we will always rewrite this query
-        // into a pinned query or the organic query. This logic remains here for backwards compatibility
+        // into a pinned/boolean query or the organic query. This logic remains here for backwards compatibility
         // with coordinator nodes running versions 8.10.0 - 8.12.1.
-        List<String> pinnedIds = pinnedIdsSupplier != null ? pinnedIdsSupplier.get() : null;
-        List<Item> pinnedDocs = pinnedDocsSupplier != null ? pinnedDocsSupplier.get() : null;
-        if ((pinnedIds != null && pinnedIds.isEmpty() == false) && (pinnedDocs != null && pinnedDocs.isEmpty() == false)) {
-            throw new IllegalArgumentException("applied rules contain both pinned ids and pinned docs, only one of ids or docs is allowed");
-        }
-        if (pinnedIds != null && pinnedIds.isEmpty() == false) {
-            PinnedQueryBuilder pinnedQueryBuilder = new PinnedQueryBuilder(organicQuery, pinnedIds.toArray(new String[0]));
-            return pinnedQueryBuilder.toQuery(context);
-        } else if (pinnedDocs != null && pinnedDocs.isEmpty() == false) {
-            PinnedQueryBuilder pinnedQueryBuilder = new PinnedQueryBuilder(organicQuery, pinnedDocs.toArray(new Item[0]));
+        List<SpecifiedDocument> pinnedDocs = pinnedDocsSupplier != null ? pinnedDocsSupplier.get() : null;
+        if (pinnedDocs != null && pinnedDocs.isEmpty() == false) {
+            PinnedQueryBuilder pinnedQueryBuilder = new PinnedQueryBuilder(organicQuery, pinnedDocs.toArray(new SpecifiedDocument[0]));
             return pinnedQueryBuilder.toQuery(context);
         } else {
             return organicQuery.toQuery(context);
@@ -196,26 +193,43 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException {
 
     @Override
     protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) {
-        if (pinnedIdsSupplier != null && pinnedDocsSupplier != null) {
-            List<String> identifiedPinnedIds = pinnedIdsSupplier.get();
-            List<Item> identifiedPinnedDocs = pinnedDocsSupplier.get();
-            if (identifiedPinnedIds == null || identifiedPinnedDocs == null) {
-                return this; // Not executed yet
-            } else if (identifiedPinnedIds.isEmpty() && identifiedPinnedDocs.isEmpty()) {
-                return organicQuery; // Nothing to pin here
-            } else if (identifiedPinnedIds.isEmpty() == false && identifiedPinnedDocs.isEmpty() == false) {
-                throw new IllegalArgumentException(
-                    "applied rules contain both pinned ids and pinned docs, only one of ids or docs is allowed"
-                );
-            } else if (identifiedPinnedIds.isEmpty() == false) {
-                return new PinnedQueryBuilder(organicQuery, truncateList(identifiedPinnedIds).toArray(new String[0]));
+
+        if (pinnedDocsSupplier != null && excludedDocsSupplier != null) {
+            List<SpecifiedDocument> identifiedPinnedDocs = pinnedDocsSupplier.get();
+            List<SpecifiedDocument> identifiedExcludedDocs = excludedDocsSupplier.get();
+
+            if (identifiedPinnedDocs == null || identifiedExcludedDocs == null) {
+                // Not executed yet
+                return this;
+            }
+
+            if (identifiedPinnedDocs.isEmpty() && identifiedExcludedDocs.isEmpty()) {
+                // Nothing to do, just return the organic query
+                return organicQuery;
+            }
+
+            if (identifiedPinnedDocs.isEmpty() == false && identifiedExcludedDocs.isEmpty()) {
+                // We have pinned IDs but nothing to exclude
+                return new PinnedQueryBuilder(organicQuery, truncateList(identifiedPinnedDocs).toArray(new SpecifiedDocument[0]));
+            }
+
+            if (identifiedPinnedDocs.isEmpty()) {
+                // We have excluded IDs but nothing to pin
+                QueryBuilder excludedDocsQueryBuilder = buildExcludedDocsQuery(identifiedExcludedDocs);
+                return new BoolQueryBuilder().must(organicQuery).mustNot(excludedDocsQueryBuilder);
             } else {
-                return new PinnedQueryBuilder(organicQuery, truncateList(identifiedPinnedDocs).toArray(new Item[0]));
+                // We have documents to both pin and exclude
+                QueryBuilder pinnedQuery = new PinnedQueryBuilder(
+                    organicQuery,
+                    truncateList(identifiedPinnedDocs).toArray(new SpecifiedDocument[0])
+                );
+                QueryBuilder excludedDocsQueryBuilder = buildExcludedDocsQuery(identifiedExcludedDocs);
+                return new BoolQueryBuilder().must(pinnedQuery).mustNot(excludedDocsQueryBuilder);
             }
         }
 
-        SetOnce<List<String>> pinnedIdsSetOnce = new SetOnce<>();
-        SetOnce<List<Item>> pinnedDocsSetOnce = new SetOnce<>();
+        SetOnce<List<SpecifiedDocument>> pinnedDocsSetOnce = new SetOnce<>();
+        SetOnce<List<SpecifiedDocument>> excludedDocsSetOnce = new SetOnce<>();
         AppliedQueryRules appliedRules = new AppliedQueryRules();
 
         // Identify matching rules and apply them as applicable
@@ -255,19 +269,40 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) {
                         }
                     }
 
-                    pinnedIdsSetOnce.set(appliedRules.pinnedIds().stream().distinct().toList());
                     pinnedDocsSetOnce.set(appliedRules.pinnedDocs().stream().distinct().toList());
+                    excludedDocsSetOnce.set(appliedRules.excludedDocs().stream().distinct().toList());
                     listener.onResponse(null);
 
                 }, listener::onFailure)
             );
         });
 
-        return new RuleQueryBuilder(organicQuery, matchCriteria, this.rulesetIds, pinnedIdsSetOnce::get, pinnedDocsSetOnce::get).boost(
+        return new RuleQueryBuilder(organicQuery, matchCriteria, this.rulesetIds, pinnedDocsSetOnce::get, excludedDocsSetOnce::get).boost(
             this.boost
         ).queryName(this.queryName);
     }
 
+    private QueryBuilder buildExcludedDocsQuery(List<SpecifiedDocument> identifiedExcludedDocs) {
+        QueryBuilder excludedDocsQueryBuilder;
+        if (identifiedExcludedDocs.stream().allMatch(item -> item.index() == null)) {
+            // Easy case - just add an ids query
+            excludedDocsQueryBuilder = QueryBuilders.idsQuery()
+                .addIds(identifiedExcludedDocs.stream().map(SpecifiedDocument::id).toArray(String[]::new));
+        } else {
+            // Here, we have to create Boolean queries for the _id and _index fields
+            excludedDocsQueryBuilder = QueryBuilders.boolQuery();
+            identifiedExcludedDocs.stream().map(item -> {
+                BoolQueryBuilder excludeQueryBuilder = QueryBuilders.boolQuery()
+                    .must(QueryBuilders.termQuery(IdFieldMapper.NAME, item.id()));
+                if (item.index() != null) {
+                    excludeQueryBuilder.must(QueryBuilders.termQuery(IndexFieldMapper.NAME, item.index()));
+                }
+                return excludeQueryBuilder;
+            }).forEach(excludeQueryBuilder -> ((BoolQueryBuilder) excludedDocsQueryBuilder).must(excludeQueryBuilder));
+        }
+        return excludedDocsQueryBuilder;
+    }
+
     private List<?> truncateList(List<?> input) {
         // PinnedQueryBuilder will return an error if we attempt to return more than the maximum number of
         // pinned hits. Here, we truncate matching rules rather than return an error.
@@ -285,13 +320,13 @@ protected boolean doEquals(RuleQueryBuilder other) {
         return Objects.equals(rulesetIds, other.rulesetIds)
             && Objects.equals(matchCriteria, other.matchCriteria)
             && Objects.equals(organicQuery, other.organicQuery)
-            && Objects.equals(pinnedIdsSupplier, other.pinnedIdsSupplier)
-            && Objects.equals(pinnedDocsSupplier, other.pinnedDocsSupplier);
+            && Objects.equals(pinnedDocsSupplier, other.pinnedDocsSupplier)
+            && Objects.equals(excludedDocsSupplier, other.excludedDocsSupplier);
     }
 
     @Override
     protected int doHashCode() {
-        return Objects.hash(rulesetIds, matchCriteria, organicQuery, pinnedIdsSupplier, pinnedDocsSupplier);
+        return Objects.hash(rulesetIds, matchCriteria, organicQuery, pinnedDocsSupplier, excludedDocsSupplier);
     }
 
     private static final ConstructingObjectParser<RuleQueryBuilder, Void> PARSER = new ConstructingObjectParser<>(
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportDeleteQueryRuleAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportDeleteQueryRuleAction.java
index 4d7f510c9a7dd..c477914d7f081 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportDeleteQueryRuleAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportDeleteQueryRuleAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.rules.QueryRulesIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportDeleteQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportDeleteQueryRulesetAction.java
index f939d48d3ba7f..2a1af90634b4c 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportDeleteQueryRulesetAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportDeleteQueryRulesetAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.rules.QueryRulesIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportGetQueryRuleAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportGetQueryRuleAction.java
index 5bcef9ba5fc74..1003e3ee695dc 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportGetQueryRuleAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportGetQueryRuleAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.rules.QueryRulesIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportGetQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportGetQueryRulesetAction.java
index e492a40a8c454..2e3e0458ecd62 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportGetQueryRulesetAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportGetQueryRulesetAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.rules.QueryRulesIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportListQueryRulesetsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportListQueryRulesetsAction.java
index b87ed30e88647..ee81f016262c2 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportListQueryRulesetsAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportListQueryRulesetsAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.rules.QueryRulesIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportPutQueryRuleAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportPutQueryRuleAction.java
index 69a568ff3b1a1..fab22f555817d 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportPutQueryRuleAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportPutQueryRuleAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.rules.QueryRule;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportPutQueryRulesetAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportPutQueryRulesetAction.java
index d5834a70b3158..d4c4be801c64b 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportPutQueryRulesetAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/action/TransportPutQueryRulesetAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.rules.QueryRulesIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/TemplateParamValidator.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/TemplateParamValidator.java
index f6b9140014747..9009cc5d9570e 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/TemplateParamValidator.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/TemplateParamValidator.java
@@ -15,11 +15,14 @@
 import com.networknt.schema.SpecVersion;
 import com.networknt.schema.ValidationMessage;
 
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -28,6 +31,9 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -79,7 +85,21 @@ private static void validateWithSchema(JsonSchema jsonSchema, JsonNode jsonNode)
     }
 
     public void validate(Map<String, Object> templateParams) throws ValidationException {
-        validateWithSchema(this.jsonSchema, OBJECT_MAPPER.valueToTree(templateParams));
+
+        JsonNode secondParam = null;
+        try {
+            SpecialPermission.check();
+            secondParam = AccessController.doPrivileged((PrivilegedExceptionAction<JsonNode>) () -> {
+                return OBJECT_MAPPER.valueToTree(templateParams);
+            });
+        } catch (PrivilegedActionException e) {
+            throw new ElasticsearchStatusException(
+                "failed to convert parameters while validating",
+                RestStatus.INTERNAL_SERVER_ERROR,
+                e.getCause()
+            );
+        }
+        validateWithSchema(this.jsonSchema, secondParam);
     }
 
     @Override
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportDeleteSearchApplicationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportDeleteSearchApplicationAction.java
index 15e5b9be1104a..88d3f5576f294 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportDeleteSearchApplicationAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportDeleteSearchApplicationAction.java
@@ -13,10 +13,10 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.search.SearchApplicationIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportGetSearchApplicationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportGetSearchApplicationAction.java
index 0c5c3b3b05d56..25de30bedd498 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportGetSearchApplicationAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportGetSearchApplicationAction.java
@@ -12,11 +12,11 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.search.SearchApplication;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportListSearchApplicationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportListSearchApplicationAction.java
index 539cd5a175021..91bec7b33d8a5 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportListSearchApplicationAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportListSearchApplicationAction.java
@@ -12,10 +12,10 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.search.SearchApplicationIndexService;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportPutSearchApplicationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportPutSearchApplicationAction.java
index 76139c97fe32d..a86f2add927db 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportPutSearchApplicationAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportPutSearchApplicationAction.java
@@ -12,11 +12,11 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.application.search.SearchApplication;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportQuerySearchApplicationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportQuerySearchApplicationAction.java
index 88367cc7a61eb..838f23e73493c 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportQuerySearchApplicationAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportQuerySearchApplicationAction.java
@@ -16,12 +16,12 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportRenderSearchApplicationQueryAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportRenderSearchApplicationQueryAction.java
index 3d71d31427c09..4d519ac51ba48 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportRenderSearchApplicationQueryAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/action/TransportRenderSearchApplicationQueryAction.java
@@ -13,12 +13,12 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ent-search/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/ent-search/src/main/plugin-metadata/plugin-security.policy
new file mode 100644
index 0000000000000..4de6d5924521d
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/main/plugin-metadata/plugin-security.policy
@@ -0,0 +1,4 @@
+grant {
+  // needed for Jackson ObjectMapper to parse floats
+  permission java.lang.RuntimePermission "accessDeclaredMembers";
+};
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRuleTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRuleTests.java
index 3f65a9a6f58c5..67e7f6ac7d9e9 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRuleTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRuleTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils;
+import org.elasticsearch.xpack.searchbusinessrules.SpecifiedDocument;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -99,7 +100,22 @@ public void testToXContentValidPinnedRulesWithIds() throws IOException {
                 "ids": ["id1", "id2"]
               }
             }""");
-        testToXContentPinnedRules(content);
+        testToXContentRules(content);
+    }
+
+    public void testToXContentValidExcludedRulesWithIds() throws IOException {
+        String content = XContentHelper.stripWhitespace("""
+            {
+              "rule_id": "my_query_rule",
+              "type": "exclude",
+              "criteria": [
+                { "type": "exact", "metadata": "query_string", "values": ["foo", "bar"] }
+              ],
+              "actions": {
+                "ids": ["id1", "id2"]
+              }
+            }""");
+        testToXContentRules(content);
     }
 
     public void testToXContentValidPinnedRulesWithDocs() throws IOException {
@@ -123,10 +139,102 @@ public void testToXContentValidPinnedRulesWithDocs() throws IOException {
                 ]
               }
             }""");
-        testToXContentPinnedRules(content);
+        testToXContentRules(content);
+    }
+
+    public void testToXContentValidExcludedRulesWithDocs() throws IOException {
+        String content = XContentHelper.stripWhitespace("""
+            {
+              "rule_id": "my_query_rule",
+              "type": "exclude",
+              "criteria": [
+                { "type": "exact", "metadata": "query_string", "values": ["foo", "bar"] }
+              ],
+              "actions": {
+                "docs": [
+                  {
+                    "_index": "foo",
+                    "_id": "id1"
+                  },
+                  {
+                    "_index": "bar",
+                    "_id": "id2"
+                  }
+                ]
+              }
+            }""");
+        testToXContentRules(content);
+    }
+
+    public void testToXContentValidPinnedAndExcludedRulesWithIds() throws IOException {
+        String content = XContentHelper.stripWhitespace("""
+            {
+              "rule_id": "my_pinned_query_rule",
+              "type": "pinned",
+              "criteria": [
+                { "type": "exact", "metadata": "query_string", "values": ["foo", "bar"] }
+              ],
+              "actions": {
+                "ids": ["id1", "id2"]
+              }
+            },
+            {
+              "rule_id": "my_exclude_query_rule",
+              "type": "exlude",
+              "criteria": [
+                { "type": "exact", "metadata": "query_string", "values": ["baz"] }
+              ],
+              "actions": {
+                "ids": ["id3", "id4"]
+              }
+            }""");
+        testToXContentRules(content);
     }
 
-    private void testToXContentPinnedRules(String content) throws IOException {
+    public void testToXContentValidPinnedAndExcludedRulesWithDocs() throws IOException {
+        String content = XContentHelper.stripWhitespace("""
+            {
+              "rule_id": "my_pinned_query_rule",
+              "type": "pinned",
+              "criteria": [
+                { "type": "exact", "metadata": "query_string", "values": ["foo", "bar"] }
+              ],
+              "actions": {
+                "docs": [
+                  {
+                    "_index": "foo",
+                    "_id": "id1"
+                  },
+                  {
+                    "_index": "bar",
+                    "_id": "id2"
+                  }
+                ]
+              }
+            },
+            {
+              "rule_id": "my_exclude_query_rule",
+              "type": "exclude",
+              "criteria": [
+                { "type": "exact", "metadata": "query_string", "values": ["foo", "bar"] }
+              ],
+              "actions": {
+                "docs": [
+                  {
+                    "_index": "foo",
+                    "_id": "id3"
+                  },
+                  {
+                    "_index": "bar",
+                    "_id": "id4"
+                  }
+                ]
+              }
+            }""");
+        testToXContentRules(content);
+    }
+
+    private void testToXContentRules(String content) throws IOException {
         QueryRule queryRule = QueryRule.fromXContentBytes(new BytesArray(content), XContentType.JSON);
         boolean humanReadable = true;
         BytesReference originalBytes = toShuffledXContent(queryRule, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable);
@@ -152,7 +260,22 @@ public void testToXContentPinnedRuleWithInvalidActions() throws IOException {
         expectThrows(IllegalArgumentException.class, () -> QueryRule.fromXContentBytes(new BytesArray(content), XContentType.JSON));
     }
 
-    public void testApplyRuleWithOneCriteria() {
+    public void testToXContentExcludeRuleWithInvalidActions() throws IOException {
+        String content = XContentHelper.stripWhitespace("""
+            {
+              "rule_id": "my_query_rule",
+              "type": "exclude",
+              "criteria": [
+                { "type": "exact", "metadata": "query_string", "values": ["foo", "bar"] }
+              ],
+              "actions": {
+                  "foo": "bar"
+                }
+            }""");
+        expectThrows(IllegalArgumentException.class, () -> QueryRule.fromXContentBytes(new BytesArray(content), XContentType.JSON));
+    }
+
+    public void testApplyPinnedRuleWithOneCriteria() {
         QueryRule rule = new QueryRule(
             randomAlphaOfLength(10),
             QueryRule.QueryRuleType.PINNED,
@@ -162,14 +285,35 @@ public void testApplyRuleWithOneCriteria() {
         );
         AppliedQueryRules appliedQueryRules = new AppliedQueryRules();
         rule.applyRule(appliedQueryRules, Map.of("query", "elastic"));
-        assertEquals(List.of("id1", "id2"), appliedQueryRules.pinnedIds());
+        assertEquals(List.of(new SpecifiedDocument(null, "id1"), new SpecifiedDocument(null, "id2")), appliedQueryRules.pinnedDocs());
+        assertEquals(Collections.emptyList(), appliedQueryRules.excludedDocs());
+
+        appliedQueryRules = new AppliedQueryRules();
+        rule.applyRule(appliedQueryRules, Map.of("query", "elastic1"));
+        assertEquals(Collections.emptyList(), appliedQueryRules.pinnedDocs());
+        assertEquals(Collections.emptyList(), appliedQueryRules.excludedDocs());
+    }
+
+    public void testApplyExcludeRuleWithOneCriteria() {
+        QueryRule rule = new QueryRule(
+            randomAlphaOfLength(10),
+            QueryRule.QueryRuleType.EXCLUDE,
+            List.of(new QueryRuleCriteria(EXACT, "query", List.of("elastic"))),
+            Map.of("ids", List.of("id1", "id2")),
+            EnterpriseSearchModuleTestUtils.randomQueryRulePriority()
+        );
+        AppliedQueryRules appliedQueryRules = new AppliedQueryRules();
+        rule.applyRule(appliedQueryRules, Map.of("query", "elastic"));
+        assertEquals(List.of(new SpecifiedDocument(null, "id1"), new SpecifiedDocument(null, "id2")), appliedQueryRules.excludedDocs());
+        assertEquals(Collections.emptyList(), appliedQueryRules.pinnedDocs());
 
         appliedQueryRules = new AppliedQueryRules();
         rule.applyRule(appliedQueryRules, Map.of("query", "elastic1"));
-        assertEquals(Collections.emptyList(), appliedQueryRules.pinnedIds());
+        assertEquals(Collections.emptyList(), appliedQueryRules.excludedDocs());
+        assertEquals(Collections.emptyList(), appliedQueryRules.pinnedDocs());
     }
 
-    public void testApplyRuleWithMultipleCriteria() {
+    public void testApplyRuleWithMultipleCriteria() throws IOException {
         QueryRule rule = new QueryRule(
             randomAlphaOfLength(10),
             QueryRule.QueryRuleType.PINNED,
@@ -179,11 +323,14 @@ public void testApplyRuleWithMultipleCriteria() {
         );
         AppliedQueryRules appliedQueryRules = new AppliedQueryRules();
         rule.applyRule(appliedQueryRules, Map.of("query", "elastic - you know, for search"));
-        assertEquals(List.of("id1", "id2"), appliedQueryRules.pinnedIds());
+        assertEquals(List.of(new SpecifiedDocument(null, "id1"), new SpecifiedDocument(null, "id2")), appliedQueryRules.pinnedDocs());
+        assertEquals(Collections.emptyList(), appliedQueryRules.excludedDocs());
+        assertEquals(Collections.emptyList(), appliedQueryRules.excludedDocs());
 
         appliedQueryRules = new AppliedQueryRules();
         rule.applyRule(appliedQueryRules, Map.of("query", "elastic"));
-        assertEquals(Collections.emptyList(), appliedQueryRules.pinnedIds());
+        assertEquals(Collections.emptyList(), appliedQueryRules.pinnedDocs());
+        assertEquals(Collections.emptyList(), appliedQueryRules.excludedDocs());
     }
 
     private void assertXContent(QueryRule queryRule, boolean humanReadable) throws IOException {
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexServiceTests.java
index 36d5bb91e619d..4de5445871739 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexServiceTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexServiceTests.java
@@ -27,6 +27,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -50,7 +51,7 @@ public class QueryRulesIndexServiceTests extends ESSingleNodeTestCase {
 
     @Before
     public void setup() {
-        Set<Setting<?>> settingsSet = ClusterSettings.BUILT_IN_CLUSTER_SETTINGS;
+        Set<Setting<?>> settingsSet = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         settingsSet.addAll(QueryRulesConfig.getSettings());
         ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, settingsSet);
         this.queryRulesIndexService = new QueryRulesIndexService(client(), clusterSettings);
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/TemplateParamValidatorTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/TemplateParamValidatorTests.java
new file mode 100644
index 0000000000000..ad924a235c901
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/TemplateParamValidatorTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.search;
+
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class TemplateParamValidatorTests extends ESTestCase {
+
+    public void testValidateThrowsWhenSchemaDontMatch() {
+        Map<String, Object> failingParams = new HashMap<>();
+        failingParams.put("fail", null);
+        TemplateParamValidator validator = new TemplateParamValidator(
+            "{\"properties\":{\"title_boost\":{\"type\":\"number\"},\"additionalProperties\":false},\"required\":[\"query_string\"]}"
+        );
+
+        assertThrows(ValidationException.class, () -> validator.validate(failingParams));
+
+    }
+
+    public void testValidateWithFloats() {
+        Map<String, Object> passingParams = new HashMap<>();
+        passingParams.put("title_boost", 1.0f);
+        passingParams.put("query_string", "query_string");
+        TemplateParamValidator validator = new TemplateParamValidator(
+            "{\"properties\":{\"title_boost\":{\"type\":\"number\"},\"additionalProperties\":false},\"required\":[\"query_string\"]}"
+        );
+        // This shouldn't throw
+        // We don't have nice assertions for this in JUnit 4
+        validator.validate(passingParams);
+    }
+
+}
diff --git a/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java b/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java
index d8b887b98e647..2a29572374fa8 100644
--- a/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java
+++ b/x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.NotEqualMessageBuilder;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -68,14 +66,7 @@ public void createIndex() throws IOException {
         bwcNodes = new ArrayList<>(nodes.getBWCNodes());
 
         String mappings = readResource(EqlSearchIT.class.getResourceAsStream("/eql_mapping.json"));
-        createIndex(
-            index,
-            Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), numShards)
-                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)
-                .build(),
-            mappings
-        );
+        createIndex(index, indexSettings(numShards, numReplicas).build(), mappings);
     }
 
     @After
diff --git a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/60_no_exact.yml b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/60_no_exact.yml
new file mode 100644
index 0000000000000..133587561ec3f
--- /dev/null
+++ b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/60_no_exact.yml
@@ -0,0 +1,37 @@
+---
+setup:
+  - do:
+      indices.create:
+          index:  eql_test
+          body:
+            mappings:
+              properties:
+                some_text:
+                  type: text
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: eql_test
+              _id:    "1"
+          - event:
+              - category: process
+            "@timestamp": 2020-02-03T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: false
+            some_text: foo
+
+
+---
+
+"Case insensitive match on text field":
+  - do:
+      catch: "bad_request"
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where some_text: "foo"'
+  - match: { error.root_cause.0.type: "verification_exception" }
+  - match: { error.root_cause.0.reason: "Found 1 problem\nline 1:15: [:] cannot operate on first argument field of data type [text]: No keyword/multi-field defined exact matches for [some_text]; define one or use MATCH/QUERY instead" }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlInfoTransportAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlInfoTransportAction.java
index 4821c4eeb6410..67fe28e77990d 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlInfoTransportAction.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlInfoTransportAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.eql;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlUsageTransportAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlUsageTransportAction.java
index 82bd5a4671674..0ba5e04fff6cd 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlUsageTransportAction.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlUsageTransportAction.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java
index 0a486b77ea839..6873543ab4ec4 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java
@@ -51,6 +51,11 @@ protected TypeResolution resolveInputType(Expression e, ParamOrdinal paramOrdina
                 resolution.message(),
                 regularOperatorSymbol()
             );
+            return new TypeResolution(message);
+        }
+        resolution = TypeResolutions.isExact(e, op, paramOrdinal);
+        if (resolution.unresolved()) {
+            String message = LoggerMessageFormat.format(null, "{}", resolution.message());
             resolution = new TypeResolution(message);
         }
         return resolution;
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetResultsAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetResultsAction.java
index 7c710ed7ea927..d5aaacbeeb787 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetResultsAction.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetResultsAction.java
@@ -9,10 +9,10 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.eql.EqlAsyncActionNames;
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetStatusAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetStatusAction.java
index 19b73ddcefc8e..2db11ca223127 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetStatusAction.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlAsyncGetStatusAction.java
@@ -9,10 +9,10 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.eql.action.EqlSearchResponse;
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
index 9f87ba5c60779..51f92bcda7da4 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
@@ -24,6 +23,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlStatsAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlStatsAction.java
index 18030c3d6207a..76f3d05cdb9dc 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlStatsAction.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlStatsAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java
index 943d1275364fb..9cd6549b4be2c 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java
@@ -209,7 +209,7 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
             ActionListener<Response> listener
         ) {
             if (request instanceof OpenPointInTimeRequest) {
-                OpenPointInTimeResponse response = new OpenPointInTimeResponse(pitId);
+                OpenPointInTimeResponse response = new OpenPointInTimeResponse(pitId, 1, 1, 0, 0);
                 listener.onResponse((Response) response);
             } else if (request instanceof ClosePointInTimeRequest) {
                 ClosePointInTimeResponse response = new ClosePointInTimeResponse(true, 1);
@@ -260,6 +260,11 @@ public void readBytes(byte[] b, int offset, int len) throws IOException {
 
                     }
 
+                    @Override
+                    public int read(byte[] b, int off, int len) throws IOException {
+                        return 0;
+                    }
+
                     @Override
                     public void close() throws IOException {
 
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java
index c0e5d398d6508..f1c5d483d4002 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/PITAwareQueryClientTests.java
@@ -204,7 +204,7 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
                 assertArrayEquals(INDICES, openPIT.indices()); // indices for opening pit should be the same as for the eql query itself
 
                 openedPIT = true;
-                OpenPointInTimeResponse response = new OpenPointInTimeResponse(pitId);
+                OpenPointInTimeResponse response = new OpenPointInTimeResponse(pitId, 1, 1, 0, 0);
                 listener.onResponse((Response) response);
             } else if (request instanceof ClosePointInTimeRequest closePIT) {
                 assertTrue(openedPIT);
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java
index c001b312d5578..ecf5ef61ac49a 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java
@@ -394,7 +394,7 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
         ) {
             if (request instanceof OpenPointInTimeRequest) {
                 pitContextCounter.incrementAndGet();
-                OpenPointInTimeResponse response = new OpenPointInTimeResponse(pitId);
+                OpenPointInTimeResponse response = new OpenPointInTimeResponse(pitId, 1, 1, 0, 0);
                 listener.onResponse((Response) response);
             } else if (request instanceof ClosePointInTimeRequest) {
                 ClosePointInTimeResponse response = new ClosePointInTimeResponse(true, 1);
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/TableInfo.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/TableInfo.java
deleted file mode 100644
index 8e1e7bec94005..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/TableInfo.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core.analyzer;
-
-import org.elasticsearch.xpack.esql.core.plan.TableIdentifier;
-
-public class TableInfo {
-
-    private final TableIdentifier id;
-    private final boolean isFrozen;
-
-    public TableInfo(TableIdentifier id, boolean isFrozen) {
-        this.id = id;
-        this.isFrozen = isFrozen;
-    }
-
-    public TableIdentifier id() {
-        return id;
-    }
-
-    public boolean isFrozen() {
-        return isFrozen;
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java
index 01cc716a20547..e33f9b1c20527 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java
@@ -16,7 +16,6 @@
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Objects;
 
 import static java.util.Collections.singletonList;
 
@@ -32,7 +31,6 @@ public final class Alias extends NamedExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(NamedExpression.class, "Alias", Alias::new);
 
     private final Expression child;
-    private final String qualifier;
 
     /**
      * Postpone attribute creation until it is actually created.
@@ -41,21 +39,24 @@ public final class Alias extends NamedExpression {
     private Attribute lazyAttribute;
 
     public Alias(Source source, String name, Expression child) {
-        this(source, name, null, child, null);
+        this(source, name, child, null);
     }
 
-    public Alias(Source source, String name, String qualifier, Expression child) {
-        this(source, name, qualifier, child, null);
+    public Alias(Source source, String name, Expression child, NameId id) {
+        this(source, name, child, id, false);
     }
 
-    public Alias(Source source, String name, String qualifier, Expression child, NameId id) {
-        this(source, name, qualifier, child, id, false);
-    }
-
-    public Alias(Source source, String name, String qualifier, Expression child, NameId id, boolean synthetic) {
+    public Alias(Source source, String name, Expression child, NameId id, boolean synthetic) {
         super(source, name, singletonList(child), id, synthetic);
         this.child = child;
-        this.qualifier = qualifier;
+    }
+
+    @Deprecated
+    /**
+     * Old constructor from when this had a qualifier string. Still needed to not break serialization.
+     */
+    private Alias(Source source, String name, String qualifier, Expression child, NameId id, boolean synthetic) {
+        this(source, name, child, id, synthetic);
     }
 
     public Alias(StreamInput in) throws IOException {
@@ -73,7 +74,8 @@ public Alias(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writeString(name());
-        out.writeOptionalString(qualifier());
+        // We used to write the qualifier here. We can still do if needed in the future.
+        out.writeOptionalString(null);
         out.writeNamedWriteable(child());
         id().writeTo(out);
         out.writeBoolean(synthetic());
@@ -86,35 +88,32 @@ public String getWriteableName() {
 
     @Override
     protected NodeInfo<Alias> info() {
-        return NodeInfo.create(this, Alias::new, name(), qualifier, child, id(), synthetic());
+        return NodeInfo.create(this, Alias::new, name(), child, id(), synthetic());
     }
 
     public Alias replaceChild(Expression child) {
-        return new Alias(source(), name(), qualifier, child, id(), synthetic());
+        return new Alias(source(), name(), child, id(), synthetic());
     }
 
     @Override
     public Alias replaceChildren(List<Expression> newChildren) {
-        return new Alias(source(), name(), qualifier, newChildren.get(0), id(), synthetic());
+        return new Alias(source(), name(), newChildren.get(0), id(), synthetic());
     }
 
     public Expression child() {
         return child;
     }
 
-    public String qualifier() {
-        return qualifier;
-    }
-
-    public String qualifiedName() {
-        return qualifier == null ? name() : qualifier + "." + name();
-    }
-
     @Override
     public Nullability nullable() {
         return child.nullable();
     }
 
+    @Override
+    protected TypeResolution resolveType() {
+        return child.resolveType();
+    }
+
     @Override
     public DataType dataType() {
         return child.dataType();
@@ -124,8 +123,8 @@ public DataType dataType() {
     public Attribute toAttribute() {
         if (lazyAttribute == null) {
             lazyAttribute = resolved()
-                ? new ReferenceAttribute(source(), name(), dataType(), qualifier, nullable(), id(), synthetic())
-                : new UnresolvedAttribute(source(), name(), qualifier);
+                ? new ReferenceAttribute(source(), name(), dataType(), nullable(), id(), synthetic())
+                : new UnresolvedAttribute(source(), name());
         }
         return lazyAttribute;
     }
@@ -146,18 +145,4 @@ public String nodeString() {
     public static Expression unwrap(Expression e) {
         return e instanceof Alias as ? as.child() : e;
     }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (super.equals(obj) == false) {
-            return false;
-        }
-        Alias other = (Alias) obj;
-        return Objects.equals(qualifier, other.qualifier);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), qualifier);
-    }
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
index e89f39294a28b..3dda28a8abf9d 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.esql.core.expression;
 
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 
@@ -15,7 +14,6 @@
 import java.util.Objects;
 
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.xpack.esql.core.util.StringUtils.splitQualifiedIndex;
 
 /**
  * {@link Expression}s that can be materialized and describe properties of the derived table.
@@ -35,33 +33,19 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         return List.of(FieldAttribute.ENTRY, MetadataAttribute.ENTRY, ReferenceAttribute.ENTRY);
     }
 
-    // empty - such as a top level attribute in SELECT cause
-    // present - table name or a table name alias
-    private final String qualifier;
-    // cluster name in the qualifier (if any)
-    private final String cluster;
-
     // can the attr be null - typically used in JOINs
     private final Nullability nullability;
 
-    public Attribute(Source source, String name, String qualifier, NameId id) {
-        this(source, name, qualifier, Nullability.TRUE, id);
+    public Attribute(Source source, String name, NameId id) {
+        this(source, name, Nullability.TRUE, id);
     }
 
-    public Attribute(Source source, String name, String qualifier, Nullability nullability, NameId id) {
-        this(source, name, qualifier, nullability, id, false);
+    public Attribute(Source source, String name, Nullability nullability, NameId id) {
+        this(source, name, nullability, id, false);
     }
 
-    public Attribute(Source source, String name, String qualifier, Nullability nullability, NameId id, boolean synthetic) {
+    public Attribute(Source source, String name, Nullability nullability, NameId id, boolean synthetic) {
         super(source, name, emptyList(), id, synthetic);
-        if (qualifier != null) {
-            Tuple<String, String> splitQualifier = splitQualifiedIndex(qualifier);
-            this.cluster = splitQualifier.v1();
-            this.qualifier = splitQualifier.v2();
-        } else {
-            this.cluster = null;
-            this.qualifier = null;
-        }
         this.nullability = nullability;
     }
 
@@ -70,14 +54,6 @@ public final Expression replaceChildren(List<Expression> newChildren) {
         throw new UnsupportedOperationException("this type of node doesn't have any children to replace");
     }
 
-    public String qualifier() {
-        return qualifier;
-    }
-
-    public String qualifiedName() {
-        return qualifier == null ? name() : qualifier + "." + name();
-    }
-
     @Override
     public Nullability nullable() {
         return nullability;
@@ -89,42 +65,26 @@ public AttributeSet references() {
     }
 
     public Attribute withLocation(Source source) {
-        return Objects.equals(source(), source) ? this : clone(source, name(), dataType(), qualifier(), nullable(), id(), synthetic());
-    }
-
-    public Attribute withQualifier(String qualifier) {
-        return Objects.equals(qualifier(), qualifier)
-            ? this
-            : clone(source(), name(), dataType(), qualifier, nullable(), id(), synthetic());
+        return Objects.equals(source(), source) ? this : clone(source, name(), dataType(), nullable(), id(), synthetic());
     }
 
     public Attribute withName(String name) {
-        return Objects.equals(name(), name) ? this : clone(source(), name, dataType(), qualifier(), nullable(), id(), synthetic());
+        return Objects.equals(name(), name) ? this : clone(source(), name, dataType(), nullable(), id(), synthetic());
     }
 
     public Attribute withNullability(Nullability nullability) {
-        return Objects.equals(nullable(), nullability)
-            ? this
-            : clone(source(), name(), dataType(), qualifier(), nullability, id(), synthetic());
+        return Objects.equals(nullable(), nullability) ? this : clone(source(), name(), dataType(), nullability, id(), synthetic());
     }
 
     public Attribute withId(NameId id) {
-        return clone(source(), name(), dataType(), qualifier(), nullable(), id, synthetic());
+        return clone(source(), name(), dataType(), nullable(), id, synthetic());
     }
 
     public Attribute withDataType(DataType type) {
-        return Objects.equals(dataType(), type) ? this : clone(source(), name(), type, qualifier(), nullable(), id(), synthetic());
+        return Objects.equals(dataType(), type) ? this : clone(source(), name(), type, nullable(), id(), synthetic());
     }
 
-    protected abstract Attribute clone(
-        Source source,
-        String name,
-        DataType type,
-        String qualifier,
-        Nullability nullability,
-        NameId id,
-        boolean synthetic
-    );
+    protected abstract Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic);
 
     @Override
     public Attribute toAttribute() {
@@ -143,19 +103,19 @@ public boolean semanticEquals(Expression other) {
 
     @Override
     protected Expression canonicalize() {
-        return clone(Source.EMPTY, name(), dataType(), qualifier, nullability, id(), synthetic());
+        return clone(Source.EMPTY, name(), dataType(), nullability, id(), synthetic());
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), qualifier, nullability);
+        return Objects.hash(super.hashCode(), nullability);
     }
 
     @Override
     public boolean equals(Object obj) {
         if (super.equals(obj)) {
             Attribute other = (Attribute) obj;
-            return Objects.equals(qualifier, other.qualifier) && Objects.equals(nullability, other.nullability);
+            return Objects.equals(nullability, other.nullability);
         }
 
         return false;
@@ -163,7 +123,7 @@ public boolean equals(Object obj) {
 
     @Override
     public String toString() {
-        return qualifiedName() + "{" + label() + "}" + "#" + id();
+        return name() + "{" + label() + "}" + "#" + id();
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java
index 5824358e57525..d5c4be6e97c7e 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java
@@ -21,7 +21,7 @@
  */
 public class EmptyAttribute extends Attribute {
     public EmptyAttribute(Source source) {
-        super(source, StringUtils.EMPTY, null, null);
+        super(source, StringUtils.EMPTY, null);
     }
 
     @Override
@@ -35,15 +35,7 @@ public String getWriteableName() {
     }
 
     @Override
-    protected Attribute clone(
-        Source source,
-        String name,
-        DataType type,
-        String qualifier,
-        Nullability nullability,
-        NameId id,
-        boolean synthetic
-    ) {
+    protected Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) {
         return this;
     }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java
index df8b6732ac0d4..0be7f65d767c7 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java
@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
-import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvable;
@@ -29,7 +28,7 @@
  * a, b, ABS(c), and i are all Expressions, with ABS(c) being a Function
  * (which is a type of expression) with a single child, c.
  */
-public abstract class Expression extends Node<Expression> implements Resolvable, NamedWriteable {
+public abstract class Expression extends Node<Expression> implements Resolvable {
     public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
         for (NamedWriteableRegistry.Entry e : NamedExpression.getNamedWriteables()) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
index a3bc7ea621d8a..37f2cf863d53e 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 
 import java.io.IOException;
@@ -36,7 +37,7 @@ public class FieldAttribute extends TypedAttribute {
     static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         Attribute.class,
         "FieldAttribute",
-        FieldAttribute::new
+        FieldAttribute::readFrom
     );
 
     private final FieldAttribute parent;
@@ -48,7 +49,7 @@ public FieldAttribute(Source source, String name, EsField field) {
     }
 
     public FieldAttribute(Source source, FieldAttribute parent, String name, EsField field) {
-        this(source, parent, name, field, null, Nullability.TRUE, null, false);
+        this(source, parent, name, field, Nullability.TRUE, null, false);
     }
 
     public FieldAttribute(
@@ -56,12 +57,11 @@ public FieldAttribute(
         FieldAttribute parent,
         String name,
         EsField field,
-        String qualifier,
         Nullability nullability,
         NameId id,
         boolean synthetic
     ) {
-        this(source, parent, name, field.getDataType(), field, qualifier, nullability, id, synthetic);
+        this(source, parent, name, field.getDataType(), field, nullability, id, synthetic);
     }
 
     public FieldAttribute(
@@ -70,18 +70,35 @@ public FieldAttribute(
         String name,
         DataType type,
         EsField field,
-        String qualifier,
         Nullability nullability,
         NameId id,
         boolean synthetic
     ) {
-        super(source, name, type, qualifier, nullability, id, synthetic);
-        this.path = parent != null ? parent.fieldName() : StringUtils.EMPTY;
+        super(source, name, type, nullability, id, synthetic);
+        this.path = parent != null ? parent.name() : StringUtils.EMPTY;
         this.parent = parent;
         this.field = field;
     }
 
-    public FieldAttribute(StreamInput in) throws IOException {
+    @Deprecated
+    /**
+     * Old constructor from when this had a qualifier string. Still needed to not break serialization.
+     */
+    private FieldAttribute(
+        Source source,
+        FieldAttribute parent,
+        String name,
+        DataType type,
+        EsField field,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic
+    ) {
+        this(source, parent, name, type, field, nullability, id, synthetic);
+    }
+
+    private FieldAttribute(StreamInput in) throws IOException {
         /*
          * The funny casting dance with `(StreamInput & PlanStreamInput) in` is required
          * because we're in esql-core here and the real PlanStreamInput is in
@@ -92,10 +109,10 @@ public FieldAttribute(StreamInput in) throws IOException {
          */
         this(
             Source.readFrom((StreamInput & PlanStreamInput) in),
-            in.readOptionalWriteable(FieldAttribute::new),
+            in.readOptionalWriteable(FieldAttribute::readFrom),
             in.readString(),
             DataType.readFrom(in),
-            in.readNamedWriteable(EsField.class),
+            EsField.readFrom(in),
             in.readOptionalString(),
             in.readEnum(Nullability.class),
             NameId.readFrom((StreamInput & PlanStreamInput) in),
@@ -105,15 +122,22 @@ public FieldAttribute(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeOptionalWriteable(parent);
-        out.writeString(name());
-        dataType().writeTo(out);
-        out.writeNamedWriteable(field);
-        out.writeOptionalString(qualifier());
-        out.writeEnum(nullable());
-        id().writeTo(out);
-        out.writeBoolean(synthetic());
+        if (((PlanStreamOutput) out).writeAttributeCacheHeader(this)) {
+            Source.EMPTY.writeTo(out);
+            out.writeOptionalWriteable(parent);
+            out.writeString(name());
+            dataType().writeTo(out);
+            field.writeTo(out);
+            // We used to write the qualifier here. We can still do if needed in the future.
+            out.writeOptionalString(null);
+            out.writeEnum(nullable());
+            id().writeTo(out);
+            out.writeBoolean(synthetic());
+        }
+    }
+
+    public static FieldAttribute readFrom(StreamInput in) throws IOException {
+        return ((PlanStreamInput) in).readAttributeWithCache(FieldAttribute::new);
     }
 
     @Override
@@ -123,7 +147,28 @@ public String getWriteableName() {
 
     @Override
     protected NodeInfo<FieldAttribute> info() {
-        return NodeInfo.create(this, FieldAttribute::new, parent, name(), dataType(), field, qualifier(), nullable(), id(), synthetic());
+        return NodeInfo.create(
+            this,
+            (source, parent1, name, type, field1, qualifier, nullability, id, synthetic) -> new FieldAttribute(
+                source,
+                parent1,
+                name,
+                type,
+                field1,
+                qualifier,
+                nullability,
+                id,
+                synthetic
+            ),
+            parent,
+            name(),
+            dataType(),
+            field,
+            (String) null,
+            nullable(),
+            id(),
+            synthetic()
+        );
     }
 
     public FieldAttribute parent() {
@@ -148,11 +193,6 @@ public String fieldName() {
         return Strings.hasText(path) ? path + "." + field.getName() : field.getName();
     }
 
-    public String qualifiedPath() {
-        // return only the qualifier is there's no path
-        return qualifier() != null ? qualifier() + (Strings.hasText(path) ? "." + path : StringUtils.EMPTY) : path;
-    }
-
     public EsField.Exact getExactInfo() {
         return field.getExactInfo();
     }
@@ -166,21 +206,12 @@ public FieldAttribute exactAttribute() {
     }
 
     private FieldAttribute innerField(EsField type) {
-        return new FieldAttribute(source(), this, name() + "." + type.getName(), type, qualifier(), nullable(), id(), synthetic());
+        return new FieldAttribute(source(), this, name() + "." + type.getName(), type, nullable(), id(), synthetic());
     }
 
     @Override
-    protected Attribute clone(
-        Source source,
-        String name,
-        DataType type,
-        String qualifier,
-        Nullability nullability,
-        NameId id,
-        boolean synthetic
-    ) {
-        FieldAttribute qualifiedParent = parent != null ? (FieldAttribute) parent.withQualifier(qualifier) : null;
-        return new FieldAttribute(source, qualifiedParent, name, field, qualifier, nullability, id, synthetic);
+    protected Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) {
+        return new FieldAttribute(source, parent, name, field, nullability, id, synthetic);
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
index fd7382b0098c9..539c55ba341cf 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 
 import java.io.IOException;
 import java.util.Map;
@@ -33,7 +34,7 @@ public class MetadataAttribute extends TypedAttribute {
     static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         Attribute.class,
         "MetadataAttribute",
-        MetadataAttribute::new
+        MetadataAttribute::readFrom
     );
 
     private static final Map<String, Tuple<DataType, Boolean>> ATTRIBUTES_MAP = Map.of(
@@ -57,22 +58,38 @@ public MetadataAttribute(
         Source source,
         String name,
         DataType dataType,
-        String qualifier,
         Nullability nullability,
         NameId id,
         boolean synthetic,
         boolean searchable
     ) {
-        super(source, name, dataType, qualifier, nullability, id, synthetic);
+        super(source, name, dataType, nullability, id, synthetic);
         this.searchable = searchable;
     }
 
     public MetadataAttribute(Source source, String name, DataType dataType, boolean searchable) {
-        this(source, name, dataType, null, Nullability.TRUE, null, false, searchable);
+        this(source, name, dataType, Nullability.TRUE, null, false, searchable);
+    }
+
+    @Deprecated
+    /**
+     * Old constructor from when this had a qualifier string. Still needed to not break serialization.
+     */
+    private MetadataAttribute(
+        Source source,
+        String name,
+        DataType dataType,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic,
+        boolean searchable
+    ) {
+        this(source, name, dataType, nullability, id, synthetic, searchable);
     }
 
     @SuppressWarnings("unchecked")
-    public MetadataAttribute(StreamInput in) throws IOException {
+    private MetadataAttribute(StreamInput in) throws IOException {
         /*
          * The funny casting dance with `(StreamInput & PlanStreamInput) in` is required
          * because we're in esql-core here and the real PlanStreamInput is in
@@ -95,14 +112,21 @@ public MetadataAttribute(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeString(name());
-        dataType().writeTo(out);
-        out.writeOptionalString(qualifier());
-        out.writeEnum(nullable());
-        id().writeTo(out);
-        out.writeBoolean(synthetic());
-        out.writeBoolean(searchable);
+        if (((PlanStreamOutput) out).writeAttributeCacheHeader(this)) {
+            Source.EMPTY.writeTo(out);
+            out.writeString(name());
+            dataType().writeTo(out);
+            // We used to write the qualifier here. We can still do if needed in the future.
+            out.writeOptionalString(null);
+            out.writeEnum(nullable());
+            id().writeTo(out);
+            out.writeBoolean(synthetic());
+            out.writeBoolean(searchable);
+        }
+    }
+
+    public static MetadataAttribute readFrom(StreamInput in) throws IOException {
+        return ((PlanStreamInput) in).readAttributeWithCache(MetadataAttribute::new);
     }
 
     @Override
@@ -111,16 +135,8 @@ public String getWriteableName() {
     }
 
     @Override
-    protected MetadataAttribute clone(
-        Source source,
-        String name,
-        DataType type,
-        String qualifier,
-        Nullability nullability,
-        NameId id,
-        boolean synthetic
-    ) {
-        return new MetadataAttribute(source, name, type, qualifier, nullability, id, synthetic, searchable);
+    protected MetadataAttribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) {
+        return new MetadataAttribute(source, name, type, null, nullability, id, synthetic, searchable);
     }
 
     @Override
@@ -130,17 +146,32 @@ protected String label() {
 
     @Override
     protected NodeInfo<? extends Expression> info() {
-        return NodeInfo.create(this, MetadataAttribute::new, name(), dataType(), qualifier(), nullable(), id(), synthetic(), searchable);
+        return NodeInfo.create(
+            this,
+            (source, name, dataType, qualifier, nullability, id, synthetic, searchable1) -> new MetadataAttribute(
+                source,
+                name,
+                dataType,
+                qualifier,
+                nullability,
+                id,
+                synthetic,
+                searchable1
+            ),
+            name(),
+            dataType(),
+            (String) null,
+            nullable(),
+            id(),
+            synthetic(),
+            searchable
+        );
     }
 
     public boolean searchable() {
         return searchable;
     }
 
-    private MetadataAttribute withSource(Source source) {
-        return new MetadataAttribute(source, name(), dataType(), qualifier(), nullable(), id(), synthetic(), searchable());
-    }
-
     public static MetadataAttribute create(Source source, String name) {
         var t = ATTRIBUTES_MAP.get(name);
         return t != null ? new MetadataAttribute(source, name, t.v1(), t.v2()) : null;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java
index e3e9a60180da7..266ad8e2bb051 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java
@@ -55,6 +55,10 @@ public boolean synthetic() {
         return synthetic;
     }
 
+    /**
+     * Try to return either {@code this} if it is an {@link Attribute}, or a {@link ReferenceAttribute} to it otherwise.
+     * Return an {@link UnresolvedAttribute} if this is unresolved.
+     */
     public abstract Attribute toAttribute();
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Order.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Order.java
deleted file mode 100644
index f73051fd9662a..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Order.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.expression;
-
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
-
-import static java.util.Collections.singletonList;
-import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isExact;
-
-public class Order extends Expression {
-
-    public enum OrderDirection {
-        ASC,
-        DESC
-    }
-
-    public enum NullsPosition {
-        FIRST,
-        LAST,
-        /**
-         * Nulls position has not been specified by the user and an appropriate default will be used.
-         *
-         * The default values are chosen such that it stays compatible with previous behavior. Unfortunately, this results in
-         * inconsistencies across different types of queries (see https://github.com/elastic/elasticsearch/issues/77068).
-         */
-        ANY;
-    }
-
-    private final Expression child;
-    private final OrderDirection direction;
-    private final NullsPosition nulls;
-
-    public Order(Source source, Expression child, OrderDirection direction, NullsPosition nulls) {
-        super(source, singletonList(child));
-        this.child = child;
-        this.direction = direction;
-        this.nulls = nulls == null ? NullsPosition.ANY : nulls;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public String getWriteableName() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    protected NodeInfo<Order> info() {
-        return NodeInfo.create(this, Order::new, child, direction, nulls);
-    }
-
-    @Override
-    public Nullability nullable() {
-        return Nullability.FALSE;
-    }
-
-    @Override
-    protected TypeResolution resolveType() {
-        return isExact(child, "ORDER BY cannot be applied to field of data type [{}]: {}");
-    }
-
-    @Override
-    public DataType dataType() {
-        return child.dataType();
-    }
-
-    @Override
-    public Order replaceChildren(List<Expression> newChildren) {
-        return new Order(source(), newChildren.get(0), direction, nulls);
-    }
-
-    public Expression child() {
-        return child;
-    }
-
-    public OrderDirection direction() {
-        return direction;
-    }
-
-    public NullsPosition nullsPosition() {
-        return nulls;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(child, direction, nulls);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-
-        if (obj == null || getClass() != obj.getClass()) {
-            return false;
-        }
-
-        Order other = (Order) obj;
-        return Objects.equals(direction, other.direction) && Objects.equals(nulls, other.nulls) && Objects.equals(child, other.child);
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
index d9a70787a56ed..504e1eae8d880 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 
 import java.io.IOException;
 
@@ -23,14 +24,22 @@ public class ReferenceAttribute extends TypedAttribute {
     static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         Attribute.class,
         "ReferenceAttribute",
-        ReferenceAttribute::new
+        ReferenceAttribute::readFrom
     );
 
     public ReferenceAttribute(Source source, String name, DataType dataType) {
-        this(source, name, dataType, null, Nullability.FALSE, null, false);
+        this(source, name, dataType, Nullability.FALSE, null, false);
     }
 
-    public ReferenceAttribute(
+    public ReferenceAttribute(Source source, String name, DataType dataType, Nullability nullability, NameId id, boolean synthetic) {
+        super(source, name, dataType, nullability, id, synthetic);
+    }
+
+    @Deprecated
+    /**
+     * Old constructor from when this had a qualifier string. Still needed to not break serialization.
+     */
+    private ReferenceAttribute(
         Source source,
         String name,
         DataType dataType,
@@ -39,11 +48,11 @@ public ReferenceAttribute(
         NameId id,
         boolean synthetic
     ) {
-        super(source, name, dataType, qualifier, nullability, id, synthetic);
+        this(source, name, dataType, nullability, id, synthetic);
     }
 
     @SuppressWarnings("unchecked")
-    public ReferenceAttribute(StreamInput in) throws IOException {
+    private ReferenceAttribute(StreamInput in) throws IOException {
         /*
          * The funny casting dance with `(StreamInput & PlanStreamInput) in` is required
          * because we're in esql-core here and the real PlanStreamInput is in
@@ -65,13 +74,20 @@ public ReferenceAttribute(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeString(name());
-        dataType().writeTo(out);
-        out.writeOptionalString(qualifier());
-        out.writeEnum(nullable());
-        id().writeTo(out);
-        out.writeBoolean(synthetic());
+        if (((PlanStreamOutput) out).writeAttributeCacheHeader(this)) {
+            Source.EMPTY.writeTo(out);
+            out.writeString(name());
+            dataType().writeTo(out);
+            // We used to write the qualifier here. We can still do if needed in the future.
+            out.writeOptionalString(null);
+            out.writeEnum(nullable());
+            id().writeTo(out);
+            out.writeBoolean(synthetic());
+        }
+    }
+
+    public static ReferenceAttribute readFrom(StreamInput in) throws IOException {
+        return ((PlanStreamInput) in).readAttributeWithCache(ReferenceAttribute::new);
     }
 
     @Override
@@ -80,21 +96,30 @@ public String getWriteableName() {
     }
 
     @Override
-    protected Attribute clone(
-        Source source,
-        String name,
-        DataType dataType,
-        String qualifier,
-        Nullability nullability,
-        NameId id,
-        boolean synthetic
-    ) {
-        return new ReferenceAttribute(source, name, dataType, qualifier, nullability, id, synthetic);
+    protected Attribute clone(Source source, String name, DataType dataType, Nullability nullability, NameId id, boolean synthetic) {
+        return new ReferenceAttribute(source, name, dataType, null, nullability, id, synthetic);
     }
 
     @Override
     protected NodeInfo<ReferenceAttribute> info() {
-        return NodeInfo.create(this, ReferenceAttribute::new, name(), dataType(), qualifier(), nullable(), id(), synthetic());
+        return NodeInfo.create(
+            this,
+            (source, name, dataType, qualifier, nullability, id, synthetic) -> new ReferenceAttribute(
+                source,
+                name,
+                dataType,
+                qualifier,
+                nullability,
+                id,
+                synthetic
+            ),
+            name(),
+            dataType(),
+            (String) null,
+            nullable(),
+            id(),
+            synthetic()
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
index c3593e91c537e..ab05a71b0e1c6 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression.TypeResolution;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
 
 import java.util.Locale;
 import java.util.StringJoiner;
@@ -176,19 +177,60 @@ public static TypeResolution isType(
         ParamOrdinal paramOrd,
         String... acceptedTypes
     ) {
-        return predicate.test(e.dataType()) || e.dataType() == NULL
-            ? TypeResolution.TYPE_RESOLVED
-            : new TypeResolution(
-                format(
-                    null,
-                    "{}argument of [{}] must be [{}], found value [{}] type [{}]",
-                    paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " ",
-                    operationName,
-                    acceptedTypesForErrorMsg(acceptedTypes),
-                    name(e),
-                    e.dataType().typeName()
-                )
-            );
+        return isType(e, predicate, operationName, paramOrd, false, acceptedTypes);
+    }
+
+    public static TypeResolution isTypeOrUnionType(
+        Expression e,
+        Predicate<DataType> predicate,
+        String operationName,
+        ParamOrdinal paramOrd,
+        String... acceptedTypes
+    ) {
+        return isType(e, predicate, operationName, paramOrd, true, acceptedTypes);
+    }
+
+    public static TypeResolution isType(
+        Expression e,
+        Predicate<DataType> predicate,
+        String operationName,
+        ParamOrdinal paramOrd,
+        boolean allowUnionTypes,
+        String... acceptedTypes
+    ) {
+        if (predicate.test(e.dataType()) || e.dataType() == NULL) {
+            return TypeResolution.TYPE_RESOLVED;
+        }
+
+        // TODO: Shouldn't we perform widening of small numerical types here?
+        if (allowUnionTypes
+            && e instanceof FieldAttribute fa
+            && fa.field() instanceof InvalidMappedField imf
+            && imf.types().stream().allMatch(predicate)) {
+            return TypeResolution.TYPE_RESOLVED;
+        }
+
+        return new TypeResolution(
+            errorStringIncompatibleTypes(operationName, paramOrd, name(e), e.dataType(), acceptedTypesForErrorMsg(acceptedTypes))
+        );
+    }
+
+    private static String errorStringIncompatibleTypes(
+        String operationName,
+        ParamOrdinal paramOrd,
+        String argumentName,
+        DataType foundType,
+        String... acceptedTypes
+    ) {
+        return format(
+            null,
+            "{}argument of [{}] must be [{}], found value [{}] type [{}]",
+            paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " ",
+            operationName,
+            acceptedTypesForErrorMsg(acceptedTypes),
+            argumentName,
+            foundType.typeName()
+        );
     }
 
     private static String acceptedTypesForErrorMsg(String... acceptedTypes) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java
index bf319856f9a93..0350abef99992 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java
@@ -15,16 +15,8 @@ public abstract class TypedAttribute extends Attribute {
 
     private final DataType dataType;
 
-    protected TypedAttribute(
-        Source source,
-        String name,
-        DataType dataType,
-        String qualifier,
-        Nullability nullability,
-        NameId id,
-        boolean synthetic
-    ) {
-        super(source, name, qualifier, nullability, id, synthetic);
+    protected TypedAttribute(Source source, String name, DataType dataType, Nullability nullability, NameId id, boolean synthetic) {
+        super(source, name, nullability, id, synthetic);
         this.dataType = dataType;
     }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java
index 87ef37cb84d1f..d8a35adcbffde 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java
@@ -28,26 +28,15 @@ public UnresolvedAttribute(Source source, String name) {
         this(source, name, null);
     }
 
-    public UnresolvedAttribute(Source source, String name, String qualifier) {
-        this(source, name, qualifier, null);
-    }
-
-    public UnresolvedAttribute(Source source, String name, String qualifier, String unresolvedMessage) {
-        this(source, name, qualifier, null, unresolvedMessage, null);
+    public UnresolvedAttribute(Source source, String name, String unresolvedMessage) {
+        this(source, name, null, unresolvedMessage, null);
     }
 
     @SuppressWarnings("this-escape")
-    public UnresolvedAttribute(
-        Source source,
-        String name,
-        String qualifier,
-        NameId id,
-        String unresolvedMessage,
-        Object resolutionMetadata
-    ) {
-        super(source, name, qualifier, id);
+    public UnresolvedAttribute(Source source, String name, NameId id, String unresolvedMessage, Object resolutionMetadata) {
+        super(source, name, id);
         this.customMessage = unresolvedMessage != null;
-        this.unresolvedMsg = unresolvedMessage == null ? errorMessage(qualifiedName(), null) : unresolvedMessage;
+        this.unresolvedMsg = unresolvedMessage == null ? errorMessage(name(), null) : unresolvedMessage;
         this.resolutionMetadata = resolutionMetadata;
     }
 
@@ -63,7 +52,7 @@ public String getWriteableName() {
 
     @Override
     protected NodeInfo<UnresolvedAttribute> info() {
-        return NodeInfo.create(this, UnresolvedAttribute::new, name(), qualifier(), id(), unresolvedMsg, resolutionMetadata);
+        return NodeInfo.create(this, UnresolvedAttribute::new, name(), id(), unresolvedMsg, resolutionMetadata);
     }
 
     public Object resolutionMetadata() {
@@ -80,20 +69,17 @@ public boolean resolved() {
     }
 
     @Override
-    protected Attribute clone(
-        Source source,
-        String name,
-        DataType dataType,
-        String qualifier,
-        Nullability nullability,
-        NameId id,
-        boolean synthetic
-    ) {
+    protected Attribute clone(Source source, String name, DataType dataType, Nullability nullability, NameId id, boolean synthetic) {
         return this;
     }
 
     public UnresolvedAttribute withUnresolvedMessage(String unresolvedMessage) {
-        return new UnresolvedAttribute(source(), name(), qualifier(), id(), unresolvedMessage, resolutionMetadata());
+        return new UnresolvedAttribute(source(), name(), id(), unresolvedMessage, resolutionMetadata());
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        return new TypeResolution("unresolved attribute [" + name() + "]");
     }
 
     @Override
@@ -103,7 +89,7 @@ public DataType dataType() {
 
     @Override
     public String toString() {
-        return UNRESOLVED_PREFIX + qualifiedName();
+        return UNRESOLVED_PREFIX + name();
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java
index f3b52cfcccf90..f4dd43cddf2d0 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java
@@ -79,7 +79,7 @@ public boolean equals(Object obj) {
     }
 
     private String message() {
-        return (qualifier() != null ? qualifier().qualifiedName() + "." : "") + "*";
+        return (qualifier() != null ? qualifier().name() + "." : "") + "*";
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java
index e734f97573c1c..2ee4866d35443 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DateUtils;
 
 import java.io.IOException;
 import java.time.DateTimeException;
@@ -22,6 +21,7 @@
 import java.util.Objects;
 
 import static java.util.Arrays.asList;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.asDateTime;
 
 // BETWEEN or range - is a mix of gt(e) AND lt(e)
 public class Range extends ScalarFunction {
@@ -118,10 +118,10 @@ protected boolean areBoundariesInvalid() {
         if (DataType.isDateTime(value.dataType()) || DataType.isDateTime(lower.dataType()) || DataType.isDateTime(upper.dataType())) {
             try {
                 if (upperValue instanceof String upperString) {
-                    upperValue = DateUtils.asDateTime(upperString);
+                    upperValue = asDateTime(upperString);
                 }
                 if (lowerValue instanceof String lowerString) {
-                    lowerValue = DateUtils.asDateTime(lowerString);
+                    lowerValue = asDateTime(lowerString);
                 }
             } catch (DateTimeException e) {
                 // one of the patterns is not a normal date, it could be a date math expression
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/EsIndex.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/EsIndex.java
deleted file mode 100644
index 3e6be6a1345d7..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/EsIndex.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.index;
-
-import org.elasticsearch.xpack.esql.core.type.EsField;
-
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-
-public class EsIndex {
-
-    private final String name;
-    private final Map<String, EsField> mapping;
-    private final Set<String> concreteIndices;
-
-    public EsIndex(String name, Map<String, EsField> mapping) {
-        this(name, mapping, Set.of());
-    }
-
-    public EsIndex(String name, Map<String, EsField> mapping, Set<String> concreteIndices) {
-        assert name != null;
-        assert mapping != null;
-        this.name = name;
-        this.mapping = mapping;
-        this.concreteIndices = concreteIndices;
-    }
-
-    public String name() {
-        return name;
-    }
-
-    public Map<String, EsField> mapping() {
-        return mapping;
-    }
-
-    public Set<String> concreteIndices() {
-        return concreteIndices;
-    }
-
-    @Override
-    public String toString() {
-        return name;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(name, mapping, concreteIndices);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-
-        if (obj == null || getClass() != obj.getClass()) {
-            return false;
-        }
-
-        EsIndex other = (EsIndex) obj;
-        return Objects.equals(name, other.name)
-            && Objects.equals(mapping, other.mapping)
-            && Objects.equals(concreteIndices, other.concreteIndices);
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
index 2df3a8eba46d5..176250222512b 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
@@ -7,12 +7,10 @@
 
 package org.elasticsearch.xpack.esql.core.planner;
 
-import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
-import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.StringQueryPredicate;
@@ -32,31 +30,17 @@
 import org.elasticsearch.xpack.esql.core.querydsl.query.NotQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery;
-import org.elasticsearch.xpack.esql.core.querydsl.query.RangeQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.RegexQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.Check;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 
-import java.time.OffsetTime;
-import java.time.ZonedDateTime;
-import java.time.temporal.TemporalAccessor;
 import java.util.Arrays;
 import java.util.List;
 
 public final class ExpressionTranslators {
 
-    public static final String DATE_FORMAT = "strict_date_optional_time_nanos";
-    public static final String TIME_FORMAT = "strict_hour_minute_second_fraction";
-
-    public static Object valueOf(Expression e) {
-        if (e.foldable()) {
-            return e.fold();
-        }
-        throw new QlIllegalArgumentException("Cannot determine value for {}", e);
-    }
-
     // TODO: see whether escaping is needed
     @SuppressWarnings("rawtypes")
     public static class Likes extends ExpressionTranslator<RegexMatch> {
@@ -193,55 +177,6 @@ private static Query translate(IsNull isNull, TranslatorHandler handler) {
         }
     }
 
-    public static class Ranges extends ExpressionTranslator<Range> {
-
-        @Override
-        protected Query asQuery(Range r, TranslatorHandler handler) {
-            return doTranslate(r, handler);
-        }
-
-        public static Query doTranslate(Range r, TranslatorHandler handler) {
-            return handler.wrapFunctionQuery(r, r.value(), () -> translate(r, handler));
-        }
-
-        private static RangeQuery translate(Range r, TranslatorHandler handler) {
-            Object lower = valueOf(r.lower());
-            Object upper = valueOf(r.upper());
-            String format = null;
-
-            // for a date constant comparison, we need to use a format for the date, to make sure that the format is the same
-            // no matter the timezone provided by the user
-            DateFormatter formatter = null;
-            if (lower instanceof ZonedDateTime || upper instanceof ZonedDateTime) {
-                formatter = DateFormatter.forPattern(DATE_FORMAT);
-            } else if (lower instanceof OffsetTime || upper instanceof OffsetTime) {
-                formatter = DateFormatter.forPattern(TIME_FORMAT);
-            }
-            if (formatter != null) {
-                // RangeQueryBuilder accepts an Object as its parameter, but it will call .toString() on the ZonedDateTime
-                // instance which can have a slightly different format depending on the ZoneId used to create the ZonedDateTime
-                // Since RangeQueryBuilder can handle date as String as well, we'll format it as String and provide the format.
-                if (lower instanceof ZonedDateTime || lower instanceof OffsetTime) {
-                    lower = formatter.format((TemporalAccessor) lower);
-                }
-                if (upper instanceof ZonedDateTime || upper instanceof OffsetTime) {
-                    upper = formatter.format((TemporalAccessor) upper);
-                }
-                format = formatter.pattern();
-            }
-            return new RangeQuery(
-                r.source(),
-                handler.nameOf(r.value()),
-                lower,
-                r.includeLower(),
-                upper,
-                r.includeUpper(),
-                format,
-                r.zoneId()
-            );
-        }
-    }
-
     public static Query or(Source source, Query left, Query right) {
         return boolQuery(source, left, right, false);
     }
@@ -260,8 +195,12 @@ private static Query boolQuery(Source source, Query left, Query right, boolean i
         }
         List<Query> queries;
         // check if either side is already a bool query to an extra bool query
-        if (left instanceof BoolQuery bool && bool.isAnd() == isAnd) {
-            queries = CollectionUtils.combine(bool.queries(), right);
+        if (left instanceof BoolQuery leftBool && leftBool.isAnd() == isAnd) {
+            if (right instanceof BoolQuery rightBool && rightBool.isAnd() == isAnd) {
+                queries = CollectionUtils.combine(leftBool.queries(), rightBool.queries());
+            } else {
+                queries = CollectionUtils.combine(leftBool.queries(), right);
+            }
         } else if (right instanceof BoolQuery bool && bool.isAnd() == isAnd) {
             queries = CollectionUtils.combine(bool.queries(), left);
         } else {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java
index 1b2a59b5c6886..639d8ed68d0a3 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java
@@ -7,9 +7,11 @@
 
 package org.elasticsearch.xpack.esql.core.plugin;
 
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.Plugin;
 
 public class EsqlCorePlugin extends Plugin implements ExtensiblePlugin {
+    public static final FeatureFlag DATE_NANOS_FEATURE_FLAG = new FeatureFlag("esql_date_nanos");
 
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/container/AttributeSort.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/container/AttributeSort.java
deleted file mode 100644
index 7c87ee2d2959f..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/container/AttributeSort.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.querydsl.container;
-
-import org.elasticsearch.xpack.esql.core.expression.Attribute;
-
-import java.util.Objects;
-
-public class AttributeSort extends Sort {
-
-    private final Attribute attribute;
-
-    public AttributeSort(Attribute attribute, Direction direction, Missing missing) {
-        super(direction, missing);
-        this.attribute = attribute;
-    }
-
-    public Attribute attribute() {
-        return attribute;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(attribute, direction(), missing());
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-
-        if (obj == null || getClass() != obj.getClass()) {
-            return false;
-        }
-
-        AttributeSort other = (AttributeSort) obj;
-        return Objects.equals(direction(), other.direction())
-            && Objects.equals(missing(), other.missing())
-            && Objects.equals(attribute, other.attribute);
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/container/Sort.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/container/Sort.java
deleted file mode 100644
index e6b3926745ea1..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/container/Sort.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.querydsl.container;
-
-import org.elasticsearch.search.aggregations.bucket.composite.MissingOrder;
-import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition;
-import org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection;
-
-public abstract class Sort {
-
-    public enum Direction {
-        ASC,
-        DESC;
-
-        public static Direction from(OrderDirection dir) {
-            return dir == null || dir == OrderDirection.ASC ? ASC : DESC;
-        }
-
-        public SortOrder asOrder() {
-            return this == Direction.ASC ? SortOrder.ASC : SortOrder.DESC;
-        }
-    }
-
-    public enum Missing {
-        FIRST("_first", MissingOrder.FIRST),
-        LAST("_last", MissingOrder.LAST),
-        /**
-         * Nulls position has not been specified by the user and an appropriate default will be used.
-         *
-         * The default values are chosen such that it stays compatible with previous behavior. Unfortunately, this results in
-         * inconsistencies across different types of queries (see https://github.com/elastic/elasticsearch/issues/77068).
-         */
-        ANY(null, null);
-
-        private final String searchOrder;
-        private final MissingOrder aggregationOrder;
-
-        Missing(String searchOrder, MissingOrder aggregationOrder) {
-            this.searchOrder = searchOrder;
-            this.aggregationOrder = aggregationOrder;
-        }
-
-        public static Missing from(NullsPosition pos) {
-            return switch (pos) {
-                case FIRST -> FIRST;
-                case LAST -> LAST;
-                default -> ANY;
-            };
-        }
-
-        public String searchOrder() {
-            return searchOrder(null);
-        }
-
-        /**
-         * Preferred order of null values in non-aggregation queries.
-         */
-        public String searchOrder(Direction fallbackDirection) {
-            if (searchOrder != null) {
-                return searchOrder;
-            } else {
-                return switch (fallbackDirection) {
-                    case ASC -> LAST.searchOrder;
-                    case DESC -> FIRST.searchOrder;
-                };
-            }
-        }
-
-        /**
-         * Preferred order of null values in aggregation queries.
-         */
-        public MissingOrder aggregationOrder() {
-            return aggregationOrder;
-        }
-    }
-
-    private final Direction direction;
-    private final Missing missing;
-
-    protected Sort(Direction direction, Missing nulls) {
-        this.direction = direction;
-        this.missing = nulls;
-    }
-
-    public Direction direction() {
-        return direction;
-    }
-
-    public Missing missing() {
-        return missing;
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/session/Configuration.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/session/Configuration.java
deleted file mode 100644
index b671edf685b58..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/session/Configuration.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core.session;
-
-import java.time.Clock;
-import java.time.Duration;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.util.Objects;
-
-public class Configuration {
-
-    protected final String clusterName;
-    protected final String username;
-    protected final ZonedDateTime now;
-    protected final ZoneId zoneId;
-
-    public Configuration(ZoneId zi, String username, String clusterName) {
-        this(zi, null, username, clusterName);
-    }
-
-    protected Configuration(ZoneId zi, Instant now, String username, String clusterName) {
-        this.zoneId = zi.normalized();
-        this.now = now != null ? now.atZone(zi) : ZonedDateTime.now(Clock.tick(Clock.system(zoneId), Duration.ofNanos(1)));
-        this.username = username;
-        this.clusterName = clusterName;
-    }
-
-    public ZoneId zoneId() {
-        return zoneId;
-    }
-
-    public ZonedDateTime now() {
-        return now;
-    }
-
-    public String clusterName() {
-        return clusterName;
-    }
-
-    public String username() {
-        return username;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        Configuration that = (Configuration) o;
-        return Objects.equals(zoneId, that.zoneId)
-            && Objects.equals(now, that.now)
-            && Objects.equals(username, that.username)
-            && Objects.equals(clusterName, that.clusterName);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(zoneId, now, username, clusterName);
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Node.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Node.java
index b1fc7d59c784d..feeba39756373 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Node.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Node.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.esql.core.tree;
 
+import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 
 import java.util.ArrayList;
@@ -32,7 +33,7 @@
  *
  * @param <T> node type
  */
-public abstract class Node<T extends Node<T>> {
+public abstract class Node<T extends Node<T>> implements NamedWriteable {
     private static final int TO_STRING_MAX_PROP = 10;
     private static final int TO_STRING_MAX_WIDTH = 110;
 
@@ -265,11 +266,11 @@ protected final <E> T transformNodeProps(Class<E> typeToken, Function<? super E,
      * <p>
      * Normally, you want to use one of the static {@code create} methods to implement this.
      * <p>
-     * For {@link org.elasticsearch.xpack.esql.core.plan.QueryPlan}s, it is very important that
+     * For {@code QueryPlan}s, it is very important that
      * the properties contain all of the expressions and references relevant to this node, and
      * that all of the properties are used in the provided constructor; otherwise query plan
      * transformations like
-     * {@link org.elasticsearch.xpack.esql.core.plan.QueryPlan#transformExpressionsOnly(Function)}
+     * {@code QueryPlan#transformExpressionsOnly(Function)}
      * will not have an effect.
      */
     protected abstract NodeInfo<? extends T> info();
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java
index e53593e944632..1b00ff4537603 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java
@@ -42,13 +42,24 @@ public static <S extends StreamInput & PlanStreamInput> Source readFrom(S in) th
         if (in.readBoolean() == false) {
             return EMPTY;
         }
-        int line = in.readInt();
-        int column = in.readInt();
-        int charPositionInLine = column - 1;
+        SourcePositions positions = new SourcePositions(in);
+        int charPositionInLine = positions.column - 1;
 
-        int length = in.readInt();
-        String text = sourceText(in.sourceText(), line, column, length);
-        return new Source(new Location(line, charPositionInLine), text);
+        String text = sourceText(in.sourceText(), positions.line, positions.column, positions.length);
+        return new Source(new Location(positions.line, charPositionInLine), text);
+    }
+
+    /**
+     * Read the components of a {@link Source} and throw it away, returning
+     * {@link Source#EMPTY}. Use this when you will never use the {@link Source}
+     * and there is no chance of getting a {@link PlanStreamInput}.
+     */
+    public static Source readEmpty(StreamInput in) throws IOException {
+        if (in.readBoolean()) {
+            // Read it and throw it away because we're always returning empty.
+            new SourcePositions(in);
+        }
+        return EMPTY;
     }
 
     @Override
@@ -58,9 +69,7 @@ public void writeTo(StreamOutput out) throws IOException {
             return;
         }
         out.writeBoolean(true);
-        out.writeInt(location.getLineNumber());
-        out.writeInt(location.getColumnNumber());
-        out.writeInt(text.length());
+        new SourcePositions(location.getLineNumber(), location.getColumnNumber(), text.length()).writeTo(out);
     }
 
     // TODO: rename to location()
@@ -130,4 +139,19 @@ private static int textOffset(String query, int line, int column) {
         return offset;
     }
 
+    /**
+     * Offsets into the source string that we use for serialization.
+     */
+    private record SourcePositions(int line, int column, int length) implements Writeable {
+        SourcePositions(StreamInput in) throws IOException {
+            this(in.readInt(), in.readInt(), in.readInt());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeInt(line);
+            out.writeInt(column);
+            out.writeInt(length);
+        }
+    }
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
index f7f9d238ec5ce..979368c300e00 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
@@ -9,7 +9,11 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.FeatureFlag;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
+import org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin;
 
 import java.io.IOException;
 import java.math.BigInteger;
@@ -28,40 +32,133 @@
 import static java.util.stream.Collectors.toUnmodifiableMap;
 
 public enum DataType {
+    /**
+     * Fields of this type are unsupported by any functions and are always
+     * rendered as {@code null} in the response.
+     */
     UNSUPPORTED(builder().typeName("UNSUPPORTED").unknownSize()),
+    /**
+     * Fields that are always {@code null}, usually created with constant
+     * {@code null} values.
+     */
     NULL(builder().esType("null").estimatedSize(0)),
+    /**
+     * Fields that can either be {@code true} or {@code false}.
+     */
     BOOLEAN(builder().esType("boolean").estimatedSize(1)),
 
     /**
-     * These are numeric fields labeled as metric counters in time-series indices. Although stored
-     * internally as numeric fields, they represent cumulative metrics and must not be treated as regular
-     * numeric fields. Therefore, we define them differently and separately from their parent numeric field.
-     * These fields are strictly for use in retrieval from indices, rate aggregation, and casting to their
-     * parent numeric type.
+     * 64-bit signed numbers labeled as metric counters in time-series indices.
+     * Although stored internally as numeric fields, they represent cumulative
+     * metrics and must not be treated as regular numeric fields. Therefore,
+     * we define them differently and separately from their parent numeric field.
+     * These fields are strictly for use in retrieval from indices, rate
+     * aggregation, and casting to their parent numeric type.
      */
     COUNTER_LONG(builder().esType("counter_long").estimatedSize(Long.BYTES).docValues().counter()),
+    /**
+     * 32-bit signed numbers labeled as metric counters in time-series indices.
+     * Although stored internally as numeric fields, they represent cumulative
+     * metrics and must not be treated as regular numeric fields. Therefore,
+     * we define them differently and separately from their parent numeric field.
+     * These fields are strictly for use in retrieval from indices, rate
+     * aggregation, and casting to their parent numeric type.
+     */
     COUNTER_INTEGER(builder().esType("counter_integer").estimatedSize(Integer.BYTES).docValues().counter()),
+    /**
+     * 64-bit floating point numbers labeled as metric counters in time-series indices.
+     * Although stored internally as numeric fields, they represent cumulative
+     * metrics and must not be treated as regular numeric fields. Therefore,
+     * we define them differently and separately from their parent numeric field.
+     * These fields are strictly for use in retrieval from indices, rate
+     * aggregation, and casting to their parent numeric type.
+     */
     COUNTER_DOUBLE(builder().esType("counter_double").estimatedSize(Double.BYTES).docValues().counter()),
 
+    /**
+     * 64-bit signed numbers loaded as a java {@code long}.
+     */
     LONG(builder().esType("long").estimatedSize(Long.BYTES).wholeNumber().docValues().counter(COUNTER_LONG)),
+    /**
+     * 32-bit signed numbers loaded as a java {@code int}.
+     */
     INTEGER(builder().esType("integer").estimatedSize(Integer.BYTES).wholeNumber().docValues().counter(COUNTER_INTEGER)),
-    SHORT(builder().esType("short").estimatedSize(Short.BYTES).wholeNumber().docValues().widenSmallNumeric(INTEGER)),
-    BYTE(builder().esType("byte").estimatedSize(Byte.BYTES).wholeNumber().docValues().widenSmallNumeric(INTEGER)),
+    /**
+     * 64-bit unsigned numbers packed into a java {@code long}.
+     */
     UNSIGNED_LONG(builder().esType("unsigned_long").estimatedSize(Long.BYTES).wholeNumber().docValues()),
+    /**
+     * 64-bit floating point number loaded as a java {@code double}.
+     */
     DOUBLE(builder().esType("double").estimatedSize(Double.BYTES).rationalNumber().docValues().counter(COUNTER_DOUBLE)),
+
+    /**
+     * 16-bit signed numbers widened on load to {@link #INTEGER}.
+     * Values of this type never escape type resolution and functions,
+     * operators, and results should never encounter one.
+     */
+    SHORT(builder().esType("short").estimatedSize(Short.BYTES).wholeNumber().docValues().widenSmallNumeric(INTEGER)),
+    /**
+     * 8-bit signed numbers widened on load to {@link #INTEGER}.
+     * Values of this type never escape type resolution and functions,
+     * operators, and results should never encounter one.
+     */
+    BYTE(builder().esType("byte").estimatedSize(Byte.BYTES).wholeNumber().docValues().widenSmallNumeric(INTEGER)),
+    /**
+     * 32-bit floating point numbers widened on load to {@link #DOUBLE}.
+     * Values of this type never escape type resolution and functions,
+     * operators, and results should never encounter one.
+     */
     FLOAT(builder().esType("float").estimatedSize(Float.BYTES).rationalNumber().docValues().widenSmallNumeric(DOUBLE)),
+    /**
+     * 16-bit floating point numbers widened on load to {@link #DOUBLE}.
+     * Values of this type never escape type resolution and functions,
+     * operators, and results should never encounter one.
+     */
     HALF_FLOAT(builder().esType("half_float").estimatedSize(Float.BYTES).rationalNumber().docValues().widenSmallNumeric(DOUBLE)),
+    /**
+     * Signed 64-bit fixed point numbers converted on load to a {@link #DOUBLE}.
+     * Values of this type never escape type resolution and functions, operators,
+     * and results should never encounter one.
+     */
     SCALED_FLOAT(builder().esType("scaled_float").estimatedSize(Long.BYTES).rationalNumber().docValues().widenSmallNumeric(DOUBLE)),
 
+    /**
+     * String fields that are analyzed when the document is received but never
+     * cut into more than one token. ESQL always loads these after-analysis.
+     * Generally ESQL uses {@code keyword} fields as raw strings. So things like
+     * {@code TO_STRING} will make a {@code keyword} field.
+     */
     KEYWORD(builder().esType("keyword").unknownSize().docValues()),
+    /**
+     * String fields that are analyzed when the document is received and may be
+     * cut into more than one token. Generally ESQL only sees {@code text} fields
+     * when loaded from the index and ESQL will load these fields
+     * <strong>without</strong> analysis. The {@code MATCH} operator can be used
+     * to query these fields with analysis.
+     */
     TEXT(builder().esType("text").unknownSize()),
+    /**
+     * Millisecond precision date, stored as a 64-bit signed number.
+     */
     DATETIME(builder().esType("date").typeName("DATETIME").estimatedSize(Long.BYTES).docValues()),
-    // IP addresses, both IPv4 and IPv6, are encoded using 16 bytes.
+    /**
+     * Nanosecond precision date, stored as a 64-bit signed number.
+     */
+    DATE_NANOS(builder().esType("date_nanos").estimatedSize(Long.BYTES).docValues()),
+    /**
+     * IP addresses. IPv4 address are always
+     * <a href="https://datatracker.ietf.org/doc/html/rfc4291#section-2.5.5">embedded</a>
+     * in IPv6. These flow through the compute engine as fixed length, 16 byte
+     * {@link BytesRef}s.
+     */
     IP(builder().esType("ip").estimatedSize(16).docValues()),
+    /**
+     * A version encoded in a way that sorts using semver.
+     */
     // 8.15.2-SNAPSHOT is 15 bytes, most are shorter, some can be longer
     VERSION(builder().esType("version").estimatedSize(15).docValues()),
     OBJECT(builder().esType("object").unknownSize()),
-    NESTED(builder().esType("nested").unknownSize()),
     SOURCE(builder().esType(SourceFieldMapper.NAME).unknownSize()),
     DATE_PERIOD(builder().typeName("DATE_PERIOD").estimatedSize(3 * Integer.BYTES)),
     TIME_DURATION(builder().typeName("TIME_DURATION").estimatedSize(Integer.BYTES + Long.BYTES)),
@@ -72,10 +169,41 @@ public enum DataType {
     CARTESIAN_SHAPE(builder().esType("cartesian_shape").estimatedSize(200).docValues()),
     GEO_SHAPE(builder().esType("geo_shape").estimatedSize(200).docValues()),
 
+    /**
+     * Fields with this type represent a Lucene doc id. This field is a bit magic in that:
+     * <ul>
+     *     <li>One copy of it is always added at the start of every query</li>
+     *     <li>It is implicitly dropped before being returned to the user</li>
+     *     <li>It is not "target-able" by any functions</li>
+     *     <li>Users shouldn't know it's there at all</li>
+     *     <li>It is used as an input for things that interact with Lucene like
+     *         loading field values</li>
+     * </ul>
+     */
     DOC_DATA_TYPE(builder().esType("_doc").estimatedSize(Integer.BYTES * 3)),
+    /**
+     * Fields with this type represent values from the {@link TimeSeriesIdFieldMapper}.
+     * Every document in {@link IndexMode#TIME_SERIES} index will have a single value
+     * for this field and the segments themselves are sorted on this value.
+     */
     TSID_DATA_TYPE(builder().esType("_tsid").unknownSize().docValues()),
+    /**
+     * Fields with this type are the partial result of running a non-time-series aggregation
+     * inside alongside time-series aggregations. These fields are not parsable from the
+     * mapping and should be hidden from users.
+     */
     PARTIAL_AGG(builder().esType("partial_agg").unknownSize());
 
+    /**
+     * Types that are actively being built. These types are not returned
+     * from Elasticsearch if their associated {@link FeatureFlag} is disabled.
+     * They aren't included in generated documentation. And the tests don't
+     * check that sending them to a function produces a sane error message.
+     */
+    public static final Map<DataType, FeatureFlag> UNDER_CONSTRUCTION = Map.ofEntries(
+        Map.entry(DATE_NANOS, EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG)
+    );
+
     private final String typeName;
 
     private final String name;
@@ -162,13 +290,25 @@ public static Collection<DataType> types() {
         return TYPES;
     }
 
+    /**
+     * Resolve a type from a name. This name is sometimes user supplied,
+     * like in the case of {@code ::<typename>} and is sometimes the name
+     * used over the wire, like in {@link #readFrom(String)}.
+     */
     public static DataType fromTypeName(String name) {
         return NAME_TO_TYPE.get(name.toLowerCase(Locale.ROOT));
     }
 
     public static DataType fromEs(String name) {
         DataType type = ES_TO_TYPE.get(name);
-        return type != null ? type : UNSUPPORTED;
+        if (type == null) {
+            return UNSUPPORTED;
+        }
+        FeatureFlag underConstruction = UNDER_CONSTRUCTION.get(type);
+        if (underConstruction != null && underConstruction.isEnabled() == false) {
+            return UNSUPPORTED;
+        }
+        return type;
     }
 
     public static DataType fromJava(Object value) {
@@ -217,8 +357,12 @@ public static boolean isString(DataType t) {
         return t == KEYWORD || t == TEXT;
     }
 
+    public static boolean isPrimitiveAndSupported(DataType t) {
+        return isPrimitive(t) && t != UNSUPPORTED;
+    }
+
     public static boolean isPrimitive(DataType t) {
-        return t != OBJECT && t != NESTED && t != UNSUPPORTED;
+        return t != OBJECT;
     }
 
     public static boolean isNull(DataType t) {
@@ -233,6 +377,64 @@ public static boolean isDateTime(DataType type) {
         return type == DATETIME;
     }
 
+    public static boolean isNullOrTimeDuration(DataType t) {
+        return t == TIME_DURATION || isNull(t);
+    }
+
+    public static boolean isNullOrDatePeriod(DataType t) {
+        return t == DATE_PERIOD || isNull(t);
+    }
+
+    public static boolean isTemporalAmount(DataType t) {
+        return t == DATE_PERIOD || t == TIME_DURATION;
+    }
+
+    public static boolean isNullOrTemporalAmount(DataType t) {
+        return isTemporalAmount(t) || isNull(t);
+    }
+
+    public static boolean isDateTimeOrTemporal(DataType t) {
+        return isDateTime(t) || isTemporalAmount(t);
+    }
+
+    public static boolean areCompatible(DataType left, DataType right) {
+        if (left == right) {
+            return true;
+        } else {
+            return (left == NULL || right == NULL) || (isString(left) && isString(right)) || (left.isNumeric() && right.isNumeric());
+        }
+    }
+
+    /**
+     * Supported types that can be contained in a block.
+     */
+    public static boolean isRepresentable(DataType t) {
+        return t != OBJECT
+            && t != UNSUPPORTED
+            && t != DATE_PERIOD
+            && t != TIME_DURATION
+            && t != BYTE
+            && t != SHORT
+            && t != FLOAT
+            && t != SCALED_FLOAT
+            && t != SOURCE
+            && t != HALF_FLOAT
+            && t != PARTIAL_AGG
+            && t.isCounter() == false;
+    }
+
+    public static boolean isSpatialPoint(DataType t) {
+        return t == GEO_POINT || t == CARTESIAN_POINT;
+    }
+
+    public static boolean isSpatialGeo(DataType t) {
+        return t == GEO_POINT || t == GEO_SHAPE;
+    }
+
+    public static boolean isSpatial(DataType t) {
+        return t == GEO_POINT || t == CARTESIAN_POINT || t == GEO_SHAPE || t == CARTESIAN_SHAPE;
+    }
+
     public String nameUpper() {
         return name;
     }
@@ -245,6 +447,14 @@ public String esType() {
         return esType;
     }
 
+    /**
+     * Return the Elasticsearch field name of this type if there is one,
+     * otherwise return the ESQL specific name.
+     */
+    public String esNameIfPossible() {
+        return esType != null ? esType : typeName;
+    }
+
     /**
      * The name we give to types on the response.
      */
@@ -314,8 +524,20 @@ public void writeTo(StreamOutput out) throws IOException {
 
     public static DataType readFrom(StreamInput in) throws IOException {
         // TODO: Use our normal enum serialization pattern
-        String name = in.readString();
+        return readFrom(in.readString());
+    }
+
+    /**
+     * Resolve a {@link DataType} from a name read from a {@link StreamInput}.
+     * @throws IOException on an unknown dataType
+     */
+    public static DataType readFrom(String name) throws IOException {
         if (name.equalsIgnoreCase(DataType.DOC_DATA_TYPE.nameUpper())) {
+            /*
+             * DOC is not declared in fromTypeName because fromTypeName is
+             * exposed to users for things like `::<typename>` and we don't
+             * want folks to be able to convert to `DOC`.
+             */
             return DataType.DOC_DATA_TYPE;
         }
         DataType dataType = DataType.fromTypeName(name);
@@ -371,7 +593,8 @@ private static class Builder {
 
         /**
          * If this is a "small" numeric type this contains the type ESQL will
-         * widen it into, otherwise this is {@code null}.
+         * widen it into, otherwise this is {@code null}. "Small" numeric types
+         * aren't supported by ESQL proper and are "widened" on load.
          */
         private DataType widenSmallNumeric;
 
@@ -423,6 +646,11 @@ Builder counter() {
             return this;
         }
 
+        /**
+         * If this is a "small" numeric type this contains the type ESQL will
+         * widen it into, otherwise this is {@code null}. "Small" numeric types
+         * aren't supported by ESQL proper and are "widened" on load.
+         */
         Builder widenSmallNumeric(DataType widenSmallNumeric) {
             this.widenSmallNumeric = widenSmallNumeric;
             return this;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java
index e2d6f79a873c9..78b395503e700 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
+import org.elasticsearch.xpack.esql.core.util.DateUtils;
 import org.elasticsearch.xpack.versionfield.Version;
 
 import java.io.IOException;
@@ -37,7 +38,6 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime;
-import static org.elasticsearch.xpack.esql.core.type.DataType.isPrimitive;
 import static org.elasticsearch.xpack.esql.core.type.DataType.isString;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.UNSIGNED_LONG_MAX;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.inUnsignedLongRange;
@@ -50,85 +50,6 @@ public final class DataTypeConverter {
 
     private DataTypeConverter() {}
 
-    /**
-     * Returns the type compatible with both left and right types
-     * <p>
-     * If one of the types is null - returns another type
-     * If both types are numeric - returns type with the highest precision int &lt; long &lt; float &lt; double
-     * If one of the types is string and another numeric - returns numeric
-     */
-    public static DataType commonType(DataType left, DataType right) {
-        if (left == right) {
-            return left;
-        }
-        if (left == NULL) {
-            return right;
-        }
-        if (right == NULL) {
-            return left;
-        }
-        if (isString(left) && isString(right)) {
-            if (left == TEXT || right == TEXT) {
-                return TEXT;
-            }
-            if (left == KEYWORD) {
-                return KEYWORD;
-            }
-            return right;
-        }
-        if (left.isNumeric() && right.isNumeric()) {
-            int lsize = left.estimatedSize().orElseThrow();
-            int rsize = right.estimatedSize().orElseThrow();
-            // if one is int
-            if (left.isWholeNumber()) {
-                // promote the highest int
-                if (right.isWholeNumber()) {
-                    if (left == UNSIGNED_LONG || right == UNSIGNED_LONG) {
-                        return UNSIGNED_LONG;
-                    }
-                    return lsize > rsize ? left : right;
-                }
-                // promote the rational
-                return right;
-            }
-            // try the other side
-            if (right.isWholeNumber()) {
-                return left;
-            }
-            // promote the highest rational
-            return lsize > rsize ? left : right;
-        }
-        if (isString(left)) {
-            if (right.isNumeric()) {
-                return right;
-            }
-        }
-        if (isString(right)) {
-            if (left.isNumeric()) {
-                return left;
-            }
-        }
-
-        if (isDateTime(left) && isDateTime(right)) {
-            return DATETIME;
-        }
-
-        // none found
-        return null;
-    }
-
-    /**
-     * Returns true if the from type can be converted to the to type, false - otherwise
-     */
-    public static boolean canConvert(DataType from, DataType to) {
-        // Special handling for nulls and if conversion is not requires
-        if (from == to || from == NULL) {
-            return true;
-        }
-        // only primitives are supported so far
-        return isPrimitive(from) && isPrimitive(to) && converterFor(from, to) != null;
-    }
-
     /**
      * Get the conversion from one type to another.
      */
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeRegistry.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeRegistry.java
deleted file mode 100644
index b96c187deeb33..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeRegistry.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core.type;
-
-import org.elasticsearch.index.mapper.TimeSeriesParams;
-
-import java.util.Collection;
-
-/**
- * Central class for {@link DataType} creation and conversion.
- */
-public interface DataTypeRegistry {
-
-    //
-    // Discovery
-    //
-    Collection<DataType> dataTypes();
-
-    DataType fromEs(String typeName, TimeSeriesParams.MetricType metricType);
-
-    DataType fromJava(Object value);
-
-    boolean isUnsupported(DataType type);
-
-    //
-    // Conversion methods
-    //
-    boolean canConvert(DataType from, DataType to);
-
-    Object convert(Object value, DataType type);
-
-    DataType commonType(DataType left, DataType right);
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
index 01728954a2e1b..f829bcdea94e4 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.esql.core.type;
 
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
@@ -17,7 +16,6 @@
  * Information about a field in an ES index with the {@code date} type
  */
 public class DateEsField extends EsField {
-    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(EsField.class, "DateEsField", DateEsField::new);
 
     public static DateEsField dateEsField(String name, Map<String, EsField> properties, boolean hasDocValues) {
         return new DateEsField(name, DataType.DATETIME, properties, hasDocValues);
@@ -27,19 +25,19 @@ private DateEsField(String name, DataType dataType, Map<String, EsField> propert
         super(name, dataType, properties, hasDocValues);
     }
 
-    private DateEsField(StreamInput in) throws IOException {
-        this(in.readString(), DataType.DATETIME, in.readMap(i -> i.readNamedWriteable(EsField.class)), in.readBoolean());
+    protected DateEsField(StreamInput in) throws IOException {
+        this(in.readString(), DataType.DATETIME, in.readImmutableMap(EsField::readFrom), in.readBoolean());
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    protected void writeContent(StreamOutput out) throws IOException {
         out.writeString(getName());
-        out.writeMap(getProperties(), StreamOutput::writeNamedWriteable);
+        out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
         out.writeBoolean(isAggregatable());
     }
 
-    @Override
     public String getWriteableName() {
-        return ENTRY.name;
+        return "DateEsField";
     }
+
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateUtils.java
deleted file mode 100644
index 29d96dcb8201a..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateUtils.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core.type;
-
-import org.elasticsearch.common.time.DateFormatters;
-
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
-import java.time.format.DateTimeFormatterBuilder;
-import java.util.Locale;
-
-import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE;
-import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME;
-
-//NB: Taken from sql-proto.
-public final class DateUtils {
-
-    public static final ZoneId UTC = ZoneId.of("Z");
-
-    private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder().append(ISO_LOCAL_DATE)
-        .optionalStart()
-        .appendLiteral(' ')
-        .append(ISO_LOCAL_TIME)
-        .optionalStart()
-        .appendZoneOrOffsetId()
-        .optionalEnd()
-        .toFormatter(Locale.ROOT)
-        .withZone(UTC);
-    private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder().append(ISO_LOCAL_DATE)
-        .optionalStart()
-        .appendLiteral('T')
-        .append(ISO_LOCAL_TIME)
-        .optionalStart()
-        .appendZoneOrOffsetId()
-        .optionalEnd()
-        .toFormatter(Locale.ROOT)
-        .withZone(UTC);
-
-    private DateUtils() {}
-
-    /**
-     * Creates a datetime from the millis since epoch (thus the time-zone is UTC).
-     */
-    public static ZonedDateTime asDateTime(long millis) {
-        return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), UTC);
-    }
-
-    public static long asMillis(ZonedDateTime zonedDateTime) {
-        return zonedDateTime.toInstant().toEpochMilli();
-    }
-
-    /**
-     * Parses the given string into a DateTime using UTC as a default timezone.
-     */
-    public static ZonedDateTime asDateTime(String dateFormat) {
-        int separatorIdx = dateFormat.indexOf('-'); // Find the first `-` date separator
-        if (separatorIdx == 0) { // first char = `-` denotes a negative year
-            separatorIdx = dateFormat.indexOf('-', 1); // Find the first `-` date separator past the negative year
-        }
-        // Find the second `-` date separator and move 3 places past the dayOfYear to find the time separator
-        // e.g. 2020-06-01T10:20:30....
-        // ^
-        // +3 = ^
-        separatorIdx = dateFormat.indexOf('-', separatorIdx + 1) + 3;
-
-        // Avoid index out of bounds - it will lead to DateTimeParseException anyways
-        if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
-            return DateFormatters.from(DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL.parse(dateFormat)).withZoneSameInstant(UTC);
-        } else {
-            return DateFormatters.from(DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE.parse(dateFormat)).withZoneSameInstant(UTC);
-        }
-    }
-
-    public static String toString(ZonedDateTime dateTime) {
-        return StringUtils.toString(dateTime);
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DefaultDataTypeRegistry.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DefaultDataTypeRegistry.java
deleted file mode 100644
index 9bba9698faff7..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DefaultDataTypeRegistry.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core.type;
-
-import org.elasticsearch.index.mapper.TimeSeriesParams;
-
-import java.util.Collection;
-
-public class DefaultDataTypeRegistry implements DataTypeRegistry {
-
-    public static final DataTypeRegistry INSTANCE = new DefaultDataTypeRegistry();
-
-    private DefaultDataTypeRegistry() {}
-
-    @Override
-    public Collection<DataType> dataTypes() {
-        return DataType.types();
-    }
-
-    @Override
-    public DataType fromEs(String typeName, TimeSeriesParams.MetricType metricType) {
-        return DataType.fromEs(typeName);
-    }
-
-    @Override
-    public DataType fromJava(Object value) {
-        return DataType.fromJava(value);
-    }
-
-    @Override
-    public boolean isUnsupported(DataType type) {
-        return DataType.isUnsupported(type);
-    }
-
-    @Override
-    public boolean canConvert(DataType from, DataType to) {
-        return DataTypeConverter.canConvert(from, to);
-    }
-
-    @Override
-    public Object convert(Object value, DataType type) {
-        return DataTypeConverter.convert(value, type);
-    }
-
-    @Override
-    public DataType commonType(DataType left, DataType right) {
-        return DataTypeConverter.commonType(left, right);
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java
index 4ef20a724ab3c..899986fecd012 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java
@@ -6,34 +6,41 @@
  */
 package org.elasticsearch.xpack.esql.core.type;
 
-import org.elasticsearch.common.io.stream.NamedWriteable;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
 /**
  * Information about a field in an ES index.
  */
-public class EsField implements NamedWriteable {
-    public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
-        return List.of(
-            EsField.ENTRY,
-            DateEsField.ENTRY,
-            InvalidMappedField.ENTRY,
-            KeywordEsField.ENTRY,
-            TextEsField.ENTRY,
-            UnsupportedEsField.ENTRY
-        );
+public class EsField implements Writeable {
+
+    private static Map<String, Writeable.Reader<? extends EsField>> readers = Map.ofEntries(
+        Map.entry("EsField", EsField::new),
+        Map.entry("DateEsField", DateEsField::new),
+        Map.entry("InvalidMappedField", InvalidMappedField::new),
+        Map.entry("KeywordEsField", KeywordEsField::new),
+        Map.entry("MultiTypeEsField", MultiTypeEsField::new),
+        Map.entry("TextEsField", TextEsField::new),
+        Map.entry("UnsupportedEsField", UnsupportedEsField::new)
+    );
+
+    public static Writeable.Reader<? extends EsField> getReader(String name) {
+        Reader<? extends EsField> result = readers.get(name);
+        if (result == null) {
+            throw new IllegalArgumentException("Invalid EsField type [" + name + "]");
+        }
+        return result;
     }
 
-    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(EsField.class, "EsField", EsField::new);
-
     private final DataType esDataType;
     private final boolean aggregatable;
     private final Map<String, EsField> properties;
@@ -52,26 +59,57 @@ public EsField(String name, DataType esDataType, Map<String, EsField> properties
         this.isAlias = isAlias;
     }
 
-    public EsField(StreamInput in) throws IOException {
+    protected EsField(StreamInput in) throws IOException {
         this.name = in.readString();
-        this.esDataType = DataType.readFrom(in);
-        this.properties = in.readImmutableMap(i -> i.readNamedWriteable(EsField.class));
+        this.esDataType = readDataType(in);
+        this.properties = in.readImmutableMap(EsField::readFrom);
         this.aggregatable = in.readBoolean();
         this.isAlias = in.readBoolean();
     }
 
+    private DataType readDataType(StreamInput in) throws IOException {
+        String name = in.readString();
+        if (in.getTransportVersion().before(TransportVersions.ESQL_NESTED_UNSUPPORTED) && name.equalsIgnoreCase("NESTED")) {
+            /*
+             * The "nested" data type existed in older versions of ESQL but was
+             * entirely used to filter mappings away. Those versions will still
+             * sometimes send it inside EsField when hitting `nested` fields in
+             * indices. But the rest of ESQL will never see that type. Thus, we
+             * translate it here. We translate to UNSUPPORTED because that seems
+             * to work. We've already performed any required filtering.
+             */
+            return DataType.UNSUPPORTED;
+        }
+        return DataType.readFrom(name);
+    }
+
+    public static <A extends EsField> A readFrom(StreamInput in) throws IOException {
+        return ((PlanStreamInput) in).readEsFieldWithCache();
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
+        if (((PlanStreamOutput) out).writeEsFieldCacheHeader(this)) {
+            writeContent(out);
+        }
+    }
+
+    /**
+     * This needs to be overridden by subclasses for specific serialization
+     */
+    protected void writeContent(StreamOutput out) throws IOException {
         out.writeString(name);
-        out.writeString(esDataType.typeName());
-        out.writeMap(properties, StreamOutput::writeNamedWriteable);
+        esDataType.writeTo(out);
+        out.writeMap(properties, (o, x) -> x.writeTo(out));
         out.writeBoolean(aggregatable);
         out.writeBoolean(isAlias);
     }
 
-    @Override
+    /**
+     * This needs to be overridden by subclasses for specific serialization
+     */
     public String getWriteableName() {
-        return ENTRY.name;
+        return "EsField";
     }
 
     /**
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
index 9b088cfb19f6c..d34af0f8565c7 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.esql.core.type;
 
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
@@ -17,6 +16,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.stream.Collectors;
 
 /**
  * Representation of field mapped differently across indices.
@@ -26,11 +26,6 @@
  * It is used specifically for the 'union types' feature in ES|QL.
  */
 public class InvalidMappedField extends EsField {
-    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
-        EsField.class,
-        "InvalidMappedField",
-        InvalidMappedField::new
-    );
 
     private final String errorMessage;
     private final Map<String, Set<String>> typesToIndices;
@@ -43,10 +38,6 @@ public InvalidMappedField(String name, String errorMessage) {
         this(name, errorMessage, new TreeMap<>());
     }
 
-    public InvalidMappedField(String name) {
-        this(name, StringUtils.EMPTY, new TreeMap<>());
-    }
-
     /**
      * Constructor supporting union types, used in ES|QL.
      */
@@ -60,20 +51,23 @@ private InvalidMappedField(String name, String errorMessage, Map<String, EsField
         this.typesToIndices = typesToIndices;
     }
 
-    private InvalidMappedField(StreamInput in) throws IOException {
-        this(in.readString(), in.readString(), in.readImmutableMap(StreamInput::readString, i -> i.readNamedWriteable(EsField.class)));
+    protected InvalidMappedField(StreamInput in) throws IOException {
+        this(in.readString(), in.readString(), in.readImmutableMap(StreamInput::readString, EsField::readFrom));
+    }
+
+    public Set<DataType> types() {
+        return typesToIndices.keySet().stream().map(DataType::fromTypeName).collect(Collectors.toSet());
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    protected void writeContent(StreamOutput out) throws IOException {
         out.writeString(getName());
         out.writeString(errorMessage);
-        out.writeMap(getProperties(), StreamOutput::writeNamedWriteable);
+        out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
     }
 
-    @Override
     public String getWriteableName() {
-        return ENTRY.name;
+        return "InvalidMappedField";
     }
 
     public String errorMessage() {
@@ -125,7 +119,13 @@ private static String makeErrorMessage(Map<String, Set<String>> typesToIndices)
             errorMessage.append("[");
             errorMessage.append(e.getKey());
             errorMessage.append("] in ");
-            errorMessage.append(e.getValue());
+            if (e.getValue().size() <= 3) {
+                errorMessage.append(e.getValue());
+            } else {
+                errorMessage.append(e.getValue().stream().sorted().limit(3).collect(Collectors.toList()));
+                errorMessage.append(" and [" + (e.getValue().size() - 3) + "] other ");
+                errorMessage.append(e.getValue().size() == 4 ? "index" : "indices");
+            }
         }
         return errorMessage.toString();
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
index d856e3d9d8297..33dcebaf3dec2 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.esql.core.type;
 
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
@@ -21,11 +20,6 @@
  * Information about a field in an ES index with the {@code keyword} type.
  */
 public class KeywordEsField extends EsField {
-    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
-        EsField.class,
-        "KeywordEsField",
-        KeywordEsField::new
-    );
 
     private final int precision;
     private final boolean normalized;
@@ -63,11 +57,11 @@ protected KeywordEsField(
         this.normalized = normalized;
     }
 
-    private KeywordEsField(StreamInput in) throws IOException {
+    public KeywordEsField(StreamInput in) throws IOException {
         this(
             in.readString(),
             KEYWORD,
-            in.readMap(i -> i.readNamedWriteable(EsField.class)),
+            in.readImmutableMap(EsField::readFrom),
             in.readBoolean(),
             in.readInt(),
             in.readBoolean(),
@@ -76,18 +70,17 @@ private KeywordEsField(StreamInput in) throws IOException {
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    protected void writeContent(StreamOutput out) throws IOException {
         out.writeString(getName());
-        out.writeMap(getProperties(), StreamOutput::writeNamedWriteable);
+        out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
         out.writeBoolean(isAggregatable());
         out.writeInt(precision);
         out.writeBoolean(normalized);
         out.writeBoolean(isAlias());
     }
 
-    @Override
     public String getWriteableName() {
-        return ENTRY.name;
+        return "KeywordEsField";
     }
 
     public int getPrecision() {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/MultiTypeEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java
similarity index 86%
rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/MultiTypeEsField.java
rename to x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java
index 8b2fc926379f2..81dc77eddcdf8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/MultiTypeEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java
@@ -5,15 +5,11 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.type;
+package org.elasticsearch.xpack.esql.core.type;
 
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.EsField;
-import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -31,11 +27,6 @@
  * type conversion is done at the data node level.
  */
 public class MultiTypeEsField extends EsField {
-    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
-        EsField.class,
-        "MultiTypeEsField",
-        MultiTypeEsField::new
-    );
 
     private final Map<String, Expression> indexToConversionExpressions;
 
@@ -44,21 +35,20 @@ public MultiTypeEsField(String name, DataType dataType, boolean aggregatable, Ma
         this.indexToConversionExpressions = indexToConversionExpressions;
     }
 
-    public MultiTypeEsField(StreamInput in) throws IOException {
+    protected MultiTypeEsField(StreamInput in) throws IOException {
         this(in.readString(), DataType.readFrom(in), in.readBoolean(), in.readImmutableMap(i -> i.readNamedWriteable(Expression.class)));
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    public void writeContent(StreamOutput out) throws IOException {
         out.writeString(getName());
         out.writeString(getDataType().typeName());
         out.writeBoolean(isAggregatable());
         out.writeMap(getIndexToConversionExpressions(), (o, v) -> out.writeNamedWriteable(v));
     }
 
-    @Override
     public String getWriteableName() {
-        return ENTRY.name;
+        return "MultiTypeEsField";
     }
 
     public Map<String, Expression> getIndexToConversionExpressions() {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/StringUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/StringUtils.java
index a833a302ade0d..924a42661ed56 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/StringUtils.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/StringUtils.java
@@ -26,7 +26,7 @@
 
 //FIXME: this class comes from sql-proto
 // find a way to share it across or potentially just copy it over
-final class StringUtils {
+public final class StringUtils {
 
     public static final String EMPTY = "";
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
index c52230fa65829..0f2f136e74423 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.esql.core.type;
 
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Tuple;
@@ -23,7 +22,6 @@
  * Information about a field in an es index with the {@code text} type.
  */
 public class TextEsField extends EsField {
-    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(EsField.class, "TextEsField", TextEsField::new);
 
     public TextEsField(String name, Map<String, EsField> properties, boolean hasDocValues) {
         this(name, properties, hasDocValues, false);
@@ -33,21 +31,20 @@ public TextEsField(String name, Map<String, EsField> properties, boolean hasDocV
         super(name, TEXT, properties, hasDocValues, isAlias);
     }
 
-    private TextEsField(StreamInput in) throws IOException {
-        this(in.readString(), in.readMap(i -> i.readNamedWriteable(EsField.class)), in.readBoolean(), in.readBoolean());
+    protected TextEsField(StreamInput in) throws IOException {
+        this(in.readString(), in.readImmutableMap(EsField::readFrom), in.readBoolean(), in.readBoolean());
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    protected void writeContent(StreamOutput out) throws IOException {
         out.writeString(getName());
-        out.writeMap(getProperties(), StreamOutput::writeNamedWriteable);
+        out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
         out.writeBoolean(isAggregatable());
         out.writeBoolean(isAlias());
     }
 
-    @Override
     public String getWriteableName() {
-        return ENTRY.name;
+        return "TextEsField";
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/Types.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/Types.java
deleted file mode 100644
index 5daa2e0050543..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/Types.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.type;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.Booleans;
-import org.elasticsearch.index.mapper.TimeSeriesParams;
-
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import static java.util.Collections.emptyMap;
-import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED;
-import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
-
-public abstract class Types {
-
-    @SuppressWarnings("unchecked")
-    public static Map<String, EsField> fromEs(DataTypeRegistry typeRegistry, Map<String, Object> asMap) {
-        Map<String, Object> props = null;
-        if (asMap != null && asMap.isEmpty() == false) {
-            props = (Map<String, Object>) asMap.get("properties");
-        }
-        return props == null || props.isEmpty() ? emptyMap() : startWalking(typeRegistry, props);
-    }
-
-    private static Map<String, EsField> startWalking(DataTypeRegistry typeRegistry, Map<String, Object> mapping) {
-        Map<String, EsField> types = new LinkedHashMap<>();
-
-        if (mapping == null) {
-            return emptyMap();
-        }
-        for (Entry<String, Object> entry : mapping.entrySet()) {
-            walkMapping(typeRegistry, entry.getKey(), entry.getValue(), types);
-        }
-
-        return types;
-    }
-
-    private static DataType getType(DataTypeRegistry typeRegistry, Map<String, Object> content) {
-        if (content.containsKey("type")) {
-            String typeName = content.get("type").toString();
-            if ("constant_keyword".equals(typeName) || "wildcard".equals(typeName)) {
-                return KEYWORD;
-            }
-            final Object metricsTypeParameter = content.get(TimeSeriesParams.TIME_SERIES_METRIC_PARAM);
-            final TimeSeriesParams.MetricType metricType;
-            if (metricsTypeParameter instanceof String str) {
-                metricType = TimeSeriesParams.MetricType.fromString(str);
-            } else {
-                metricType = (TimeSeriesParams.MetricType) metricsTypeParameter;
-            }
-            try {
-                return typeRegistry.fromEs(typeName, metricType);
-            } catch (IllegalArgumentException ex) {
-                return UNSUPPORTED;
-            }
-        } else if (content.containsKey("properties")) {
-            return OBJECT;
-        } else {
-            return UNSUPPORTED;
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    private static void walkMapping(DataTypeRegistry typeRegistry, String name, Object value, Map<String, EsField> mapping) {
-        // object type - only root or nested docs supported
-        if (value instanceof Map) {
-            Map<String, Object> content = (Map<String, Object>) value;
-
-            // extract field type
-            DataType esDataType = getType(typeRegistry, content);
-            final Map<String, EsField> properties;
-            if (esDataType == OBJECT || esDataType == NESTED) {
-                properties = fromEs(typeRegistry, content);
-            } else if (content.containsKey("fields")) {
-                // Check for multifields
-                Object fields = content.get("fields");
-                if (fields instanceof Map) {
-                    properties = startWalking(typeRegistry, (Map<String, Object>) fields);
-                } else {
-                    properties = Collections.emptyMap();
-                }
-            } else {
-                properties = fromEs(typeRegistry, content);
-            }
-            boolean docValues = boolSetting(content.get("doc_values"), esDataType.hasDocValues());
-            final EsField field;
-            if (esDataType == TEXT) {
-                field = new TextEsField(name, properties, docValues);
-            } else if (esDataType == KEYWORD) {
-                int length = intSetting(content.get("ignore_above"), Short.MAX_VALUE);
-                boolean normalized = Strings.hasText(textSetting(content.get("normalizer"), null));
-                field = new KeywordEsField(name, properties, docValues, length, normalized);
-            } else if (esDataType == DATETIME) {
-                field = DateEsField.dateEsField(name, properties, docValues);
-            } else if (esDataType == UNSUPPORTED) {
-                String type = content.get("type").toString();
-                field = new UnsupportedEsField(name, type, null, properties);
-                propagateUnsupportedType(name, type, properties);
-            } else {
-                field = new EsField(name, esDataType, properties, docValues);
-            }
-            mapping.put(name, field);
-        } else {
-            throw new IllegalArgumentException("Unrecognized mapping " + value);
-        }
-    }
-
-    private static String textSetting(Object value, String defaultValue) {
-        return value == null ? defaultValue : value.toString();
-    }
-
-    private static boolean boolSetting(Object value, boolean defaultValue) {
-        return value == null ? defaultValue : Booleans.parseBoolean(value.toString(), defaultValue);
-    }
-
-    private static int intSetting(Object value, int defaultValue) {
-        return value == null ? defaultValue : Integer.parseInt(value.toString());
-    }
-
-    public static void propagateUnsupportedType(String inherited, String originalType, Map<String, EsField> properties) {
-        if (properties != null && properties.isEmpty() == false) {
-            for (Entry<String, EsField> entry : properties.entrySet()) {
-                EsField field = entry.getValue();
-                UnsupportedEsField u;
-                if (field instanceof UnsupportedEsField) {
-                    u = (UnsupportedEsField) field;
-                    u = new UnsupportedEsField(u.getName(), originalType, inherited, u.getProperties());
-                } else {
-                    u = new UnsupportedEsField(field.getName(), originalType, inherited, field.getProperties());
-                }
-                entry.setValue(u);
-                propagateUnsupportedType(inherited, originalType, u.getProperties());
-            }
-        }
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
index 13e4d6ad953a8..13ee2b42a321b 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.esql.core.type;
 
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
@@ -20,11 +19,6 @@
  * All the subfields (properties) of an unsupported type are also be unsupported.
  */
 public class UnsupportedEsField extends EsField {
-    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
-        EsField.class,
-        "UnsupportedEsField",
-        UnsupportedEsField::new
-    );
 
     private final String originalType;
     private final String inherited; // for fields belonging to parents (or grandparents) that have an unsupported type
@@ -40,20 +34,19 @@ public UnsupportedEsField(String name, String originalType, String inherited, Ma
     }
 
     public UnsupportedEsField(StreamInput in) throws IOException {
-        this(in.readString(), in.readString(), in.readOptionalString(), in.readMap(i -> i.readNamedWriteable(EsField.class)));
+        this(in.readString(), in.readString(), in.readOptionalString(), in.readImmutableMap(EsField::readFrom));
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    public void writeContent(StreamOutput out) throws IOException {
         out.writeString(getName());
         out.writeString(getOriginalType());
         out.writeOptionalString(getInherited());
-        out.writeMap(getProperties(), StreamOutput::writeNamedWriteable);
+        out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
     }
 
-    @Override
     public String getWriteableName() {
-        return ENTRY.name;
+        return "UnsupportedEsField";
     }
 
     public String getOriginalType() {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java
index fa39a502ae1e8..280cf172a8a58 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java
@@ -12,6 +12,7 @@
 
 import java.sql.Timestamp;
 import java.time.Duration;
+import java.time.Instant;
 import java.time.OffsetTime;
 import java.time.Period;
 import java.time.ZoneId;
@@ -23,6 +24,7 @@
 import java.util.concurrent.TimeUnit;
 
 import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE;
+import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME;
 import static java.time.temporal.ChronoField.HOUR_OF_DAY;
 import static java.time.temporal.ChronoField.MINUTE_OF_HOUR;
 import static java.time.temporal.ChronoField.NANO_OF_SECOND;
@@ -34,6 +36,24 @@
 public class DateUtils {
 
     public static final ZoneId UTC = ZoneId.of("Z");
+    private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder().append(ISO_LOCAL_DATE)
+        .optionalStart()
+        .appendLiteral('T')
+        .append(ISO_LOCAL_TIME)
+        .optionalStart()
+        .appendZoneOrOffsetId()
+        .optionalEnd()
+        .toFormatter(Locale.ROOT)
+        .withZone(UTC);
+    private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder().append(ISO_LOCAL_DATE)
+        .optionalStart()
+        .appendLiteral(' ')
+        .append(ISO_LOCAL_TIME)
+        .optionalStart()
+        .appendZoneOrOffsetId()
+        .optionalEnd()
+        .toFormatter(Locale.ROOT)
+        .withZone(UTC);
 
     public static final String EMPTY = "";
 
@@ -146,4 +166,41 @@ public static String toString(Object value) {
     private static String indent(long timeUnit) {
         return timeUnit < 10 ? "0" + timeUnit : Long.toString(timeUnit);
     }
+
+    /**
+     * Creates a datetime from the millis since epoch (thus the time-zone is UTC).
+     */
+    public static ZonedDateTime asDateTime(long millis) {
+        return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), UTC);
+    }
+
+    public static long asMillis(ZonedDateTime zonedDateTime) {
+        return zonedDateTime.toInstant().toEpochMilli();
+    }
+
+    /**
+     * Parses the given string into a DateTime using UTC as a default timezone.
+     */
+    public static ZonedDateTime asDateTime(String dateFormat) {
+        int separatorIdx = dateFormat.indexOf('-'); // Find the first `-` date separator
+        if (separatorIdx == 0) { // first char = `-` denotes a negative year
+            separatorIdx = dateFormat.indexOf('-', 1); // Find the first `-` date separator past the negative year
+        }
+        // Find the second `-` date separator and move 3 places past the dayOfYear to find the time separator
+        // e.g. 2020-06-01T10:20:30....
+        // ^
+        // +3 = ^
+        separatorIdx = dateFormat.indexOf('-', separatorIdx + 1) + 3;
+
+        // Avoid index out of bounds - it will lead to DateTimeParseException anyways
+        if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
+            return DateFormatters.from(DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL.parse(dateFormat)).withZoneSameInstant(UTC);
+        } else {
+            return DateFormatters.from(DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE.parse(dateFormat)).withZoneSameInstant(UTC);
+        }
+    }
+
+    public static String toString(ZonedDateTime dateTime) {
+        return org.elasticsearch.xpack.esql.core.type.StringUtils.toString(dateTime);
+    }
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
index df8fac06dd478..471c9476ad31d 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
@@ -7,8 +7,12 @@
 
 package org.elasticsearch.xpack.esql.core.util;
 
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.EsField;
 
 import java.io.IOException;
 
@@ -33,4 +37,14 @@ public interface PlanStreamInput {
      * the same result.
      */
     NameId mapNameId(long id) throws IOException;
+
+    /**
+     * Reads an Attribute using the attribute cache.
+     * @param constructor the constructor needed to build the actual attribute when read from the wire
+     * @return An attribute; this will generally be the same type as the provided constructor
+     * @throws IOException
+     */
+    <A extends Attribute> A readAttributeWithCache(CheckedFunction<StreamInput, A, IOException> constructor) throws IOException;
+
+    <A extends EsField> A readEsFieldWithCache() throws IOException;
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java
new file mode 100644
index 0000000000000..4c30cb66e9f86
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.util;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+
+import java.io.IOException;
+
+public interface PlanStreamOutput {
+
+    /**
+     * Writes a cache header for an {@link Attribute} and caches it if it is not already in the cache.
+     * In that case, the attribute will have to serialize itself into this stream immediately after this method call.
+     * @param attribute The attribute to serialize
+     * @return true if the attribute needs to serialize itself, false otherwise (ie. if already cached)
+     * @throws IOException
+     */
+    boolean writeAttributeCacheHeader(Attribute attribute) throws IOException;
+
+    /**
+     * Writes a cache header for an {@link org.elasticsearch.xpack.esql.core.type.EsField} and caches it if it is not already in the cache.
+     * In that case, the field will have to serialize itself into this stream immediately after this method call.
+     * @param field The EsField to serialize
+     * @return true if the attribute needs to serialize itself, false otherwise (ie. if already cached)
+     * @throws IOException
+     */
+    boolean writeEsFieldCacheHeader(EsField field) throws IOException;
+}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/Queries.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/Queries.java
index 9403c3c6a0bc0..759d7b80acc22 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/Queries.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/Queries.java
@@ -50,7 +50,7 @@ public static QueryBuilder combine(Clause clause, List<QueryBuilder> queries) {
             if (firstQuery == null) {
                 firstQuery = query;
                 if (firstQuery instanceof BoolQueryBuilder bqb) {
-                    bool = bqb;
+                    bool = bqb.shallowCopy();
                 }
             }
             // at least two entries, start copying
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SourceUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SourceUtils.java
deleted file mode 100644
index 6bed7f06fa2f5..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SourceUtils.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core.util;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xpack.esql.core.tree.Location;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-
-import java.io.IOException;
-
-public final class SourceUtils {
-
-    private SourceUtils() {}
-
-    /**
-     * Write a {@link Source} including the text in it.
-     * @deprecated replace with {@link Source#writeTo}.
-     *             That's not binary compatible so the replacement is complex.
-     */
-    @Deprecated
-    public static void writeSource(StreamOutput out, Source source) throws IOException {
-        out.writeInt(source.source().getLineNumber());
-        out.writeInt(source.source().getColumnNumber());
-        out.writeString(source.text());
-    }
-
-    /**
-     * Read a {@link Source} including the text in it.
-     * @deprecated replace with {@link Source#readFrom(StreamInput)}.
-     *             That's not binary compatible so the replacement is complex.
-     */
-    @Deprecated
-    public static Source readSource(StreamInput in) throws IOException {
-        int line = in.readInt();
-        int column = in.readInt();
-        int charPositionInLine = column - 1;
-
-        String text = in.readString();
-        return new Source(new Location(line, charPositionInLine), text);
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java
index 4ba3658697c0d..cd0ade2054ce6 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java
@@ -398,20 +398,41 @@ public static boolean isQualified(String indexWildcard) {
 
     public static boolean isInteger(String value) {
         for (char c : value.trim().toCharArray()) {
-            if (Character.isDigit(c) == false) {
+            if (isDigit(c) == false) {
                 return false;
             }
         }
         return true;
     }
 
+    private static boolean isLetter(char c) {
+        return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
+    }
+
+    private static boolean isDigit(char c) {
+        return c >= '0' && c <= '9';
+    }
+
+    private static boolean isUnderscore(char c) {
+        return c == '_';
+    }
+
+    private static boolean isLetterOrDigitOrUnderscore(char c) {
+        return isLetter(c) || isDigit(c) || isUnderscore(c);
+    }
+
+    private static boolean isLetterOrUnderscore(char c) {
+        return isLetter(c) || isUnderscore(c);
+    }
+
     public static boolean isValidParamName(String value) {
-        // A valid name starts with a letter and contain only letter, digit or _
-        if (Character.isLetter(value.charAt(0)) == false) {
+        // A valid name starts with a letter or _
+        if (isLetterOrUnderscore(value.charAt(0)) == false) {
             return false;
         }
-        for (char c : value.trim().toCharArray()) {
-            if (Character.isLetterOrDigit(c) == false && c != '_') {
+        // contain only letter, digit or _
+        for (char c : value.toCharArray()) {
+            if (isLetterOrDigitOrUnderscore(c) == false) {
                 return false;
             }
         }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java
index c077af4026974..511c7f4b1d2f8 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java
@@ -19,8 +19,8 @@
 
 import static java.util.Arrays.asList;
 import static java.util.stream.Collectors.toList;
-import static org.elasticsearch.xpack.esql.core.TestUtils.fieldAttribute;
-import static org.elasticsearch.xpack.esql.core.TestUtils.of;
+import static org.elasticsearch.xpack.esql.core.util.TestUtils.fieldAttribute;
+import static org.elasticsearch.xpack.esql.core.util.TestUtils.of;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.contains;
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttributeTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttributeTests.java
index e7cd38b8f938a..6ac8377a2d8b9 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttributeTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttributeTests.java
@@ -18,11 +18,10 @@ public class UnresolvedAttributeTests extends AbstractNodeTestCase<UnresolvedAtt
     public static UnresolvedAttribute randomUnresolvedAttribute() {
         Source source = SourceTests.randomSource();
         String name = randomAlphaOfLength(5);
-        String qualifier = randomQualifier();
         NameId id = randomBoolean() ? null : new NameId();
         String unresolvedMessage = randomUnresolvedMessage();
         Object resolutionMetadata = new Object();
-        return new UnresolvedAttribute(source, name, qualifier, id, unresolvedMessage, resolutionMetadata);
+        return new UnresolvedAttribute(source, name, id, unresolvedMessage, resolutionMetadata);
     }
 
     /**
@@ -53,7 +52,6 @@ protected UnresolvedAttribute mutate(UnresolvedAttribute a) {
                 () -> new UnresolvedAttribute(
                     a.source(),
                     randomValueOtherThan(a.name(), () -> randomAlphaOfLength(5)),
-                    a.qualifier(),
                     a.id(),
                     a.unresolvedMessage(),
                     a.resolutionMetadata()
@@ -61,20 +59,11 @@ protected UnresolvedAttribute mutate(UnresolvedAttribute a) {
                 () -> new UnresolvedAttribute(
                     a.source(),
                     a.name(),
-                    randomValueOtherThan(a.qualifier(), UnresolvedAttributeTests::randomQualifier),
-                    a.id(),
-                    a.unresolvedMessage(),
-                    a.resolutionMetadata()
-                ),
-                () -> new UnresolvedAttribute(
-                    a.source(),
-                    a.name(),
-                    a.qualifier(),
                     a.id(),
                     randomValueOtherThan(a.unresolvedMessage(), () -> randomUnresolvedMessage()),
                     a.resolutionMetadata()
                 ),
-                () -> new UnresolvedAttribute(a.source(), a.name(), a.qualifier(), a.id(), a.unresolvedMessage(), new Object())
+                () -> new UnresolvedAttribute(a.source(), a.name(), a.id(), a.unresolvedMessage(), new Object())
             )
         );
         return option.get();
@@ -82,7 +71,7 @@ protected UnresolvedAttribute mutate(UnresolvedAttribute a) {
 
     @Override
     protected UnresolvedAttribute copy(UnresolvedAttribute a) {
-        return new UnresolvedAttribute(a.source(), a.name(), a.qualifier(), a.id(), a.unresolvedMessage(), a.resolutionMetadata());
+        return new UnresolvedAttribute(a.source(), a.name(), a.id(), a.unresolvedMessage(), a.resolutionMetadata());
     }
 
     @Override
@@ -91,31 +80,25 @@ public void testTransform() {
 
         String newName = randomValueOtherThan(a.name(), () -> randomAlphaOfLength(5));
         assertEquals(
-            new UnresolvedAttribute(a.source(), newName, a.qualifier(), a.id(), a.unresolvedMessage(), a.resolutionMetadata()),
+            new UnresolvedAttribute(a.source(), newName, a.id(), a.unresolvedMessage(), a.resolutionMetadata()),
             a.transformPropertiesOnly(Object.class, v -> Objects.equals(v, a.name()) ? newName : v)
         );
 
-        String newQualifier = randomValueOtherThan(a.qualifier(), UnresolvedAttributeTests::randomQualifier);
-        assertEquals(
-            new UnresolvedAttribute(a.source(), a.name(), newQualifier, a.id(), a.unresolvedMessage(), a.resolutionMetadata()),
-            a.transformPropertiesOnly(Object.class, v -> Objects.equals(v, a.qualifier()) ? newQualifier : v)
-        );
-
         NameId newId = new NameId();
         assertEquals(
-            new UnresolvedAttribute(a.source(), a.name(), a.qualifier(), newId, a.unresolvedMessage(), a.resolutionMetadata()),
+            new UnresolvedAttribute(a.source(), a.name(), newId, a.unresolvedMessage(), a.resolutionMetadata()),
             a.transformPropertiesOnly(Object.class, v -> Objects.equals(v, a.id()) ? newId : v)
         );
 
         String newMessage = randomValueOtherThan(a.unresolvedMessage(), UnresolvedAttributeTests::randomUnresolvedMessage);
         assertEquals(
-            new UnresolvedAttribute(a.source(), a.name(), a.qualifier(), a.id(), newMessage, a.resolutionMetadata()),
+            new UnresolvedAttribute(a.source(), a.name(), a.id(), newMessage, a.resolutionMetadata()),
             a.transformPropertiesOnly(Object.class, v -> Objects.equals(v, a.unresolvedMessage()) ? newMessage : v)
         );
 
         Object newMeta = new Object();
         assertEquals(
-            new UnresolvedAttribute(a.source(), a.name(), a.qualifier(), a.id(), a.unresolvedMessage(), newMeta),
+            new UnresolvedAttribute(a.source(), a.name(), a.id(), a.unresolvedMessage(), newMeta),
             a.transformPropertiesOnly(Object.class, v -> Objects.equals(v, a.resolutionMetadata()) ? newMeta : v)
         );
     }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java
index 6009ca774f8cd..ed4c6282368ca 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DateUtils;
+import org.elasticsearch.xpack.esql.core.util.DateUtils;
 
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
index 789e9a22d39d1..d323174d2d3d9 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.esql.core.TestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
@@ -17,14 +16,15 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.TestUtils;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 
-import static org.elasticsearch.xpack.esql.core.TestUtils.of;
-import static org.elasticsearch.xpack.esql.core.TestUtils.rangeOf;
 import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.util.TestUtils.of;
+import static org.elasticsearch.xpack.esql.core.util.TestUtils.rangeOf;
 
 public class OptimizerRulesTests extends ESTestCase {
 
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/tree/NodeTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/tree/NodeTests.java
index 61ff9cdc809c3..7c738ee42c921 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/tree/NodeTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/tree/NodeTests.java
@@ -6,9 +6,11 @@
  */
 package org.elasticsearch.xpack.esql.core.tree;
 
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -67,6 +69,16 @@ public Dummy(Source source, List<Dummy> children, String thing) {
             this.thing = thing;
         }
 
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getWriteableName() {
+            throw new UnsupportedOperationException();
+        }
+
         public String thing() {
             return thing;
         }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/tree/SourceTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/tree/SourceTests.java
index b53d04bbb1e74..bc95f99e36037 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/tree/SourceTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/tree/SourceTests.java
@@ -6,13 +6,17 @@
  */
 package org.elasticsearch.xpack.esql.core.tree;
 
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.function.Function;
 
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.hamcrest.Matchers.equalTo;
 
 public class SourceTests extends ESTestCase {
     public static Source randomSource() {
@@ -42,4 +46,16 @@ public void testEqualsAndHashCode() {
             SourceTests::mutate
         );
     }
+
+    public void testReadEmpty() throws IOException {
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            randomSource().writeTo(out);
+            int test = randomInt();
+            out.writeInt(test);
+            try (StreamInput in = out.bytes().streamInput()) {
+                assertThat(Source.readEmpty(in), equalTo(Source.EMPTY));
+                assertThat(in.readInt(), equalTo(test));
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/AbstractEsFieldTypeTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/AbstractEsFieldTypeTests.java
deleted file mode 100644
index a415c529894c3..0000000000000
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/AbstractEsFieldTypeTests.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core.type;
-
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.test.AbstractNamedWriteableTestCase;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.TreeMap;
-
-public abstract class AbstractEsFieldTypeTests<T extends EsField> extends AbstractNamedWriteableTestCase<EsField> {
-    public static EsField randomAnyEsField(int maxDepth) {
-        return switch (between(0, 5)) {
-            case 0 -> EsFieldTests.randomEsField(maxDepth);
-            case 1 -> DateEsFieldTests.randomDateEsField(maxDepth);
-            case 2 -> InvalidMappedFieldTests.randomInvalidMappedField(maxDepth);
-            case 3 -> KeywordEsFieldTests.randomKeywordEsField(maxDepth);
-            case 4 -> TextEsFieldTests.randomTextEsField(maxDepth);
-            case 5 -> UnsupportedEsFieldTests.randomUnsupportedEsField(maxDepth);
-            default -> throw new IllegalArgumentException();
-        };
-    }
-
-    @Override
-    protected abstract T createTestInstance();
-
-    protected abstract T mutate(T instance);
-
-    /**
-     * Generate sub-properties.
-     * @param maxDepth the maximum number of levels of properties to make
-     */
-    static Map<String, EsField> randomProperties(int maxDepth) {
-        if (maxDepth < 0) {
-            throw new IllegalArgumentException("depth must be >= 0");
-        }
-        if (maxDepth == 0 || randomBoolean()) {
-            return Map.of();
-        }
-        int targetSize = between(1, 5);
-        Map<String, EsField> properties = new TreeMap<>();
-        while (properties.size() < targetSize) {
-            properties.put(randomAlphaOfLength(properties.size() + 1), randomAnyEsField(maxDepth - 1));
-        }
-        return properties;
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    protected final T mutateInstance(EsField instance) throws IOException {
-        return mutate((T) instance);
-    }
-
-    @Override
-    protected final NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(EsField.getNamedWriteables());
-    }
-
-    @Override
-    protected final Class<EsField> categoryClass() {
-        return EsField.class;
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TypesTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TypesTests.java
deleted file mode 100644
index 1974eb3669f4b..0000000000000
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TypesTests.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.type;
-
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xcontent.json.JsonXContent;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Map;
-
-import static java.util.Collections.emptyMap;
-import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED;
-import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-
-public class TypesTests extends ESTestCase {
-
-    public void testNullMap() {
-        Map<String, EsField> fromEs = Types.fromEs(DefaultDataTypeRegistry.INSTANCE, null);
-        assertThat(fromEs.isEmpty(), is(true));
-    }
-
-    public void testEmptyMap() {
-        Map<String, EsField> fromEs = Types.fromEs(DefaultDataTypeRegistry.INSTANCE, emptyMap());
-        assertThat(fromEs.isEmpty(), is(true));
-    }
-
-    public void testBasicMapping() {
-        Map<String, EsField> mapping = loadMapping("mapping-basic.json");
-        assertThat(mapping.size(), is(7));
-        assertThat(mapping.get("emp_no").getDataType(), is(INTEGER));
-        assertThat(mapping.get("first_name"), instanceOf(TextEsField.class));
-        assertThat(mapping.get("last_name").getDataType(), is(TEXT));
-        assertThat(mapping.get("gender").getDataType(), is(KEYWORD));
-        assertThat(mapping.get("salary").getDataType(), is(INTEGER));
-        assertThat(mapping.get("_meta_field").getDataType(), is(KEYWORD));
-    }
-
-    public void testDefaultStringMapping() {
-        Map<String, EsField> mapping = loadMapping("mapping-default-string.json");
-
-        assertThat(mapping.size(), is(1));
-        assertThat(mapping.get("dep_no").getDataType(), is(TEXT));
-    }
-
-    public void testTextField() {
-        Map<String, EsField> mapping = loadMapping("mapping-text.json");
-
-        assertThat(mapping.size(), is(1));
-        EsField type = mapping.get("full_name");
-        assertThat(type, instanceOf(TextEsField.class));
-        assertThat(type.isAggregatable(), is(false));
-        TextEsField ttype = (TextEsField) type;
-        assertThat(ttype.isAggregatable(), is(false));
-    }
-
-    public void testKeywordField() {
-        Map<String, EsField> mapping = loadMapping("mapping-keyword.json");
-
-        assertThat(mapping.size(), is(1));
-        EsField field = mapping.get("full_name");
-        assertThat(field, instanceOf(KeywordEsField.class));
-        assertThat(field.isAggregatable(), is(true));
-    }
-
-    public void testDateField() {
-        Map<String, EsField> mapping = loadMapping("mapping-date.json");
-
-        assertThat(mapping.size(), is(1));
-        EsField field = mapping.get("date");
-        assertThat(field.getDataType(), is(DATETIME));
-        assertThat(field.isAggregatable(), is(true));
-    }
-
-    public void testDateNoFormat() {
-        Map<String, EsField> mapping = loadMapping("mapping-date-no-format.json");
-
-        assertThat(mapping.size(), is(1));
-        EsField field = mapping.get("date");
-        assertThat(field.getDataType(), is(DATETIME));
-        assertThat(field.isAggregatable(), is(true));
-        assertThat(field, is(instanceOf(DateEsField.class)));
-    }
-
-    public void testDateMulti() {
-        Map<String, EsField> mapping = loadMapping("mapping-date-multi.json");
-
-        assertThat(mapping.size(), is(1));
-        EsField field = mapping.get("date");
-        assertThat(field.getDataType(), is(DATETIME));
-        assertThat(field.isAggregatable(), is(true));
-        assertThat(field, is(instanceOf(DateEsField.class)));
-    }
-
-    public void testDocValueField() {
-        Map<String, EsField> mapping = loadMapping("mapping-docvalues.json");
-
-        assertThat(mapping.size(), is(1));
-        EsField field = mapping.get("session_id");
-        assertThat(field, instanceOf(KeywordEsField.class));
-        // assertThat(field.getPrecision(), is(15));
-        assertThat(field.isAggregatable(), is(false));
-    }
-
-    public void testDottedField() {
-        Map<String, EsField> mapping = loadMapping("mapping-object.json");
-
-        assertThat(mapping.size(), is(2));
-        EsField field = mapping.get("manager");
-        assertThat(DataType.isPrimitive(field.getDataType()), is(false));
-        assertThat(field.getDataType(), is(OBJECT));
-        Map<String, EsField> children = field.getProperties();
-        assertThat(children.size(), is(2));
-        EsField names = children.get("name");
-        children = names.getProperties();
-        assertThat(children.size(), is(2));
-        assertThat(children.get("first").getDataType(), is(TEXT));
-    }
-
-    public void testMultiField() {
-        Map<String, EsField> mapping = loadMapping("mapping-multi-field.json");
-
-        assertThat(mapping.size(), is(1));
-        EsField field = mapping.get("text");
-        assertThat(DataType.isPrimitive(field.getDataType()), is(true));
-        assertThat(field.getDataType(), is(TEXT));
-        Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(4));
-        assertThat(fields.get("raw").getDataType(), is(KEYWORD));
-        assertThat(fields.get("english").getDataType(), is(TEXT));
-        assertThat(fields.get("wildcard").getDataType(), is(KEYWORD));
-    }
-
-    public void testMultiFieldTooManyOptions() {
-        Map<String, EsField> mapping = loadMapping("mapping-multi-field.json");
-
-        assertThat(mapping.size(), is(1));
-        EsField field = mapping.get("text");
-        assertThat(DataType.isPrimitive(field.getDataType()), is(true));
-        assertThat(field, instanceOf(TextEsField.class));
-        Map<String, EsField> fields = field.getProperties();
-        assertThat(fields.size(), is(4));
-        assertThat(fields.get("raw").getDataType(), is(KEYWORD));
-        assertThat(fields.get("english").getDataType(), is(TEXT));
-        assertThat(fields.get("wildcard").getDataType(), is(KEYWORD));
-    }
-
-    public void testNestedDoc() {
-        Map<String, EsField> mapping = loadMapping("mapping-nested.json");
-
-        assertThat(mapping.size(), is(1));
-        EsField field = mapping.get("dep");
-        assertThat(DataType.isPrimitive(field.getDataType()), is(false));
-        assertThat(field.getDataType(), is(NESTED));
-        Map<String, EsField> children = field.getProperties();
-        assertThat(children.size(), is(4));
-        assertThat(children.get("dep_name").getDataType(), is(TEXT));
-        assertThat(children.get("start_date").getDataType(), is(DATETIME));
-    }
-
-    public void testIpField() {
-        Map<String, EsField> mapping = loadMapping("mapping-ip.json");
-        assertThat(mapping.size(), is(1));
-        EsField dt = mapping.get("ip_addr");
-        assertThat(dt.getDataType().typeName(), is("ip"));
-    }
-
-    public void testVersionField() {
-        Map<String, EsField> mapping = loadMapping("mapping-version.json");
-        assertThat(mapping.size(), is(1));
-        EsField dt = mapping.get("version_number");
-        assertThat(dt.getDataType().typeName(), is("version"));
-    }
-
-    public void testConstantKeywordField() {
-        Map<String, EsField> mapping = loadMapping("mapping-constant-keyword.json");
-        assertThat(mapping.size(), is(1));
-        EsField dt = mapping.get("full_name");
-        assertThat(dt.getDataType().typeName(), is("keyword"));
-    }
-
-    public void testWildcardField() {
-        Map<String, EsField> mapping = loadMapping("mapping-wildcard.json");
-        assertThat(mapping.size(), is(1));
-        EsField dt = mapping.get("full_name");
-        assertThat(dt.getDataType().typeName(), is("keyword"));
-    }
-
-    public void testUnsupportedTypes() {
-        Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
-        EsField dt = mapping.get("range");
-        assertThat(dt.getDataType().typeName(), is("unsupported"));
-        dt = mapping.get("time_frame");
-        assertThat(dt.getDataType().typeName(), is("unsupported"));
-        dt = mapping.get("flat");
-        assertThat(dt.getDataType().typeName(), is("unsupported"));
-    }
-
-    public static Map<String, EsField> loadMapping(String name) {
-        return loadMapping(DefaultDataTypeRegistry.INSTANCE, name, null);
-    }
-
-    public static Map<String, EsField> loadMapping(String name, boolean ordered) {
-        return loadMapping(DefaultDataTypeRegistry.INSTANCE, name, ordered);
-    }
-
-    public static Map<String, EsField> loadMapping(DataTypeRegistry registry, String name) {
-        return loadMapping(registry, name, null);
-    }
-
-    public static Map<String, EsField> loadMapping(DataTypeRegistry registry, String name, Boolean ordered) {
-        InputStream stream = TypesTests.class.getResourceAsStream("/" + name);
-        assertNotNull("Could not find mapping resource:" + name, stream);
-        return loadMapping(registry, stream, ordered);
-    }
-
-    private static Map<String, EsField> loadMapping(DataTypeRegistry registry, InputStream stream, Boolean ordered) {
-        boolean order = ordered != null ? ordered.booleanValue() : randomBoolean();
-        try (InputStream in = stream) {
-            Map<String, Object> map = XContentHelper.convertToMap(JsonXContent.jsonXContent, in, order);
-            return Types.fromEs(registry, map);
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/QueriesTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/QueriesTests.java
index c5f4eb2ba8283..8dde968640c21 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/QueriesTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/QueriesTests.java
@@ -14,6 +14,7 @@
 
 import static java.util.Arrays.asList;
 import static org.hamcrest.Matchers.everyItem;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.in;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.sameInstance;
@@ -92,13 +93,27 @@ public void testCombineBoolQueries() {
 
         assertThat(combination, instanceOf(BoolQueryBuilder.class));
         var bool = (BoolQueryBuilder) combination;
+        assertBoolQueryMerge(queries, bool, clause);
+    }
 
-        var clauseList = clause.innerQueries.apply(bool);
+    private void assertBoolQueryMerge(QueryBuilder[] queries, BoolQueryBuilder bool, Queries.Clause clause) {
+        BoolQueryBuilder first = (BoolQueryBuilder) queries[0];
+        for (QueryBuilder b : first.must()) {
+            assertThat(bool.must(), hasItem(b));
+        }
+        for (QueryBuilder b : first.mustNot()) {
+            assertThat(bool.mustNot(), hasItem(b));
+        }
+        for (QueryBuilder b : first.should()) {
+            assertThat(bool.should(), hasItem(b));
+        }
+        for (QueryBuilder b : first.filter()) {
+            assertThat(bool.filter(), hasItem(b));
+        }
 
-        for (QueryBuilder query : queries) {
-            if (query != bool) {
-                assertThat(query, in(clauseList));
-            }
+        var clauseList = clause.innerQueries.apply(bool);
+        for (int i = 1; i < queries.length; i++) {
+            assertThat(queries[i], in(clauseList));
         }
     }
 
@@ -118,10 +133,11 @@ public void testCombineMixedBoolAndNonBoolQueries() {
         assertThat(combination, instanceOf(BoolQueryBuilder.class));
         var bool = (BoolQueryBuilder) combination;
 
-        var clauseList = clause.innerQueries.apply(bool);
-
-        for (QueryBuilder query : queries) {
-            if (query != bool) {
+        if (queries[0] instanceof BoolQueryBuilder) {
+            assertBoolQueryMerge(queries, bool, clause);
+        } else {
+            var clauseList = clause.innerQueries.apply(bool);
+            for (QueryBuilder query : queries) {
                 assertThat(query, in(clauseList));
             }
         }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java
new file mode 100644
index 0000000000000..9f8e23cb15a97
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.util;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
+import static org.elasticsearch.test.ESTestCase.randomBoolean;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
+import static org.elasticsearch.test.ESTestCase.randomZone;
+import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+
+public final class TestUtils {
+    private TestUtils() {}
+
+    public static Literal of(Object value) {
+        return of(Source.EMPTY, value);
+    }
+
+    /**
+     * Utility method for creating 'in-line' Literals (out of values instead of expressions).
+     */
+    public static Literal of(Source source, Object value) {
+        if (value instanceof Literal) {
+            return (Literal) value;
+        }
+        return new Literal(source, value, DataType.fromJava(value));
+    }
+
+    public static Range rangeOf(Expression value, Expression lower, boolean includeLower, Expression upper, boolean includeUpper) {
+        return new Range(EMPTY, value, lower, includeLower, upper, includeUpper, randomZone());
+    }
+
+    public static RLike rlike(Expression left, String exp) {
+        return new RLike(EMPTY, left, new RLikePattern(exp));
+    }
+
+    public static FieldAttribute fieldAttribute() {
+        return fieldAttribute(randomAlphaOfLength(10), randomFrom(DataType.types()));
+    }
+
+    public static FieldAttribute fieldAttribute(String name, DataType type) {
+        return new FieldAttribute(EMPTY, name, new EsField(name, type, emptyMap(), randomBoolean()));
+    }
+
+    public static FieldAttribute getFieldAttribute(String name) {
+        return getFieldAttribute(name, INTEGER);
+    }
+
+    public static FieldAttribute getFieldAttribute(String name, DataType dataType) {
+        return new FieldAttribute(EMPTY, name, new EsField(name + "f", dataType, emptyMap(), true));
+    }
+}
diff --git a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestNode.java b/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestNode.java
deleted file mode 100644
index 6571c5e60497e..0000000000000
--- a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestNode.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core;
-
-import org.apache.http.HttpHost;
-import org.elasticsearch.TransportVersion;
-import org.elasticsearch.core.Nullable;
-
-public record TestNode(String id, String version, @Nullable TransportVersion transportVersion, HttpHost publishAddress) {
-    @Override
-    public String toString() {
-        return "Node{" + "id='" + id + '\'' + ", version=" + version + '}';
-    }
-}
diff --git a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestNodes.java b/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestNodes.java
deleted file mode 100644
index 9201860ee038b..0000000000000
--- a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestNodes.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core;
-
-import org.elasticsearch.TransportVersion;
-
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-public final class TestNodes extends HashMap<String, TestNode> {
-
-    private final String bwcNodesVersion;
-
-    TestNodes(String bwcNodesVersion) {
-        this.bwcNodesVersion = bwcNodesVersion;
-    }
-
-    public void add(TestNode node) {
-        put(node.id(), node);
-    }
-
-    public List<TestNode> getNewNodes() {
-        return values().stream().filter(n -> n.version().equals(bwcNodesVersion) == false).collect(Collectors.toList());
-    }
-
-    public List<TestNode> getBWCNodes() {
-        return values().stream().filter(n -> n.version().equals(bwcNodesVersion)).collect(Collectors.toList());
-    }
-
-    public TransportVersion getBWCTransportVersion() {
-        if (isEmpty()) {
-            throw new IllegalStateException("no nodes available");
-        }
-        // there will be either at least one node with version <8.8.0, and so a mapped TransportVersion will be set,
-        // or all >=8.8.0,so TransportVersion will always be there
-        return values().stream().map(TestNode::transportVersion).filter(Objects::nonNull).min(Comparator.naturalOrder()).get();
-    }
-
-    @Override
-    public String toString() {
-        return "Nodes{" + values().stream().map(TestNode::toString).collect(Collectors.joining("\n")) + '}';
-    }
-}
diff --git a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestUtils.java b/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestUtils.java
deleted file mode 100644
index c587821e82986..0000000000000
--- a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestUtils.java
+++ /dev/null
@@ -1,444 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core;
-
-import org.apache.http.HttpHost;
-import org.elasticsearch.TransportVersion;
-import org.elasticsearch.Version;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.PathUtils;
-import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.Tuple;
-import org.elasticsearch.test.rest.ObjectPath;
-import org.elasticsearch.xcontent.json.JsonXContent;
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
-import org.elasticsearch.xpack.esql.core.expression.Literal;
-import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.EsField;
-import org.elasticsearch.xpack.esql.core.util.StringUtils;
-import org.hamcrest.Description;
-import org.hamcrest.Matcher;
-import org.hamcrest.TypeSafeDiagnosingMatcher;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.net.URLConnection;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.FileVisitOption;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.time.ZoneId;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.jar.JarInputStream;
-import java.util.regex.Pattern;
-import java.util.zip.ZipEntry;
-
-import static java.util.Collections.emptyMap;
-import static org.elasticsearch.cluster.ClusterState.VERSION_INTRODUCING_TRANSPORT_VERSIONS;
-import static org.elasticsearch.test.ESTestCase.between;
-import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
-import static org.elasticsearch.test.ESTestCase.randomBoolean;
-import static org.elasticsearch.test.ESTestCase.randomFrom;
-import static org.elasticsearch.test.ESTestCase.randomZone;
-import static org.elasticsearch.xpack.esql.core.TestUtils.StringContainsRegex.containsRegex;
-import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
-import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
-import static org.hamcrest.Matchers.containsString;
-import static org.junit.Assert.assertEquals;
-
-public final class TestUtils {
-
-    public static final ZoneId UTC = ZoneId.of("Z");
-    public static final Configuration TEST_CFG = new Configuration(UTC, null, null);
-
-    private static final String MATCHER_TYPE_CONTAINS = "CONTAINS";
-    private static final String MATCHER_TYPE_REGEX = "REGEX";
-
-    private TestUtils() {}
-
-    public static Configuration randomConfiguration() {
-        return new Configuration(randomZone(), randomAlphaOfLength(10), randomAlphaOfLength(10));
-    }
-
-    public static Configuration randomConfiguration(ZoneId zoneId) {
-        return new Configuration(zoneId, randomAlphaOfLength(10), randomAlphaOfLength(10));
-    }
-
-    public static Literal of(Object value) {
-        return of(Source.EMPTY, value);
-    }
-
-    /**
-     * Utility method for creating 'in-line' Literals (out of values instead of expressions).
-     */
-    public static Literal of(Source source, Object value) {
-        if (value instanceof Literal) {
-            return (Literal) value;
-        }
-        return new Literal(source, value, DataType.fromJava(value));
-    }
-
-    public static Range rangeOf(Expression value, Expression lower, boolean includeLower, Expression upper, boolean includeUpper) {
-        return new Range(EMPTY, value, lower, includeLower, upper, includeUpper, randomZone());
-    }
-
-    public static WildcardLike wildcardLike(Expression left, String exp) {
-        return new WildcardLike(EMPTY, left, new WildcardPattern(exp));
-    }
-
-    public static RLike rlike(Expression left, String exp) {
-        return new RLike(EMPTY, left, new RLikePattern(exp));
-    }
-
-    public static FieldAttribute fieldAttribute() {
-        return fieldAttribute(randomAlphaOfLength(10), randomFrom(DataType.types()));
-    }
-
-    public static FieldAttribute fieldAttribute(String name, DataType type) {
-        return new FieldAttribute(EMPTY, name, new EsField(name, type, emptyMap(), randomBoolean()));
-    }
-
-    //
-    // Common methods / assertions
-    //
-
-    public static void assertNoSearchContexts(RestClient client) throws IOException {
-        Map<String, Object> stats = searchStats(client);
-        @SuppressWarnings("unchecked")
-        Map<String, Object> indicesStats = (Map<String, Object>) stats.get("indices");
-        for (String index : indicesStats.keySet()) {
-            if (index.startsWith(".") == false) { // We are not interested in internal indices
-                assertEquals(index + " should have no search contexts", 0, getOpenContexts(stats, index));
-            }
-        }
-    }
-
-    public static int getNumberOfSearchContexts(RestClient client, String index) throws IOException {
-        return getOpenContexts(searchStats(client), index);
-    }
-
-    private static Map<String, Object> searchStats(RestClient client) throws IOException {
-        Response response = client.performRequest(new Request("GET", "/_stats/search"));
-        try (InputStream content = response.getEntity().getContent()) {
-            return XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false);
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    private static int getOpenContexts(Map<String, Object> stats, String index) {
-        stats = (Map<String, Object>) stats.get("indices");
-        stats = (Map<String, Object>) stats.get(index);
-        stats = (Map<String, Object>) stats.get("total");
-        stats = (Map<String, Object>) stats.get("search");
-        return (Integer) stats.get("open_contexts");
-    }
-
-    //
-    // Classpath
-    //
-    /**
-     * Returns the classpath resources matching a simple pattern ("*.csv").
-     * It supports folders separated by "/" (e.g. "/some/folder/*.txt").
-     *
-     * Currently able to resolve resources inside the classpath either from:
-     * folders in the file-system (typically IDEs) or
-     * inside jars (gradle).
-     */
-    @SuppressForbidden(reason = "classpath discovery")
-    public static List<URL> classpathResources(String pattern) throws IOException {
-        while (pattern.startsWith("/")) {
-            pattern = pattern.substring(1);
-        }
-
-        Tuple<String, String> split = pathAndName(pattern);
-
-        // the root folder searched inside the classpath - default is the root classpath
-        // default file match
-        final String root = split.v1();
-        final String filePattern = split.v2();
-
-        String[] resources = System.getProperty("java.class.path").split(System.getProperty("path.separator"));
-
-        List<URL> matches = new ArrayList<>();
-
-        for (String resource : resources) {
-            Path path = PathUtils.get(resource);
-
-            // check whether we're dealing with a jar
-            // Java 7 java.nio.fileFileSystem can be used on top of ZIPs/JARs but consumes more memory
-            // hence the use of the JAR API
-            if (path.toString().endsWith(".jar")) {
-                try (JarInputStream jar = jarInputStream(path.toUri().toURL())) {
-                    ZipEntry entry = null;
-                    while ((entry = jar.getNextEntry()) != null) {
-                        String name = entry.getName();
-                        Tuple<String, String> entrySplit = pathAndName(name);
-                        if (root.equals(entrySplit.v1()) && Regex.simpleMatch(filePattern, entrySplit.v2())) {
-                            matches.add(new URL("jar:" + path.toUri() + "!/" + name));
-                        }
-                    }
-                }
-            }
-            // normal file access
-            else if (Files.isDirectory(path)) {
-                Files.walkFileTree(path, EnumSet.allOf(FileVisitOption.class), 1, new SimpleFileVisitor<>() {
-                    @Override
-                    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                        if (Regex.simpleMatch(filePattern, file.toString())) {
-                            matches.add(file.toUri().toURL());
-                        }
-                        return FileVisitResult.CONTINUE;
-                    }
-                });
-            }
-        }
-        return matches;
-    }
-
-    @SuppressForbidden(reason = "need to open stream")
-    public static InputStream inputStream(URL resource) throws IOException {
-        URLConnection con = resource.openConnection();
-        // do not to cache files (to avoid keeping file handles around)
-        con.setUseCaches(false);
-        return con.getInputStream();
-    }
-
-    @SuppressForbidden(reason = "need to open jar")
-    public static JarInputStream jarInputStream(URL resource) throws IOException {
-        return new JarInputStream(inputStream(resource));
-    }
-
-    public static BufferedReader reader(URL resource) throws IOException {
-        return new BufferedReader(new InputStreamReader(inputStream(resource), StandardCharsets.UTF_8));
-    }
-
-    public static Tuple<String, String> pathAndName(String string) {
-        String folder = StringUtils.EMPTY;
-        String file = string;
-        int lastIndexOf = string.lastIndexOf('/');
-        if (lastIndexOf > 0) {
-            folder = string.substring(0, lastIndexOf - 1);
-            if (lastIndexOf + 1 < string.length()) {
-                file = string.substring(lastIndexOf + 1);
-            }
-        }
-        return new Tuple<>(folder, file);
-    }
-
-    public static TestNodes buildNodeAndVersions(RestClient client, String bwcNodesVersion) throws IOException {
-        Response response = client.performRequest(new Request("GET", "_nodes"));
-        ObjectPath objectPath = ObjectPath.createFromResponse(response);
-        Map<String, Object> nodesAsMap = objectPath.evaluate("nodes");
-        TestNodes nodes = new TestNodes(bwcNodesVersion);
-        for (String id : nodesAsMap.keySet()) {
-            String nodeVersion = objectPath.evaluate("nodes." + id + ".version");
-
-            Object tvField;
-            TransportVersion transportVersion = null;
-            if ((tvField = objectPath.evaluate("nodes." + id + ".transport_version")) != null) {
-                // this json might be from a node <8.8.0, but about a node >=8.8.0
-                // in which case the transport_version field won't exist. Just ignore it for now.
-                transportVersion = TransportVersion.fromString(tvField.toString());
-            } else { // no transport_version field
-                // this json might be from a node <8.8.0, but about a node >=8.8.0
-                // In that case the transport_version field won't exist. Just ignore it for now.
-                Version version = Version.fromString(nodeVersion);
-                if (version.before(VERSION_INTRODUCING_TRANSPORT_VERSIONS)) {
-                    transportVersion = TransportVersion.fromId(version.id);
-                }
-            }
-
-            nodes.add(
-                new TestNode(
-                    id,
-                    nodeVersion,
-                    transportVersion,
-                    HttpHost.create(objectPath.evaluate("nodes." + id + ".http.publish_address"))
-                )
-            );
-        }
-        return nodes;
-    }
-
-    public static String readResource(InputStream input) throws IOException {
-        StringBuilder builder = new StringBuilder();
-        try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8))) {
-            String line = reader.readLine();
-            while (line != null) {
-                if (line.trim().startsWith("//") == false) {
-                    builder.append(line);
-                    builder.append('\n');
-                }
-                line = reader.readLine();
-            }
-            return builder.toString();
-        }
-    }
-
-    public static Map<String, Object> randomRuntimeMappings() {
-        int count = between(1, 100);
-        Map<String, Object> runtimeFields = Maps.newMapWithExpectedSize(count);
-        while (runtimeFields.size() < count) {
-            int size = between(1, 10);
-            Map<String, Object> config = Maps.newMapWithExpectedSize(size);
-            while (config.size() < size) {
-                config.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
-            }
-            runtimeFields.put(randomAlphaOfLength(5), config);
-        }
-        return runtimeFields;
-    }
-
-    public static Collection<Object[]> readSpec(Class<?> clazz, String testFileName) throws Exception {
-        ArrayList<Object[]> arr = new ArrayList<>();
-        Map<String, Integer> testNames = new LinkedHashMap<>();
-
-        try (
-            InputStream is = clazz.getResourceAsStream(testFileName);
-            BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))
-        ) {
-            int lineNumber = 0;
-            String line;
-            boolean done = false;
-            String name = null;
-            String query = null;
-            ArrayList<Matcher<String>> matchers = new ArrayList<>(8);
-
-            StringBuilder sb = new StringBuilder();
-
-            while ((line = reader.readLine()) != null) {
-                lineNumber++;
-                line = line.trim();
-
-                if (line.isEmpty() || line.startsWith("//")) {
-                    continue;
-                }
-
-                if (name == null) {
-                    name = line;
-                    Integer previousName = testNames.put(name, lineNumber);
-                    if (previousName != null) {
-                        throw new IllegalArgumentException(
-                            "Duplicate test name '" + line + "' at line " + lineNumber + " (previously seen at line " + previousName + ")"
-                        );
-                    }
-                }
-
-                else if (query == null) {
-                    sb.append(line).append(' ');
-                    if (line.endsWith(";")) {
-                        sb.setLength(sb.length() - 2);
-                        query = sb.toString();
-                        sb.setLength(0);
-                    }
-                }
-
-                else {
-                    if (line.endsWith(";")) {
-                        line = line.substring(0, line.length() - 1);
-                        done = true;
-                    }
-
-                    if (line.isEmpty() == false) {
-                        String[] matcherAndExpectation = line.split("[ \\t]+", 2);
-                        if (matcherAndExpectation.length == 1) {
-                            matchers.add(containsString(matcherAndExpectation[0]));
-                        } else if (matcherAndExpectation.length == 2) {
-                            String matcherType = matcherAndExpectation[0];
-                            String expectation = matcherAndExpectation[1];
-                            switch (matcherType.toUpperCase(Locale.ROOT)) {
-                                case MATCHER_TYPE_CONTAINS -> matchers.add(containsString(expectation));
-                                case MATCHER_TYPE_REGEX -> matchers.add(containsRegex(expectation));
-                                default -> throw new IllegalArgumentException(
-                                    "unsupported matcher on line " + testFileName + ":" + lineNumber + ": " + matcherType
-                                );
-                            }
-                        }
-                    }
-
-                    if (done) {
-                        // Add and zero out for the next spec
-                        arr.add(new Object[] { testFileName, name, query, matchers });
-                        name = null;
-                        query = null;
-                        matchers = new ArrayList<>(8);
-                        done = false;
-                    }
-                }
-            }
-
-            if (name != null) {
-                throw new IllegalStateException("Read a test [" + name + "] without a body at the end of [" + testFileName + "]");
-            }
-        }
-        return arr;
-    }
-
-    public static FieldAttribute getFieldAttribute(String name) {
-        return getFieldAttribute(name, INTEGER);
-    }
-
-    public static FieldAttribute getFieldAttribute(String name, DataType dataType) {
-        return new FieldAttribute(EMPTY, name, new EsField(name + "f", dataType, emptyMap(), true));
-    }
-
-    // Matcher which extends the functionality of org.hamcrest.Matchers.matchesPattern(String)}
-    // by allowing to match detected regex groups later on in the pattern, e.g.:
-    // "(?<id>.+?)"....... \k<id>....."}
-    public static class StringContainsRegex extends TypeSafeDiagnosingMatcher<String> {
-
-        private final Pattern pattern;
-
-        protected StringContainsRegex(Pattern pattern) {
-            this.pattern = pattern;
-        }
-
-        @Override
-        public void describeTo(Description description) {
-            description.appendText("a string containing the pattern ").appendValue(pattern);
-        }
-
-        @Override
-        protected boolean matchesSafely(String actual, Description mismatchDescription) {
-            if (pattern.matcher(actual).find() == false) {
-                mismatchDescription.appendText("the string was ").appendValue(actual);
-                return false;
-            }
-            return true;
-        }
-
-        public static Matcher<String> containsRegex(String regex) {
-            return new StringContainsRegex(Pattern.compile(regex));
-        }
-    }
-}
diff --git a/x-pack/plugin/esql/arrow/build.gradle b/x-pack/plugin/esql/arrow/build.gradle
index e8ae4b049cf7d..3bf637080841f 100644
--- a/x-pack/plugin/esql/arrow/build.gradle
+++ b/x-pack/plugin/esql/arrow/build.gradle
@@ -56,6 +56,6 @@ tasks.named("thirdPartyAudit").configure {
   )
 }
 
-test {
+tasks.named("test").configure {
   jvmArgs('--add-opens=java.base/java.nio=ALL-UNNAMED')
 }
diff --git a/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java b/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java
index 8c2243284a538..7a8328060a390 100644
--- a/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java
+++ b/x-pack/plugin/esql/arrow/src/main/java/org/elasticsearch/xpack/esql/arrow/ArrowResponse.java
@@ -326,7 +326,7 @@ protected void encodeChunk(int sizeHint, RecyclerBytesStreamOutput out) throws I
      */
     static final Map<String, BlockConverter> ESQL_CONVERTERS = Map.ofEntries(
         // For reference:
-        // - EsqlDataTypes: list of ESQL data types (not all are present in outputs)
+        // - DataType: list of ESQL data types (not all are present in outputs)
         // - PositionToXContent: conversions for ESQL JSON output
         // - EsqlDataTypeConverter: conversions to ESQL datatypes
         // Missing: multi-valued values
diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle
index 8803fd81147ef..26cf53b334b1e 100644
--- a/x-pack/plugin/esql/build.gradle
+++ b/x-pack/plugin/esql/build.gradle
@@ -1,5 +1,6 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask;
+import org.elasticsearch.gradle.internal.util.SourceDirectoryCommandLineArgumentProvider;
 
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
@@ -49,8 +50,10 @@ dependencies {
   internalClusterTestImplementation project(":modules:mapper-extras")
 }
 
+def projectDirectory = project.layout.projectDirectory
+def generatedSourceDir = projectDirectory.dir("src/main/generated")
 tasks.named("compileJava").configure {
-  options.compilerArgs.addAll(["-s", "src/main/generated"])
+  options.compilerArgumentProviders.add(new SourceDirectoryCommandLineArgumentProvider(generatedSourceDir))
   // IntelliJ sticks generated files here and we can't stop it....
   exclude { it.file.toString().contains("src/main/generated-src/generated") }
 }
diff --git a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java
index 69db6a1310c9e..444dbcc1b9e58 100644
--- a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java
+++ b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java
@@ -57,4 +57,10 @@
 
     IntermediateState[] value() default {};
 
+    /**
+     * Exceptions thrown by the `combine*(...)` methods to catch and convert
+     * into a warning and turn into a null value.
+     */
+    Class<? extends Exception>[] warnExceptions() default {};
+
 }
diff --git a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java
index 0216ea07e5c7c..8d81b60e20e4d 100644
--- a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java
+++ b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java
@@ -22,6 +22,12 @@
 
     IntermediateState[] value() default {};
 
+    /**
+     * Exceptions thrown by the `combine*(...)` methods to catch and convert
+     * into a warning and turn into a null value.
+     */
+    Class<? extends Exception>[] warnExceptions() default {};
+
     /**
      * If {@code true} then the @timestamp LongVector will be appended to the input blocks of the aggregation function.
      */
diff --git a/x-pack/plugin/esql/compute/build.gradle b/x-pack/plugin/esql/compute/build.gradle
index e5816d0b7c78b..971bfd39c231f 100644
--- a/x-pack/plugin/esql/compute/build.gradle
+++ b/x-pack/plugin/esql/compute/build.gradle
@@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.internal.util.SourceDirectoryCommandLineArgumentProvider;
+
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.string-templates'
 apply plugin: 'elasticsearch.publish'
@@ -16,13 +18,15 @@ dependencies {
   testImplementation(project(xpackModule('esql-core')))
 }
 
+def projectDirectory = project.layout.projectDirectory
+def generatedSourceDir = projectDirectory.dir("src/main/generated")
 tasks.named("compileJava").configure {
-  options.compilerArgs.addAll(["-s", "src/main/generated"])
+  options.compilerArgumentProviders.add(new SourceDirectoryCommandLineArgumentProvider(generatedSourceDir))
 }
 
 tasks.named('checkstyleMain').configure {
   source = "src/main/java"
-  excludes = [ "**/*.java.st" ]
+  excludes = ["**/*.java.st"]
 }
 
 tasks.named("spotlessJava") { dependsOn stringTemplates }
@@ -36,499 +40,568 @@ spotless {
   }
 }
 
-def prop(Type, type, Wrapper, TYPE, BYTES, Array, Hash) {
+def prop(Name, Type, type, Wrapper, TYPE, BYTES, Array, Hash) {
   return [
-    "Type" : Type,
-    "type" : type,
-    "Wrapper": Wrapper,
-    "TYPE" : TYPE,
-    "BYTES" : BYTES,
-    "Array" : Array,
-    "Hash" : Hash,
+    // Name of the DataType. Use in DataType names
+    "Name"    : Name,
+    // PascalCased type. Use in ElementType names
+    "Type"    : Type,
+    // Variable type. May be a primitive
+    "type"    : type,
+    // Wrapper type. Only for primitive types
+    "Wrapper" : Wrapper,
+    // SCREAMING_SNAKE_CASE type. Use in ElementType names
+    "TYPE"    : TYPE,
+    "BYTES"   : BYTES,
+    "Array"   : Array,
+    "Hash"    : Hash,
 
-    "int" : type == "int" ? "true" : "",
-    "float" : type == "float" ? "true" : "",
-    "long" : type == "long" ? "true" : "",
-    "double" : type == "double" ? "true" : "",
-    "BytesRef" : type == "BytesRef" ? "true" : "",
-    "boolean" : type == "boolean" ? "true" : "",
+    "int"     : Name == "Int" ? "true" : "",
+    "float"   : Name == "Float" ? "true" : "",
+    "long"    : Name == "Long" ? "true" : "",
+    "double"  : Name == "Double" ? "true" : "",
+    "boolean" : Name == "Boolean" ? "true" : "",
+    "BytesRef": Name == "BytesRef" ? "true" : "",
+    "Ip"      : Name == "Ip" ? "true" : "",
   ]
 }
 
 tasks.named('stringTemplates').configure {
-  var intProperties      = prop("Int", "int", "Integer", "INT", "Integer.BYTES", "IntArray", "LongHash")
-  var floatProperties    = prop("Float", "float", "Float", "FLOAT", "Float.BYTES", "FloatArray", "LongHash")
-  var longProperties     = prop("Long", "long", "Long", "LONG", "Long.BYTES", "LongArray", "LongHash")
-  var doubleProperties   = prop("Double", "double", "Double", "DOUBLE", "Double.BYTES", "DoubleArray", "LongHash")
-  var bytesRefProperties = prop("BytesRef", "BytesRef", "", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "", "BytesRefHash")
-  var booleanProperties  = prop("Boolean", "boolean", "Boolean", "BOOLEAN", "Byte.BYTES", "BitArray", "")
+  var intProperties = prop("Int", "Int", "int", "Integer", "INT", "Integer.BYTES", "IntArray", "LongHash")
+  var floatProperties = prop("Float", "Float", "float", "Float", "FLOAT", "Float.BYTES", "FloatArray", "LongHash")
+  var longProperties = prop("Long", "Long", "long", "Long", "LONG", "Long.BYTES", "LongArray", "LongHash")
+  var doubleProperties = prop("Double", "Double", "double", "Double", "DOUBLE", "Double.BYTES", "DoubleArray", "LongHash")
+  var booleanProperties = prop("Boolean", "Boolean", "boolean", "Boolean", "BOOLEAN", "Byte.BYTES", "BitArray", "")
+  var bytesRefProperties = prop(
+    "BytesRef",
+    "BytesRef",
+    "BytesRef",
+    "",
+    "BYTES_REF",
+    "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF",
+    "",
+    "BytesRefHash"
+  )
+  var ipProperties = prop("Ip", "BytesRef", "BytesRef", "", "BYTES_REF", "16", "", "")
 
   // primitive vectors
   File vectorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  vectorInputFile
+    it.inputFile = vectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/IntVector.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  vectorInputFile
+    it.inputFile = vectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/FloatVector.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  vectorInputFile
+    it.inputFile = vectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/LongVector.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  vectorInputFile
+    it.inputFile = vectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/DoubleVector.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  vectorInputFile
+    it.inputFile = vectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/BytesRefVector.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  vectorInputFile
+    it.inputFile = vectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanVector.java"
   }
   // array vector implementations
   File arrayVectorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  arrayVectorInputFile
+    it.inputFile = arrayVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/IntArrayVector.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  arrayVectorInputFile
+    it.inputFile = arrayVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/FloatArrayVector.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  arrayVectorInputFile
+    it.inputFile = arrayVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/LongArrayVector.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  arrayVectorInputFile
+    it.inputFile = arrayVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/DoubleArrayVector.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  arrayVectorInputFile
+    it.inputFile = arrayVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/BytesRefArrayVector.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  arrayVectorInputFile
+    it.inputFile = arrayVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanArrayVector.java"
   }
   // BigArray vector implementations
   File bigArrayVectorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  bigArrayVectorInputFile
+    it.inputFile = bigArrayVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/IntBigArrayVector.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  bigArrayVectorInputFile
+    it.inputFile = bigArrayVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/FloatBigArrayVector.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  bigArrayVectorInputFile
+    it.inputFile = bigArrayVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/LongBigArrayVector.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  bigArrayVectorInputFile
+    it.inputFile = bigArrayVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/DoubleBigArrayVector.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  bigArrayVectorInputFile
+    it.inputFile = bigArrayVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanBigArrayVector.java"
   }
   // constant vectors
   File constantVectorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  constantVectorInputFile
+    it.inputFile = constantVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/ConstantIntVector.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  constantVectorInputFile
+    it.inputFile = constantVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/ConstantFloatVector.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  constantVectorInputFile
+    it.inputFile = constantVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/ConstantLongVector.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  constantVectorInputFile
+    it.inputFile = constantVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/ConstantDoubleVector.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  constantVectorInputFile
+    it.inputFile = constantVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/ConstantBytesRefVector.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  constantVectorInputFile
+    it.inputFile = constantVectorInputFile
     it.outputFile = "org/elasticsearch/compute/data/ConstantBooleanVector.java"
   }
   // primitive blocks
   File blockInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-Block.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  blockInputFile
+    it.inputFile = blockInputFile
     it.outputFile = "org/elasticsearch/compute/data/IntBlock.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  blockInputFile
+    it.inputFile = blockInputFile
     it.outputFile = "org/elasticsearch/compute/data/FloatBlock.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  blockInputFile
+    it.inputFile = blockInputFile
     it.outputFile = "org/elasticsearch/compute/data/LongBlock.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  blockInputFile
+    it.inputFile = blockInputFile
     it.outputFile = "org/elasticsearch/compute/data/DoubleBlock.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  blockInputFile
+    it.inputFile = blockInputFile
     it.outputFile = "org/elasticsearch/compute/data/BytesRefBlock.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  blockInputFile
+    it.inputFile = blockInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanBlock.java"
   }
   // array blocks
   File arrayBlockInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  arrayBlockInputFile
+    it.inputFile = arrayBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/IntArrayBlock.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  arrayBlockInputFile
+    it.inputFile = arrayBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/FloatArrayBlock.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  arrayBlockInputFile
+    it.inputFile = arrayBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/LongArrayBlock.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  arrayBlockInputFile
+    it.inputFile = arrayBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/DoubleArrayBlock.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  arrayBlockInputFile
+    it.inputFile = arrayBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/BytesRefArrayBlock.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  arrayBlockInputFile
+    it.inputFile = arrayBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanArrayBlock.java"
   }
   // BigArray block implementations
   File bigArrayBlockInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  bigArrayBlockInputFile
+    it.inputFile = bigArrayBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/IntBigArrayBlock.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  bigArrayBlockInputFile
+    it.inputFile = bigArrayBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/FloatBigArrayBlock.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  bigArrayBlockInputFile
+    it.inputFile = bigArrayBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/LongBigArrayBlock.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  bigArrayBlockInputFile
+    it.inputFile = bigArrayBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/DoubleBigArrayBlock.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  bigArrayBlockInputFile
+    it.inputFile = bigArrayBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanBigArrayBlock.java"
   }
   // vector blocks
   File vectorBlockInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  vectorBlockInputFile
+    it.inputFile = vectorBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/IntVectorBlock.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  vectorBlockInputFile
+    it.inputFile = vectorBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/FloatVectorBlock.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  vectorBlockInputFile
+    it.inputFile = vectorBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/LongVectorBlock.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  vectorBlockInputFile
+    it.inputFile = vectorBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/DoubleVectorBlock.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  vectorBlockInputFile
+    it.inputFile = vectorBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/BytesRefVectorBlock.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  vectorBlockInputFile
+    it.inputFile = vectorBlockInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanVectorBlock.java"
   }
   // block builders
   File blockBuildersInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-BlockBuilder.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  blockBuildersInputFile
+    it.inputFile = blockBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/IntBlockBuilder.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  blockBuildersInputFile
+    it.inputFile = blockBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/FloatBlockBuilder.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  blockBuildersInputFile
+    it.inputFile = blockBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/LongBlockBuilder.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  blockBuildersInputFile
+    it.inputFile = blockBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/DoubleBlockBuilder.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  blockBuildersInputFile
+    it.inputFile = blockBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/BytesRefBlockBuilder.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  blockBuildersInputFile
+    it.inputFile = blockBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanBlockBuilder.java"
   }
   // vector builders
   File vectorBuildersInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-VectorBuilder.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  vectorBuildersInputFile
+    it.inputFile = vectorBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/IntVectorBuilder.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  vectorBuildersInputFile
+    it.inputFile = vectorBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/FloatVectorBuilder.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  vectorBuildersInputFile
+    it.inputFile = vectorBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/LongVectorBuilder.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  vectorBuildersInputFile
+    it.inputFile = vectorBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/DoubleVectorBuilder.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  vectorBuildersInputFile
+    it.inputFile = vectorBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/BytesRefVectorBuilder.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  vectorBuildersInputFile
+    it.inputFile = vectorBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanVectorBuilder.java"
   }
   File vectorFixedBuildersInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-VectorFixedBuilder.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  vectorFixedBuildersInputFile
+    it.inputFile = vectorFixedBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/IntVectorFixedBuilder.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  vectorFixedBuildersInputFile
+    it.inputFile = vectorFixedBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/FloatVectorFixedBuilder.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  vectorFixedBuildersInputFile
+    it.inputFile = vectorFixedBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/LongVectorFixedBuilder.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  vectorFixedBuildersInputFile
+    it.inputFile = vectorFixedBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/DoubleVectorFixedBuilder.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  vectorFixedBuildersInputFile
+    it.inputFile = vectorFixedBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanVectorFixedBuilder.java"
   }
   File stateInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-State.java.st")
   template {
     it.properties = booleanProperties
-    it.inputFile =  stateInputFile
+    it.inputFile = stateInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/BooleanState.java"
   }
   template {
     it.properties = intProperties
-    it.inputFile =  stateInputFile
+    it.inputFile = stateInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/IntState.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  stateInputFile
+    it.inputFile = stateInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/LongState.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  stateInputFile
+    it.inputFile = stateInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/FloatState.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  stateInputFile
+    it.inputFile = stateInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/DoubleState.java"
   }
+  File fallibleStateInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleState.java.st")
+  template {
+    it.properties = booleanProperties
+    it.inputFile =  fallibleStateInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/BooleanFallibleState.java"
+  }
+  template {
+    it.properties = intProperties
+    it.inputFile =  fallibleStateInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/IntFallibleState.java"
+  }
+  template {
+    it.properties = longProperties
+    it.inputFile =  fallibleStateInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/LongFallibleState.java"
+  }
+  template {
+    it.properties = floatProperties
+    it.inputFile =  fallibleStateInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/FloatFallibleState.java"
+  }
+  template {
+    it.properties = doubleProperties
+    it.inputFile =  fallibleStateInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/DoubleFallibleState.java"
+  }
   // block lookups
   File lookupInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-Lookup.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  lookupInputFile
+    it.inputFile = lookupInputFile
     it.outputFile = "org/elasticsearch/compute/data/IntLookup.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  lookupInputFile
+    it.inputFile = lookupInputFile
     it.outputFile = "org/elasticsearch/compute/data/LongLookup.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  lookupInputFile
+    it.inputFile = lookupInputFile
     it.outputFile = "org/elasticsearch/compute/data/FloatLookup.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  lookupInputFile
+    it.inputFile = lookupInputFile
     it.outputFile = "org/elasticsearch/compute/data/DoubleLookup.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  lookupInputFile
+    it.inputFile = lookupInputFile
     it.outputFile = "org/elasticsearch/compute/data/BytesRefLookup.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  lookupInputFile
+    it.inputFile = lookupInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanLookup.java"
   }
   File arrayStateInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st")
   template {
     it.properties = booleanProperties
-    it.inputFile =  arrayStateInputFile
+    it.inputFile = arrayStateInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/BooleanArrayState.java"
   }
   template {
     it.properties = intProperties
-    it.inputFile =  arrayStateInputFile
+    it.inputFile = arrayStateInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/IntArrayState.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  arrayStateInputFile
+    it.inputFile = arrayStateInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/LongArrayState.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  arrayStateInputFile
+    it.inputFile = arrayStateInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/DoubleArrayState.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  arrayStateInputFile
+    it.inputFile = arrayStateInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/FloatArrayState.java"
   }
+  File fallibleArrayStateInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleArrayState.java.st")
+  template {
+    it.properties = booleanProperties
+    it.inputFile =  fallibleArrayStateInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/BooleanFallibleArrayState.java"
+  }
+  template {
+    it.properties = intProperties
+    it.inputFile =  fallibleArrayStateInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/IntFallibleArrayState.java"
+  }
+  template {
+    it.properties = longProperties
+    it.inputFile =  fallibleArrayStateInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/LongFallibleArrayState.java"
+  }
+  template {
+    it.properties = doubleProperties
+    it.inputFile =  fallibleArrayStateInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/DoubleFallibleArrayState.java"
+  }
+  template {
+    it.properties = floatProperties
+    it.inputFile =  fallibleArrayStateInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/FloatFallibleArrayState.java"
+  }
   File valuesAggregatorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  valuesAggregatorInputFile
+    it.inputFile = valuesAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/ValuesIntAggregator.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  valuesAggregatorInputFile
+    it.inputFile = valuesAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/ValuesLongAggregator.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  valuesAggregatorInputFile
+    it.inputFile = valuesAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  valuesAggregatorInputFile
+    it.inputFile = valuesAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/ValuesDoubleAggregator.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  valuesAggregatorInputFile
+    it.inputFile = valuesAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java"
   }
 
   File rateAggregatorInputFile = file("src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  rateAggregatorInputFile
+    it.inputFile = rateAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/RateIntAggregator.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  rateAggregatorInputFile
+    it.inputFile = rateAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/RateLongAggregator.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  rateAggregatorInputFile
+    it.inputFile = rateAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/RateFloatAggregator.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  rateAggregatorInputFile
+    it.inputFile = rateAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/RateDoubleAggregator.java"
   }
 
@@ -536,180 +609,190 @@ tasks.named('stringTemplates').configure {
   File topAggregatorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  topAggregatorInputFile
+    it.inputFile = topAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/TopIntAggregator.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  topAggregatorInputFile
+    it.inputFile = topAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/TopLongAggregator.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  topAggregatorInputFile
+    it.inputFile = topAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/TopFloatAggregator.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  topAggregatorInputFile
+    it.inputFile = topAggregatorInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/TopDoubleAggregator.java"
   }
+  template {
+    it.properties = booleanProperties
+    it.inputFile = topAggregatorInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/TopBooleanAggregator.java"
+  }
+  template {
+    it.properties = ipProperties
+    it.inputFile = topAggregatorInputFile
+    it.outputFile = "org/elasticsearch/compute/aggregation/TopIpAggregator.java"
+  }
 
   File multivalueDedupeInputFile = file("src/main/java/org/elasticsearch/compute/operator/mvdedupe/X-MultivalueDedupe.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  multivalueDedupeInputFile
+    it.inputFile = multivalueDedupeInputFile
     it.outputFile = "org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeInt.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  multivalueDedupeInputFile
+    it.inputFile = multivalueDedupeInputFile
     it.outputFile = "org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeLong.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  multivalueDedupeInputFile
+    it.inputFile = multivalueDedupeInputFile
     it.outputFile = "org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeDouble.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  multivalueDedupeInputFile
+    it.inputFile = multivalueDedupeInputFile
     it.outputFile = "org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeBytesRef.java"
   }
   File blockHashInputFile = file("src/main/java/org/elasticsearch/compute/aggregation/blockhash/X-BlockHash.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  blockHashInputFile
+    it.inputFile = blockHashInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  blockHashInputFile
+    it.inputFile = blockHashInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  blockHashInputFile
+    it.inputFile = blockHashInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java"
   }
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  blockHashInputFile
+    it.inputFile = blockHashInputFile
     it.outputFile = "org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java"
   }
   File keyExtractorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/operator/topn/X-KeyExtractor.java.st")
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  keyExtractorInputFile
+    it.inputFile = keyExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/KeyExtractorForBytesRef.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  keyExtractorInputFile
+    it.inputFile = keyExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/KeyExtractorForBoolean.java"
   }
   template {
     it.properties = intProperties
-    it.inputFile =  keyExtractorInputFile
+    it.inputFile = keyExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/KeyExtractorForInt.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  keyExtractorInputFile
+    it.inputFile = keyExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/KeyExtractorForLong.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  keyExtractorInputFile
+    it.inputFile = keyExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/KeyExtractorForFloat.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  keyExtractorInputFile
+    it.inputFile = keyExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/KeyExtractorForDouble.java"
   }
   File valueExtractorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/operator/topn/X-ValueExtractor.java.st")
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  valueExtractorInputFile
+    it.inputFile = valueExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ValueExtractorForBytesRef.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  valueExtractorInputFile
+    it.inputFile = valueExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ValueExtractorForBoolean.java"
   }
   template {
     it.properties = intProperties
-    it.inputFile =  valueExtractorInputFile
+    it.inputFile = valueExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ValueExtractorForInt.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  valueExtractorInputFile
+    it.inputFile = valueExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ValueExtractorForLong.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  valueExtractorInputFile
+    it.inputFile = valueExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ValueExtractorForFloat.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  valueExtractorInputFile
+    it.inputFile = valueExtractorInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ValueExtractorForDouble.java"
   }
   File resultBuilderInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/operator/topn/X-ResultBuilder.java.st")
   template {
     it.properties = bytesRefProperties
-    it.inputFile =  resultBuilderInputFile
+    it.inputFile = resultBuilderInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ResultBuilderForBytesRef.java"
   }
   template {
     it.properties = booleanProperties
-    it.inputFile =  resultBuilderInputFile
+    it.inputFile = resultBuilderInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ResultBuilderForBoolean.java"
   }
   template {
     it.properties = intProperties
-    it.inputFile =  resultBuilderInputFile
+    it.inputFile = resultBuilderInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ResultBuilderForInt.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  resultBuilderInputFile
+    it.inputFile = resultBuilderInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ResultBuilderForLong.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  resultBuilderInputFile
+    it.inputFile = resultBuilderInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ResultBuilderForDouble.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  resultBuilderInputFile
+    it.inputFile = resultBuilderInputFile
     it.outputFile = "org/elasticsearch/compute/operator/topn/ResultBuilderForFloat.java"
   }
 
   File bucketedSortInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/sort/X-BucketedSort.java.st")
   template {
     it.properties = intProperties
-    it.inputFile =  bucketedSortInputFile
+    it.inputFile = bucketedSortInputFile
     it.outputFile = "org/elasticsearch/compute/data/sort/IntBucketedSort.java"
   }
   template {
     it.properties = longProperties
-    it.inputFile =  bucketedSortInputFile
+    it.inputFile = bucketedSortInputFile
     it.outputFile = "org/elasticsearch/compute/data/sort/LongBucketedSort.java"
   }
   template {
     it.properties = floatProperties
-    it.inputFile =  bucketedSortInputFile
+    it.inputFile = bucketedSortInputFile
     it.outputFile = "org/elasticsearch/compute/data/sort/FloatBucketedSort.java"
   }
   template {
     it.properties = doubleProperties
-    it.inputFile =  bucketedSortInputFile
+    it.inputFile = bucketedSortInputFile
     it.outputFile = "org/elasticsearch/compute/data/sort/DoubleBucketedSort.java"
   }
 }
diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorFunctionSupplierImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorFunctionSupplierImplementer.java
index 3f031db2978f9..f11ccbced6fbe 100644
--- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorFunctionSupplierImplementer.java
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorFunctionSupplierImplementer.java
@@ -10,6 +10,7 @@
 import com.squareup.javapoet.ClassName;
 import com.squareup.javapoet.JavaFile;
 import com.squareup.javapoet.MethodSpec;
+import com.squareup.javapoet.TypeName;
 import com.squareup.javapoet.TypeSpec;
 
 import org.elasticsearch.compute.ann.Aggregator;
@@ -31,6 +32,7 @@
 import static org.elasticsearch.compute.gen.Types.AGGREGATOR_FUNCTION_SUPPLIER;
 import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT;
 import static org.elasticsearch.compute.gen.Types.LIST_INTEGER;
+import static org.elasticsearch.compute.gen.Types.STRING;
 
 /**
  * Implements "AggregationFunctionSupplier" from a class annotated with both
@@ -40,6 +42,7 @@ public class AggregatorFunctionSupplierImplementer {
     private final TypeElement declarationType;
     private final AggregatorImplementer aggregatorImplementer;
     private final GroupingAggregatorImplementer groupingAggregatorImplementer;
+    private final boolean hasWarnings;
     private final List<Parameter> createParameters;
     private final ClassName implementation;
 
@@ -47,11 +50,13 @@ public AggregatorFunctionSupplierImplementer(
         Elements elements,
         TypeElement declarationType,
         AggregatorImplementer aggregatorImplementer,
-        GroupingAggregatorImplementer groupingAggregatorImplementer
+        GroupingAggregatorImplementer groupingAggregatorImplementer,
+        boolean hasWarnings
     ) {
         this.declarationType = declarationType;
         this.aggregatorImplementer = aggregatorImplementer;
         this.groupingAggregatorImplementer = groupingAggregatorImplementer;
+        this.hasWarnings = hasWarnings;
 
         Set<Parameter> createParameters = new LinkedHashSet<>();
         if (aggregatorImplementer != null) {
@@ -86,6 +91,11 @@ private TypeSpec type() {
         builder.addModifiers(Modifier.PUBLIC, Modifier.FINAL);
         builder.addSuperinterface(AGGREGATOR_FUNCTION_SUPPLIER);
 
+        if (hasWarnings) {
+            builder.addField(TypeName.INT, "warningsLineNumber");
+            builder.addField(TypeName.INT, "warningsColumnNumber");
+            builder.addField(STRING, "warningsSourceText");
+        }
         createParameters.stream().forEach(p -> p.declareField(builder));
         builder.addMethod(ctor());
         if (aggregatorImplementer != null) {
@@ -100,6 +110,14 @@ private TypeSpec type() {
 
     private MethodSpec ctor() {
         MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC);
+        if (hasWarnings) {
+            builder.addParameter(TypeName.INT, "warningsLineNumber");
+            builder.addParameter(TypeName.INT, "warningsColumnNumber");
+            builder.addParameter(STRING, "warningsSourceText");
+            builder.addStatement("this.warningsLineNumber = warningsLineNumber");
+            builder.addStatement("this.warningsColumnNumber = warningsColumnNumber");
+            builder.addStatement("this.warningsSourceText = warningsSourceText");
+        }
         createParameters.stream().forEach(p -> p.buildCtor(builder));
         return builder.build();
     }
@@ -114,30 +132,48 @@ private MethodSpec unsupportedNonGroupingAggregator() {
     }
 
     private MethodSpec aggregator() {
-        MethodSpec.Builder builder = MethodSpec.methodBuilder("aggregator")
-            .addParameter(DRIVER_CONTEXT, "driverContext")
-            .returns(aggregatorImplementer.implementation());
+        MethodSpec.Builder builder = MethodSpec.methodBuilder("aggregator");
         builder.addAnnotation(Override.class).addModifiers(Modifier.PUBLIC);
+        builder.addParameter(DRIVER_CONTEXT, "driverContext");
+        builder.returns(aggregatorImplementer.implementation());
+
+        if (hasWarnings) {
+            builder.addStatement(
+                "var warnings = Warnings.createWarnings(driverContext.warningsMode(), "
+                    + "warningsLineNumber, warningsColumnNumber, warningsSourceText)"
+            );
+        }
+
         builder.addStatement(
             "return $T.create($L)",
             aggregatorImplementer.implementation(),
-            Stream.concat(Stream.of("driverContext, channels"), aggregatorImplementer.createParameters().stream().map(Parameter::name))
-                .collect(Collectors.joining(", "))
+            Stream.concat(
+                Stream.concat(hasWarnings ? Stream.of("warnings") : Stream.of(), Stream.of("driverContext, channels")),
+                aggregatorImplementer.createParameters().stream().map(Parameter::name)
+            ).collect(Collectors.joining(", "))
         );
 
         return builder.build();
     }
 
     private MethodSpec groupingAggregator() {
-        MethodSpec.Builder builder = MethodSpec.methodBuilder("groupingAggregator")
-            .addParameter(DRIVER_CONTEXT, "driverContext")
-            .returns(groupingAggregatorImplementer.implementation());
+        MethodSpec.Builder builder = MethodSpec.methodBuilder("groupingAggregator");
         builder.addAnnotation(Override.class).addModifiers(Modifier.PUBLIC);
+        builder.addParameter(DRIVER_CONTEXT, "driverContext");
+        builder.returns(groupingAggregatorImplementer.implementation());
+
+        if (hasWarnings) {
+            builder.addStatement(
+                "var warnings = Warnings.createWarnings(driverContext.warningsMode(), "
+                    + "warningsLineNumber, warningsColumnNumber, warningsSourceText)"
+            );
+        }
+
         builder.addStatement(
             "return $T.create($L)",
             groupingAggregatorImplementer.implementation(),
             Stream.concat(
-                Stream.of("channels, driverContext"),
+                Stream.concat(hasWarnings ? Stream.of("warnings") : Stream.of(), Stream.of("channels, driverContext")),
                 groupingAggregatorImplementer.createParameters().stream().map(Parameter::name)
             ).collect(Collectors.joining(", "))
         );
diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java
index b3d32a82cc7a9..67ce0cf709704 100644
--- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java
@@ -21,10 +21,15 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
+import java.util.Objects;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import javax.lang.model.element.ExecutableElement;
 import javax.lang.model.element.Modifier;
 import javax.lang.model.element.TypeElement;
+import javax.lang.model.type.TypeMirror;
 import javax.lang.model.util.Elements;
 
 import static java.util.stream.Collectors.joining;
@@ -40,6 +45,7 @@
 import static org.elasticsearch.compute.gen.Types.BYTES_REF;
 import static org.elasticsearch.compute.gen.Types.BYTES_REF_BLOCK;
 import static org.elasticsearch.compute.gen.Types.BYTES_REF_VECTOR;
+import static org.elasticsearch.compute.gen.Types.COMPUTE_WARNINGS;
 import static org.elasticsearch.compute.gen.Types.DOUBLE_BLOCK;
 import static org.elasticsearch.compute.gen.Types.DOUBLE_VECTOR;
 import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT;
@@ -68,6 +74,7 @@
  */
 public class AggregatorImplementer {
     private final TypeElement declarationType;
+    private final List<TypeMirror> warnExceptions;
     private final ExecutableElement init;
     private final ExecutableElement combine;
     private final ExecutableElement combineValueCount;
@@ -76,18 +83,28 @@ public class AggregatorImplementer {
     private final ClassName implementation;
     private final TypeName stateType;
     private final boolean stateTypeHasSeen;
+    private final boolean stateTypeHasFailed;
     private final boolean valuesIsBytesRef;
     private final List<IntermediateStateDesc> intermediateState;
     private final List<Parameter> createParameters;
 
-    public AggregatorImplementer(Elements elements, TypeElement declarationType, IntermediateState[] interStateAnno) {
+    public AggregatorImplementer(
+        Elements elements,
+        TypeElement declarationType,
+        IntermediateState[] interStateAnno,
+        List<TypeMirror> warnExceptions
+    ) {
         this.declarationType = declarationType;
+        this.warnExceptions = warnExceptions;
 
         this.init = findRequiredMethod(declarationType, new String[] { "init", "initSingle" }, e -> true);
         this.stateType = choseStateType();
-        stateTypeHasSeen = elements.getAllMembers(elements.getTypeElement(stateType.toString()))
+        this.stateTypeHasSeen = elements.getAllMembers(elements.getTypeElement(stateType.toString()))
             .stream()
             .anyMatch(e -> e.toString().equals("seen()"));
+        this.stateTypeHasFailed = elements.getAllMembers(elements.getTypeElement(stateType.toString()))
+            .stream()
+            .anyMatch(e -> e.toString().equals("failed()"));
 
         this.combine = findRequiredMethod(declarationType, new String[] { "combine" }, e -> {
             if (e.getParameters().size() == 0) {
@@ -102,7 +119,7 @@ public AggregatorImplementer(Elements elements, TypeElement declarationType, Int
         this.createParameters = init.getParameters()
             .stream()
             .map(Parameter::from)
-            .filter(f -> false == f.type().equals(BIG_ARRAYS))
+            .filter(f -> false == f.type().equals(BIG_ARRAYS) && false == f.type().equals(DRIVER_CONTEXT))
             .toList();
 
         this.implementation = ClassName.get(
@@ -126,7 +143,10 @@ private TypeName choseStateType() {
         if (false == initReturn.isPrimitive()) {
             return initReturn;
         }
-        return ClassName.get("org.elasticsearch.compute.aggregation", firstUpper(initReturn.toString()) + "State");
+        if (warnExceptions.isEmpty()) {
+            return ClassName.get("org.elasticsearch.compute.aggregation", firstUpper(initReturn.toString()) + "State");
+        }
+        return ClassName.get("org.elasticsearch.compute.aggregation", firstUpper(initReturn.toString()) + "FallibleState");
     }
 
     static String valueType(ExecutableElement init, ExecutableElement combine) {
@@ -202,6 +222,11 @@ private TypeSpec type() {
                 .initializer(initInterState())
                 .build()
         );
+
+        if (warnExceptions.isEmpty() == false) {
+            builder.addField(COMPUTE_WARNINGS, "warnings", Modifier.PRIVATE, Modifier.FINAL);
+        }
+
         builder.addField(DRIVER_CONTEXT, "driverContext", Modifier.PRIVATE, Modifier.FINAL);
         builder.addField(stateType, "state", Modifier.PRIVATE, Modifier.FINAL);
         builder.addField(LIST_INTEGER, "channels", Modifier.PRIVATE, Modifier.FINAL);
@@ -228,17 +253,26 @@ private TypeSpec type() {
     private MethodSpec create() {
         MethodSpec.Builder builder = MethodSpec.methodBuilder("create");
         builder.addModifiers(Modifier.PUBLIC, Modifier.STATIC).returns(implementation);
+        if (warnExceptions.isEmpty() == false) {
+            builder.addParameter(COMPUTE_WARNINGS, "warnings");
+        }
         builder.addParameter(DRIVER_CONTEXT, "driverContext");
         builder.addParameter(LIST_INTEGER, "channels");
         for (Parameter p : createParameters) {
             builder.addParameter(p.type(), p.name());
         }
         if (createParameters.isEmpty()) {
-            builder.addStatement("return new $T(driverContext, channels, $L)", implementation, callInit());
+            builder.addStatement(
+                "return new $T($LdriverContext, channels, $L)",
+                implementation,
+                warnExceptions.isEmpty() ? "" : "warnings, ",
+                callInit()
+            );
         } else {
             builder.addStatement(
-                "return new $T(driverContext, channels, $L, $L)",
+                "return new $T($LdriverContext, channels, $L, $L)",
                 implementation,
+                warnExceptions.isEmpty() ? "" : "warnings, ",
                 callInit(),
                 createParameters.stream().map(p -> p.name()).collect(joining(", "))
             );
@@ -275,16 +309,22 @@ private CodeBlock initInterState() {
 
     private MethodSpec ctor() {
         MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC);
+        if (warnExceptions.isEmpty() == false) {
+            builder.addParameter(COMPUTE_WARNINGS, "warnings");
+        }
         builder.addParameter(DRIVER_CONTEXT, "driverContext");
         builder.addParameter(LIST_INTEGER, "channels");
         builder.addParameter(stateType, "state");
+
+        if (warnExceptions.isEmpty() == false) {
+            builder.addStatement("this.warnings = warnings");
+        }
         builder.addStatement("this.driverContext = driverContext");
         builder.addStatement("this.channels = channels");
         builder.addStatement("this.state = state");
 
         for (Parameter p : createParameters()) {
-            builder.addParameter(p.type(), p.name());
-            builder.addStatement("this.$N = $N", p.name(), p.name());
+            p.buildCtor(builder);
         }
         return builder.build();
     }
@@ -306,6 +346,11 @@ private MethodSpec intermediateBlockCount() {
     private MethodSpec addRawInput() {
         MethodSpec.Builder builder = MethodSpec.methodBuilder("addRawInput");
         builder.addAnnotation(Override.class).addModifiers(Modifier.PUBLIC).addParameter(PAGE, "page");
+        if (stateTypeHasFailed) {
+            builder.beginControlFlow("if (state.failed())");
+            builder.addStatement("return");
+            builder.endControlFlow();
+        }
         builder.addStatement("$T block = page.getBlock(channels.get(0))", valueBlockType(init, combine));
         builder.addStatement("$T vector = block.asVector()", valueVectorType(init, combine));
         builder.beginControlFlow("if (vector != null)").addStatement("addRawVector(vector)");
@@ -366,20 +411,27 @@ private MethodSpec addRawBlock() {
     }
 
     private void combineRawInput(MethodSpec.Builder builder, String blockVariable) {
+        TypeName returnType = TypeName.get(combine.getReturnType());
+        if (warnExceptions.isEmpty() == false) {
+            builder.beginControlFlow("try");
+        }
         if (valuesIsBytesRef) {
             combineRawInputForBytesRef(builder, blockVariable);
-            return;
-        }
-        TypeName returnType = TypeName.get(combine.getReturnType());
-        if (returnType.isPrimitive()) {
+        } else if (returnType.isPrimitive()) {
             combineRawInputForPrimitive(returnType, builder, blockVariable);
-            return;
-        }
-        if (returnType == TypeName.VOID) {
+        } else if (returnType == TypeName.VOID) {
             combineRawInputForVoid(builder, blockVariable);
-            return;
+        } else {
+            throw new IllegalArgumentException("combine must return void or a primitive");
+        }
+        if (warnExceptions.isEmpty() == false) {
+            String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)";
+            builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray());
+            builder.addStatement("warnings.registerException(e)");
+            builder.addStatement("state.failed(true)");
+            builder.addStatement("return");
+            builder.endControlFlow();
         }
-        throw new IllegalArgumentException("combine must return void or a primitive");
     }
 
     private void combineRawInputForPrimitive(TypeName returnType, MethodSpec.Builder builder, String blockVariable) {
@@ -423,16 +475,37 @@ private MethodSpec addIntermediateInput() {
             }
             builder.addStatement("$T.combineIntermediate(state, " + intermediateStateRowAccess() + ")", declarationType);
         } else if (hasPrimitiveState()) {
-            assert intermediateState.size() == 2;
-            assert intermediateState.get(1).name().equals("seen");
-            builder.beginControlFlow("if (seen.getBoolean(0))");
-            {
-                var state = intermediateState.get(0);
-                var s = "state.$L($T.combine(state.$L(), " + state.name() + "." + vectorAccessorName(state.elementType()) + "(0)))";
-                builder.addStatement(s, primitiveStateMethod(), declarationType, primitiveStateMethod());
-                builder.addStatement("state.seen(true)");
+            if (warnExceptions.isEmpty()) {
+                assert intermediateState.size() == 2;
+                assert intermediateState.get(1).name().equals("seen");
+                builder.beginControlFlow("if (seen.getBoolean(0))");
+            } else {
+                assert intermediateState.size() == 3;
+                assert intermediateState.get(1).name().equals("seen");
+                assert intermediateState.get(2).name().equals("failed");
+                builder.beginControlFlow("if (failed.getBoolean(0))");
+                {
+                    builder.addStatement("state.failed(true)");
+                    builder.addStatement("state.seen(true)");
+                }
+                builder.nextControlFlow("else if (seen.getBoolean(0))");
+            }
+
+            if (warnExceptions.isEmpty() == false) {
+                builder.beginControlFlow("try");
+            }
+            var state = intermediateState.get(0);
+            var s = "state.$L($T.combine(state.$L(), " + state.name() + "." + vectorAccessorName(state.elementType()) + "(0)))";
+            builder.addStatement(s, primitiveStateMethod(), declarationType, primitiveStateMethod());
+            builder.addStatement("state.seen(true)");
+            if (warnExceptions.isEmpty() == false) {
+                String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)";
+                builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray());
+                builder.addStatement("warnings.registerException(e)");
+                builder.addStatement("state.failed(true)");
                 builder.endControlFlow();
             }
+            builder.endControlFlow();
         } else {
             throw new IllegalArgumentException("Don't know how to combine intermediate input. Define combineIntermediate");
         }
@@ -445,15 +518,15 @@ String intermediateStateRowAccess() {
 
     private String primitiveStateMethod() {
         switch (stateType.toString()) {
-            case "org.elasticsearch.compute.aggregation.BooleanState":
+            case "org.elasticsearch.compute.aggregation.BooleanState", "org.elasticsearch.compute.aggregation.BooleanFallibleState":
                 return "booleanValue";
-            case "org.elasticsearch.compute.aggregation.IntState":
+            case "org.elasticsearch.compute.aggregation.IntState", "org.elasticsearch.compute.aggregation.IntFallibleState":
                 return "intValue";
-            case "org.elasticsearch.compute.aggregation.LongState":
+            case "org.elasticsearch.compute.aggregation.LongState", "org.elasticsearch.compute.aggregation.LongFallibleState":
                 return "longValue";
-            case "org.elasticsearch.compute.aggregation.DoubleState":
+            case "org.elasticsearch.compute.aggregation.DoubleState", "org.elasticsearch.compute.aggregation.DoubleFallibleState":
                 return "doubleValue";
-            case "org.elasticsearch.compute.aggregation.FloatState":
+            case "org.elasticsearch.compute.aggregation.FloatState", "org.elasticsearch.compute.aggregation.FloatFallibleState":
                 return "floatValue";
             default:
                 throw new IllegalArgumentException(
@@ -480,8 +553,11 @@ private MethodSpec evaluateFinal() {
             .addParameter(BLOCK_ARRAY, "blocks")
             .addParameter(TypeName.INT, "offset")
             .addParameter(DRIVER_CONTEXT, "driverContext");
-        if (stateTypeHasSeen) {
-            builder.beginControlFlow("if (state.seen() == false)");
+        if (stateTypeHasSeen || stateTypeHasFailed) {
+            var condition = Stream.of(stateTypeHasSeen ? "state.seen() == false" : null, stateTypeHasFailed ? "state.failed()" : null)
+                .filter(Objects::nonNull)
+                .collect(joining(" || "));
+            builder.beginControlFlow("if ($L)", condition);
             builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantNullBlock(1)", BLOCK);
             builder.addStatement("return");
             builder.endControlFlow();
@@ -496,19 +572,19 @@ private MethodSpec evaluateFinal() {
 
     private void primitiveStateToResult(MethodSpec.Builder builder) {
         switch (stateType.toString()) {
-            case "org.elasticsearch.compute.aggregation.BooleanState":
+            case "org.elasticsearch.compute.aggregation.BooleanState", "org.elasticsearch.compute.aggregation.BooleanFallibleState":
                 builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantBooleanBlockWith(state.booleanValue(), 1)");
                 return;
-            case "org.elasticsearch.compute.aggregation.IntState":
+            case "org.elasticsearch.compute.aggregation.IntState", "org.elasticsearch.compute.aggregation.IntFallibleState":
                 builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantIntBlockWith(state.intValue(), 1)");
                 return;
-            case "org.elasticsearch.compute.aggregation.LongState":
+            case "org.elasticsearch.compute.aggregation.LongState", "org.elasticsearch.compute.aggregation.LongFallibleState":
                 builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantLongBlockWith(state.longValue(), 1)");
                 return;
-            case "org.elasticsearch.compute.aggregation.DoubleState":
+            case "org.elasticsearch.compute.aggregation.DoubleState", "org.elasticsearch.compute.aggregation.DoubleFallibleState":
                 builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantDoubleBlockWith(state.doubleValue(), 1)");
                 return;
-            case "org.elasticsearch.compute.aggregation.FloatState":
+            case "org.elasticsearch.compute.aggregation.FloatState", "org.elasticsearch.compute.aggregation.FloatFallibleState":
                 builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantFloatBlockWith(state.floatValue(), 1)");
                 return;
             default:
@@ -534,13 +610,12 @@ private MethodSpec close() {
         return builder.build();
     }
 
+    private static final Pattern PRIMITIVE_STATE_PATTERN = Pattern.compile(
+        "org.elasticsearch.compute.aggregation.(Boolean|Int|Long|Double|Float)(Fallible)?State"
+    );
+
     private boolean hasPrimitiveState() {
-        return switch (stateType.toString()) {
-            case "org.elasticsearch.compute.aggregation.BooleanState", "org.elasticsearch.compute.aggregation.IntState",
-                "org.elasticsearch.compute.aggregation.LongState", "org.elasticsearch.compute.aggregation.DoubleState",
-                "org.elasticsearch.compute.aggregation.FloatState" -> true;
-            default -> false;
-        };
+        return PRIMITIVE_STATE_PATTERN.matcher(stateType.toString()).matches();
     }
 
     record IntermediateStateDesc(String name, String elementType, boolean block) {
diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java
index d07b24047b7e2..4b1f946a1d176 100644
--- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java
@@ -80,9 +80,14 @@ public boolean process(Set<? extends TypeElement> set, RoundEnvironment roundEnv
         }
         for (TypeElement aggClass : annotatedClasses) {
             AggregatorImplementer implementer = null;
+            var warnExceptionsTypes = Annotations.listAttributeValues(
+                aggClass,
+                Set.of(Aggregator.class, GroupingAggregator.class),
+                "warnExceptions"
+            );
             if (aggClass.getAnnotation(Aggregator.class) != null) {
                 IntermediateState[] intermediateState = aggClass.getAnnotation(Aggregator.class).value();
-                implementer = new AggregatorImplementer(env.getElementUtils(), aggClass, intermediateState);
+                implementer = new AggregatorImplementer(env.getElementUtils(), aggClass, intermediateState, warnExceptionsTypes);
                 write(aggClass, "aggregator", implementer.sourceFile(), env);
             }
             GroupingAggregatorImplementer groupingAggregatorImplementer = null;
@@ -96,6 +101,7 @@ public boolean process(Set<? extends TypeElement> set, RoundEnvironment roundEnv
                     env.getElementUtils(),
                     aggClass,
                     intermediateState,
+                    warnExceptionsTypes,
                     includeTimestamps
                 );
                 write(aggClass, "grouping aggregator", groupingAggregatorImplementer.sourceFile(), env);
@@ -104,8 +110,13 @@ public boolean process(Set<? extends TypeElement> set, RoundEnvironment roundEnv
                 write(
                     aggClass,
                     "aggregator function supplier",
-                    new AggregatorFunctionSupplierImplementer(env.getElementUtils(), aggClass, implementer, groupingAggregatorImplementer)
-                        .sourceFile(),
+                    new AggregatorFunctionSupplierImplementer(
+                        env.getElementUtils(),
+                        aggClass,
+                        implementer,
+                        groupingAggregatorImplementer,
+                        warnExceptionsTypes.isEmpty() == false
+                    ).sourceFile(),
                     env
                 );
             }
diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Annotations.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Annotations.java
new file mode 100644
index 0000000000000..d3892f7d2a40b
--- /dev/null
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Annotations.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.gen;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import javax.lang.model.element.AnnotationValue;
+import javax.lang.model.element.Element;
+import javax.lang.model.type.TypeMirror;
+
+public class Annotations {
+    private Annotations() {}
+
+    /**
+     * Returns the values of the requested attribute, from all the matching annotations on the given element.
+     *
+     * @param element the element to inspect
+     * @param annotations the annotations to look for
+     * @param attributeName the attribute to extract
+     */
+    public static List<TypeMirror> listAttributeValues(Element element, Set<Class<?>> annotations, String attributeName) {
+        List<TypeMirror> result = new ArrayList<>();
+        for (var mirror : element.getAnnotationMirrors()) {
+            String annotationType = mirror.getAnnotationType().toString();
+            if (annotations.stream().anyMatch(a -> a.getName().equals(annotationType))) {
+                for (var e : mirror.getElementValues().entrySet()) {
+                    if (false == e.getKey().getSimpleName().toString().equals(attributeName)) {
+                        continue;
+                    }
+                    for (var v : (List<?>) e.getValue().getValue()) {
+                        result.add((TypeMirror) ((AnnotationValue) v).getValue());
+                    }
+                }
+            }
+        }
+        return result;
+    }
+}
diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConsumeProcessor.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConsumeProcessor.java
index 1c28acfd6a23b..b76b1cc7ea74b 100644
--- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConsumeProcessor.java
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConsumeProcessor.java
@@ -36,7 +36,7 @@ public Set<String> getSupportedOptions() {
     public Set<String> getSupportedAnnotationTypes() {
         return Set.of(
             "org.elasticsearch.core.Nullable",
-            "org.elasticsearch.common.inject.Inject",
+            "org.elasticsearch.injection.guice.Inject",
             "org.elasticsearch.xpack.esql.expression.function.FunctionInfo",
             "org.elasticsearch.xpack.esql.expression.function.Param",
             "org.elasticsearch.rest.ServerlessScope",
diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java
index e1456328e7f64..629a45574ebbb 100644
--- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java
@@ -784,7 +784,7 @@ public void unpackValues(MethodSpec.Builder builder, boolean blockStyle) {
 
         @Override
         public void buildInvocation(StringBuilder pattern, List<Object> args, boolean blockStyle) {
-            pattern.append("$L");
+            pattern.append("this.$L");
             args.add(name);
         }
 
diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorProcessor.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorProcessor.java
index ea3ee938298de..09012c7b3a48a 100644
--- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorProcessor.java
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorProcessor.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.compute.ann.Evaluator;
 import org.elasticsearch.compute.ann.MvEvaluator;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
@@ -21,11 +20,9 @@
 import javax.annotation.processing.RoundEnvironment;
 import javax.lang.model.SourceVersion;
 import javax.lang.model.element.AnnotationMirror;
-import javax.lang.model.element.AnnotationValue;
 import javax.lang.model.element.Element;
 import javax.lang.model.element.ExecutableElement;
 import javax.lang.model.element.TypeElement;
-import javax.lang.model.type.TypeMirror;
 import javax.tools.Diagnostic;
 
 /**
@@ -69,6 +66,11 @@ public Iterable<? extends Completion> getCompletions(
     public boolean process(Set<? extends TypeElement> set, RoundEnvironment roundEnvironment) {
         for (TypeElement ann : set) {
             for (Element evaluatorMethod : roundEnvironment.getElementsAnnotatedWith(ann)) {
+                var warnExceptionsTypes = Annotations.listAttributeValues(
+                    evaluatorMethod,
+                    Set.of(Evaluator.class, MvEvaluator.class, ConvertEvaluator.class),
+                    "warnExceptions"
+                );
                 Evaluator evaluatorAnn = evaluatorMethod.getAnnotation(Evaluator.class);
                 if (evaluatorAnn != null) {
                     try {
@@ -80,7 +82,7 @@ public boolean process(Set<? extends TypeElement> set, RoundEnvironment roundEnv
                                 env.getTypeUtils(),
                                 (ExecutableElement) evaluatorMethod,
                                 evaluatorAnn.extraName(),
-                                warnExceptions(evaluatorMethod)
+                                warnExceptionsTypes
                             ).sourceFile(),
                             env
                         );
@@ -102,7 +104,7 @@ public boolean process(Set<? extends TypeElement> set, RoundEnvironment roundEnv
                                 mvEvaluatorAnn.finish(),
                                 mvEvaluatorAnn.single(),
                                 mvEvaluatorAnn.ascending(),
-                                warnExceptions(evaluatorMethod)
+                                warnExceptionsTypes
                             ).sourceFile(),
                             env
                         );
@@ -121,7 +123,7 @@ public boolean process(Set<? extends TypeElement> set, RoundEnvironment roundEnv
                                 env.getElementUtils(),
                                 (ExecutableElement) evaluatorMethod,
                                 convertEvaluatorAnn.extraName(),
-                                warnExceptions(evaluatorMethod)
+                                warnExceptionsTypes
                             ).sourceFile(),
                             env
                         );
@@ -134,25 +136,4 @@ public boolean process(Set<? extends TypeElement> set, RoundEnvironment roundEnv
         }
         return true;
     }
-
-    private static List<TypeMirror> warnExceptions(Element evaluatorMethod) {
-        List<TypeMirror> result = new ArrayList<>();
-        for (var mirror : evaluatorMethod.getAnnotationMirrors()) {
-            String annotationType = mirror.getAnnotationType().toString();
-            if (annotationType.equals(Evaluator.class.getName())
-                || annotationType.equals(MvEvaluator.class.getName())
-                || annotationType.equals(ConvertEvaluator.class.getName())) {
-
-                for (var e : mirror.getElementValues().entrySet()) {
-                    if (false == e.getKey().getSimpleName().toString().equals("warnExceptions")) {
-                        continue;
-                    }
-                    for (var v : (List<?>) e.getValue().getValue()) {
-                        result.add((TypeMirror) ((AnnotationValue) v).getValue());
-                    }
-                }
-            }
-        }
-        return result;
-    }
 }
diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java
index 79df41f304c06..3dffbcf84eb78 100644
--- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java
@@ -22,11 +22,13 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.function.Consumer;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import javax.lang.model.element.ExecutableElement;
 import javax.lang.model.element.Modifier;
 import javax.lang.model.element.TypeElement;
+import javax.lang.model.type.TypeMirror;
 import javax.lang.model.util.Elements;
 
 import static java.util.stream.Collectors.joining;
@@ -38,6 +40,7 @@
 import static org.elasticsearch.compute.gen.Types.BIG_ARRAYS;
 import static org.elasticsearch.compute.gen.Types.BLOCK_ARRAY;
 import static org.elasticsearch.compute.gen.Types.BYTES_REF;
+import static org.elasticsearch.compute.gen.Types.COMPUTE_WARNINGS;
 import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT;
 import static org.elasticsearch.compute.gen.Types.ELEMENT_TYPE;
 import static org.elasticsearch.compute.gen.Types.GROUPING_AGGREGATOR_FUNCTION;
@@ -63,6 +66,7 @@
  */
 public class GroupingAggregatorImplementer {
     private final TypeElement declarationType;
+    private final List<TypeMirror> warnExceptions;
     private final ExecutableElement init;
     private final ExecutableElement combine;
     private final ExecutableElement combineStates;
@@ -79,9 +83,11 @@ public GroupingAggregatorImplementer(
         Elements elements,
         TypeElement declarationType,
         IntermediateState[] interStateAnno,
+        List<TypeMirror> warnExceptions,
         boolean includeTimestampVector
     ) {
         this.declarationType = declarationType;
+        this.warnExceptions = warnExceptions;
 
         this.init = findRequiredMethod(declarationType, new String[] { "init", "initGrouping" }, e -> true);
         this.stateType = choseStateType();
@@ -129,7 +135,10 @@ private TypeName choseStateType() {
         }
         String head = initReturn.toString().substring(0, 1).toUpperCase(Locale.ROOT);
         String tail = initReturn.toString().substring(1);
-        return ClassName.get("org.elasticsearch.compute.aggregation", head + tail + "ArrayState");
+        if (warnExceptions.isEmpty()) {
+            return ClassName.get("org.elasticsearch.compute.aggregation", head + tail + "ArrayState");
+        }
+        return ClassName.get("org.elasticsearch.compute.aggregation", head + tail + "FallibleArrayState");
     }
 
     public JavaFile sourceFile() {
@@ -154,6 +163,9 @@ private TypeSpec type() {
                 .build()
         );
         builder.addField(stateType, "state", Modifier.PRIVATE, Modifier.FINAL);
+        if (warnExceptions.isEmpty() == false) {
+            builder.addField(COMPUTE_WARNINGS, "warnings", Modifier.PRIVATE, Modifier.FINAL);
+        }
         builder.addField(LIST_INTEGER, "channels", Modifier.PRIVATE, Modifier.FINAL);
         builder.addField(DRIVER_CONTEXT, "driverContext", Modifier.PRIVATE, Modifier.FINAL);
 
@@ -182,17 +194,26 @@ private TypeSpec type() {
     private MethodSpec create() {
         MethodSpec.Builder builder = MethodSpec.methodBuilder("create");
         builder.addModifiers(Modifier.PUBLIC, Modifier.STATIC).returns(implementation);
+        if (warnExceptions.isEmpty() == false) {
+            builder.addParameter(COMPUTE_WARNINGS, "warnings");
+        }
         builder.addParameter(LIST_INTEGER, "channels");
         builder.addParameter(DRIVER_CONTEXT, "driverContext");
         for (Parameter p : createParameters) {
             builder.addParameter(p.type(), p.name());
         }
         if (createParameters.isEmpty()) {
-            builder.addStatement("return new $T(channels, $L, driverContext)", implementation, callInit());
+            builder.addStatement(
+                "return new $T($Lchannels, $L, driverContext)",
+                implementation,
+                warnExceptions.isEmpty() ? "" : "warnings, ",
+                callInit()
+            );
         } else {
             builder.addStatement(
-                "return new $T(channels, $L, driverContext, $L)",
+                "return new $T($Lchannels, $L, driverContext, $L)",
                 implementation,
+                warnExceptions.isEmpty() ? "" : "warnings, ",
                 callInit(),
                 createParameters.stream().map(p -> p.name()).collect(joining(", "))
             );
@@ -235,9 +256,15 @@ private CodeBlock initInterState() {
 
     private MethodSpec ctor() {
         MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC);
+        if (warnExceptions.isEmpty() == false) {
+            builder.addParameter(COMPUTE_WARNINGS, "warnings");
+        }
         builder.addParameter(LIST_INTEGER, "channels");
         builder.addParameter(stateType, "state");
         builder.addParameter(DRIVER_CONTEXT, "driverContext");
+        if (warnExceptions.isEmpty() == false) {
+            builder.addStatement("this.warnings = warnings");
+        }
         builder.addStatement("this.channels = channels");
         builder.addStatement("this.state = state");
         builder.addStatement("this.driverContext = driverContext");
@@ -337,16 +364,21 @@ private MethodSpec addRawInputLoop(TypeName groupsType, TypeName valuesType) {
         builder.beginControlFlow("for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++)");
         {
             if (groupsIsBlock) {
-                // TODO we can drop this once we stop sending null group keys
                 builder.beginControlFlow("if (groups.isNull(groupPosition))");
                 builder.addStatement("continue");
                 builder.endControlFlow();
                 builder.addStatement("int groupStart = groups.getFirstValueIndex(groupPosition)");
                 builder.addStatement("int groupEnd = groupStart + groups.getValueCount(groupPosition)");
                 builder.beginControlFlow("for (int g = groupStart; g < groupEnd; g++)");
-                builder.addStatement("int groupId = Math.toIntExact(groups.getInt(g))");
+                builder.addStatement("int groupId = groups.getInt(g)");
             } else {
-                builder.addStatement("int groupId = Math.toIntExact(groups.getInt(groupPosition))");
+                builder.addStatement("int groupId = groups.getInt(groupPosition)");
+            }
+
+            if (warnExceptions.isEmpty() == false) {
+                builder.beginControlFlow("if (state.hasFailed(groupId))");
+                builder.addStatement("continue");
+                builder.endControlFlow();
             }
 
             if (valuesIsBlock) {
@@ -371,31 +403,35 @@ private MethodSpec addRawInputLoop(TypeName groupsType, TypeName valuesType) {
     }
 
     private void combineRawInput(MethodSpec.Builder builder, String blockVariable, String offsetVariable) {
-        if (valuesIsBytesRef) {
-            combineRawInputForBytesRef(builder, blockVariable, offsetVariable);
-            return;
-        }
-        if (includeTimestampVector) {
-            combineRawInputWithTimestamp(builder, offsetVariable);
-            return;
-        }
         TypeName valueType = TypeName.get(combine.getParameters().get(combine.getParameters().size() - 1).asType());
-        if (valueType.isPrimitive() == false) {
-            throw new IllegalArgumentException("second parameter to combine must be a primitive");
-        }
         String secondParameterGetter = "get"
             + valueType.toString().substring(0, 1).toUpperCase(Locale.ROOT)
             + valueType.toString().substring(1);
         TypeName returnType = TypeName.get(combine.getReturnType());
-        if (returnType.isPrimitive()) {
-            combineRawInputForPrimitive(builder, secondParameterGetter, blockVariable, offsetVariable);
-            return;
+
+        if (warnExceptions.isEmpty() == false) {
+            builder.beginControlFlow("try");
         }
-        if (returnType == TypeName.VOID) {
+        if (valuesIsBytesRef) {
+            combineRawInputForBytesRef(builder, blockVariable, offsetVariable);
+        } else if (includeTimestampVector) {
+            combineRawInputWithTimestamp(builder, offsetVariable);
+        } else if (valueType.isPrimitive() == false) {
+            throw new IllegalArgumentException("second parameter to combine must be a primitive");
+        } else if (returnType.isPrimitive()) {
+            combineRawInputForPrimitive(builder, secondParameterGetter, blockVariable, offsetVariable);
+        } else if (returnType == TypeName.VOID) {
             combineRawInputForVoid(builder, secondParameterGetter, blockVariable, offsetVariable);
-            return;
+        } else {
+            throw new IllegalArgumentException("combine must return void or a primitive");
+        }
+        if (warnExceptions.isEmpty() == false) {
+            String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)";
+            builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray());
+            builder.addStatement("warnings.registerException(e)");
+            builder.addStatement("state.setFailed(groupId)");
+            builder.endControlFlow();
         }
-        throw new IllegalArgumentException("combine must return void or a primitive");
     }
 
     private void combineRawInputForPrimitive(
@@ -479,22 +515,42 @@ private MethodSpec addIntermediateInput() {
         }
         builder.beginControlFlow("for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++)");
         {
-            builder.addStatement("int groupId = Math.toIntExact(groups.getInt(groupPosition))");
+            builder.addStatement("int groupId = groups.getInt(groupPosition)");
             if (hasPrimitiveState()) {
-                assert intermediateState.size() == 2;
-                assert intermediateState.get(1).name().equals("seen");
-                builder.beginControlFlow("if (seen.getBoolean(groupPosition + positionOffset))");
-                {
-                    var name = intermediateState.get(0).name();
-                    var m = vectorAccessorName(intermediateState.get(0).elementType());
-                    builder.addStatement(
-                        "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.$L(groupPosition + positionOffset)))",
-                        declarationType,
-                        name,
-                        m
-                    );
+                if (warnExceptions.isEmpty()) {
+                    assert intermediateState.size() == 2;
+                    assert intermediateState.get(1).name().equals("seen");
+                    builder.beginControlFlow("if (seen.getBoolean(groupPosition + positionOffset))");
+                } else {
+                    assert intermediateState.size() == 3;
+                    assert intermediateState.get(1).name().equals("seen");
+                    assert intermediateState.get(2).name().equals("failed");
+                    builder.beginControlFlow("if (failed.getBoolean(groupPosition + positionOffset))");
+                    {
+                        builder.addStatement("state.setFailed(groupId)");
+                    }
+                    builder.nextControlFlow("else if (seen.getBoolean(groupPosition + positionOffset))");
+                }
+
+                if (warnExceptions.isEmpty() == false) {
+                    builder.beginControlFlow("try");
+                }
+                var name = intermediateState.get(0).name();
+                var vectorAccessor = vectorAccessorName(intermediateState.get(0).elementType());
+                builder.addStatement(
+                    "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.$L(groupPosition + positionOffset)))",
+                    declarationType,
+                    name,
+                    vectorAccessor
+                );
+                if (warnExceptions.isEmpty() == false) {
+                    String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)";
+                    builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray());
+                    builder.addStatement("warnings.registerException(e)");
+                    builder.addStatement("state.setFailed(groupId)");
                     builder.endControlFlow();
                 }
+                builder.endControlFlow();
             } else {
                 builder.addStatement("$T.combineIntermediate(state, groupId, " + intermediateStateRowAccess() + ")", declarationType);
             }
@@ -582,12 +638,11 @@ private MethodSpec close() {
         return builder.build();
     }
 
+    private static final Pattern PRIMITIVE_STATE_PATTERN = Pattern.compile(
+        "org.elasticsearch.compute.aggregation.(Boolean|Int|Long|Double|Float)(Fallible)?ArrayState"
+    );
+
     private boolean hasPrimitiveState() {
-        return switch (stateType.toString()) {
-            case "org.elasticsearch.compute.aggregation.BooleanArrayState", "org.elasticsearch.compute.aggregation.IntArrayState",
-                "org.elasticsearch.compute.aggregation.LongArrayState", "org.elasticsearch.compute.aggregation.DoubleArrayState",
-                "org.elasticsearch.compute.aggregation.FloatArrayState" -> true;
-            default -> false;
-        };
+        return PRIMITIVE_STATE_PATTERN.matcher(stateType.toString()).matches();
     }
 }
diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java
index 3150741ddcb05..096d0b86e6cff 100644
--- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java
@@ -27,6 +27,8 @@ public class Types {
     private static final String OPERATOR_PACKAGE = PACKAGE + ".operator";
     private static final String DATA_PACKAGE = PACKAGE + ".data";
 
+    static final TypeName STRING = ClassName.get("java.lang", "String");
+
     static final TypeName LIST_INTEGER = ParameterizedTypeName.get(ClassName.get(List.class), TypeName.INT.box());
 
     static final ClassName PAGE = ClassName.get(DATA_PACKAGE, "Page");
@@ -34,6 +36,7 @@ public class Types {
     static final TypeName BLOCK_ARRAY = ArrayTypeName.of(BLOCK);
     static final ClassName VECTOR = ClassName.get(DATA_PACKAGE, "Vector");
 
+    static final ClassName CIRCUIT_BREAKER = ClassName.get("org.elasticsearch.common.breaker", "CircuitBreaker");
     static final ClassName BIG_ARRAYS = ClassName.get("org.elasticsearch.common.util", "BigArrays");
 
     static final ClassName BOOLEAN_BLOCK = ClassName.get(DATA_PACKAGE, "BooleanBlock");
@@ -127,6 +130,11 @@ public class Types {
     );
 
     static final ClassName WARNINGS = ClassName.get("org.elasticsearch.xpack.esql.expression.function", "Warnings");
+    /**
+     * Warnings class used in compute module.
+     * It uses no external dependencies (Like Warnings and Source).
+     */
+    static final ClassName COMPUTE_WARNINGS = ClassName.get("org.elasticsearch.compute.aggregation", "Warnings");
 
     static final ClassName SOURCE = ClassName.get("org.elasticsearch.xpack.esql.core.tree", "Source");
 
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleArrayState.java
new file mode 100644
index 0000000000000..6367fdfb6617e
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleArrayState.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.BitArray;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.core.Releasables;
+
+/**
+ * Aggregator state for an array of booleans, that also tracks failures.
+ * It is created in a mode where it won't track
+ * the {@code groupId}s that are sent to it and it is the
+ * responsibility of the caller to only fetch values for {@code groupId}s
+ * that it has sent using the {@code selected} parameter when building the
+ * results. This is fine when there are no {@code null} values in the input
+ * data. But once there are null values in the input data it is
+ * <strong>much</strong> more convenient to only send non-null values and
+ * the tracking built into the grouping code can't track that. In that case
+ * call {@link #enableGroupIdTracking} to transition the state into a mode
+ * where it'll track which {@code groupIds} have been written.
+ * <p>
+ * This class is generated. Do not edit it.
+ * </p>
+ */
+final class BooleanFallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState {
+    private final boolean init;
+
+    private BitArray values;
+    private int size;
+
+    BooleanFallibleArrayState(BigArrays bigArrays, boolean init) {
+        super(bigArrays);
+        this.values = new BitArray(1, bigArrays);
+        this.size = 1;
+        this.values.set(0, init);
+        this.init = init;
+    }
+
+    boolean get(int groupId) {
+        return values.get(groupId);
+    }
+
+    boolean getOrDefault(int groupId) {
+        return groupId < size ? values.get(groupId) : init;
+    }
+
+    void set(int groupId, boolean value) {
+        ensureCapacity(groupId);
+        values.set(groupId, value);
+        trackGroupId(groupId);
+    }
+
+    Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) {
+        if (false == trackingGroupIds() && false == anyFailure()) {
+            try (var builder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount())) {
+                for (int i = 0; i < selected.getPositionCount(); i++) {
+                    builder.appendBoolean(i, values.get(selected.getInt(i)));
+                }
+                return builder.build().asBlock();
+            }
+        }
+        try (BooleanBlock.Builder builder = driverContext.blockFactory().newBooleanBlockBuilder(selected.getPositionCount())) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (hasValue(group) && !hasFailed(group)) {
+                    builder.appendBoolean(values.get(group));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
+    private void ensureCapacity(int groupId) {
+        if (groupId >= size) {
+            values.fill(size, groupId + 1, init);
+            size = groupId + 1;
+        }
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(
+        Block[] blocks,
+        int offset,
+        IntVector selected,
+        org.elasticsearch.compute.operator.DriverContext driverContext
+    ) {
+        assert blocks.length >= offset + 3;
+        try (
+            var valuesBuilder = driverContext.blockFactory().newBooleanBlockBuilder(selected.getPositionCount());
+            var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount());
+            var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount())
+        ) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (group < size) {
+                    valuesBuilder.appendBoolean(values.get(group));
+                } else {
+                    valuesBuilder.appendBoolean(false); // TODO can we just use null?
+                }
+                hasValueBuilder.appendBoolean(i, hasValue(group));
+                hasFailedBuilder.appendBoolean(i, hasFailed(group));
+            }
+            blocks[offset + 0] = valuesBuilder.build();
+            blocks[offset + 1] = hasValueBuilder.build().asBlock();
+            blocks[offset + 2] = hasFailedBuilder.build().asBlock();
+        }
+    }
+
+    @Override
+    public void close() {
+        Releasables.close(values, super::close);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleState.java
new file mode 100644
index 0000000000000..073f31c390a6f
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleState.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * Aggregator state for a single boolean.
+ * It stores a third boolean to store if the aggregation failed.
+ * This class is generated. Do not edit it.
+ */
+final class BooleanFallibleState implements AggregatorState {
+    private boolean value;
+    private boolean seen;
+    private boolean failed;
+
+    BooleanFallibleState(boolean init) {
+        this.value = init;
+    }
+
+    boolean booleanValue() {
+        return value;
+    }
+
+    void booleanValue(boolean value) {
+        this.value = value;
+    }
+
+    boolean seen() {
+        return seen;
+    }
+
+    void seen(boolean seen) {
+        this.seen = seen;
+    }
+
+    boolean failed() {
+        return failed;
+    }
+
+    void failed(boolean failed) {
+        this.failed = failed;
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+        assert blocks.length >= offset + 3;
+        blocks[offset + 0] = driverContext.blockFactory().newConstantBooleanBlockWith(value, 1);
+        blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1);
+        blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1);
+    }
+
+    @Override
+    public void close() {}
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanState.java
index 7d225c7c06a72..ba4d133dee553 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanState.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanState.java
@@ -18,10 +18,6 @@ final class BooleanState implements AggregatorState {
     private boolean value;
     private boolean seen;
 
-    BooleanState() {
-        this(false);
-    }
-
     BooleanState(boolean init) {
         this.value = init;
     }
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleArrayState.java
new file mode 100644
index 0000000000000..dd1d60f7bd246
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleArrayState.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.DoubleArray;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.core.Releasables;
+
+/**
+ * Aggregator state for an array of doubles, that also tracks failures.
+ * It is created in a mode where it won't track
+ * the {@code groupId}s that are sent to it and it is the
+ * responsibility of the caller to only fetch values for {@code groupId}s
+ * that it has sent using the {@code selected} parameter when building the
+ * results. This is fine when there are no {@code null} values in the input
+ * data. But once there are null values in the input data it is
+ * <strong>much</strong> more convenient to only send non-null values and
+ * the tracking built into the grouping code can't track that. In that case
+ * call {@link #enableGroupIdTracking} to transition the state into a mode
+ * where it'll track which {@code groupIds} have been written.
+ * <p>
+ * This class is generated. Do not edit it.
+ * </p>
+ */
+final class DoubleFallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState {
+    private final double init;
+
+    private DoubleArray values;
+
+    DoubleFallibleArrayState(BigArrays bigArrays, double init) {
+        super(bigArrays);
+        this.values = bigArrays.newDoubleArray(1, false);
+        this.values.set(0, init);
+        this.init = init;
+    }
+
+    double get(int groupId) {
+        return values.get(groupId);
+    }
+
+    double getOrDefault(int groupId) {
+        return groupId < values.size() ? values.get(groupId) : init;
+    }
+
+    void set(int groupId, double value) {
+        ensureCapacity(groupId);
+        values.set(groupId, value);
+        trackGroupId(groupId);
+    }
+
+    Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) {
+        if (false == trackingGroupIds() && false == anyFailure()) {
+            try (var builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(selected.getPositionCount())) {
+                for (int i = 0; i < selected.getPositionCount(); i++) {
+                    builder.appendDouble(i, values.get(selected.getInt(i)));
+                }
+                return builder.build().asBlock();
+            }
+        }
+        try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(selected.getPositionCount())) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (hasValue(group) && !hasFailed(group)) {
+                    builder.appendDouble(values.get(group));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
+    private void ensureCapacity(int groupId) {
+        if (groupId >= values.size()) {
+            long prevSize = values.size();
+            values = bigArrays.grow(values, groupId + 1);
+            values.fill(prevSize, values.size(), init);
+        }
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(
+        Block[] blocks,
+        int offset,
+        IntVector selected,
+        org.elasticsearch.compute.operator.DriverContext driverContext
+    ) {
+        assert blocks.length >= offset + 3;
+        try (
+            var valuesBuilder = driverContext.blockFactory().newDoubleBlockBuilder(selected.getPositionCount());
+            var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount());
+            var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount())
+        ) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (group < values.size()) {
+                    valuesBuilder.appendDouble(values.get(group));
+                } else {
+                    valuesBuilder.appendDouble(0); // TODO can we just use null?
+                }
+                hasValueBuilder.appendBoolean(i, hasValue(group));
+                hasFailedBuilder.appendBoolean(i, hasFailed(group));
+            }
+            blocks[offset + 0] = valuesBuilder.build();
+            blocks[offset + 1] = hasValueBuilder.build().asBlock();
+            blocks[offset + 2] = hasFailedBuilder.build().asBlock();
+        }
+    }
+
+    @Override
+    public void close() {
+        Releasables.close(values, super::close);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleState.java
new file mode 100644
index 0000000000000..4cdeddec724bf
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleState.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * Aggregator state for a single double.
+ * It stores a third boolean to store if the aggregation failed.
+ * This class is generated. Do not edit it.
+ */
+final class DoubleFallibleState implements AggregatorState {
+    private double value;
+    private boolean seen;
+    private boolean failed;
+
+    DoubleFallibleState(double init) {
+        this.value = init;
+    }
+
+    double doubleValue() {
+        return value;
+    }
+
+    void doubleValue(double value) {
+        this.value = value;
+    }
+
+    boolean seen() {
+        return seen;
+    }
+
+    void seen(boolean seen) {
+        this.seen = seen;
+    }
+
+    boolean failed() {
+        return failed;
+    }
+
+    void failed(boolean failed) {
+        this.failed = failed;
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+        assert blocks.length >= offset + 3;
+        blocks[offset + 0] = driverContext.blockFactory().newConstantDoubleBlockWith(value, 1);
+        blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1);
+        blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1);
+    }
+
+    @Override
+    public void close() {}
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleState.java
index f1c92c685bcab..90ecc2c1d3c03 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleState.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleState.java
@@ -18,10 +18,6 @@ final class DoubleState implements AggregatorState {
     private double value;
     private boolean seen;
 
-    DoubleState() {
-        this(0);
-    }
-
     DoubleState(double init) {
         this.value = init;
     }
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleArrayState.java
new file mode 100644
index 0000000000000..055cf345033c5
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleArrayState.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.FloatArray;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.FloatBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.core.Releasables;
+
+/**
+ * Aggregator state for an array of floats, that also tracks failures.
+ * It is created in a mode where it won't track
+ * the {@code groupId}s that are sent to it and it is the
+ * responsibility of the caller to only fetch values for {@code groupId}s
+ * that it has sent using the {@code selected} parameter when building the
+ * results. This is fine when there are no {@code null} values in the input
+ * data. But once there are null values in the input data it is
+ * <strong>much</strong> more convenient to only send non-null values and
+ * the tracking built into the grouping code can't track that. In that case
+ * call {@link #enableGroupIdTracking} to transition the state into a mode
+ * where it'll track which {@code groupIds} have been written.
+ * <p>
+ * This class is generated. Do not edit it.
+ * </p>
+ */
+final class FloatFallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState {
+    private final float init;
+
+    private FloatArray values;
+
+    FloatFallibleArrayState(BigArrays bigArrays, float init) {
+        super(bigArrays);
+        this.values = bigArrays.newFloatArray(1, false);
+        this.values.set(0, init);
+        this.init = init;
+    }
+
+    float get(int groupId) {
+        return values.get(groupId);
+    }
+
+    float getOrDefault(int groupId) {
+        return groupId < values.size() ? values.get(groupId) : init;
+    }
+
+    void set(int groupId, float value) {
+        ensureCapacity(groupId);
+        values.set(groupId, value);
+        trackGroupId(groupId);
+    }
+
+    Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) {
+        if (false == trackingGroupIds() && false == anyFailure()) {
+            try (var builder = driverContext.blockFactory().newFloatVectorFixedBuilder(selected.getPositionCount())) {
+                for (int i = 0; i < selected.getPositionCount(); i++) {
+                    builder.appendFloat(i, values.get(selected.getInt(i)));
+                }
+                return builder.build().asBlock();
+            }
+        }
+        try (FloatBlock.Builder builder = driverContext.blockFactory().newFloatBlockBuilder(selected.getPositionCount())) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (hasValue(group) && !hasFailed(group)) {
+                    builder.appendFloat(values.get(group));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
+    private void ensureCapacity(int groupId) {
+        if (groupId >= values.size()) {
+            long prevSize = values.size();
+            values = bigArrays.grow(values, groupId + 1);
+            values.fill(prevSize, values.size(), init);
+        }
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(
+        Block[] blocks,
+        int offset,
+        IntVector selected,
+        org.elasticsearch.compute.operator.DriverContext driverContext
+    ) {
+        assert blocks.length >= offset + 3;
+        try (
+            var valuesBuilder = driverContext.blockFactory().newFloatBlockBuilder(selected.getPositionCount());
+            var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount());
+            var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount())
+        ) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (group < values.size()) {
+                    valuesBuilder.appendFloat(values.get(group));
+                } else {
+                    valuesBuilder.appendFloat(0); // TODO can we just use null?
+                }
+                hasValueBuilder.appendBoolean(i, hasValue(group));
+                hasFailedBuilder.appendBoolean(i, hasFailed(group));
+            }
+            blocks[offset + 0] = valuesBuilder.build();
+            blocks[offset + 1] = hasValueBuilder.build().asBlock();
+            blocks[offset + 2] = hasFailedBuilder.build().asBlock();
+        }
+    }
+
+    @Override
+    public void close() {
+        Releasables.close(values, super::close);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleState.java
new file mode 100644
index 0000000000000..b050c86258dcd
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleState.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * Aggregator state for a single float.
+ * It stores a third boolean to store if the aggregation failed.
+ * This class is generated. Do not edit it.
+ */
+final class FloatFallibleState implements AggregatorState {
+    private float value;
+    private boolean seen;
+    private boolean failed;
+
+    FloatFallibleState(float init) {
+        this.value = init;
+    }
+
+    float floatValue() {
+        return value;
+    }
+
+    void floatValue(float value) {
+        this.value = value;
+    }
+
+    boolean seen() {
+        return seen;
+    }
+
+    void seen(boolean seen) {
+        this.seen = seen;
+    }
+
+    boolean failed() {
+        return failed;
+    }
+
+    void failed(boolean failed) {
+        this.failed = failed;
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+        assert blocks.length >= offset + 3;
+        blocks[offset + 0] = driverContext.blockFactory().newConstantFloatBlockWith(value, 1);
+        blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1);
+        blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1);
+    }
+
+    @Override
+    public void close() {}
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java
index 81bdd39e51b6e..6f608271b6e42 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java
@@ -18,10 +18,6 @@ final class FloatState implements AggregatorState {
     private float value;
     private boolean seen;
 
-    FloatState() {
-        this(0);
-    }
-
     FloatState(float init) {
         this.value = init;
     }
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleArrayState.java
new file mode 100644
index 0000000000000..e45d84720ca1a
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleArrayState.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.IntArray;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.core.Releasables;
+
+/**
+ * Aggregator state for an array of ints, that also tracks failures.
+ * It is created in a mode where it won't track
+ * the {@code groupId}s that are sent to it and it is the
+ * responsibility of the caller to only fetch values for {@code groupId}s
+ * that it has sent using the {@code selected} parameter when building the
+ * results. This is fine when there are no {@code null} values in the input
+ * data. But once there are null values in the input data it is
+ * <strong>much</strong> more convenient to only send non-null values and
+ * the tracking built into the grouping code can't track that. In that case
+ * call {@link #enableGroupIdTracking} to transition the state into a mode
+ * where it'll track which {@code groupIds} have been written.
+ * <p>
+ * This class is generated. Do not edit it.
+ * </p>
+ */
+final class IntFallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState {
+    private final int init;
+
+    private IntArray values;
+
+    IntFallibleArrayState(BigArrays bigArrays, int init) {
+        super(bigArrays);
+        this.values = bigArrays.newIntArray(1, false);
+        this.values.set(0, init);
+        this.init = init;
+    }
+
+    int get(int groupId) {
+        return values.get(groupId);
+    }
+
+    int getOrDefault(int groupId) {
+        return groupId < values.size() ? values.get(groupId) : init;
+    }
+
+    void set(int groupId, int value) {
+        ensureCapacity(groupId);
+        values.set(groupId, value);
+        trackGroupId(groupId);
+    }
+
+    Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) {
+        if (false == trackingGroupIds() && false == anyFailure()) {
+            try (var builder = driverContext.blockFactory().newIntVectorFixedBuilder(selected.getPositionCount())) {
+                for (int i = 0; i < selected.getPositionCount(); i++) {
+                    builder.appendInt(i, values.get(selected.getInt(i)));
+                }
+                return builder.build().asBlock();
+            }
+        }
+        try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount())) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (hasValue(group) && !hasFailed(group)) {
+                    builder.appendInt(values.get(group));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
+    private void ensureCapacity(int groupId) {
+        if (groupId >= values.size()) {
+            long prevSize = values.size();
+            values = bigArrays.grow(values, groupId + 1);
+            values.fill(prevSize, values.size(), init);
+        }
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(
+        Block[] blocks,
+        int offset,
+        IntVector selected,
+        org.elasticsearch.compute.operator.DriverContext driverContext
+    ) {
+        assert blocks.length >= offset + 3;
+        try (
+            var valuesBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount());
+            var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount());
+            var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount())
+        ) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (group < values.size()) {
+                    valuesBuilder.appendInt(values.get(group));
+                } else {
+                    valuesBuilder.appendInt(0); // TODO can we just use null?
+                }
+                hasValueBuilder.appendBoolean(i, hasValue(group));
+                hasFailedBuilder.appendBoolean(i, hasFailed(group));
+            }
+            blocks[offset + 0] = valuesBuilder.build();
+            blocks[offset + 1] = hasValueBuilder.build().asBlock();
+            blocks[offset + 2] = hasFailedBuilder.build().asBlock();
+        }
+    }
+
+    @Override
+    public void close() {
+        Releasables.close(values, super::close);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleState.java
new file mode 100644
index 0000000000000..360f3fdb009e4
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleState.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * Aggregator state for a single int.
+ * It stores a third boolean to store if the aggregation failed.
+ * This class is generated. Do not edit it.
+ */
+final class IntFallibleState implements AggregatorState {
+    private int value;
+    private boolean seen;
+    private boolean failed;
+
+    IntFallibleState(int init) {
+        this.value = init;
+    }
+
+    int intValue() {
+        return value;
+    }
+
+    void intValue(int value) {
+        this.value = value;
+    }
+
+    boolean seen() {
+        return seen;
+    }
+
+    void seen(boolean seen) {
+        this.seen = seen;
+    }
+
+    boolean failed() {
+        return failed;
+    }
+
+    void failed(boolean failed) {
+        this.failed = failed;
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+        assert blocks.length >= offset + 3;
+        blocks[offset + 0] = driverContext.blockFactory().newConstantIntBlockWith(value, 1);
+        blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1);
+        blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1);
+    }
+
+    @Override
+    public void close() {}
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntState.java
index e7db40eccf9c8..c539c576ef36d 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntState.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntState.java
@@ -18,10 +18,6 @@ final class IntState implements AggregatorState {
     private int value;
     private boolean seen;
 
-    IntState() {
-        this(0);
-    }
-
     IntState(int init) {
         this.value = init;
     }
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleArrayState.java
new file mode 100644
index 0000000000000..cb69579906871
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleArrayState.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.LongArray;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.core.Releasables;
+
+/**
+ * Aggregator state for an array of longs, that also tracks failures.
+ * It is created in a mode where it won't track
+ * the {@code groupId}s that are sent to it and it is the
+ * responsibility of the caller to only fetch values for {@code groupId}s
+ * that it has sent using the {@code selected} parameter when building the
+ * results. This is fine when there are no {@code null} values in the input
+ * data. But once there are null values in the input data it is
+ * <strong>much</strong> more convenient to only send non-null values and
+ * the tracking built into the grouping code can't track that. In that case
+ * call {@link #enableGroupIdTracking} to transition the state into a mode
+ * where it'll track which {@code groupIds} have been written.
+ * <p>
+ * This class is generated. Do not edit it.
+ * </p>
+ */
+final class LongFallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState {
+    private final long init;
+
+    private LongArray values;
+
+    LongFallibleArrayState(BigArrays bigArrays, long init) {
+        super(bigArrays);
+        this.values = bigArrays.newLongArray(1, false);
+        this.values.set(0, init);
+        this.init = init;
+    }
+
+    long get(int groupId) {
+        return values.get(groupId);
+    }
+
+    long getOrDefault(int groupId) {
+        return groupId < values.size() ? values.get(groupId) : init;
+    }
+
+    void set(int groupId, long value) {
+        ensureCapacity(groupId);
+        values.set(groupId, value);
+        trackGroupId(groupId);
+    }
+
+    void increment(int groupId, long value) {
+        ensureCapacity(groupId);
+        values.increment(groupId, value);
+        trackGroupId(groupId);
+    }
+
+    Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) {
+        if (false == trackingGroupIds() && false == anyFailure()) {
+            try (var builder = driverContext.blockFactory().newLongVectorFixedBuilder(selected.getPositionCount())) {
+                for (int i = 0; i < selected.getPositionCount(); i++) {
+                    builder.appendLong(i, values.get(selected.getInt(i)));
+                }
+                return builder.build().asBlock();
+            }
+        }
+        try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount())) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (hasValue(group) && !hasFailed(group)) {
+                    builder.appendLong(values.get(group));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
+    private void ensureCapacity(int groupId) {
+        if (groupId >= values.size()) {
+            long prevSize = values.size();
+            values = bigArrays.grow(values, groupId + 1);
+            values.fill(prevSize, values.size(), init);
+        }
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(
+        Block[] blocks,
+        int offset,
+        IntVector selected,
+        org.elasticsearch.compute.operator.DriverContext driverContext
+    ) {
+        assert blocks.length >= offset + 3;
+        try (
+            var valuesBuilder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount());
+            var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount());
+            var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount())
+        ) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (group < values.size()) {
+                    valuesBuilder.appendLong(values.get(group));
+                } else {
+                    valuesBuilder.appendLong(0); // TODO can we just use null?
+                }
+                hasValueBuilder.appendBoolean(i, hasValue(group));
+                hasFailedBuilder.appendBoolean(i, hasFailed(group));
+            }
+            blocks[offset + 0] = valuesBuilder.build();
+            blocks[offset + 1] = hasValueBuilder.build().asBlock();
+            blocks[offset + 2] = hasFailedBuilder.build().asBlock();
+        }
+    }
+
+    @Override
+    public void close() {
+        Releasables.close(values, super::close);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleState.java
new file mode 100644
index 0000000000000..98669ef627d04
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleState.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * Aggregator state for a single long.
+ * It stores a third boolean to store if the aggregation failed.
+ * This class is generated. Do not edit it.
+ */
+final class LongFallibleState implements AggregatorState {
+    private long value;
+    private boolean seen;
+    private boolean failed;
+
+    LongFallibleState(long init) {
+        this.value = init;
+    }
+
+    long longValue() {
+        return value;
+    }
+
+    void longValue(long value) {
+        this.value = value;
+    }
+
+    boolean seen() {
+        return seen;
+    }
+
+    void seen(boolean seen) {
+        this.seen = seen;
+    }
+
+    boolean failed() {
+        return failed;
+    }
+
+    void failed(boolean failed) {
+        this.failed = failed;
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+        assert blocks.length >= offset + 3;
+        blocks[offset + 0] = driverContext.blockFactory().newConstantLongBlockWith(value, 1);
+        blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1);
+        blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1);
+    }
+
+    @Override
+    public void close() {}
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongState.java
index da78b649782d5..e9d97dcfe7fc1 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongState.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongState.java
@@ -18,10 +18,6 @@ final class LongState implements AggregatorState {
     private long value;
     private boolean seen;
 
-    LongState() {
-        this(0);
-    }
-
     LongState(long init) {
         this.value = init;
     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java
similarity index 98%
rename from x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java
rename to x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java
index 97d8cf0915b05..32391c4827303 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopBooleanAggregator.java
@@ -23,6 +23,9 @@
 
 /**
  * Aggregates the top N field values for boolean.
+ * <p>
+ *     This class is generated. Edit `X-TopAggregator.java.st` to edit this file.
+ * </p>
  */
 @Aggregator({ @IntermediateState(name = "top", type = "BOOLEAN_BLOCK") })
 @GroupingAggregator
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java
index 3bd76b79d62f2..d9a7a302f07c8 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopDoubleAggregator.java
@@ -23,6 +23,9 @@
 
 /**
  * Aggregates the top N field values for double.
+ * <p>
+ *     This class is generated. Edit `X-TopAggregator.java.st` to edit this file.
+ * </p>
  */
 @Aggregator({ @IntermediateState(name = "top", type = "DOUBLE_BLOCK") })
 @GroupingAggregator
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java
index 066c82e9448fb..8b65261e10f46 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopFloatAggregator.java
@@ -23,6 +23,9 @@
 
 /**
  * Aggregates the top N field values for float.
+ * <p>
+ *     This class is generated. Edit `X-TopAggregator.java.st` to edit this file.
+ * </p>
  */
 @Aggregator({ @IntermediateState(name = "top", type = "FLOAT_BLOCK") })
 @GroupingAggregator
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java
index 2f5149c594d94..5c6b79f710af5 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIntAggregator.java
@@ -23,6 +23,9 @@
 
 /**
  * Aggregates the top N field values for int.
+ * <p>
+ *     This class is generated. Edit `X-TopAggregator.java.st` to edit this file.
+ * </p>
  */
 @Aggregator({ @IntermediateState(name = "top", type = "INT_BLOCK") })
 @GroupingAggregator
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIpAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIpAggregator.java
new file mode 100644
index 0000000000000..219f7385b56df
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopIpAggregator.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.compute.ann.Aggregator;
+import org.elasticsearch.compute.ann.GroupingAggregator;
+import org.elasticsearch.compute.ann.IntermediateState;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.sort.IpBucketedSort;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.search.sort.SortOrder;
+
+/**
+ * Aggregates the top N field values for BytesRef.
+ * <p>
+ *     This class is generated. Edit `X-TopAggregator.java.st` to edit this file.
+ * </p>
+ */
+@Aggregator({ @IntermediateState(name = "top", type = "BYTES_REF_BLOCK") })
+@GroupingAggregator
+class TopIpAggregator {
+    public static SingleState initSingle(BigArrays bigArrays, int limit, boolean ascending) {
+        return new SingleState(bigArrays, limit, ascending);
+    }
+
+    public static void combine(SingleState state, BytesRef v) {
+        state.add(v);
+    }
+
+    public static void combineIntermediate(SingleState state, BytesRefBlock values) {
+        int start = values.getFirstValueIndex(0);
+        int end = start + values.getValueCount(0);
+        var scratch = new BytesRef();
+        for (int i = start; i < end; i++) {
+            combine(state, values.getBytesRef(i, scratch));
+        }
+    }
+
+    public static Block evaluateFinal(SingleState state, DriverContext driverContext) {
+        return state.toBlock(driverContext.blockFactory());
+    }
+
+    public static GroupingState initGrouping(BigArrays bigArrays, int limit, boolean ascending) {
+        return new GroupingState(bigArrays, limit, ascending);
+    }
+
+    public static void combine(GroupingState state, int groupId, BytesRef v) {
+        state.add(groupId, v);
+    }
+
+    public static void combineIntermediate(GroupingState state, int groupId, BytesRefBlock values, int valuesPosition) {
+        int start = values.getFirstValueIndex(valuesPosition);
+        int end = start + values.getValueCount(valuesPosition);
+        var scratch = new BytesRef();
+        for (int i = start; i < end; i++) {
+            combine(state, groupId, values.getBytesRef(i, scratch));
+        }
+    }
+
+    public static void combineStates(GroupingState current, int groupId, GroupingState state, int statePosition) {
+        current.merge(groupId, state, statePosition);
+    }
+
+    public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) {
+        return state.toBlock(driverContext.blockFactory(), selected);
+    }
+
+    public static class GroupingState implements Releasable {
+        private final IpBucketedSort sort;
+
+        private GroupingState(BigArrays bigArrays, int limit, boolean ascending) {
+            this.sort = new IpBucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit);
+        }
+
+        public void add(int groupId, BytesRef value) {
+            sort.collect(value, groupId);
+        }
+
+        public void merge(int groupId, GroupingState other, int otherGroupId) {
+            sort.merge(groupId, other.sort, otherGroupId);
+        }
+
+        void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) {
+            blocks[offset] = toBlock(driverContext.blockFactory(), selected);
+        }
+
+        Block toBlock(BlockFactory blockFactory, IntVector selected) {
+            return sort.toBlock(blockFactory, selected);
+        }
+
+        void enableGroupIdTracking(SeenGroupIds seen) {
+            // we figure out seen values from nulls on the values block
+        }
+
+        @Override
+        public void close() {
+            Releasables.closeExpectNoException(sort);
+        }
+    }
+
+    public static class SingleState implements Releasable {
+        private final GroupingState internalState;
+
+        private SingleState(BigArrays bigArrays, int limit, boolean ascending) {
+            this.internalState = new GroupingState(bigArrays, limit, ascending);
+        }
+
+        public void add(BytesRef value) {
+            internalState.add(0, value);
+        }
+
+        public void merge(GroupingState other) {
+            internalState.merge(0, other, 0);
+        }
+
+        void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+            blocks[offset] = toBlock(driverContext.blockFactory());
+        }
+
+        Block toBlock(BlockFactory blockFactory) {
+            try (var intValues = blockFactory.newConstantIntVector(0, 1)) {
+                return internalState.toBlock(blockFactory, intValues);
+            }
+        }
+
+        @Override
+        public void close() {
+            Releasables.closeExpectNoException(internalState);
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java
index d6bafaa30c425..44cef8df7257b 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopLongAggregator.java
@@ -23,6 +23,9 @@
 
 /**
  * Aggregates the top N field values for long.
+ * <p>
+ *     This class is generated. Edit `X-TopAggregator.java.st` to edit this file.
+ * </p>
  */
 @Aggregator({ @IntermediateState(name = "top", type = "LONG_BLOCK") })
 @GroupingAggregator
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java
index 736b320a9dde8..602fd29433193 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregator.java
@@ -100,7 +100,7 @@ Block toBlock(BlockFactory blockFactory) {
             }
             BytesRef scratch = new BytesRef();
             if (values.size() == 1) {
-                return blockFactory.newConstantBytesRefBlockWith(values.get(0, scratch), 1);
+                return blockFactory.newConstantBytesRefBlockWith(BytesRef.deepCopyOf(values.get(0, scratch)), 1);
             }
             try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder((int) values.size())) {
                 builder.beginPositionEntry();
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java
index 254d56f849768..3d600bec1bd65 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java
@@ -85,6 +85,27 @@ public BooleanVector asVector() {
         return null;
     }
 
+    @Override
+    public ToMask toMask() {
+        if (getPositionCount() == 0) {
+            return new ToMask(blockFactory().newConstantBooleanVector(false, 0), false);
+        }
+        try (BooleanVector.FixedBuilder builder = blockFactory().newBooleanVectorFixedBuilder(getPositionCount())) {
+            boolean hasMv = false;
+            for (int p = 0; p < getPositionCount(); p++) {
+                builder.appendBoolean(switch (getValueCount(p)) {
+                    case 0 -> false;
+                    case 1 -> getBoolean(getFirstValueIndex(p));
+                    default -> {
+                        hasMv = true;
+                        yield false;
+                    }
+                });
+            }
+            return new ToMask(builder.build(), hasMv);
+        }
+    }
+
     @Override
     public boolean getBoolean(int valueIndex) {
         return vector.getBoolean(valueIndex);
@@ -114,6 +135,47 @@ public BooleanBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public BooleanBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (BooleanBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (BooleanBlock.Builder builder = blockFactory().newBooleanBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.appendBoolean(getBoolean(start));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.appendBoolean(getBoolean(i));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<BooleanBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new BooleanLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java
index a91999a49c16b..b44ad180a66ff 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java
@@ -93,6 +93,32 @@ public BooleanVector filter(int... positions) {
         }
     }
 
+    @Override
+    public BooleanBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new BooleanVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new BooleanVectorBlock(this);
+            }
+            return (BooleanBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (BooleanBlock.Builder builder = blockFactory().newBooleanBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendBoolean(getBoolean(p));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<BooleanBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new BooleanLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java
index aac728236b136..f353512eb93b7 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java
@@ -86,6 +86,27 @@ public BooleanVector asVector() {
         return null;
     }
 
+    @Override
+    public ToMask toMask() {
+        if (getPositionCount() == 0) {
+            return new ToMask(blockFactory().newConstantBooleanVector(false, 0), false);
+        }
+        try (BooleanVector.FixedBuilder builder = blockFactory().newBooleanVectorFixedBuilder(getPositionCount())) {
+            boolean hasMv = false;
+            for (int p = 0; p < getPositionCount(); p++) {
+                builder.appendBoolean(switch (getValueCount(p)) {
+                    case 0 -> false;
+                    case 1 -> getBoolean(getFirstValueIndex(p));
+                    default -> {
+                        hasMv = true;
+                        yield false;
+                    }
+                });
+            }
+            return new ToMask(builder.build(), hasMv);
+        }
+    }
+
     @Override
     public boolean getBoolean(int valueIndex) {
         return vector.getBoolean(valueIndex);
@@ -115,6 +136,47 @@ public BooleanBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public BooleanBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (BooleanBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (BooleanBlock.Builder builder = blockFactory().newBooleanBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.appendBoolean(getBoolean(start));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.appendBoolean(getBoolean(i));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<BooleanBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new BooleanLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java
index 9215cd0d9bbda..f6bd6e978bc7e 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java
@@ -89,6 +89,32 @@ public BooleanVector filter(int... positions) {
         return new BooleanBigArrayVector(filtered, positions.length, blockFactory);
     }
 
+    @Override
+    public BooleanBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new BooleanVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new BooleanVectorBlock(this);
+            }
+            return (BooleanBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (BooleanBlock.Builder builder = blockFactory().newBooleanBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendBoolean(getBoolean(p));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<BooleanBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new BooleanLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java
index 8ae2984018640..5d2d6c97a11f1 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBlock.java
@@ -37,9 +37,19 @@ public sealed interface BooleanBlock extends Block permits BooleanArrayBlock, Bo
     @Override
     BooleanVector asVector();
 
+    /**
+     * Convert this to a {@link BooleanVector "mask"} that's appropriate for
+     * passing to {@link #keepMask}. Null and multivalued positions will be
+     * converted to {@code false}.
+     */
+    ToMask toMask();
+
     @Override
     BooleanBlock filter(int... positions);
 
+    @Override
+    BooleanBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends BooleanBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -97,10 +107,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (vector != null) {
             out.writeByte(SERIALIZE_BLOCK_VECTOR);
             vector.writeTo(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof BooleanArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BooleanArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_ARRAY);
             b.writeArrayBlock(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof BooleanBigArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BooleanBigArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY);
             b.writeArrayBlock(out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java
index 5cf900cfc4a71..5e3157a107fa5 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java
@@ -29,6 +29,9 @@ public sealed interface BooleanVector extends Vector permits ConstantBooleanVect
     @Override
     BooleanVector filter(int... positions);
 
+    @Override
+    BooleanBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends BooleanBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -101,10 +104,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (isConstant() && positions > 0) {
             out.writeByte(SERIALIZE_VECTOR_CONSTANT);
             out.writeBoolean(getBoolean(0));
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof BooleanArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BooleanArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_ARRAY);
             v.writeArrayVector(positions, out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof BooleanBigArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BooleanBigArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY);
             v.writeArrayVector(positions, out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java
index 193e6ea5d8965..1544cc3355cd0 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java
@@ -31,6 +31,12 @@ public BooleanVector asVector() {
         return vector;
     }
 
+    @Override
+    public ToMask toMask() {
+        vector.incRef();
+        return new ToMask(vector, false);
+    }
+
     @Override
     public boolean getBoolean(int valueIndex) {
         return vector.getBoolean(valueIndex);
@@ -51,6 +57,11 @@ public BooleanBlock filter(int... positions) {
         return vector.filter(positions).asBlock();
     }
 
+    @Override
+    public BooleanBlock keepMask(BooleanVector mask) {
+        return vector.keepMask(mask);
+    }
+
     @Override
     public ReleasableIterator<? extends BooleanBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return vector.lookup(positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java
index 1372f7b94b78c..f50135aa51dd4 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java
@@ -123,6 +123,48 @@ public BytesRefBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public BytesRefBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (BytesRefBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        BytesRef scratch = new BytesRef();
+        try (BytesRefBlock.Builder builder = blockFactory().newBytesRefBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.appendBytesRef(getBytesRef(start, scratch));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.appendBytesRef(getBytesRef(i, scratch));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<BytesRefBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new BytesRefLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java
index 61bbfb5ebbd02..e72c900718735 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java
@@ -93,6 +93,33 @@ public BytesRefVector filter(int... positions) {
         }
     }
 
+    @Override
+    public BytesRefBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new BytesRefVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new BytesRefVectorBlock(this);
+            }
+            return (BytesRefBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        BytesRef scratch = new BytesRef();
+        try (BytesRefBlock.Builder builder = blockFactory().newBytesRefBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendBytesRef(getBytesRef(p, scratch));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<BytesRefBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new BytesRefLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java
index d7c28a24482e0..6fe45f33a7df6 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java
@@ -50,6 +50,9 @@ public sealed interface BytesRefBlock extends Block permits BytesRefArrayBlock,
     @Override
     BytesRefBlock filter(int... positions);
 
+    @Override
+    BytesRefBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends BytesRefBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -107,10 +110,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (vector != null) {
             out.writeByte(SERIALIZE_BLOCK_VECTOR);
             vector.writeTo(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof BytesRefArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BytesRefArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_ARRAY);
             b.writeArrayBlock(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_ORDINAL_BLOCK) && this instanceof OrdinalBytesRefBlock b && b.isDense()) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof OrdinalBytesRefBlock b && b.isDense()) {
             out.writeByte(SERIALIZE_BLOCK_ORDINAL);
             b.writeOrdinalBlock(out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVector.java
index 3739dccb0f956..a3432a2913fde 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVector.java
@@ -36,6 +36,9 @@ public sealed interface BytesRefVector extends Vector permits ConstantBytesRefVe
     @Override
     BytesRefVector filter(int... positions);
 
+    @Override
+    BytesRefBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends BytesRefBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -108,10 +111,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (isConstant() && positions > 0) {
             out.writeByte(SERIALIZE_VECTOR_CONSTANT);
             out.writeBytesRef(getBytesRef(0, new BytesRef()));
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof BytesRefArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof BytesRefArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_ARRAY);
             v.writeArrayVector(positions, out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_ORDINAL_BLOCK) && this instanceof OrdinalBytesRefVector v && v.isDense()) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof OrdinalBytesRefVector v && v.isDense()) {
             out.writeByte(SERIALIZE_VECTOR_ORDINAL);
             v.writeOrdinalVector(out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java
index 16a8fc0888096..6c0334e147aa0 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java
@@ -62,6 +62,11 @@ public BytesRefBlock filter(int... positions) {
         return vector.filter(positions).asBlock();
     }
 
+    @Override
+    public BytesRefBlock keepMask(BooleanVector mask) {
+        return vector.keepMask(mask);
+    }
+
     @Override
     public ReleasableIterator<? extends BytesRefBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return vector.lookup(positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java
index 1f6786f64e0a9..1c886eb7c2dab 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java
@@ -41,6 +41,32 @@ public BooleanVector filter(int... positions) {
         return blockFactory().newConstantBooleanVector(value, positions.length);
     }
 
+    @Override
+    public BooleanBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new BooleanVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new BooleanVectorBlock(this);
+            }
+            return (BooleanBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (BooleanBlock.Builder builder = blockFactory().newBooleanBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendBoolean(value);
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<BooleanBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         if (positions.getPositionCount() == 0) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBytesRefVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBytesRefVector.java
index 33967d66374c1..216b7db26b6fb 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBytesRefVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBytesRefVector.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.ReleasableIterator;
+import org.elasticsearch.core.Releasables;
 
 /**
  * Vector implementation that stores a constant BytesRef value.
@@ -47,6 +48,35 @@ public BytesRefVector filter(int... positions) {
         return blockFactory().newConstantBytesRefVector(value, positions.length);
     }
 
+    @Override
+    public BytesRefBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new BytesRefVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new BytesRefVectorBlock(this);
+            }
+            return (BytesRefBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        IntBlock ordinals = null;
+        BytesRefVector bytes = null;
+        try {
+            try (IntVector unmaskedOrdinals = blockFactory().newConstantIntVector(0, getPositionCount())) {
+                ordinals = unmaskedOrdinals.keepMask(mask);
+            }
+            bytes = blockFactory().newConstantBytesRefVector(value, getPositionCount());
+            OrdinalBytesRefBlock result = new OrdinalBytesRefBlock(ordinals, bytes);
+            ordinals = null;
+            bytes = null;
+            return result;
+        } finally {
+            Releasables.close(ordinals, bytes);
+        }
+    }
+
     @Override
     public ReleasableIterator<BytesRefBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         if (positions.getPositionCount() == 0) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantDoubleVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantDoubleVector.java
index 1ddf31d753d43..3c91eaf7171b5 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantDoubleVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantDoubleVector.java
@@ -41,6 +41,32 @@ public DoubleVector filter(int... positions) {
         return blockFactory().newConstantDoubleVector(value, positions.length);
     }
 
+    @Override
+    public DoubleBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new DoubleVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new DoubleVectorBlock(this);
+            }
+            return (DoubleBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (DoubleBlock.Builder builder = blockFactory().newDoubleBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendDouble(value);
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<DoubleBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         if (positions.getPositionCount() == 0) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantFloatVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantFloatVector.java
index 2e674288eac92..a9ee5e811da64 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantFloatVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantFloatVector.java
@@ -41,6 +41,32 @@ public FloatVector filter(int... positions) {
         return blockFactory().newConstantFloatVector(value, positions.length);
     }
 
+    @Override
+    public FloatBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new FloatVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new FloatVectorBlock(this);
+            }
+            return (FloatBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (FloatBlock.Builder builder = blockFactory().newFloatBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendFloat(value);
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<FloatBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         if (positions.getPositionCount() == 0) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantIntVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantIntVector.java
index e8fb8cb39ceb4..20c1b07a3f102 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantIntVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantIntVector.java
@@ -41,6 +41,32 @@ public IntVector filter(int... positions) {
         return blockFactory().newConstantIntVector(value, positions.length);
     }
 
+    @Override
+    public IntBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new IntVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new IntVectorBlock(this);
+            }
+            return (IntBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (IntBlock.Builder builder = blockFactory().newIntBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendInt(value);
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<IntBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         if (positions.getPositionCount() == 0) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantLongVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantLongVector.java
index b997cbbe22849..6e3decb9ad540 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantLongVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantLongVector.java
@@ -41,6 +41,32 @@ public LongVector filter(int... positions) {
         return blockFactory().newConstantLongVector(value, positions.length);
     }
 
+    @Override
+    public LongBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new LongVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new LongVectorBlock(this);
+            }
+            return (LongBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (LongBlock.Builder builder = blockFactory().newLongBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendLong(value);
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<LongBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         if (positions.getPositionCount() == 0) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java
index 4d923e4ca77c8..eceec30348749 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java
@@ -114,6 +114,47 @@ public DoubleBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public DoubleBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (DoubleBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (DoubleBlock.Builder builder = blockFactory().newDoubleBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.appendDouble(getDouble(start));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.appendDouble(getDouble(i));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<DoubleBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new DoubleLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java
index e7c1d342133d5..6426f8f39ecd9 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java
@@ -92,6 +92,32 @@ public DoubleVector filter(int... positions) {
         }
     }
 
+    @Override
+    public DoubleBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new DoubleVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new DoubleVectorBlock(this);
+            }
+            return (DoubleBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (DoubleBlock.Builder builder = blockFactory().newDoubleBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendDouble(getDouble(p));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<DoubleBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new DoubleLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java
index 203856f88c4ce..8dc9efa755c17 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java
@@ -115,6 +115,47 @@ public DoubleBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public DoubleBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (DoubleBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (DoubleBlock.Builder builder = blockFactory().newDoubleBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.appendDouble(getDouble(start));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.appendDouble(getDouble(i));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<DoubleBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new DoubleLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayVector.java
index d558eabd2dd4c..e98d0da358cdf 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayVector.java
@@ -88,6 +88,32 @@ public DoubleVector filter(int... positions) {
         return new DoubleBigArrayVector(filtered, positions.length, blockFactory);
     }
 
+    @Override
+    public DoubleBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new DoubleVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new DoubleVectorBlock(this);
+            }
+            return (DoubleBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (DoubleBlock.Builder builder = blockFactory().newDoubleBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendDouble(getDouble(p));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<DoubleBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new DoubleLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java
index 95f318703df62..395ccd412fabb 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBlock.java
@@ -40,6 +40,9 @@ public sealed interface DoubleBlock extends Block permits DoubleArrayBlock, Doub
     @Override
     DoubleBlock filter(int... positions);
 
+    @Override
+    DoubleBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends DoubleBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -97,10 +100,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (vector != null) {
             out.writeByte(SERIALIZE_BLOCK_VECTOR);
             vector.writeTo(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof DoubleArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof DoubleArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_ARRAY);
             b.writeArrayBlock(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof DoubleBigArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof DoubleBigArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY);
             b.writeArrayBlock(out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVector.java
index 10d4f4abe5f6a..ec4b9cadc074e 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVector.java
@@ -29,6 +29,9 @@ public sealed interface DoubleVector extends Vector permits ConstantDoubleVector
     @Override
     DoubleVector filter(int... positions);
 
+    @Override
+    DoubleBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends DoubleBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -102,10 +105,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (isConstant() && positions > 0) {
             out.writeByte(SERIALIZE_VECTOR_CONSTANT);
             out.writeDouble(getDouble(0));
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof DoubleArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof DoubleArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_ARRAY);
             v.writeArrayVector(positions, out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof DoubleBigArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof DoubleBigArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY);
             v.writeArrayVector(positions, out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java
index 24887bebcd838..7e680f8e59ff0 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java
@@ -51,6 +51,11 @@ public DoubleBlock filter(int... positions) {
         return vector.filter(positions).asBlock();
     }
 
+    @Override
+    public DoubleBlock keepMask(BooleanVector mask) {
+        return vector.keepMask(mask);
+    }
+
     @Override
     public ReleasableIterator<? extends DoubleBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return vector.lookup(positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayBlock.java
index b2666a9d86926..56f0cedb5f15e 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayBlock.java
@@ -114,6 +114,47 @@ public FloatBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public FloatBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (FloatBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (FloatBlock.Builder builder = blockFactory().newFloatBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.appendFloat(getFloat(start));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.appendFloat(getFloat(i));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<FloatBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new FloatLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayVector.java
index 7e5e0eef436ff..fd47f40463552 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayVector.java
@@ -92,6 +92,32 @@ public FloatVector filter(int... positions) {
         }
     }
 
+    @Override
+    public FloatBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new FloatVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new FloatVectorBlock(this);
+            }
+            return (FloatBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (FloatBlock.Builder builder = blockFactory().newFloatBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendFloat(getFloat(p));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<FloatBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new FloatLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayBlock.java
index 693823636043a..a2191ba9fc69a 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayBlock.java
@@ -115,6 +115,47 @@ public FloatBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public FloatBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (FloatBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (FloatBlock.Builder builder = blockFactory().newFloatBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.appendFloat(getFloat(start));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.appendFloat(getFloat(i));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<FloatBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new FloatLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayVector.java
index 2de1019103522..4e481bc6bcaaf 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayVector.java
@@ -76,6 +76,32 @@ public FloatVector filter(int... positions) {
         return new FloatBigArrayVector(filtered, positions.length, blockFactory);
     }
 
+    @Override
+    public FloatBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new FloatVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new FloatVectorBlock(this);
+            }
+            return (FloatBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (FloatBlock.Builder builder = blockFactory().newFloatBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendFloat(getFloat(p));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<FloatBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new FloatLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlock.java
index 3d2def604a61e..633c9f309901a 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlock.java
@@ -40,6 +40,9 @@ public sealed interface FloatBlock extends Block permits FloatArrayBlock, FloatV
     @Override
     FloatBlock filter(int... positions);
 
+    @Override
+    FloatBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends FloatBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -97,10 +100,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (vector != null) {
             out.writeByte(SERIALIZE_BLOCK_VECTOR);
             vector.writeTo(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof FloatArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof FloatArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_ARRAY);
             b.writeArrayBlock(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof FloatBigArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof FloatBigArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY);
             b.writeArrayBlock(out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVector.java
index 5fd2ae7b9c719..5f49efd2779ea 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVector.java
@@ -29,6 +29,9 @@ public sealed interface FloatVector extends Vector permits ConstantFloatVector,
     @Override
     FloatVector filter(int... positions);
 
+    @Override
+    FloatBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends FloatBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -101,10 +104,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (isConstant() && positions > 0) {
             out.writeByte(SERIALIZE_VECTOR_CONSTANT);
             out.writeFloat(getFloat(0));
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof FloatArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof FloatArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_ARRAY);
             v.writeArrayVector(positions, out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof FloatBigArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof FloatBigArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY);
             v.writeArrayVector(positions, out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorBlock.java
index 62a56dc6833d1..d5df1e8ea6e57 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorBlock.java
@@ -51,6 +51,11 @@ public FloatBlock filter(int... positions) {
         return vector.filter(positions).asBlock();
     }
 
+    @Override
+    public FloatBlock keepMask(BooleanVector mask) {
+        return vector.keepMask(mask);
+    }
+
     @Override
     public ReleasableIterator<? extends FloatBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return vector.lookup(positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java
index 6231e8f9c5a10..2e10d09e1a410 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java
@@ -114,6 +114,47 @@ public IntBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public IntBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (IntBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (IntBlock.Builder builder = blockFactory().newIntBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.appendInt(getInt(start));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.appendInt(getInt(i));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<IntBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new IntLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java
index e9d9a6b3fb958..c3d6691dec3ca 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java
@@ -102,6 +102,32 @@ public IntVector filter(int... positions) {
         }
     }
 
+    @Override
+    public IntBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new IntVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new IntVectorBlock(this);
+            }
+            return (IntBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (IntBlock.Builder builder = blockFactory().newIntBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendInt(getInt(p));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<IntBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new IntLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java
index a1e84db8c4f27..247720489a3fd 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java
@@ -115,6 +115,47 @@ public IntBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public IntBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (IntBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (IntBlock.Builder builder = blockFactory().newIntBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.appendInt(getInt(start));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.appendInt(getInt(i));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<IntBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new IntLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayVector.java
index df8298b87237e..b6c73205ad12e 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayVector.java
@@ -128,6 +128,32 @@ public IntVector filter(int... positions) {
         return new IntBigArrayVector(filtered, positions.length, blockFactory);
     }
 
+    @Override
+    public IntBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new IntVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new IntVectorBlock(this);
+            }
+            return (IntBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (IntBlock.Builder builder = blockFactory().newIntBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendInt(getInt(p));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<IntBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new IntLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java
index 21d40170151a5..7c77d9965391e 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBlock.java
@@ -40,6 +40,9 @@ public sealed interface IntBlock extends Block permits IntArrayBlock, IntVectorB
     @Override
     IntBlock filter(int... positions);
 
+    @Override
+    IntBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends IntBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -97,10 +100,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (vector != null) {
             out.writeByte(SERIALIZE_BLOCK_VECTOR);
             vector.writeTo(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof IntArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof IntArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_ARRAY);
             b.writeArrayBlock(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof IntBigArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof IntBigArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY);
             b.writeArrayBlock(out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVector.java
index 384d5813d5750..da1a92b21c657 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVector.java
@@ -29,6 +29,9 @@ public sealed interface IntVector extends Vector permits ConstantIntVector, IntA
     @Override
     IntVector filter(int... positions);
 
+    @Override
+    IntBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends IntBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -111,10 +114,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (isConstant() && positions > 0) {
             out.writeByte(SERIALIZE_VECTOR_CONSTANT);
             out.writeInt(getInt(0));
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof IntArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof IntArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_ARRAY);
             v.writeArrayVector(positions, out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof IntBigArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof IntBigArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY);
             v.writeArrayVector(positions, out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java
index ae28fb9f6ffa6..57e69a3aa9acc 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java
@@ -51,6 +51,11 @@ public IntBlock filter(int... positions) {
         return vector.filter(positions).asBlock();
     }
 
+    @Override
+    public IntBlock keepMask(BooleanVector mask) {
+        return vector.keepMask(mask);
+    }
+
     @Override
     public ReleasableIterator<? extends IntBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return vector.lookup(positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java
index d8357e5d367cc..776fa363f6080 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java
@@ -114,6 +114,47 @@ public LongBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public LongBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (LongBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (LongBlock.Builder builder = blockFactory().newLongBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.appendLong(getLong(start));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.appendLong(getLong(i));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<LongBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new LongLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java
index 5fa904dcf1acc..748f25a0005fb 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java
@@ -92,6 +92,32 @@ public LongVector filter(int... positions) {
         }
     }
 
+    @Override
+    public LongBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new LongVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new LongVectorBlock(this);
+            }
+            return (LongBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (LongBlock.Builder builder = blockFactory().newLongBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendLong(getLong(p));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<LongBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new LongLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java
index 0ccd4ab368659..ec600d02bc720 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java
@@ -115,6 +115,47 @@ public LongBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public LongBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (LongBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (LongBlock.Builder builder = blockFactory().newLongBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.appendLong(getLong(start));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.appendLong(getLong(i));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<LongBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new LongLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayVector.java
index a7828788169ca..dae27331afc1e 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayVector.java
@@ -88,6 +88,32 @@ public LongVector filter(int... positions) {
         return new LongBigArrayVector(filtered, positions.length, blockFactory);
     }
 
+    @Override
+    public LongBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new LongVectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new LongVectorBlock(this);
+            }
+            return (LongBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try (LongBlock.Builder builder = blockFactory().newLongBlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.appendLong(getLong(p));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<LongBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new LongLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java
index 5a11ee8e2a6e3..6c88da8860ca7 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBlock.java
@@ -40,6 +40,9 @@ public sealed interface LongBlock extends Block permits LongArrayBlock, LongVect
     @Override
     LongBlock filter(int... positions);
 
+    @Override
+    LongBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends LongBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -97,10 +100,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (vector != null) {
             out.writeByte(SERIALIZE_BLOCK_VECTOR);
             vector.writeTo(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof LongArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof LongArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_ARRAY);
             b.writeArrayBlock(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof LongBigArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof LongBigArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY);
             b.writeArrayBlock(out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVector.java
index a74146b692e31..3773e602b8d03 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVector.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVector.java
@@ -29,6 +29,9 @@ public sealed interface LongVector extends Vector permits ConstantLongVector, Lo
     @Override
     LongVector filter(int... positions);
 
+    @Override
+    LongBlock keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends LongBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -102,10 +105,10 @@ default void writeTo(StreamOutput out) throws IOException {
         if (isConstant() && positions > 0) {
             out.writeByte(SERIALIZE_VECTOR_CONSTANT);
             out.writeLong(getLong(0));
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof LongArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof LongArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_ARRAY);
             v.writeArrayVector(positions, out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof LongBigArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof LongBigArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY);
             v.writeArrayVector(positions, out);
         } else {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java
index 01921e1195f4a..edf58dae1c756 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java
@@ -51,6 +51,11 @@ public LongBlock filter(int... positions) {
         return vector.filter(positions).asBlock();
     }
 
+    @Override
+    public LongBlock keepMask(BooleanVector mask) {
+        return vector.keepMask(mask);
+    }
+
     @Override
     public ReleasableIterator<? extends LongBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return vector.lookup(positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBooleanGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBooleanGroupingAggregatorFunction.java
index a12677e70e8a9..98e57b71db416 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBooleanGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBooleanGroupingAggregatorFunction.java
@@ -90,7 +90,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, BooleanBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -104,7 +104,7 @@ private void addRawInput(int positionOffset, IntVector groups, BooleanBlock valu
 
   private void addRawInput(int positionOffset, IntVector groups, BooleanVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctBooleanAggregator.combine(state, groupId, values.getBoolean(groupPosition + positionOffset));
     }
   }
@@ -117,7 +117,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BooleanBlock value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -138,7 +138,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BooleanVector valu
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         CountDistinctBooleanAggregator.combine(state, groupId, values.getBoolean(groupPosition + positionOffset));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector tbit = ((BooleanBlock) tbitUncast).asVector();
     assert fbit.getPositionCount() == tbit.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctBooleanAggregator.combineIntermediate(state, groupId, fbit.getBoolean(groupPosition + positionOffset), tbit.getBoolean(groupPosition + positionOffset));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java
index 4879df5cf1c2c..35fd83598b9d6 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java
@@ -94,7 +94,7 @@ public void add(int positionOffset, IntVector groupIds) {
   private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -109,7 +109,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock val
   private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
@@ -123,7 +123,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock valu
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -145,7 +145,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector val
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         CountDistinctBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
       }
     }
@@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java
index 1e0ce58377f9e..894b81b311363 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java
@@ -95,7 +95,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -109,7 +109,7 @@ private void addRawInput(int positionOffset, IntVector groups, DoubleBlock value
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
     }
   }
@@ -122,7 +122,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleBlock values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -143,7 +143,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleVector value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         CountDistinctDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java
index 60c1755b88c6a..5f6b4211e6c5e 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java
@@ -95,7 +95,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -109,7 +109,7 @@ private void addRawInput(int positionOffset, IntVector groups, FloatBlock values
 
   private void addRawInput(int positionOffset, IntVector groups, FloatVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
     }
   }
@@ -122,7 +122,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -143,7 +143,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatVector values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         CountDistinctFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java
index 99e6ace52b256..83300393e560d 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java
@@ -93,7 +93,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, IntBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -107,7 +107,7 @@ private void addRawInput(int positionOffset, IntVector groups, IntBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, IntVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset));
     }
   }
@@ -120,7 +120,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) {
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -141,7 +141,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         CountDistinctIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset));
       }
     }
@@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java
index 85f823296c886..44e9fefb3161c 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java
@@ -95,7 +95,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, LongBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -109,7 +109,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, LongVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
     }
   }
@@ -122,7 +122,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -143,7 +143,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         CountDistinctLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanGroupingAggregatorFunction.java
index f404fccd45d51..084e346a7b093 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanGroupingAggregatorFunction.java
@@ -90,7 +90,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, BooleanBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -104,7 +104,7 @@ private void addRawInput(int positionOffset, IntVector groups, BooleanBlock valu
 
   private void addRawInput(int positionOffset, IntVector groups, BooleanVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, MaxBooleanAggregator.combine(state.getOrDefault(groupId), values.getBoolean(groupPosition + positionOffset)));
     }
   }
@@ -117,7 +117,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BooleanBlock value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -138,7 +138,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BooleanVector valu
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, MaxBooleanAggregator.combine(state.getOrDefault(groupId), values.getBoolean(groupPosition + positionOffset)));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert max.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, MaxBooleanAggregator.combine(state.getOrDefault(groupId), max.getBoolean(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java
new file mode 100644
index 0000000000000..62897c61ea80e
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java
@@ -0,0 +1,133 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.compute.aggregation;
+
+import java.lang.Integer;
+import java.lang.Override;
+import java.lang.String;
+import java.lang.StringBuilder;
+import java.util.List;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.ElementType;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * {@link AggregatorFunction} implementation for {@link MaxBytesRefAggregator}.
+ * This class is generated. Do not edit it.
+ */
+public final class MaxBytesRefAggregatorFunction implements AggregatorFunction {
+  private static final List<IntermediateStateDesc> INTERMEDIATE_STATE_DESC = List.of(
+      new IntermediateStateDesc("max", ElementType.BYTES_REF),
+      new IntermediateStateDesc("seen", ElementType.BOOLEAN)  );
+
+  private final DriverContext driverContext;
+
+  private final MaxBytesRefAggregator.SingleState state;
+
+  private final List<Integer> channels;
+
+  public MaxBytesRefAggregatorFunction(DriverContext driverContext, List<Integer> channels,
+      MaxBytesRefAggregator.SingleState state) {
+    this.driverContext = driverContext;
+    this.channels = channels;
+    this.state = state;
+  }
+
+  public static MaxBytesRefAggregatorFunction create(DriverContext driverContext,
+      List<Integer> channels) {
+    return new MaxBytesRefAggregatorFunction(driverContext, channels, MaxBytesRefAggregator.initSingle(driverContext));
+  }
+
+  public static List<IntermediateStateDesc> intermediateStateDesc() {
+    return INTERMEDIATE_STATE_DESC;
+  }
+
+  @Override
+  public int intermediateBlockCount() {
+    return INTERMEDIATE_STATE_DESC.size();
+  }
+
+  @Override
+  public void addRawInput(Page page) {
+    BytesRefBlock block = page.getBlock(channels.get(0));
+    BytesRefVector vector = block.asVector();
+    if (vector != null) {
+      addRawVector(vector);
+    } else {
+      addRawBlock(block);
+    }
+  }
+
+  private void addRawVector(BytesRefVector vector) {
+    BytesRef scratch = new BytesRef();
+    for (int i = 0; i < vector.getPositionCount(); i++) {
+      MaxBytesRefAggregator.combine(state, vector.getBytesRef(i, scratch));
+    }
+  }
+
+  private void addRawBlock(BytesRefBlock block) {
+    BytesRef scratch = new BytesRef();
+    for (int p = 0; p < block.getPositionCount(); p++) {
+      if (block.isNull(p)) {
+        continue;
+      }
+      int start = block.getFirstValueIndex(p);
+      int end = start + block.getValueCount(p);
+      for (int i = start; i < end; i++) {
+        MaxBytesRefAggregator.combine(state, block.getBytesRef(i, scratch));
+      }
+    }
+  }
+
+  @Override
+  public void addIntermediateInput(Page page) {
+    assert channels.size() == intermediateBlockCount();
+    assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size();
+    Block maxUncast = page.getBlock(channels.get(0));
+    if (maxUncast.areAllValuesNull()) {
+      return;
+    }
+    BytesRefVector max = ((BytesRefBlock) maxUncast).asVector();
+    assert max.getPositionCount() == 1;
+    Block seenUncast = page.getBlock(channels.get(1));
+    if (seenUncast.areAllValuesNull()) {
+      return;
+    }
+    BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
+    assert seen.getPositionCount() == 1;
+    BytesRef scratch = new BytesRef();
+    MaxBytesRefAggregator.combineIntermediate(state, max.getBytesRef(0, scratch), seen.getBoolean(0));
+  }
+
+  @Override
+  public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+    state.toIntermediate(blocks, offset, driverContext);
+  }
+
+  @Override
+  public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) {
+    blocks[offset] = MaxBytesRefAggregator.evaluateFinal(state, driverContext);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(getClass().getSimpleName()).append("[");
+    sb.append("channels=").append(channels);
+    sb.append("]");
+    return sb.toString();
+  }
+
+  @Override
+  public void close() {
+    state.close();
+  }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionSupplier.java
new file mode 100644
index 0000000000000..7c8af2e0c7e6d
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionSupplier.java
@@ -0,0 +1,38 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.compute.aggregation;
+
+import java.lang.Integer;
+import java.lang.Override;
+import java.lang.String;
+import java.util.List;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * {@link AggregatorFunctionSupplier} implementation for {@link MaxBytesRefAggregator}.
+ * This class is generated. Do not edit it.
+ */
+public final class MaxBytesRefAggregatorFunctionSupplier implements AggregatorFunctionSupplier {
+  private final List<Integer> channels;
+
+  public MaxBytesRefAggregatorFunctionSupplier(List<Integer> channels) {
+    this.channels = channels;
+  }
+
+  @Override
+  public MaxBytesRefAggregatorFunction aggregator(DriverContext driverContext) {
+    return MaxBytesRefAggregatorFunction.create(driverContext, channels);
+  }
+
+  @Override
+  public MaxBytesRefGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) {
+    return MaxBytesRefGroupingAggregatorFunction.create(channels, driverContext);
+  }
+
+  @Override
+  public String describe() {
+    return "max of bytes";
+  }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java
new file mode 100644
index 0000000000000..a50cf8593a6e1
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java
@@ -0,0 +1,210 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.compute.aggregation;
+
+import java.lang.Integer;
+import java.lang.Override;
+import java.lang.String;
+import java.lang.StringBuilder;
+import java.util.List;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.ElementType;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * {@link GroupingAggregatorFunction} implementation for {@link MaxBytesRefAggregator}.
+ * This class is generated. Do not edit it.
+ */
+public final class MaxBytesRefGroupingAggregatorFunction implements GroupingAggregatorFunction {
+  private static final List<IntermediateStateDesc> INTERMEDIATE_STATE_DESC = List.of(
+      new IntermediateStateDesc("max", ElementType.BYTES_REF),
+      new IntermediateStateDesc("seen", ElementType.BOOLEAN)  );
+
+  private final MaxBytesRefAggregator.GroupingState state;
+
+  private final List<Integer> channels;
+
+  private final DriverContext driverContext;
+
+  public MaxBytesRefGroupingAggregatorFunction(List<Integer> channels,
+      MaxBytesRefAggregator.GroupingState state, DriverContext driverContext) {
+    this.channels = channels;
+    this.state = state;
+    this.driverContext = driverContext;
+  }
+
+  public static MaxBytesRefGroupingAggregatorFunction create(List<Integer> channels,
+      DriverContext driverContext) {
+    return new MaxBytesRefGroupingAggregatorFunction(channels, MaxBytesRefAggregator.initGrouping(driverContext), driverContext);
+  }
+
+  public static List<IntermediateStateDesc> intermediateStateDesc() {
+    return INTERMEDIATE_STATE_DESC;
+  }
+
+  @Override
+  public int intermediateBlockCount() {
+    return INTERMEDIATE_STATE_DESC.size();
+  }
+
+  @Override
+  public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds,
+      Page page) {
+    BytesRefBlock valuesBlock = page.getBlock(channels.get(0));
+    BytesRefVector valuesVector = valuesBlock.asVector();
+    if (valuesVector == null) {
+      if (valuesBlock.mayHaveNulls()) {
+        state.enableGroupIdTracking(seenGroupIds);
+      }
+      return new GroupingAggregatorFunction.AddInput() {
+        @Override
+        public void add(int positionOffset, IntBlock groupIds) {
+          addRawInput(positionOffset, groupIds, valuesBlock);
+        }
+
+        @Override
+        public void add(int positionOffset, IntVector groupIds) {
+          addRawInput(positionOffset, groupIds, valuesBlock);
+        }
+      };
+    }
+    return new GroupingAggregatorFunction.AddInput() {
+      @Override
+      public void add(int positionOffset, IntBlock groupIds) {
+        addRawInput(positionOffset, groupIds, valuesVector);
+      }
+
+      @Override
+      public void add(int positionOffset, IntVector groupIds) {
+        addRawInput(positionOffset, groupIds, valuesVector);
+      }
+    };
+  }
+
+  private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      int groupId = groups.getInt(groupPosition);
+      if (values.isNull(groupPosition + positionOffset)) {
+        continue;
+      }
+      int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset);
+      int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset);
+      for (int v = valuesStart; v < valuesEnd; v++) {
+        MaxBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch));
+      }
+    }
+  }
+
+  private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      int groupId = groups.getInt(groupPosition);
+      MaxBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
+    }
+  }
+
+  private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      if (groups.isNull(groupPosition)) {
+        continue;
+      }
+      int groupStart = groups.getFirstValueIndex(groupPosition);
+      int groupEnd = groupStart + groups.getValueCount(groupPosition);
+      for (int g = groupStart; g < groupEnd; g++) {
+        int groupId = groups.getInt(g);
+        if (values.isNull(groupPosition + positionOffset)) {
+          continue;
+        }
+        int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset);
+        int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset);
+        for (int v = valuesStart; v < valuesEnd; v++) {
+          MaxBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch));
+        }
+      }
+    }
+  }
+
+  private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      if (groups.isNull(groupPosition)) {
+        continue;
+      }
+      int groupStart = groups.getFirstValueIndex(groupPosition);
+      int groupEnd = groupStart + groups.getValueCount(groupPosition);
+      for (int g = groupStart; g < groupEnd; g++) {
+        int groupId = groups.getInt(g);
+        MaxBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
+      }
+    }
+  }
+
+  @Override
+  public void addIntermediateInput(int positionOffset, IntVector groups, Page page) {
+    state.enableGroupIdTracking(new SeenGroupIds.Empty());
+    assert channels.size() == intermediateBlockCount();
+    Block maxUncast = page.getBlock(channels.get(0));
+    if (maxUncast.areAllValuesNull()) {
+      return;
+    }
+    BytesRefVector max = ((BytesRefBlock) maxUncast).asVector();
+    Block seenUncast = page.getBlock(channels.get(1));
+    if (seenUncast.areAllValuesNull()) {
+      return;
+    }
+    BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
+    assert max.getPositionCount() == seen.getPositionCount();
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      int groupId = groups.getInt(groupPosition);
+      MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(groupPosition + positionOffset, scratch), seen.getBoolean(groupPosition + positionOffset));
+    }
+  }
+
+  @Override
+  public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) {
+    if (input.getClass() != getClass()) {
+      throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass());
+    }
+    MaxBytesRefAggregator.GroupingState inState = ((MaxBytesRefGroupingAggregatorFunction) input).state;
+    state.enableGroupIdTracking(new SeenGroupIds.Empty());
+    MaxBytesRefAggregator.combineStates(state, groupId, inState, position);
+  }
+
+  @Override
+  public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) {
+    state.toIntermediate(blocks, offset, selected, driverContext);
+  }
+
+  @Override
+  public void evaluateFinal(Block[] blocks, int offset, IntVector selected,
+      DriverContext driverContext) {
+    blocks[offset] = MaxBytesRefAggregator.evaluateFinal(state, selected, driverContext);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(getClass().getSimpleName()).append("[");
+    sb.append("channels=").append(channels);
+    sb.append("]");
+    return sb.toString();
+  }
+
+  @Override
+  public void close() {
+    state.close();
+  }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxDoubleGroupingAggregatorFunction.java
index da93320eaf96e..b874bc43dc238 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxDoubleGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxDoubleGroupingAggregatorFunction.java
@@ -92,7 +92,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, DoubleBlock value
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, MaxDoubleAggregator.combine(state.getOrDefault(groupId), values.getDouble(groupPosition + positionOffset)));
     }
   }
@@ -119,7 +119,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleBlock values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -140,7 +140,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleVector value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, MaxDoubleAggregator.combine(state.getOrDefault(groupId), values.getDouble(groupPosition + positionOffset)));
       }
     }
@@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert max.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, MaxDoubleAggregator.combine(state.getOrDefault(groupId), max.getDouble(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunction.java
index 85708792732a7..f3ebd468ebc72 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunction.java
@@ -92,7 +92,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, FloatBlock values
 
   private void addRawInput(int positionOffset, IntVector groups, FloatVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset)));
     }
   }
@@ -119,7 +119,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -140,7 +140,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatVector values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset)));
       }
     }
@@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert max.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), max.getFloat(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIntGroupingAggregatorFunction.java
index c8b1b6910c0aa..8b364e7a02e96 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIntGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIntGroupingAggregatorFunction.java
@@ -90,7 +90,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, IntBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -104,7 +104,7 @@ private void addRawInput(int positionOffset, IntVector groups, IntBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, IntVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, MaxIntAggregator.combine(state.getOrDefault(groupId), values.getInt(groupPosition + positionOffset)));
     }
   }
@@ -117,7 +117,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) {
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -138,7 +138,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, MaxIntAggregator.combine(state.getOrDefault(groupId), values.getInt(groupPosition + positionOffset)));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert max.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, MaxIntAggregator.combine(state.getOrDefault(groupId), max.getInt(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java
index 2b6f4ac0c8c26..a722d95f3b108 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java
@@ -30,14 +30,14 @@ public final class MaxIpGroupingAggregatorFunction implements GroupingAggregator
       new IntermediateStateDesc("max", ElementType.BYTES_REF),
       new IntermediateStateDesc("seen", ElementType.BOOLEAN)  );
 
-  private final IpArrayState state;
+  private final MaxIpAggregator.GroupingState state;
 
   private final List<Integer> channels;
 
   private final DriverContext driverContext;
 
-  public MaxIpGroupingAggregatorFunction(List<Integer> channels, IpArrayState state,
-      DriverContext driverContext) {
+  public MaxIpGroupingAggregatorFunction(List<Integer> channels,
+      MaxIpAggregator.GroupingState state, DriverContext driverContext) {
     this.channels = channels;
     this.state = state;
     this.driverContext = driverContext;
@@ -94,7 +94,7 @@ public void add(int positionOffset, IntVector groupIds) {
   private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -109,7 +109,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock val
   private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MaxIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
@@ -123,7 +123,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock valu
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -145,7 +145,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector val
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         MaxIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
       }
     }
@@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     assert max.getPositionCount() == seen.getPositionCount();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(groupPosition + positionOffset, scratch), seen.getBoolean(groupPosition + positionOffset));
     }
   }
@@ -178,7 +178,7 @@ public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction inpu
     if (input.getClass() != getClass()) {
       throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass());
     }
-    IpArrayState inState = ((MaxIpGroupingAggregatorFunction) input).state;
+    MaxIpAggregator.GroupingState inState = ((MaxIpGroupingAggregatorFunction) input).state;
     state.enableGroupIdTracking(new SeenGroupIds.Empty());
     MaxIpAggregator.combineStates(state, groupId, inState, position);
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxLongGroupingAggregatorFunction.java
index 41d893f9bbf0c..fee2f5a9c2e7c 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxLongGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxLongGroupingAggregatorFunction.java
@@ -92,7 +92,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, LongBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, LongVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, MaxLongAggregator.combine(state.getOrDefault(groupId), values.getLong(groupPosition + positionOffset)));
     }
   }
@@ -119,7 +119,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -140,7 +140,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, MaxLongAggregator.combine(state.getOrDefault(groupId), values.getLong(groupPosition + positionOffset)));
       }
     }
@@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert max.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, MaxLongAggregator.combine(state.getOrDefault(groupId), max.getLong(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java
index e08488685d2cb..836248428f231 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java
@@ -92,7 +92,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, DoubleBlock value
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MedianAbsoluteDeviationDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
     }
   }
@@ -119,7 +119,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleBlock values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -140,7 +140,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleVector value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         MedianAbsoluteDeviationDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
       }
     }
@@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java
index 84646476fcee0..7a67f0d3449f0 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java
@@ -92,7 +92,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, FloatBlock values
 
   private void addRawInput(int positionOffset, IntVector groups, FloatVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MedianAbsoluteDeviationFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
     }
   }
@@ -119,7 +119,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -140,7 +140,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatVector values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         MedianAbsoluteDeviationFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
       }
     }
@@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java
index 02866ee15b961..315034a28ff8f 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java
@@ -90,7 +90,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, IntBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -104,7 +104,7 @@ private void addRawInput(int positionOffset, IntVector groups, IntBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, IntVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MedianAbsoluteDeviationIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset));
     }
   }
@@ -117,7 +117,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) {
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -138,7 +138,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         MedianAbsoluteDeviationIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset));
       }
     }
@@ -155,7 +155,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java
index 36c40e10e54d5..af0374012be52 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java
@@ -92,7 +92,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, LongBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, LongVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MedianAbsoluteDeviationLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
     }
   }
@@ -119,7 +119,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -140,7 +140,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         MedianAbsoluteDeviationLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
       }
     }
@@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBooleanGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBooleanGroupingAggregatorFunction.java
index 6175cad3924e2..45e677ee25b56 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBooleanGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBooleanGroupingAggregatorFunction.java
@@ -90,7 +90,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, BooleanBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -104,7 +104,7 @@ private void addRawInput(int positionOffset, IntVector groups, BooleanBlock valu
 
   private void addRawInput(int positionOffset, IntVector groups, BooleanVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, MinBooleanAggregator.combine(state.getOrDefault(groupId), values.getBoolean(groupPosition + positionOffset)));
     }
   }
@@ -117,7 +117,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BooleanBlock value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -138,7 +138,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BooleanVector valu
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, MinBooleanAggregator.combine(state.getOrDefault(groupId), values.getBoolean(groupPosition + positionOffset)));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert min.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, MinBooleanAggregator.combine(state.getOrDefault(groupId), min.getBoolean(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java
new file mode 100644
index 0000000000000..3346dd762f17f
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java
@@ -0,0 +1,133 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.compute.aggregation;
+
+import java.lang.Integer;
+import java.lang.Override;
+import java.lang.String;
+import java.lang.StringBuilder;
+import java.util.List;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.ElementType;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * {@link AggregatorFunction} implementation for {@link MinBytesRefAggregator}.
+ * This class is generated. Do not edit it.
+ */
+public final class MinBytesRefAggregatorFunction implements AggregatorFunction {
+  private static final List<IntermediateStateDesc> INTERMEDIATE_STATE_DESC = List.of(
+      new IntermediateStateDesc("min", ElementType.BYTES_REF),
+      new IntermediateStateDesc("seen", ElementType.BOOLEAN)  );
+
+  private final DriverContext driverContext;
+
+  private final MinBytesRefAggregator.SingleState state;
+
+  private final List<Integer> channels;
+
+  public MinBytesRefAggregatorFunction(DriverContext driverContext, List<Integer> channels,
+      MinBytesRefAggregator.SingleState state) {
+    this.driverContext = driverContext;
+    this.channels = channels;
+    this.state = state;
+  }
+
+  public static MinBytesRefAggregatorFunction create(DriverContext driverContext,
+      List<Integer> channels) {
+    return new MinBytesRefAggregatorFunction(driverContext, channels, MinBytesRefAggregator.initSingle(driverContext));
+  }
+
+  public static List<IntermediateStateDesc> intermediateStateDesc() {
+    return INTERMEDIATE_STATE_DESC;
+  }
+
+  @Override
+  public int intermediateBlockCount() {
+    return INTERMEDIATE_STATE_DESC.size();
+  }
+
+  @Override
+  public void addRawInput(Page page) {
+    BytesRefBlock block = page.getBlock(channels.get(0));
+    BytesRefVector vector = block.asVector();
+    if (vector != null) {
+      addRawVector(vector);
+    } else {
+      addRawBlock(block);
+    }
+  }
+
+  private void addRawVector(BytesRefVector vector) {
+    BytesRef scratch = new BytesRef();
+    for (int i = 0; i < vector.getPositionCount(); i++) {
+      MinBytesRefAggregator.combine(state, vector.getBytesRef(i, scratch));
+    }
+  }
+
+  private void addRawBlock(BytesRefBlock block) {
+    BytesRef scratch = new BytesRef();
+    for (int p = 0; p < block.getPositionCount(); p++) {
+      if (block.isNull(p)) {
+        continue;
+      }
+      int start = block.getFirstValueIndex(p);
+      int end = start + block.getValueCount(p);
+      for (int i = start; i < end; i++) {
+        MinBytesRefAggregator.combine(state, block.getBytesRef(i, scratch));
+      }
+    }
+  }
+
+  @Override
+  public void addIntermediateInput(Page page) {
+    assert channels.size() == intermediateBlockCount();
+    assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size();
+    Block minUncast = page.getBlock(channels.get(0));
+    if (minUncast.areAllValuesNull()) {
+      return;
+    }
+    BytesRefVector min = ((BytesRefBlock) minUncast).asVector();
+    assert min.getPositionCount() == 1;
+    Block seenUncast = page.getBlock(channels.get(1));
+    if (seenUncast.areAllValuesNull()) {
+      return;
+    }
+    BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
+    assert seen.getPositionCount() == 1;
+    BytesRef scratch = new BytesRef();
+    MinBytesRefAggregator.combineIntermediate(state, min.getBytesRef(0, scratch), seen.getBoolean(0));
+  }
+
+  @Override
+  public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+    state.toIntermediate(blocks, offset, driverContext);
+  }
+
+  @Override
+  public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) {
+    blocks[offset] = MinBytesRefAggregator.evaluateFinal(state, driverContext);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(getClass().getSimpleName()).append("[");
+    sb.append("channels=").append(channels);
+    sb.append("]");
+    return sb.toString();
+  }
+
+  @Override
+  public void close() {
+    state.close();
+  }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionSupplier.java
new file mode 100644
index 0000000000000..cb6ab0d06d401
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionSupplier.java
@@ -0,0 +1,38 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.compute.aggregation;
+
+import java.lang.Integer;
+import java.lang.Override;
+import java.lang.String;
+import java.util.List;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * {@link AggregatorFunctionSupplier} implementation for {@link MinBytesRefAggregator}.
+ * This class is generated. Do not edit it.
+ */
+public final class MinBytesRefAggregatorFunctionSupplier implements AggregatorFunctionSupplier {
+  private final List<Integer> channels;
+
+  public MinBytesRefAggregatorFunctionSupplier(List<Integer> channels) {
+    this.channels = channels;
+  }
+
+  @Override
+  public MinBytesRefAggregatorFunction aggregator(DriverContext driverContext) {
+    return MinBytesRefAggregatorFunction.create(driverContext, channels);
+  }
+
+  @Override
+  public MinBytesRefGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) {
+    return MinBytesRefGroupingAggregatorFunction.create(channels, driverContext);
+  }
+
+  @Override
+  public String describe() {
+    return "min of bytes";
+  }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java
new file mode 100644
index 0000000000000..e092dd93210f6
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java
@@ -0,0 +1,210 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.compute.aggregation;
+
+import java.lang.Integer;
+import java.lang.Override;
+import java.lang.String;
+import java.lang.StringBuilder;
+import java.util.List;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.ElementType;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * {@link GroupingAggregatorFunction} implementation for {@link MinBytesRefAggregator}.
+ * This class is generated. Do not edit it.
+ */
+public final class MinBytesRefGroupingAggregatorFunction implements GroupingAggregatorFunction {
+  private static final List<IntermediateStateDesc> INTERMEDIATE_STATE_DESC = List.of(
+      new IntermediateStateDesc("min", ElementType.BYTES_REF),
+      new IntermediateStateDesc("seen", ElementType.BOOLEAN)  );
+
+  private final MinBytesRefAggregator.GroupingState state;
+
+  private final List<Integer> channels;
+
+  private final DriverContext driverContext;
+
+  public MinBytesRefGroupingAggregatorFunction(List<Integer> channels,
+      MinBytesRefAggregator.GroupingState state, DriverContext driverContext) {
+    this.channels = channels;
+    this.state = state;
+    this.driverContext = driverContext;
+  }
+
+  public static MinBytesRefGroupingAggregatorFunction create(List<Integer> channels,
+      DriverContext driverContext) {
+    return new MinBytesRefGroupingAggregatorFunction(channels, MinBytesRefAggregator.initGrouping(driverContext), driverContext);
+  }
+
+  public static List<IntermediateStateDesc> intermediateStateDesc() {
+    return INTERMEDIATE_STATE_DESC;
+  }
+
+  @Override
+  public int intermediateBlockCount() {
+    return INTERMEDIATE_STATE_DESC.size();
+  }
+
+  @Override
+  public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds,
+      Page page) {
+    BytesRefBlock valuesBlock = page.getBlock(channels.get(0));
+    BytesRefVector valuesVector = valuesBlock.asVector();
+    if (valuesVector == null) {
+      if (valuesBlock.mayHaveNulls()) {
+        state.enableGroupIdTracking(seenGroupIds);
+      }
+      return new GroupingAggregatorFunction.AddInput() {
+        @Override
+        public void add(int positionOffset, IntBlock groupIds) {
+          addRawInput(positionOffset, groupIds, valuesBlock);
+        }
+
+        @Override
+        public void add(int positionOffset, IntVector groupIds) {
+          addRawInput(positionOffset, groupIds, valuesBlock);
+        }
+      };
+    }
+    return new GroupingAggregatorFunction.AddInput() {
+      @Override
+      public void add(int positionOffset, IntBlock groupIds) {
+        addRawInput(positionOffset, groupIds, valuesVector);
+      }
+
+      @Override
+      public void add(int positionOffset, IntVector groupIds) {
+        addRawInput(positionOffset, groupIds, valuesVector);
+      }
+    };
+  }
+
+  private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      int groupId = groups.getInt(groupPosition);
+      if (values.isNull(groupPosition + positionOffset)) {
+        continue;
+      }
+      int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset);
+      int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset);
+      for (int v = valuesStart; v < valuesEnd; v++) {
+        MinBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch));
+      }
+    }
+  }
+
+  private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      int groupId = groups.getInt(groupPosition);
+      MinBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
+    }
+  }
+
+  private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      if (groups.isNull(groupPosition)) {
+        continue;
+      }
+      int groupStart = groups.getFirstValueIndex(groupPosition);
+      int groupEnd = groupStart + groups.getValueCount(groupPosition);
+      for (int g = groupStart; g < groupEnd; g++) {
+        int groupId = groups.getInt(g);
+        if (values.isNull(groupPosition + positionOffset)) {
+          continue;
+        }
+        int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset);
+        int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset);
+        for (int v = valuesStart; v < valuesEnd; v++) {
+          MinBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch));
+        }
+      }
+    }
+  }
+
+  private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      if (groups.isNull(groupPosition)) {
+        continue;
+      }
+      int groupStart = groups.getFirstValueIndex(groupPosition);
+      int groupEnd = groupStart + groups.getValueCount(groupPosition);
+      for (int g = groupStart; g < groupEnd; g++) {
+        int groupId = groups.getInt(g);
+        MinBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
+      }
+    }
+  }
+
+  @Override
+  public void addIntermediateInput(int positionOffset, IntVector groups, Page page) {
+    state.enableGroupIdTracking(new SeenGroupIds.Empty());
+    assert channels.size() == intermediateBlockCount();
+    Block minUncast = page.getBlock(channels.get(0));
+    if (minUncast.areAllValuesNull()) {
+      return;
+    }
+    BytesRefVector min = ((BytesRefBlock) minUncast).asVector();
+    Block seenUncast = page.getBlock(channels.get(1));
+    if (seenUncast.areAllValuesNull()) {
+      return;
+    }
+    BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
+    assert min.getPositionCount() == seen.getPositionCount();
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      int groupId = groups.getInt(groupPosition);
+      MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(groupPosition + positionOffset, scratch), seen.getBoolean(groupPosition + positionOffset));
+    }
+  }
+
+  @Override
+  public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) {
+    if (input.getClass() != getClass()) {
+      throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass());
+    }
+    MinBytesRefAggregator.GroupingState inState = ((MinBytesRefGroupingAggregatorFunction) input).state;
+    state.enableGroupIdTracking(new SeenGroupIds.Empty());
+    MinBytesRefAggregator.combineStates(state, groupId, inState, position);
+  }
+
+  @Override
+  public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) {
+    state.toIntermediate(blocks, offset, selected, driverContext);
+  }
+
+  @Override
+  public void evaluateFinal(Block[] blocks, int offset, IntVector selected,
+      DriverContext driverContext) {
+    blocks[offset] = MinBytesRefAggregator.evaluateFinal(state, selected, driverContext);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(getClass().getSimpleName()).append("[");
+    sb.append("channels=").append(channels);
+    sb.append("]");
+    return sb.toString();
+  }
+
+  @Override
+  public void close() {
+    state.close();
+  }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinDoubleGroupingAggregatorFunction.java
index 7d0374b3d21f7..970a8a7597514 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinDoubleGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinDoubleGroupingAggregatorFunction.java
@@ -92,7 +92,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, DoubleBlock value
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, MinDoubleAggregator.combine(state.getOrDefault(groupId), values.getDouble(groupPosition + positionOffset)));
     }
   }
@@ -119,7 +119,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleBlock values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -140,7 +140,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleVector value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, MinDoubleAggregator.combine(state.getOrDefault(groupId), values.getDouble(groupPosition + positionOffset)));
       }
     }
@@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert min.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, MinDoubleAggregator.combine(state.getOrDefault(groupId), min.getDouble(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunction.java
index 2f00bbf1335ed..4e8b4cc9417c8 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunction.java
@@ -92,7 +92,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, FloatBlock values
 
   private void addRawInput(int positionOffset, IntVector groups, FloatVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset)));
     }
   }
@@ -119,7 +119,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -140,7 +140,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatVector values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset)));
       }
     }
@@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert min.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), min.getFloat(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIntGroupingAggregatorFunction.java
index 6625fd327237b..6e976a582a892 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIntGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIntGroupingAggregatorFunction.java
@@ -90,7 +90,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, IntBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -104,7 +104,7 @@ private void addRawInput(int positionOffset, IntVector groups, IntBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, IntVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, MinIntAggregator.combine(state.getOrDefault(groupId), values.getInt(groupPosition + positionOffset)));
     }
   }
@@ -117,7 +117,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) {
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -138,7 +138,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, MinIntAggregator.combine(state.getOrDefault(groupId), values.getInt(groupPosition + positionOffset)));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert min.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, MinIntAggregator.combine(state.getOrDefault(groupId), min.getInt(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java
index 61cc2d91a017b..146515d363af7 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java
@@ -30,14 +30,14 @@ public final class MinIpGroupingAggregatorFunction implements GroupingAggregator
       new IntermediateStateDesc("max", ElementType.BYTES_REF),
       new IntermediateStateDesc("seen", ElementType.BOOLEAN)  );
 
-  private final IpArrayState state;
+  private final MinIpAggregator.GroupingState state;
 
   private final List<Integer> channels;
 
   private final DriverContext driverContext;
 
-  public MinIpGroupingAggregatorFunction(List<Integer> channels, IpArrayState state,
-      DriverContext driverContext) {
+  public MinIpGroupingAggregatorFunction(List<Integer> channels,
+      MinIpAggregator.GroupingState state, DriverContext driverContext) {
     this.channels = channels;
     this.state = state;
     this.driverContext = driverContext;
@@ -94,7 +94,7 @@ public void add(int positionOffset, IntVector groupIds) {
   private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -109,7 +109,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock val
   private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MinIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
@@ -123,7 +123,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock valu
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -145,7 +145,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector val
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         MinIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
       }
     }
@@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     assert max.getPositionCount() == seen.getPositionCount();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(groupPosition + positionOffset, scratch), seen.getBoolean(groupPosition + positionOffset));
     }
   }
@@ -178,7 +178,7 @@ public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction inpu
     if (input.getClass() != getClass()) {
       throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass());
     }
-    IpArrayState inState = ((MinIpGroupingAggregatorFunction) input).state;
+    MinIpAggregator.GroupingState inState = ((MinIpGroupingAggregatorFunction) input).state;
     state.enableGroupIdTracking(new SeenGroupIds.Empty());
     MinIpAggregator.combineStates(state, groupId, inState, position);
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinLongGroupingAggregatorFunction.java
index f0c3727d7db0b..a3db9a2704660 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinLongGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinLongGroupingAggregatorFunction.java
@@ -92,7 +92,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, LongBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, LongVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, MinLongAggregator.combine(state.getOrDefault(groupId), values.getLong(groupPosition + positionOffset)));
     }
   }
@@ -119,7 +119,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -140,7 +140,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, MinLongAggregator.combine(state.getOrDefault(groupId), values.getLong(groupPosition + positionOffset)));
       }
     }
@@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert min.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, MinLongAggregator.combine(state.getOrDefault(groupId), min.getLong(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java
index 9d486b9614dab..871e93a72d900 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java
@@ -95,7 +95,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -109,7 +109,7 @@ private void addRawInput(int positionOffset, IntVector groups, DoubleBlock value
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       PercentileDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
     }
   }
@@ -122,7 +122,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleBlock values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -143,7 +143,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleVector value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         PercentileDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java
index 564e0e90018c2..8b0f28b2632d1 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java
@@ -95,7 +95,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -109,7 +109,7 @@ private void addRawInput(int positionOffset, IntVector groups, FloatBlock values
 
   private void addRawInput(int positionOffset, IntVector groups, FloatVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       PercentileFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
     }
   }
@@ -122,7 +122,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -143,7 +143,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatVector values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         PercentileFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java
index 8c2bd7091143f..fc1031dcbe0d0 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java
@@ -93,7 +93,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, IntBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -107,7 +107,7 @@ private void addRawInput(int positionOffset, IntVector groups, IntBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, IntVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       PercentileIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset));
     }
   }
@@ -120,7 +120,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) {
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -141,7 +141,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         PercentileIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset));
       }
     }
@@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java
index c1c332ba0094d..1b14f02356b8f 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java
@@ -95,7 +95,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, LongBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -109,7 +109,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, LongVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       PercentileLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
     }
   }
@@ -122,7 +122,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -143,7 +143,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         PercentileLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector();
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateDoubleGroupingAggregatorFunction.java
index 8d9e011891e95..c85cf78a39c45 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateDoubleGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateDoubleGroupingAggregatorFunction.java
@@ -103,7 +103,7 @@ public void add(int positionOffset, IntVector groupIds) {
   private void addRawInput(int positionOffset, IntVector groups, DoubleBlock values,
       LongVector timestamps) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -118,7 +118,7 @@ private void addRawInput(int positionOffset, IntVector groups, DoubleBlock value
   private void addRawInput(int positionOffset, IntVector groups, DoubleVector values,
       LongVector timestamps) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       var valuePosition = groupPosition + positionOffset;
       RateDoubleAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getDouble(valuePosition));
     }
@@ -133,7 +133,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleBlock values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -155,7 +155,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleVector value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         var valuePosition = groupPosition + positionOffset;
         RateDoubleAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getDouble(valuePosition));
       }
@@ -183,7 +183,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     DoubleVector resets = ((DoubleBlock) resetsUncast).asVector();
     assert timestamps.getPositionCount() == values.getPositionCount() && timestamps.getPositionCount() == resets.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       RateDoubleAggregator.combineIntermediate(state, groupId, timestamps, values, resets.getDouble(groupPosition + positionOffset), groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatGroupingAggregatorFunction.java
index 40f53741bf3da..a5d2131a2445a 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatGroupingAggregatorFunction.java
@@ -105,7 +105,7 @@ public void add(int positionOffset, IntVector groupIds) {
   private void addRawInput(int positionOffset, IntVector groups, FloatBlock values,
       LongVector timestamps) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -120,7 +120,7 @@ private void addRawInput(int positionOffset, IntVector groups, FloatBlock values
   private void addRawInput(int positionOffset, IntVector groups, FloatVector values,
       LongVector timestamps) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       var valuePosition = groupPosition + positionOffset;
       RateFloatAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getFloat(valuePosition));
     }
@@ -135,7 +135,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values,
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -157,7 +157,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatVector values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         var valuePosition = groupPosition + positionOffset;
         RateFloatAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getFloat(valuePosition));
       }
@@ -185,7 +185,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     DoubleVector resets = ((DoubleBlock) resetsUncast).asVector();
     assert timestamps.getPositionCount() == values.getPositionCount() && timestamps.getPositionCount() == resets.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       RateFloatAggregator.combineIntermediate(state, groupId, timestamps, values, resets.getDouble(groupPosition + positionOffset), groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateIntGroupingAggregatorFunction.java
index 6bd4b833dc9e6..0fb0b05c11164 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateIntGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateIntGroupingAggregatorFunction.java
@@ -103,7 +103,7 @@ public void add(int positionOffset, IntVector groupIds) {
   private void addRawInput(int positionOffset, IntVector groups, IntBlock values,
       LongVector timestamps) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -118,7 +118,7 @@ private void addRawInput(int positionOffset, IntVector groups, IntBlock values,
   private void addRawInput(int positionOffset, IntVector groups, IntVector values,
       LongVector timestamps) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       var valuePosition = groupPosition + positionOffset;
       RateIntAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getInt(valuePosition));
     }
@@ -133,7 +133,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntBlock values,
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -155,7 +155,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntVector values,
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         var valuePosition = groupPosition + positionOffset;
         RateIntAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getInt(valuePosition));
       }
@@ -183,7 +183,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     DoubleVector resets = ((DoubleBlock) resetsUncast).asVector();
     assert timestamps.getPositionCount() == values.getPositionCount() && timestamps.getPositionCount() == resets.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       RateIntAggregator.combineIntermediate(state, groupId, timestamps, values, resets.getDouble(groupPosition + positionOffset), groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateLongGroupingAggregatorFunction.java
index 27318d6496737..82297b618b03e 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateLongGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateLongGroupingAggregatorFunction.java
@@ -103,7 +103,7 @@ public void add(int positionOffset, IntVector groupIds) {
   private void addRawInput(int positionOffset, IntVector groups, LongBlock values,
       LongVector timestamps) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -118,7 +118,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values,
   private void addRawInput(int positionOffset, IntVector groups, LongVector values,
       LongVector timestamps) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       var valuePosition = groupPosition + positionOffset;
       RateLongAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getLong(valuePosition));
     }
@@ -133,7 +133,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values,
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -155,7 +155,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values,
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         var valuePosition = groupPosition + positionOffset;
         RateLongAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getLong(valuePosition));
       }
@@ -183,7 +183,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     DoubleVector resets = ((DoubleBlock) resetsUncast).asVector();
     assert timestamps.getPositionCount() == values.getPositionCount() && timestamps.getPositionCount() == resets.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       RateLongAggregator.combineIntermediate(state, groupId, timestamps, values, resets.getDouble(groupPosition + positionOffset), groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumDoubleGroupingAggregatorFunction.java
index 5085cfc3bebcf..4f0bcae66ee4a 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumDoubleGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumDoubleGroupingAggregatorFunction.java
@@ -93,7 +93,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -107,7 +107,7 @@ private void addRawInput(int positionOffset, IntVector groups, DoubleBlock value
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SumDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
     }
   }
@@ -120,7 +120,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleBlock values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -141,7 +141,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleVector value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         SumDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
       }
     }
@@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert value.getPositionCount() == delta.getPositionCount() && value.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SumDoubleAggregator.combineIntermediate(state, groupId, value.getDouble(groupPosition + positionOffset), delta.getDouble(groupPosition + positionOffset), seen.getBoolean(groupPosition + positionOffset));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunction.java
index c69ce16f0bccb..2f4165dfeadfa 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunction.java
@@ -95,7 +95,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -109,7 +109,7 @@ private void addRawInput(int positionOffset, IntVector groups, FloatBlock values
 
   private void addRawInput(int positionOffset, IntVector groups, FloatVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SumFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
     }
   }
@@ -122,7 +122,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -143,7 +143,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatVector values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         SumFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
       }
     }
@@ -170,7 +170,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert value.getPositionCount() == delta.getPositionCount() && value.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SumFloatAggregator.combineIntermediate(state, groupId, value.getDouble(groupPosition + positionOffset), delta.getDouble(groupPosition + positionOffset), seen.getBoolean(groupPosition + positionOffset));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumIntGroupingAggregatorFunction.java
index 6891fe548908f..95d380c455bf4 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumIntGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumIntGroupingAggregatorFunction.java
@@ -92,7 +92,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, IntBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, IntBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, IntVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, SumIntAggregator.combine(state.getOrDefault(groupId), values.getInt(groupPosition + positionOffset)));
     }
   }
@@ -119,7 +119,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) {
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -140,7 +140,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, SumIntAggregator.combine(state.getOrDefault(groupId), values.getInt(groupPosition + positionOffset)));
       }
     }
@@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert sum.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, SumIntAggregator.combine(state.getOrDefault(groupId), sum.getLong(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumLongGroupingAggregatorFunction.java
index 507aa343aa74e..324d8f53e65cb 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumLongGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumLongGroupingAggregatorFunction.java
@@ -92,7 +92,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, LongBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, LongVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       state.set(groupId, SumLongAggregator.combine(state.getOrDefault(groupId), values.getLong(groupPosition + positionOffset)));
     }
   }
@@ -119,7 +119,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -140,7 +140,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         state.set(groupId, SumLongAggregator.combine(state.getOrDefault(groupId), values.getLong(groupPosition + positionOffset)));
       }
     }
@@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BooleanVector seen = ((BooleanBlock) seenUncast).asVector();
     assert sum.getPositionCount() == seen.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (seen.getBoolean(groupPosition + positionOffset)) {
         state.set(groupId, SumLongAggregator.combine(state.getOrDefault(groupId), sum.getLong(groupPosition + positionOffset)));
       }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanGroupingAggregatorFunction.java
index 53b5149e4da7e..d169c456329b7 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanGroupingAggregatorFunction.java
@@ -96,7 +96,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, BooleanBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -110,7 +110,7 @@ private void addRawInput(int positionOffset, IntVector groups, BooleanBlock valu
 
   private void addRawInput(int positionOffset, IntVector groups, BooleanVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       TopBooleanAggregator.combine(state, groupId, values.getBoolean(groupPosition + positionOffset));
     }
   }
@@ -123,7 +123,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BooleanBlock value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -144,7 +144,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BooleanVector valu
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         TopBooleanAggregator.combine(state, groupId, values.getBoolean(groupPosition + positionOffset));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     }
     BooleanBlock top = (BooleanBlock) topUncast;
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       TopBooleanAggregator.combineIntermediate(state, groupId, top, groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopDoubleGroupingAggregatorFunction.java
index c54dce5715846..07da387f88ce6 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopDoubleGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopDoubleGroupingAggregatorFunction.java
@@ -96,7 +96,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -110,7 +110,7 @@ private void addRawInput(int positionOffset, IntVector groups, DoubleBlock value
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       TopDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
     }
   }
@@ -123,7 +123,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleBlock values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -144,7 +144,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleVector value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         TopDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     }
     DoubleBlock top = (DoubleBlock) topUncast;
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       TopDoubleAggregator.combineIntermediate(state, groupId, top, groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopFloatGroupingAggregatorFunction.java
index 4c00f4d2c237d..369fa7401e508 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopFloatGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopFloatGroupingAggregatorFunction.java
@@ -96,7 +96,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -110,7 +110,7 @@ private void addRawInput(int positionOffset, IntVector groups, FloatBlock values
 
   private void addRawInput(int positionOffset, IntVector groups, FloatVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       TopFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
     }
   }
@@ -123,7 +123,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -144,7 +144,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatVector values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         TopFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     }
     FloatBlock top = (FloatBlock) topUncast;
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       TopFloatAggregator.combineIntermediate(state, groupId, top, groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIntGroupingAggregatorFunction.java
index 37384238b7297..04b53fe6aab69 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIntGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIntGroupingAggregatorFunction.java
@@ -94,7 +94,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, IntBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -108,7 +108,7 @@ private void addRawInput(int positionOffset, IntVector groups, IntBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, IntVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       TopIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset));
     }
   }
@@ -121,7 +121,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) {
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -142,7 +142,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         TopIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset));
       }
     }
@@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     }
     IntBlock top = (IntBlock) topUncast;
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       TopIntAggregator.combineIntermediate(state, groupId, top, groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java
new file mode 100644
index 0000000000000..43f4d78d59cd9
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java
@@ -0,0 +1,130 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.compute.aggregation;
+
+import java.lang.Integer;
+import java.lang.Override;
+import java.lang.String;
+import java.lang.StringBuilder;
+import java.util.List;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.ElementType;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * {@link AggregatorFunction} implementation for {@link TopIpAggregator}.
+ * This class is generated. Do not edit it.
+ */
+public final class TopIpAggregatorFunction implements AggregatorFunction {
+  private static final List<IntermediateStateDesc> INTERMEDIATE_STATE_DESC = List.of(
+      new IntermediateStateDesc("top", ElementType.BYTES_REF)  );
+
+  private final DriverContext driverContext;
+
+  private final TopIpAggregator.SingleState state;
+
+  private final List<Integer> channels;
+
+  private final int limit;
+
+  private final boolean ascending;
+
+  public TopIpAggregatorFunction(DriverContext driverContext, List<Integer> channels,
+      TopIpAggregator.SingleState state, int limit, boolean ascending) {
+    this.driverContext = driverContext;
+    this.channels = channels;
+    this.state = state;
+    this.limit = limit;
+    this.ascending = ascending;
+  }
+
+  public static TopIpAggregatorFunction create(DriverContext driverContext, List<Integer> channels,
+      int limit, boolean ascending) {
+    return new TopIpAggregatorFunction(driverContext, channels, TopIpAggregator.initSingle(driverContext.bigArrays(), limit, ascending), limit, ascending);
+  }
+
+  public static List<IntermediateStateDesc> intermediateStateDesc() {
+    return INTERMEDIATE_STATE_DESC;
+  }
+
+  @Override
+  public int intermediateBlockCount() {
+    return INTERMEDIATE_STATE_DESC.size();
+  }
+
+  @Override
+  public void addRawInput(Page page) {
+    BytesRefBlock block = page.getBlock(channels.get(0));
+    BytesRefVector vector = block.asVector();
+    if (vector != null) {
+      addRawVector(vector);
+    } else {
+      addRawBlock(block);
+    }
+  }
+
+  private void addRawVector(BytesRefVector vector) {
+    BytesRef scratch = new BytesRef();
+    for (int i = 0; i < vector.getPositionCount(); i++) {
+      TopIpAggregator.combine(state, vector.getBytesRef(i, scratch));
+    }
+  }
+
+  private void addRawBlock(BytesRefBlock block) {
+    BytesRef scratch = new BytesRef();
+    for (int p = 0; p < block.getPositionCount(); p++) {
+      if (block.isNull(p)) {
+        continue;
+      }
+      int start = block.getFirstValueIndex(p);
+      int end = start + block.getValueCount(p);
+      for (int i = start; i < end; i++) {
+        TopIpAggregator.combine(state, block.getBytesRef(i, scratch));
+      }
+    }
+  }
+
+  @Override
+  public void addIntermediateInput(Page page) {
+    assert channels.size() == intermediateBlockCount();
+    assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size();
+    Block topUncast = page.getBlock(channels.get(0));
+    if (topUncast.areAllValuesNull()) {
+      return;
+    }
+    BytesRefBlock top = (BytesRefBlock) topUncast;
+    assert top.getPositionCount() == 1;
+    BytesRef scratch = new BytesRef();
+    TopIpAggregator.combineIntermediate(state, top);
+  }
+
+  @Override
+  public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+    state.toIntermediate(blocks, offset, driverContext);
+  }
+
+  @Override
+  public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) {
+    blocks[offset] = TopIpAggregator.evaluateFinal(state, driverContext);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(getClass().getSimpleName()).append("[");
+    sb.append("channels=").append(channels);
+    sb.append("]");
+    return sb.toString();
+  }
+
+  @Override
+  public void close() {
+    state.close();
+  }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionSupplier.java
new file mode 100644
index 0000000000000..8f630c0306170
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionSupplier.java
@@ -0,0 +1,44 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.compute.aggregation;
+
+import java.lang.Integer;
+import java.lang.Override;
+import java.lang.String;
+import java.util.List;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * {@link AggregatorFunctionSupplier} implementation for {@link TopIpAggregator}.
+ * This class is generated. Do not edit it.
+ */
+public final class TopIpAggregatorFunctionSupplier implements AggregatorFunctionSupplier {
+  private final List<Integer> channels;
+
+  private final int limit;
+
+  private final boolean ascending;
+
+  public TopIpAggregatorFunctionSupplier(List<Integer> channels, int limit, boolean ascending) {
+    this.channels = channels;
+    this.limit = limit;
+    this.ascending = ascending;
+  }
+
+  @Override
+  public TopIpAggregatorFunction aggregator(DriverContext driverContext) {
+    return TopIpAggregatorFunction.create(driverContext, channels, limit, ascending);
+  }
+
+  @Override
+  public TopIpGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) {
+    return TopIpGroupingAggregatorFunction.create(channels, driverContext, limit, ascending);
+  }
+
+  @Override
+  public String describe() {
+    return "top of ips";
+  }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java
new file mode 100644
index 0000000000000..272b4827b5817
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java
@@ -0,0 +1,208 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.compute.aggregation;
+
+import java.lang.Integer;
+import java.lang.Override;
+import java.lang.String;
+import java.lang.StringBuilder;
+import java.util.List;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.ElementType;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * {@link GroupingAggregatorFunction} implementation for {@link TopIpAggregator}.
+ * This class is generated. Do not edit it.
+ */
+public final class TopIpGroupingAggregatorFunction implements GroupingAggregatorFunction {
+  private static final List<IntermediateStateDesc> INTERMEDIATE_STATE_DESC = List.of(
+      new IntermediateStateDesc("top", ElementType.BYTES_REF)  );
+
+  private final TopIpAggregator.GroupingState state;
+
+  private final List<Integer> channels;
+
+  private final DriverContext driverContext;
+
+  private final int limit;
+
+  private final boolean ascending;
+
+  public TopIpGroupingAggregatorFunction(List<Integer> channels,
+      TopIpAggregator.GroupingState state, DriverContext driverContext, int limit,
+      boolean ascending) {
+    this.channels = channels;
+    this.state = state;
+    this.driverContext = driverContext;
+    this.limit = limit;
+    this.ascending = ascending;
+  }
+
+  public static TopIpGroupingAggregatorFunction create(List<Integer> channels,
+      DriverContext driverContext, int limit, boolean ascending) {
+    return new TopIpGroupingAggregatorFunction(channels, TopIpAggregator.initGrouping(driverContext.bigArrays(), limit, ascending), driverContext, limit, ascending);
+  }
+
+  public static List<IntermediateStateDesc> intermediateStateDesc() {
+    return INTERMEDIATE_STATE_DESC;
+  }
+
+  @Override
+  public int intermediateBlockCount() {
+    return INTERMEDIATE_STATE_DESC.size();
+  }
+
+  @Override
+  public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds,
+      Page page) {
+    BytesRefBlock valuesBlock = page.getBlock(channels.get(0));
+    BytesRefVector valuesVector = valuesBlock.asVector();
+    if (valuesVector == null) {
+      if (valuesBlock.mayHaveNulls()) {
+        state.enableGroupIdTracking(seenGroupIds);
+      }
+      return new GroupingAggregatorFunction.AddInput() {
+        @Override
+        public void add(int positionOffset, IntBlock groupIds) {
+          addRawInput(positionOffset, groupIds, valuesBlock);
+        }
+
+        @Override
+        public void add(int positionOffset, IntVector groupIds) {
+          addRawInput(positionOffset, groupIds, valuesBlock);
+        }
+      };
+    }
+    return new GroupingAggregatorFunction.AddInput() {
+      @Override
+      public void add(int positionOffset, IntBlock groupIds) {
+        addRawInput(positionOffset, groupIds, valuesVector);
+      }
+
+      @Override
+      public void add(int positionOffset, IntVector groupIds) {
+        addRawInput(positionOffset, groupIds, valuesVector);
+      }
+    };
+  }
+
+  private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      int groupId = groups.getInt(groupPosition);
+      if (values.isNull(groupPosition + positionOffset)) {
+        continue;
+      }
+      int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset);
+      int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset);
+      for (int v = valuesStart; v < valuesEnd; v++) {
+        TopIpAggregator.combine(state, groupId, values.getBytesRef(v, scratch));
+      }
+    }
+  }
+
+  private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      int groupId = groups.getInt(groupPosition);
+      TopIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
+    }
+  }
+
+  private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      if (groups.isNull(groupPosition)) {
+        continue;
+      }
+      int groupStart = groups.getFirstValueIndex(groupPosition);
+      int groupEnd = groupStart + groups.getValueCount(groupPosition);
+      for (int g = groupStart; g < groupEnd; g++) {
+        int groupId = groups.getInt(g);
+        if (values.isNull(groupPosition + positionOffset)) {
+          continue;
+        }
+        int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset);
+        int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset);
+        for (int v = valuesStart; v < valuesEnd; v++) {
+          TopIpAggregator.combine(state, groupId, values.getBytesRef(v, scratch));
+        }
+      }
+    }
+  }
+
+  private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector values) {
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      if (groups.isNull(groupPosition)) {
+        continue;
+      }
+      int groupStart = groups.getFirstValueIndex(groupPosition);
+      int groupEnd = groupStart + groups.getValueCount(groupPosition);
+      for (int g = groupStart; g < groupEnd; g++) {
+        int groupId = groups.getInt(g);
+        TopIpAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
+      }
+    }
+  }
+
+  @Override
+  public void addIntermediateInput(int positionOffset, IntVector groups, Page page) {
+    state.enableGroupIdTracking(new SeenGroupIds.Empty());
+    assert channels.size() == intermediateBlockCount();
+    Block topUncast = page.getBlock(channels.get(0));
+    if (topUncast.areAllValuesNull()) {
+      return;
+    }
+    BytesRefBlock top = (BytesRefBlock) topUncast;
+    BytesRef scratch = new BytesRef();
+    for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
+      int groupId = groups.getInt(groupPosition);
+      TopIpAggregator.combineIntermediate(state, groupId, top, groupPosition + positionOffset);
+    }
+  }
+
+  @Override
+  public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) {
+    if (input.getClass() != getClass()) {
+      throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass());
+    }
+    TopIpAggregator.GroupingState inState = ((TopIpGroupingAggregatorFunction) input).state;
+    state.enableGroupIdTracking(new SeenGroupIds.Empty());
+    TopIpAggregator.combineStates(state, groupId, inState, position);
+  }
+
+  @Override
+  public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) {
+    state.toIntermediate(blocks, offset, selected, driverContext);
+  }
+
+  @Override
+  public void evaluateFinal(Block[] blocks, int offset, IntVector selected,
+      DriverContext driverContext) {
+    blocks[offset] = TopIpAggregator.evaluateFinal(state, selected, driverContext);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(getClass().getSimpleName()).append("[");
+    sb.append("channels=").append(channels);
+    sb.append("]");
+    return sb.toString();
+  }
+
+  @Override
+  public void close() {
+    state.close();
+  }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopLongGroupingAggregatorFunction.java
index 7b199b2a81389..9d1ed395c5964 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopLongGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopLongGroupingAggregatorFunction.java
@@ -96,7 +96,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, LongBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -110,7 +110,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, LongVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       TopLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
     }
   }
@@ -123,7 +123,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -144,7 +144,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         TopLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
       }
     }
@@ -160,7 +160,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     }
     LongBlock top = (LongBlock) topUncast;
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       TopLongAggregator.combineIntermediate(state, groupId, top, groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBooleanGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBooleanGroupingAggregatorFunction.java
index 16e92a7c69ca8..062a49dbf4f7c 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBooleanGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBooleanGroupingAggregatorFunction.java
@@ -89,7 +89,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, BooleanBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -103,7 +103,7 @@ private void addRawInput(int positionOffset, IntVector groups, BooleanBlock valu
 
   private void addRawInput(int positionOffset, IntVector groups, BooleanVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesBooleanAggregator.combine(state, groupId, values.getBoolean(groupPosition + positionOffset));
     }
   }
@@ -116,7 +116,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BooleanBlock value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -137,7 +137,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BooleanVector valu
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         ValuesBooleanAggregator.combine(state, groupId, values.getBoolean(groupPosition + positionOffset));
       }
     }
@@ -153,7 +153,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     }
     BooleanBlock values = (BooleanBlock) valuesUncast;
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesBooleanAggregator.combineIntermediate(state, groupId, values, groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefGroupingAggregatorFunction.java
index f9a51fcc52221..0a929913e9fde 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefGroupingAggregatorFunction.java
@@ -91,7 +91,7 @@ public void add(int positionOffset, IntVector groupIds) {
   private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -106,7 +106,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock val
   private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
@@ -120,7 +120,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock valu
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -142,7 +142,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector val
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         ValuesBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
       }
     }
@@ -159,7 +159,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     BytesRefBlock values = (BytesRefBlock) valuesUncast;
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesBytesRefAggregator.combineIntermediate(state, groupId, values, groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesDoubleGroupingAggregatorFunction.java
index 11a0eb96c6a8e..b8ca2d2b9665b 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesDoubleGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesDoubleGroupingAggregatorFunction.java
@@ -89,7 +89,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -103,7 +103,7 @@ private void addRawInput(int positionOffset, IntVector groups, DoubleBlock value
 
   private void addRawInput(int positionOffset, IntVector groups, DoubleVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
     }
   }
@@ -116,7 +116,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleBlock values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -137,7 +137,7 @@ private void addRawInput(int positionOffset, IntBlock groups, DoubleVector value
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         ValuesDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset));
       }
     }
@@ -153,7 +153,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     }
     DoubleBlock values = (DoubleBlock) valuesUncast;
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesDoubleAggregator.combineIntermediate(state, groupId, values, groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunction.java
index 54cc06072cd24..0c4e9c32328c7 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunction.java
@@ -89,7 +89,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -103,7 +103,7 @@ private void addRawInput(int positionOffset, IntVector groups, FloatBlock values
 
   private void addRawInput(int positionOffset, IntVector groups, FloatVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
     }
   }
@@ -116,7 +116,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -137,7 +137,7 @@ private void addRawInput(int positionOffset, IntBlock groups, FloatVector values
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         ValuesFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset));
       }
     }
@@ -153,7 +153,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     }
     FloatBlock values = (FloatBlock) valuesUncast;
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesFloatAggregator.combineIntermediate(state, groupId, values, groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesIntGroupingAggregatorFunction.java
index 67722cd1318c0..95e527c018cd1 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesIntGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesIntGroupingAggregatorFunction.java
@@ -87,7 +87,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, IntBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -101,7 +101,7 @@ private void addRawInput(int positionOffset, IntVector groups, IntBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, IntVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset));
     }
   }
@@ -114,7 +114,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) {
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -135,7 +135,7 @@ private void addRawInput(int positionOffset, IntBlock groups, IntVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         ValuesIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset));
       }
     }
@@ -151,7 +151,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     }
     IntBlock values = (IntBlock) valuesUncast;
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesIntAggregator.combineIntermediate(state, groupId, values, groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesLongGroupingAggregatorFunction.java
index 06508ce360ba4..a7963447037a8 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesLongGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesLongGroupingAggregatorFunction.java
@@ -89,7 +89,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, LongBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -103,7 +103,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, LongVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
     }
   }
@@ -116,7 +116,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -137,7 +137,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         ValuesLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
       }
     }
@@ -153,7 +153,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     }
     LongBlock values = (LongBlock) valuesUncast;
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       ValuesLongAggregator.combineIntermediate(state, groupId, values, groupPosition + positionOffset);
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointDocValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointDocValuesGroupingAggregatorFunction.java
index 795207b245023..dc3c1cf2917ec 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointDocValuesGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointDocValuesGroupingAggregatorFunction.java
@@ -98,7 +98,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, LongBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -112,7 +112,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, LongVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SpatialCentroidCartesianPointDocValuesAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
     }
   }
@@ -125,7 +125,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -146,7 +146,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         SpatialCentroidCartesianPointDocValuesAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
       }
     }
@@ -183,7 +183,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     LongVector count = ((LongBlock) countUncast).asVector();
     assert xVal.getPositionCount() == xDel.getPositionCount() && xVal.getPositionCount() == yVal.getPositionCount() && xVal.getPositionCount() == yDel.getPositionCount() && xVal.getPositionCount() == count.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SpatialCentroidCartesianPointDocValuesAggregator.combineIntermediate(state, groupId, xVal.getDouble(groupPosition + positionOffset), xDel.getDouble(groupPosition + positionOffset), yVal.getDouble(groupPosition + positionOffset), yDel.getDouble(groupPosition + positionOffset), count.getLong(groupPosition + positionOffset));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointSourceValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointSourceValuesGroupingAggregatorFunction.java
index 12c0f24ef43e3..0d1378ce988f3 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointSourceValuesGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointSourceValuesGroupingAggregatorFunction.java
@@ -102,7 +102,7 @@ public void add(int positionOffset, IntVector groupIds) {
   private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -117,7 +117,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock val
   private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SpatialCentroidCartesianPointSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
@@ -131,7 +131,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock valu
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -153,7 +153,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector val
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         SpatialCentroidCartesianPointSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
       }
     }
@@ -190,7 +190,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     LongVector count = ((LongBlock) countUncast).asVector();
     assert xVal.getPositionCount() == xDel.getPositionCount() && xVal.getPositionCount() == yVal.getPositionCount() && xVal.getPositionCount() == yDel.getPositionCount() && xVal.getPositionCount() == count.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SpatialCentroidCartesianPointSourceValuesAggregator.combineIntermediate(state, groupId, xVal.getDouble(groupPosition + positionOffset), xDel.getDouble(groupPosition + positionOffset), yVal.getDouble(groupPosition + positionOffset), yDel.getDouble(groupPosition + positionOffset), count.getLong(groupPosition + positionOffset));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointDocValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointDocValuesGroupingAggregatorFunction.java
index 2447939d56db9..f5604e9e23200 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointDocValuesGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointDocValuesGroupingAggregatorFunction.java
@@ -98,7 +98,7 @@ public void add(int positionOffset, IntVector groupIds) {
 
   private void addRawInput(int positionOffset, IntVector groups, LongBlock values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -112,7 +112,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values)
 
   private void addRawInput(int positionOffset, IntVector groups, LongVector values) {
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SpatialCentroidGeoPointDocValuesAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
     }
   }
@@ -125,7 +125,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -146,7 +146,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values)
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         SpatialCentroidGeoPointDocValuesAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset));
       }
     }
@@ -183,7 +183,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     LongVector count = ((LongBlock) countUncast).asVector();
     assert xVal.getPositionCount() == xDel.getPositionCount() && xVal.getPositionCount() == yVal.getPositionCount() && xVal.getPositionCount() == yDel.getPositionCount() && xVal.getPositionCount() == count.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SpatialCentroidGeoPointDocValuesAggregator.combineIntermediate(state, groupId, xVal.getDouble(groupPosition + positionOffset), xDel.getDouble(groupPosition + positionOffset), yVal.getDouble(groupPosition + positionOffset), yDel.getDouble(groupPosition + positionOffset), count.getLong(groupPosition + positionOffset));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointSourceValuesGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointSourceValuesGroupingAggregatorFunction.java
index 075f8749503b8..b3caeef925a73 100644
--- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointSourceValuesGroupingAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointSourceValuesGroupingAggregatorFunction.java
@@ -102,7 +102,7 @@ public void add(int positionOffset, IntVector groupIds) {
   private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       if (values.isNull(groupPosition + positionOffset)) {
         continue;
       }
@@ -117,7 +117,7 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock val
   private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) {
     BytesRef scratch = new BytesRef();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SpatialCentroidGeoPointSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
     }
   }
@@ -131,7 +131,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock valu
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         if (values.isNull(groupPosition + positionOffset)) {
           continue;
         }
@@ -153,7 +153,7 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector val
       int groupStart = groups.getFirstValueIndex(groupPosition);
       int groupEnd = groupStart + groups.getValueCount(groupPosition);
       for (int g = groupStart; g < groupEnd; g++) {
-        int groupId = Math.toIntExact(groups.getInt(g));
+        int groupId = groups.getInt(g);
         SpatialCentroidGeoPointSourceValuesAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch));
       }
     }
@@ -190,7 +190,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page
     LongVector count = ((LongBlock) countUncast).asVector();
     assert xVal.getPositionCount() == xDel.getPositionCount() && xVal.getPositionCount() == yVal.getPositionCount() && xVal.getPositionCount() == yDel.getPositionCount() && xVal.getPositionCount() == count.getPositionCount();
     for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) {
-      int groupId = Math.toIntExact(groups.getInt(groupPosition));
+      int groupId = groups.getInt(groupPosition);
       SpatialCentroidGeoPointSourceValuesAggregator.combineIntermediate(state, groupId, xVal.getDouble(groupPosition + positionOffset), xDel.getDouble(groupPosition + positionOffset), yVal.getDouble(groupPosition + positionOffset), yDel.getDouble(groupPosition + positionOffset), count.getLong(groupPosition + positionOffset));
     }
   }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java
index 0dc008cb22396..f9962922cc4a7 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java
@@ -12,6 +12,13 @@
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 
+/**
+ * Base class for array states that track which group ids have been set.
+ * Most of this class subclasses are autogenerated.
+ * <p>
+ *     Most of this class subclasses are autogenerated.
+ * </p>
+ */
 public class AbstractArrayState implements Releasable {
     protected final BigArrays bigArrays;
 
@@ -21,7 +28,7 @@ public AbstractArrayState(BigArrays bigArrays) {
         this.bigArrays = bigArrays;
     }
 
-    final boolean hasValue(int groupId) {
+    boolean hasValue(int groupId) {
         return seen == null || seen.get(groupId);
     }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractFallibleArrayState.java
new file mode 100644
index 0000000000000..d5ad3189e2f9e
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractFallibleArrayState.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.BitArray;
+import org.elasticsearch.core.Releasables;
+
+/**
+ * Base class that extends {@link AbstractArrayState} to add failure tracking.
+ * That is, when a group id fails, it is marked as failed in the state.
+ * <p>
+ *     Most of this class subclasses are autogenerated.
+ * </p>
+ */
+public class AbstractFallibleArrayState extends AbstractArrayState {
+    private BitArray failed;
+
+    public AbstractFallibleArrayState(BigArrays bigArrays) {
+        super(bigArrays);
+    }
+
+    final boolean hasFailed(int groupId) {
+        return failed != null && failed.get(groupId);
+    }
+
+    protected final boolean anyFailure() {
+        return failed != null;
+    }
+
+    protected final void setFailed(int groupId) {
+        if (failed == null) {
+            failed = new BitArray(groupId + 1, bigArrays);
+        }
+        failed.set(groupId);
+    }
+
+    @Override
+    public void close() {
+        super.close();
+        Releasables.close(failed);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/BytesRefArrayState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/BytesRefArrayState.java
new file mode 100644
index 0000000000000..eb0a992c8610f
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/BytesRefArrayState.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ObjectArray;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+
+/**
+ * Aggregator state for an array of BytesRefs. It is created in a mode where it
+ * won't track the {@code groupId}s that are sent to it and it is the
+ * responsibility of the caller to only fetch values for {@code groupId}s
+ * that it has sent using the {@code selected} parameter when building the
+ * results. This is fine when there are no {@code null} values in the input
+ * data. But once there are null values in the input data it is
+ * <strong>much</strong> more convenient to only send non-null values and
+ * the tracking built into the grouping code can't track that. In that case
+ * call {@link #enableGroupIdTracking} to transition the state into a mode
+ * where it'll track which {@code groupIds} have been written.
+ * <p>
+ *     This class is a specialized version of the {@code X-ArrayState.java.st} template.
+ * </p>
+ */
+public final class BytesRefArrayState implements GroupingAggregatorState, Releasable {
+    private final BigArrays bigArrays;
+    private final CircuitBreaker breaker;
+    private final String breakerLabel;
+    private ObjectArray<BreakingBytesRefBuilder> values;
+    /**
+     * If false, no group id is expected to have nulls.
+     * If true, they may have nulls.
+     */
+    private boolean groupIdTrackingEnabled;
+
+    BytesRefArrayState(BigArrays bigArrays, CircuitBreaker breaker, String breakerLabel) {
+        this.bigArrays = bigArrays;
+        this.breaker = breaker;
+        this.breakerLabel = breakerLabel;
+        this.values = bigArrays.newObjectArray(0);
+    }
+
+    BytesRef get(int groupId) {
+        return values.get(groupId).bytesRefView();
+    }
+
+    void set(int groupId, BytesRef value) {
+        ensureCapacity(groupId);
+
+        var currentBuilder = values.get(groupId);
+        if (currentBuilder == null) {
+            currentBuilder = new BreakingBytesRefBuilder(breaker, breakerLabel, value.length);
+            values.set(groupId, currentBuilder);
+        }
+
+        currentBuilder.copyBytes(value);
+    }
+
+    Block toValuesBlock(IntVector selected, DriverContext driverContext) {
+        if (false == groupIdTrackingEnabled) {
+            try (var builder = driverContext.blockFactory().newBytesRefVectorBuilder(selected.getPositionCount())) {
+                for (int i = 0; i < selected.getPositionCount(); i++) {
+                    int group = selected.getInt(i);
+                    var value = get(group);
+                    builder.appendBytesRef(value);
+                }
+                return builder.build().asBlock();
+            }
+        }
+        try (var builder = driverContext.blockFactory().newBytesRefBlockBuilder(selected.getPositionCount())) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (hasValue(group)) {
+                    var value = get(group);
+                    builder.appendBytesRef(value);
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
+    private void ensureCapacity(int groupId) {
+        var minSize = groupId + 1;
+        if (minSize > values.size()) {
+            long prevSize = values.size();
+            values = bigArrays.grow(values, minSize);
+        }
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) {
+        assert blocks.length >= offset + 2;
+        try (
+            var valuesBuilder = driverContext.blockFactory().newBytesRefVectorBuilder(selected.getPositionCount());
+            var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount())
+        ) {
+            var emptyBytesRef = new BytesRef();
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (hasValue(group)) {
+                    var value = get(group);
+                    valuesBuilder.appendBytesRef(value);
+                } else {
+                    valuesBuilder.appendBytesRef(emptyBytesRef); // TODO can we just use null?
+                }
+                hasValueBuilder.appendBoolean(i, hasValue(group));
+            }
+            blocks[offset] = valuesBuilder.build().asBlock();
+            blocks[offset + 1] = hasValueBuilder.build().asBlock();
+        }
+    }
+
+    boolean hasValue(int groupId) {
+        return groupId < values.size() && values.get(groupId) != null;
+    }
+
+    /**
+     * Switches this array state into tracking which group ids are set. This is
+     * idempotent and fast if already tracking so it's safe to, say, call it once
+     * for every block of values that arrives containing {@code null}.
+     *
+     * <p>
+     *     This class tracks seen group IDs differently from {@code AbstractArrayState}, as it just
+     *     stores a flag to know if optimizations can be made.
+     * </p>
+     */
+    void enableGroupIdTracking(SeenGroupIds seenGroupIds) {
+        this.groupIdTrackingEnabled = true;
+    }
+
+    @Override
+    public void close() {
+        for (int i = 0; i < values.size(); i++) {
+            Releasables.closeWhileHandlingException(values.get(i));
+        }
+
+        Releasables.close(values);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountAggregatorFunction.java
index 13a4204edfd8f..c32f6f4703a79 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountAggregatorFunction.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountAggregatorFunction.java
@@ -52,7 +52,7 @@ public static List<IntermediateStateDesc> intermediateStateDesc() {
     private final boolean countAll;
 
     public static CountAggregatorFunction create(List<Integer> inputChannels) {
-        return new CountAggregatorFunction(inputChannels, new LongState());
+        return new CountAggregatorFunction(inputChannels, new LongState(0));
     }
 
     private CountAggregatorFunction(List<Integer> channels, LongState state) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregator.java
new file mode 100644
index 0000000000000..144214f93571e
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregator.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.compute.ann.Aggregator;
+import org.elasticsearch.compute.ann.GroupingAggregator;
+import org.elasticsearch.compute.ann.IntermediateState;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+
+/**
+ * Aggregator for `Max`, that works with BytesRef values.
+ * Gets the biggest BytesRef value, based on its bytes natural order (Delegated to {@link BytesRef#compareTo}).
+ */
+@Aggregator({ @IntermediateState(name = "max", type = "BYTES_REF"), @IntermediateState(name = "seen", type = "BOOLEAN") })
+@GroupingAggregator
+class MaxBytesRefAggregator {
+    private static boolean isBetter(BytesRef value, BytesRef otherValue) {
+        return value.compareTo(otherValue) > 0;
+    }
+
+    public static SingleState initSingle(DriverContext driverContext) {
+        return new SingleState(driverContext.breaker());
+    }
+
+    public static void combine(SingleState state, BytesRef value) {
+        state.add(value);
+    }
+
+    public static void combineIntermediate(SingleState state, BytesRef value, boolean seen) {
+        if (seen) {
+            combine(state, value);
+        }
+    }
+
+    public static Block evaluateFinal(SingleState state, DriverContext driverContext) {
+        return state.toBlock(driverContext);
+    }
+
+    public static GroupingState initGrouping(DriverContext driverContext) {
+        return new GroupingState(driverContext.bigArrays(), driverContext.breaker());
+    }
+
+    public static void combine(GroupingState state, int groupId, BytesRef value) {
+        state.add(groupId, value);
+    }
+
+    public static void combineIntermediate(GroupingState state, int groupId, BytesRef value, boolean seen) {
+        if (seen) {
+            state.add(groupId, value);
+        }
+    }
+
+    public static void combineStates(GroupingState state, int groupId, GroupingState otherState, int otherGroupId) {
+        state.combine(groupId, otherState, otherGroupId);
+    }
+
+    public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) {
+        return state.toBlock(selected, driverContext);
+    }
+
+    public static class GroupingState implements Releasable {
+        private final BytesRefArrayState internalState;
+
+        private GroupingState(BigArrays bigArrays, CircuitBreaker breaker) {
+            this.internalState = new BytesRefArrayState(bigArrays, breaker, "max_bytes_ref_grouping_aggregator");
+        }
+
+        public void add(int groupId, BytesRef value) {
+            if (internalState.hasValue(groupId) == false || isBetter(value, internalState.get(groupId))) {
+                internalState.set(groupId, value);
+            }
+        }
+
+        public void combine(int groupId, GroupingState otherState, int otherGroupId) {
+            if (otherState.internalState.hasValue(otherGroupId)) {
+                add(groupId, otherState.internalState.get(otherGroupId));
+            }
+        }
+
+        void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) {
+            internalState.toIntermediate(blocks, offset, selected, driverContext);
+        }
+
+        Block toBlock(IntVector selected, DriverContext driverContext) {
+            return internalState.toValuesBlock(selected, driverContext);
+        }
+
+        void enableGroupIdTracking(SeenGroupIds seen) {
+            internalState.enableGroupIdTracking(seen);
+        }
+
+        @Override
+        public void close() {
+            Releasables.close(internalState);
+        }
+    }
+
+    public static class SingleState implements Releasable {
+        private final BreakingBytesRefBuilder internalState;
+        private boolean seen;
+
+        private SingleState(CircuitBreaker breaker) {
+            this.internalState = new BreakingBytesRefBuilder(breaker, "max_bytes_ref_aggregator");
+            this.seen = false;
+        }
+
+        public void add(BytesRef value) {
+            if (seen == false || isBetter(value, internalState.bytesRefView())) {
+                seen = true;
+
+                internalState.grow(value.length);
+                internalState.setLength(value.length);
+
+                System.arraycopy(value.bytes, value.offset, internalState.bytes(), 0, value.length);
+            }
+        }
+
+        void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+            blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1);
+            blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1);
+        }
+
+        Block toBlock(DriverContext driverContext) {
+            if (seen == false) {
+                return driverContext.blockFactory().newConstantNullBlock(1);
+            }
+
+            return driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1);
+        }
+
+        @Override
+        public void close() {
+            Releasables.close(internalState);
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregator.java
new file mode 100644
index 0000000000000..830900702a371
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregator.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.compute.ann.Aggregator;
+import org.elasticsearch.compute.ann.GroupingAggregator;
+import org.elasticsearch.compute.ann.IntermediateState;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+
+/**
+ * Aggregator for `Min`, that works with BytesRef values.
+ * Gets the smallest BytesRef value, based on its bytes natural order (Delegated to {@link BytesRef#compareTo}).
+ */
+@Aggregator({ @IntermediateState(name = "min", type = "BYTES_REF"), @IntermediateState(name = "seen", type = "BOOLEAN") })
+@GroupingAggregator
+class MinBytesRefAggregator {
+    private static boolean isBetter(BytesRef value, BytesRef otherValue) {
+        return value.compareTo(otherValue) < 0;
+    }
+
+    public static SingleState initSingle(DriverContext driverContext) {
+        return new SingleState(driverContext.breaker());
+    }
+
+    public static void combine(SingleState state, BytesRef value) {
+        state.add(value);
+    }
+
+    public static void combineIntermediate(SingleState state, BytesRef value, boolean seen) {
+        if (seen) {
+            combine(state, value);
+        }
+    }
+
+    public static Block evaluateFinal(SingleState state, DriverContext driverContext) {
+        return state.toBlock(driverContext);
+    }
+
+    public static GroupingState initGrouping(DriverContext driverContext) {
+        return new GroupingState(driverContext.bigArrays(), driverContext.breaker());
+    }
+
+    public static void combine(GroupingState state, int groupId, BytesRef value) {
+        state.add(groupId, value);
+    }
+
+    public static void combineIntermediate(GroupingState state, int groupId, BytesRef value, boolean seen) {
+        if (seen) {
+            state.add(groupId, value);
+        }
+    }
+
+    public static void combineStates(GroupingState state, int groupId, GroupingState otherState, int otherGroupId) {
+        state.combine(groupId, otherState, otherGroupId);
+    }
+
+    public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) {
+        return state.toBlock(selected, driverContext);
+    }
+
+    public static class GroupingState implements Releasable {
+        private final BytesRefArrayState internalState;
+
+        private GroupingState(BigArrays bigArrays, CircuitBreaker breaker) {
+            this.internalState = new BytesRefArrayState(bigArrays, breaker, "min_bytes_ref_grouping_aggregator");
+        }
+
+        public void add(int groupId, BytesRef value) {
+            if (internalState.hasValue(groupId) == false || isBetter(value, internalState.get(groupId))) {
+                internalState.set(groupId, value);
+            }
+        }
+
+        public void combine(int groupId, GroupingState otherState, int otherGroupId) {
+            if (otherState.internalState.hasValue(otherGroupId)) {
+                add(groupId, otherState.internalState.get(otherGroupId));
+            }
+        }
+
+        void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) {
+            internalState.toIntermediate(blocks, offset, selected, driverContext);
+        }
+
+        Block toBlock(IntVector selected, DriverContext driverContext) {
+            return internalState.toValuesBlock(selected, driverContext);
+        }
+
+        void enableGroupIdTracking(SeenGroupIds seen) {
+            internalState.enableGroupIdTracking(seen);
+        }
+
+        @Override
+        public void close() {
+            Releasables.close(internalState);
+        }
+    }
+
+    public static class SingleState implements Releasable {
+        private final BreakingBytesRefBuilder internalState;
+        private boolean seen;
+
+        private SingleState(CircuitBreaker breaker) {
+            this.internalState = new BreakingBytesRefBuilder(breaker, "min_bytes_ref_aggregator");
+            this.seen = false;
+        }
+
+        public void add(BytesRef value) {
+            if (seen == false || isBetter(value, internalState.bytesRefView())) {
+                seen = true;
+
+                internalState.grow(value.length);
+                internalState.setLength(value.length);
+
+                System.arraycopy(value.bytes, value.offset, internalState.bytes(), 0, value.length);
+            }
+        }
+
+        void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+            blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1);
+            blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1);
+        }
+
+        Block toBlock(DriverContext driverContext) {
+            if (seen == false) {
+                return driverContext.blockFactory().newConstantNullBlock(1);
+            }
+
+            return driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1);
+        }
+
+        @Override
+        public void close() {
+            Releasables.close(internalState);
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/Warnings.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/Warnings.java
new file mode 100644
index 0000000000000..eb2255a4e349b
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/Warnings.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.compute.operator.DriverContext;
+
+import static org.elasticsearch.common.logging.HeaderWarning.addWarning;
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
+
+/**
+ * Utilities to collect warnings for running an executor.
+ */
+public class Warnings {
+    static final int MAX_ADDED_WARNINGS = 20;
+
+    private final String location;
+    private final String first;
+
+    private int addedWarnings;
+
+    public static final Warnings NOOP_WARNINGS = new Warnings(-1, -2, "") {
+        @Override
+        public void registerException(Exception exception) {
+            // this space intentionally left blank
+        }
+    };
+
+    /**
+     * Create a new warnings object based on the given mode
+     * @param warningsMode The warnings collection strategy to use
+     * @param lineNumber The line number of the source text. Same as `source.getLineNumber()`
+     * @param columnNumber The column number of the source text. Same as `source.getColumnNumber()`
+     * @param sourceText The source text that caused the warning. Same as `source.text()`
+     * @return A warnings collector object
+     */
+    public static Warnings createWarnings(DriverContext.WarningsMode warningsMode, int lineNumber, int columnNumber, String sourceText) {
+        switch (warningsMode) {
+            case COLLECT -> {
+                return new Warnings(lineNumber, columnNumber, sourceText);
+            }
+            case IGNORE -> {
+                return NOOP_WARNINGS;
+            }
+        }
+        throw new IllegalStateException("Unreachable");
+    }
+
+    public Warnings(int lineNumber, int columnNumber, String sourceText) {
+        location = format("Line {}:{}: ", lineNumber, columnNumber);
+        first = format(
+            null,
+            "{}evaluation of [{}] failed, treating result as null. Only first {} failures recorded.",
+            location,
+            sourceText,
+            MAX_ADDED_WARNINGS
+        );
+    }
+
+    public void registerException(Exception exception) {
+        if (addedWarnings < MAX_ADDED_WARNINGS) {
+            if (addedWarnings == 0) {
+                addWarning(first);
+            }
+            // location needs to be added to the exception too, since the headers are deduplicated
+            addWarning(location + exception.getClass().getName() + ": " + exception.getMessage());
+            addedWarnings++;
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleArrayState.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleArrayState.java.st
new file mode 100644
index 0000000000000..3c57ab948a79f
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleArrayState.java.st
@@ -0,0 +1,166 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.common.util.BigArrays;
+$if(boolean)$
+import org.elasticsearch.common.util.BitArray;
+$else$
+import org.elasticsearch.common.util.$Type$Array;
+$endif$
+import org.elasticsearch.compute.data.Block;
+$if(long)$
+import org.elasticsearch.compute.data.IntVector;
+$endif$
+import org.elasticsearch.compute.data.$Type$Block;
+$if(int)$
+import org.elasticsearch.compute.data.$Type$Vector;
+$endif$
+$if(boolean||double||float)$
+import org.elasticsearch.compute.data.IntVector;
+$endif$
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.core.Releasables;
+
+/**
+ * Aggregator state for an array of $type$s, that also tracks failures.
+ * It is created in a mode where it won't track
+ * the {@code groupId}s that are sent to it and it is the
+ * responsibility of the caller to only fetch values for {@code groupId}s
+ * that it has sent using the {@code selected} parameter when building the
+ * results. This is fine when there are no {@code null} values in the input
+ * data. But once there are null values in the input data it is
+ * <strong>much</strong> more convenient to only send non-null values and
+ * the tracking built into the grouping code can't track that. In that case
+ * call {@link #enableGroupIdTracking} to transition the state into a mode
+ * where it'll track which {@code groupIds} have been written.
+ * <p>
+ * This class is generated. Do not edit it.
+ * </p>
+ */
+final class $Type$FallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState {
+    private final $type$ init;
+
+$if(boolean)$
+    private BitArray values;
+    private int size;
+
+$else$
+    private $Type$Array values;
+$endif$
+
+    $Type$FallibleArrayState(BigArrays bigArrays, $type$ init) {
+        super(bigArrays);
+$if(boolean)$
+        this.values = new BitArray(1, bigArrays);
+        this.size = 1;
+$else$
+        this.values = bigArrays.new$Type$Array(1, false);
+$endif$
+        this.values.set(0, init);
+        this.init = init;
+    }
+
+    $type$ get(int groupId) {
+        return values.get(groupId);
+    }
+
+    $type$ getOrDefault(int groupId) {
+$if(boolean)$
+        return groupId < size ? values.get(groupId) : init;
+$else$
+        return groupId < values.size() ? values.get(groupId) : init;
+$endif$
+    }
+
+    void set(int groupId, $type$ value) {
+        ensureCapacity(groupId);
+        values.set(groupId, value);
+        trackGroupId(groupId);
+    }
+
+$if(long)$
+    void increment(int groupId, long value) {
+        ensureCapacity(groupId);
+        values.increment(groupId, value);
+        trackGroupId(groupId);
+    }
+$endif$
+
+    Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) {
+        if (false == trackingGroupIds() && false == anyFailure()) {
+            try (var builder = driverContext.blockFactory().new$Type$VectorFixedBuilder(selected.getPositionCount())) {
+                for (int i = 0; i < selected.getPositionCount(); i++) {
+                    builder.append$Type$(i, values.get(selected.getInt(i)));
+                }
+                return builder.build().asBlock();
+            }
+        }
+        try ($Type$Block.Builder builder = driverContext.blockFactory().new$Type$BlockBuilder(selected.getPositionCount())) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (hasValue(group) && !hasFailed(group)) {
+                    builder.append$Type$(values.get(group));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
+    private void ensureCapacity(int groupId) {
+$if(boolean)$
+        if (groupId >= size) {
+            values.fill(size, groupId + 1, init);
+            size = groupId + 1;
+        }
+$else$
+        if (groupId >= values.size()) {
+            long prevSize = values.size();
+            values = bigArrays.grow(values, groupId + 1);
+            values.fill(prevSize, values.size(), init);
+        }
+$endif$
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(
+        Block[] blocks,
+        int offset,
+        IntVector selected,
+        org.elasticsearch.compute.operator.DriverContext driverContext
+    ) {
+        assert blocks.length >= offset + 3;
+        try (
+            var valuesBuilder = driverContext.blockFactory().new$Type$BlockBuilder(selected.getPositionCount());
+            var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount());
+            var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount())
+        ) {
+            for (int i = 0; i < selected.getPositionCount(); i++) {
+                int group = selected.getInt(i);
+                if (group < $if(boolean)$size$else$values.size()$endif$) {
+                    valuesBuilder.append$Type$(values.get(group));
+                } else {
+                    valuesBuilder.append$Type$($if(boolean)$false$else$0$endif$); // TODO can we just use null?
+                }
+                hasValueBuilder.appendBoolean(i, hasValue(group));
+                hasFailedBuilder.appendBoolean(i, hasFailed(group));
+            }
+            blocks[offset + 0] = valuesBuilder.build();
+            blocks[offset + 1] = hasValueBuilder.build().asBlock();
+            blocks[offset + 2] = hasFailedBuilder.build().asBlock();
+        }
+    }
+
+    @Override
+    public void close() {
+        Releasables.close(values, super::close);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleState.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleState.java.st
new file mode 100644
index 0000000000000..27609383e4f61
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleState.java.st
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.operator.DriverContext;
+
+/**
+ * Aggregator state for a single $type$.
+ * It stores a third boolean to store if the aggregation failed.
+ * This class is generated. Do not edit it.
+ */
+final class $Type$FallibleState implements AggregatorState {
+    private $type$ value;
+    private boolean seen;
+    private boolean failed;
+
+    $Type$FallibleState($type$ init) {
+        this.value = init;
+    }
+
+    $type$ $type$Value() {
+        return value;
+    }
+
+    void $type$Value($type$ value) {
+        this.value = value;
+    }
+
+    boolean seen() {
+        return seen;
+    }
+
+    void seen(boolean seen) {
+        this.seen = seen;
+    }
+
+    boolean failed() {
+        return failed;
+    }
+
+    void failed(boolean failed) {
+        this.failed = failed;
+    }
+
+    /** Extracts an intermediate view of the contents of this state.  */
+    @Override
+    public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) {
+        assert blocks.length >= offset + 3;
+        blocks[offset + 0] = driverContext.blockFactory().newConstant$Type$BlockWith(value, 1);
+        blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1);
+        blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1);
+    }
+
+    @Override
+    public void close() {}
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-State.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-State.java.st
index 2d2d706c9454f..7e0949c86faaa 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-State.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-State.java.st
@@ -18,14 +18,6 @@ final class $Type$State implements AggregatorState {
     private $type$ value;
     private boolean seen;
 
-    $Type$State() {
-$if(boolean)$
-        this(false);
-$else$
-        this(0);
-$endif$
-    }
-
     $Type$State($type$ init) {
         this.value = init;
     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st
index 41d0224f37214..b97d26ee6147d 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopAggregator.java.st
@@ -7,6 +7,9 @@
 
 package org.elasticsearch.compute.aggregation;
 
+$if(Ip)$
+import org.apache.lucene.util.BytesRef;
+$endif$
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.ann.Aggregator;
 import org.elasticsearch.compute.ann.GroupingAggregator;
@@ -20,7 +23,7 @@ import org.elasticsearch.compute.data.IntVector;
 $if(long)$
 import org.elasticsearch.compute.data.$Type$Block;
 $endif$
-import org.elasticsearch.compute.data.sort.$Type$BucketedSort;
+import org.elasticsearch.compute.data.sort.$Name$BucketedSort;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
@@ -28,10 +31,13 @@ import org.elasticsearch.search.sort.SortOrder;
 
 /**
  * Aggregates the top N field values for $type$.
+ * <p>
+ *     This class is generated. Edit `X-TopAggregator.java.st` to edit this file.
+ * </p>
  */
 @Aggregator({ @IntermediateState(name = "top", type = "$TYPE$_BLOCK") })
 @GroupingAggregator
-class Top$Type$Aggregator {
+class Top$Name$Aggregator {
     public static SingleState initSingle(BigArrays bigArrays, int limit, boolean ascending) {
         return new SingleState(bigArrays, limit, ascending);
     }
@@ -43,9 +49,16 @@ class Top$Type$Aggregator {
     public static void combineIntermediate(SingleState state, $Type$Block values) {
         int start = values.getFirstValueIndex(0);
         int end = start + values.getValueCount(0);
+$if(Ip)$
+        var scratch = new BytesRef();
+        for (int i = start; i < end; i++) {
+            combine(state, values.get$Type$(i, scratch));
+        }
+$else$
         for (int i = start; i < end; i++) {
             combine(state, values.get$Type$(i));
         }
+$endif$
     }
 
     public static Block evaluateFinal(SingleState state, DriverContext driverContext) {
@@ -63,9 +76,16 @@ class Top$Type$Aggregator {
     public static void combineIntermediate(GroupingState state, int groupId, $Type$Block values, int valuesPosition) {
         int start = values.getFirstValueIndex(valuesPosition);
         int end = start + values.getValueCount(valuesPosition);
+$if(Ip)$
+        var scratch = new BytesRef();
+        for (int i = start; i < end; i++) {
+            combine(state, groupId, values.get$Type$(i, scratch));
+        }
+$else$
         for (int i = start; i < end; i++) {
             combine(state, groupId, values.get$Type$(i));
         }
+$endif$
     }
 
     public static void combineStates(GroupingState current, int groupId, GroupingState state, int statePosition) {
@@ -77,10 +97,10 @@ class Top$Type$Aggregator {
     }
 
     public static class GroupingState implements Releasable {
-        private final $Type$BucketedSort sort;
+        private final $Name$BucketedSort sort;
 
         private GroupingState(BigArrays bigArrays, int limit, boolean ascending) {
-            this.sort = new $Type$BucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit);
+            this.sort = new $Name$BucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit);
         }
 
         public void add(int groupId, $type$ value) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st
index ea62dcf295825..a8884c58116f3 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st
@@ -192,7 +192,7 @@ $elseif(double)$
 $elseif(int)$
                 return blockFactory.newConstantIntBlockWith((int) values.get(0), 1);
 $elseif(BytesRef)$
-                return blockFactory.newConstantBytesRefBlockWith(values.get(0, scratch), 1);
+                return blockFactory.newConstantBytesRefBlockWith(BytesRef.deepCopyOf(values.get(0, scratch)), 1);
 $endif$
             }
             try ($Type$Block.Builder builder = blockFactory.new$Type$BlockBuilder((int) values.size())) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/table/RowInTableLookup.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/table/RowInTableLookup.java
index 4fa582e761e18..8bc9bacf8ff21 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/table/RowInTableLookup.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/table/RowInTableLookup.java
@@ -37,6 +37,9 @@ public abstract sealed class RowInTableLookup implements Releasable permits Empt
     public abstract String toString();
 
     public static RowInTableLookup build(BlockFactory blockFactory, Block[] keys) {
+        if (keys.length < 1) {
+            throw new IllegalArgumentException("expected [keys] to be non-empty");
+        }
         int positions = keys[0].getPositionCount();
         for (int k = 0; k < keys.length; k++) {
             if (positions != keys[k].getPositionCount()) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java
index 302defa03473f..9e48bc13cdafa 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java
@@ -139,6 +139,14 @@ public interface Block extends Accountable, BlockLoader.Block, NamedWriteable, R
      */
     Block filter(int... positions);
 
+    /**
+     * Build a {@link Block} with the same values as this {@linkplain Block}, but replacing
+     * all values for which {@code mask.getBooleanValue(position)} returns
+     * {@code false} with {@code null}. The {@code mask} vector must be at least
+     * as long as this {@linkplain Block}.
+     */
+    Block keepMask(BooleanVector mask);
+
     /**
      * Builds an Iterator of new {@link Block}s with the same {@link #elementType}
      * as this Block whose values are copied from positions in this Block. It has the
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java
index a697a3f6c15fa..3df389135e9d3 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java
@@ -87,6 +87,8 @@ public static Block[] fromListRow(BlockFactory blockFactory, List<Object> row, i
                             } else {
                                 wrapper.builder.mvOrdering(Block.MvOrdering.DEDUPLICATED_UNORDERD);
                             }
+                        } else if (isAscending(listVal) && random.nextBoolean()) {
+                            wrapper.builder.mvOrdering(Block.MvOrdering.SORTED_ASCENDING);
                         }
                         blocks[i] = wrapper.builder.build();
                     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java
index dbbcf905a2e57..b83e2d1efc259 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java
@@ -161,6 +161,23 @@ public CompositeBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public Block keepMask(BooleanVector mask) {
+        CompositeBlock result = null;
+        final Block[] masked = new Block[blocks.length];
+        try {
+            for (int i = 0; i < blocks.length; i++) {
+                masked[i] = blocks[i].keepMask(mask);
+            }
+            result = new CompositeBlock(masked);
+            return result;
+        } finally {
+            if (result == null) {
+                Releasables.close(masked);
+            }
+        }
+    }
+
     @Override
     public ReleasableIterator<? extends Block> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         // TODO: support this
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java
index 876cbc499bec6..3d61613ba70e9 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java
@@ -48,6 +48,11 @@ public OrdinalBytesRefBlock asOrdinals() {
         return null;
     }
 
+    @Override
+    public ToMask toMask() {
+        return new ToMask(blockFactory.newConstantBooleanVector(false, positionCount), false);
+    }
+
     @Override
     public boolean isNull(int position) {
         return true;
@@ -83,6 +88,11 @@ public ConstantNullBlock filter(int... positions) {
         return (ConstantNullBlock) blockFactory().newConstantNullBlock(positions.length);
     }
 
+    @Override
+    public ConstantNullBlock keepMask(BooleanVector mask) {
+        return (ConstantNullBlock) blockFactory().newConstantNullBlock(getPositionCount());
+    }
+
     @Override
     public ReleasableIterator<ConstantNullBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return ReleasableIterator.single((ConstantNullBlock) positions.blockFactory().newConstantNullBlock(positions.getPositionCount()));
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java
index b053267ba0e0f..9bcd8a19c0132 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java
@@ -54,6 +54,12 @@ public ConstantNullVector filter(int... positions) {
         throw new UnsupportedOperationException("null vector");
     }
 
+    @Override
+    public ConstantNullBlock keepMask(BooleanVector mask) {
+        assert false : "null vector";
+        throw new UnsupportedOperationException("null vector");
+    }
+
     @Override
     public ReleasableIterator<ConstantNullBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         assert false : "null vector";
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java
index da9ca2bbae270..e0e10dbadd130 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java
@@ -50,6 +50,11 @@ public Block filter(int... positions) {
         return new DocBlock(asVector().filter(positions));
     }
 
+    @Override
+    public Block keepMask(BooleanVector mask) {
+        return vector.keepMask(mask);
+    }
+
     @Override
     public ReleasableIterator<? extends Block> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         throw new UnsupportedOperationException("can't lookup values from DocBlock");
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java
index 33f5797f60df8..82777f8941060 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java
@@ -237,6 +237,11 @@ public DocVector filter(int... positions) {
         }
     }
 
+    @Override
+    public DocBlock keepMask(BooleanVector mask) {
+        throw new UnsupportedOperationException("can't mask DocVector because it can't contain nulls");
+    }
+
     @Override
     public ReleasableIterator<? extends Block> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         throw new UnsupportedOperationException("can't lookup values from DocVector");
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java
index 52c84c80610d2..f38c6d70991f9 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java
@@ -13,40 +13,42 @@
  * The type of elements in {@link Block} and {@link Vector}
  */
 public enum ElementType {
-    BOOLEAN(BlockFactory::newBooleanBlockBuilder),
-    INT(BlockFactory::newIntBlockBuilder),
-    LONG(BlockFactory::newLongBlockBuilder),
-    FLOAT(BlockFactory::newFloatBlockBuilder),
-    DOUBLE(BlockFactory::newDoubleBlockBuilder),
+    BOOLEAN("Boolean", BlockFactory::newBooleanBlockBuilder),
+    INT("Int", BlockFactory::newIntBlockBuilder),
+    LONG("Long", BlockFactory::newLongBlockBuilder),
+    FLOAT("Float", BlockFactory::newFloatBlockBuilder),
+    DOUBLE("Double", BlockFactory::newDoubleBlockBuilder),
     /**
      * Blocks containing only null values.
      */
-    NULL((blockFactory, estimatedSize) -> new ConstantNullBlock.Builder(blockFactory)),
+    NULL("Null", (blockFactory, estimatedSize) -> new ConstantNullBlock.Builder(blockFactory)),
 
-    BYTES_REF(BlockFactory::newBytesRefBlockBuilder),
+    BYTES_REF("BytesRef", BlockFactory::newBytesRefBlockBuilder),
 
     /**
      * Blocks that reference individual lucene documents.
      */
-    DOC(DocBlock::newBlockBuilder),
+    DOC("Doc", DocBlock::newBlockBuilder),
 
     /**
      * Composite blocks which contain array of sub-blocks.
      */
-    COMPOSITE((blockFactory, estimatedSize) -> { throw new UnsupportedOperationException("can't build composite blocks"); }),
+    COMPOSITE("Composite", (blockFactory, estimatedSize) -> { throw new UnsupportedOperationException("can't build composite blocks"); }),
 
     /**
      * Intermediate blocks which don't support retrieving elements.
      */
-    UNKNOWN((blockFactory, estimatedSize) -> { throw new UnsupportedOperationException("can't build null blocks"); });
+    UNKNOWN("Unknown", (blockFactory, estimatedSize) -> { throw new UnsupportedOperationException("can't build null blocks"); });
 
     private interface BuilderSupplier {
         Block.Builder newBlockBuilder(BlockFactory blockFactory, int estimatedSize);
     }
 
+    private final String pascalCaseName;
     private final BuilderSupplier builder;
 
-    ElementType(BuilderSupplier builder) {
+    ElementType(String pascalCaseName, BuilderSupplier builder) {
+        this.pascalCaseName = pascalCaseName;
         this.builder = builder;
     }
 
@@ -78,4 +80,8 @@ public static ElementType fromJava(Class<?> type) {
         }
         return elementType;
     }
+
+    public String pascalCaseName() {
+        return pascalCaseName;
+    }
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefBlock.java
index b8ac46e790acd..863f89827207e 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefBlock.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefBlock.java
@@ -125,6 +125,32 @@ public BytesRefBlock filter(int... positions) {
         }
     }
 
+    @Override
+    public BytesRefBlock keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return (BytesRefBlock) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        OrdinalBytesRefBlock result = null;
+        IntBlock filteredOrdinals = ordinals.keepMask(mask);
+        try {
+            result = new OrdinalBytesRefBlock(filteredOrdinals, bytes);
+            bytes.incRef();
+        } finally {
+            if (result == null) {
+                filteredOrdinals.close();
+            }
+        }
+        return result;
+    }
+
     @Override
     public ReleasableIterator<BytesRefBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new BytesRefLookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefVector.java
index ec0c7efa715ad..98a60dbcb7c59 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefVector.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefVector.java
@@ -122,6 +122,15 @@ public BytesRefVector filter(int... positions) {
         }
     }
 
+    @Override
+    public BytesRefBlock keepMask(BooleanVector mask) {
+        /*
+         * The implementation in OrdinalBytesRefBlock is quite fast and
+         * amounts to the same thing so we can just reuse it.
+         */
+        return asBlock().keepMask(mask);
+    }
+
     @Override
     public ReleasableIterator<? extends BytesRefBlock> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new BytesRefLookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ToMask.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ToMask.java
new file mode 100644
index 0000000000000..5b71679048e21
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ToMask.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.data;
+
+import org.elasticsearch.core.Releasable;
+
+/**
+ * Result from calling {@link BooleanBlock#toMask}. {@link #close closing} this will
+ * close the contained {@link #mask()}. If you want to keep a reference to it then you'll
+ * have to {@link Block#incRef()} it.
+ */
+public record ToMask(BooleanVector mask, boolean hadMultivaluedFields) implements Releasable {
+    @Override
+    public void close() {
+        mask.close();
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java
index 9a5688685374d..62373c717d609 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java
@@ -37,6 +37,14 @@ public interface Vector extends Accountable, RefCounted, Releasable {
      */
     Vector filter(int... positions);
 
+    /**
+     * Build a {@link Block} the same values as this {@link Vector}, but replacing
+     * all values for which {@code mask.getBooleanValue(position)} returns
+     * {@code false} with {@code null}. The {@code mask} vector must be at least
+     * as long as this {@linkplain Vector}.
+     */
+    Block keepMask(BooleanVector mask);
+
     /**
      * Builds an Iterator of new {@link Block}s with the same {@link #elementType}
      * as this {@link Vector} whose values are copied from positions in this Vector.
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st
index 1f4285686f03a..e855e6d6296d8 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st
@@ -101,6 +101,28 @@ $if(BytesRef)$
     public OrdinalBytesRefBlock asOrdinals() {
         return null;
     }
+
+$elseif(boolean)$
+    @Override
+    public ToMask toMask() {
+        if (getPositionCount() == 0) {
+            return new ToMask(blockFactory().newConstantBooleanVector(false, 0), false);
+        }
+        try (BooleanVector.FixedBuilder builder = blockFactory().newBooleanVectorFixedBuilder(getPositionCount())) {
+            boolean hasMv = false;
+            for (int p = 0; p < getPositionCount(); p++) {
+                builder.appendBoolean(switch (getValueCount(p)) {
+                    case 0 -> false;
+                    case 1 -> getBoolean(getFirstValueIndex(p));
+                    default -> {
+                        hasMv = true;
+                        yield false;
+                    }
+                });
+            }
+            return new ToMask(builder.build(), hasMv);
+        }
+    }
 $endif$
 
     @Override
@@ -140,6 +162,50 @@ $endif$
         }
     }
 
+    @Override
+    public $Type$Block keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return ($Type$Block) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+$if(BytesRef)$
+        BytesRef scratch = new BytesRef();
+$endif$
+        try ($Type$Block.Builder builder = blockFactory().new$Type$BlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.append$Type$(get$Type$(start$if(BytesRef)$, scratch$endif$));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.append$Type$(get$Type$(i$if(BytesRef)$, scratch$endif$));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<$Type$Block> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new $Type$Lookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st
index d594d32898d36..9b5b8b65e8c66 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st
@@ -172,6 +172,35 @@ $endif$
         }
     }
 
+    @Override
+    public $Type$Block keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new $Type$VectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new $Type$VectorBlock(this);
+            }
+            return ($Type$Block) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+$if(BytesRef)$
+        BytesRef scratch = new BytesRef();
+$endif$
+        try ($Type$Block.Builder builder = blockFactory().new$Type$BlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.append$Type$(get$Type$(p$if(BytesRef)$, scratch$endif$));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<$Type$Block> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new $Type$Lookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st
index 66bdcc5d39fb0..23632bf41349c 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st
@@ -86,6 +86,29 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty
         return null;
     }
 
+$if(boolean)$
+    @Override
+    public ToMask toMask() {
+        if (getPositionCount() == 0) {
+            return new ToMask(blockFactory().newConstantBooleanVector(false, 0), false);
+        }
+        try (BooleanVector.FixedBuilder builder = blockFactory().newBooleanVectorFixedBuilder(getPositionCount())) {
+            boolean hasMv = false;
+            for (int p = 0; p < getPositionCount(); p++) {
+                builder.appendBoolean(switch (getValueCount(p)) {
+                    case 0 -> false;
+                    case 1 -> getBoolean(getFirstValueIndex(p));
+                    default -> {
+                        hasMv = true;
+                        yield false;
+                    }
+                });
+            }
+            return new ToMask(builder.build(), hasMv);
+        }
+    }
+$endif$
+
     @Override
     public $type$ get$Type$(int valueIndex) {
         return vector.get$Type$(valueIndex);
@@ -115,6 +138,50 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty
         }
     }
 
+    @Override
+    public $Type$Block keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return this;
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return this;
+            }
+            return ($Type$Block) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+$if(BytesRef)$
+        BytesRef scratch = new BytesRef();
+$endif$
+        try ($Type$Block.Builder builder = blockFactory().new$Type$BlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (false == mask.getBoolean(p)) {
+                    builder.appendNull();
+                    continue;
+                }
+                int valueCount = getValueCount(p);
+                if (valueCount == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+                int start = getFirstValueIndex(p);
+                if (valueCount == 1) {
+                    builder.append$Type$(get$Type$(start$if(BytesRef)$, scratch$endif$));
+                    continue;
+                }
+                int end = start + valueCount;
+                builder.beginPositionEntry();
+                for (int i = start; i < end; i++) {
+                    builder.append$Type$(get$Type$(i$if(BytesRef)$, scratch$endif$));
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<$Type$Block> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new $Type$Lookup(this, positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st
index 85bf2b086e3b2..b509313b079dd 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st
@@ -158,6 +158,32 @@ $endif$
         return new $Type$BigArrayVector(filtered, positions.length, blockFactory);
     }
 
+    @Override
+    public $Type$Block keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new $Type$VectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new $Type$VectorBlock(this);
+            }
+            return ($Type$Block) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+        try ($Type$Block.Builder builder = blockFactory().new$Type$BlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.append$Type$(get$Type$(p));
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public ReleasableIterator<$Type$Block> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return new $Type$Lookup(asBlock(), positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st
index dc6f4ee1003cf..67e4ac4bb334f 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st
@@ -63,11 +63,22 @@ $if(BytesRef)$
      * returns null. Callers must not release the returned block as no extra reference is retained by this method.
      */
     OrdinalBytesRefBlock asOrdinals();
-$endif$
 
+$elseif(boolean)$
+    /**
+     * Convert this to a {@link BooleanVector "mask"} that's appropriate for
+     * passing to {@link #keepMask}. Null and multivalued positions will be
+     * converted to {@code false}.
+     */
+    ToMask toMask();
+
+$endif$
     @Override
     $Type$Block filter(int... positions);
 
+    @Override
+    $Type$Block keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends $Type$Block> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -129,15 +140,15 @@ $endif$
         if (vector != null) {
             out.writeByte(SERIALIZE_BLOCK_VECTOR);
             vector.writeTo(out);
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof $Type$ArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof $Type$ArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_ARRAY);
             b.writeArrayBlock(out);
 $if(BytesRef)$
-        } else if (version.onOrAfter(TransportVersions.ESQL_ORDINAL_BLOCK) && this instanceof OrdinalBytesRefBlock b && b.isDense()) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof OrdinalBytesRefBlock b && b.isDense()) {
             out.writeByte(SERIALIZE_BLOCK_ORDINAL);
             b.writeOrdinalBlock(out);
 $else$
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof $Type$BigArrayBlock b) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof $Type$BigArrayBlock b) {
             out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY);
             b.writeArrayBlock(out);
 $endif$
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st
index 42c34128121a8..72999ffd96fae 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st
@@ -12,8 +12,14 @@ import org.apache.lucene.util.BytesRef;
 $endif$
 import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.common.unit.ByteSizeValue;
+$if(BytesRef)$
+import org.elasticsearch.core.ReleasableIterator;
+import org.elasticsearch.core.Releasables;
+
+$else$
 import org.elasticsearch.core.ReleasableIterator;
 
+$endif$
 /**
  * Vector implementation that stores a constant $type$ value.
  * This class is generated. Do not edit it.
@@ -60,6 +66,49 @@ $endif$
         return blockFactory().newConstant$Type$Vector(value, positions.length);
     }
 
+    @Override
+    public $Type$Block keepMask(BooleanVector mask) {
+        if (getPositionCount() == 0) {
+            incRef();
+            return new $Type$VectorBlock(this);
+        }
+        if (mask.isConstant()) {
+            if (mask.getBoolean(0)) {
+                incRef();
+                return new $Type$VectorBlock(this);
+            }
+            return ($Type$Block) blockFactory().newConstantNullBlock(getPositionCount());
+        }
+$if(BytesRef)$
+        IntBlock ordinals = null;
+        BytesRefVector bytes = null;
+        try {
+            try (IntVector unmaskedOrdinals = blockFactory().newConstantIntVector(0, getPositionCount())) {
+                ordinals = unmaskedOrdinals.keepMask(mask);
+            }
+            bytes = blockFactory().newConstantBytesRefVector(value, getPositionCount());
+            OrdinalBytesRefBlock result = new OrdinalBytesRefBlock(ordinals, bytes);
+            ordinals = null;
+            bytes = null;
+            return result;
+        } finally {
+            Releasables.close(ordinals, bytes);
+        }
+$else$
+        try ($Type$Block.Builder builder = blockFactory().new$Type$BlockBuilder(getPositionCount())) {
+            // TODO if X-ArrayBlock used BooleanVector for it's null mask then we could shuffle references here.
+            for (int p = 0; p < getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    builder.append$Type$(value);
+                } else {
+                    builder.appendNull();
+                }
+            }
+            return builder.build();
+        }
+$endif$
+    }
+
     @Override
     public ReleasableIterator<$Type$Block> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         if (positions.getPositionCount() == 0) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st
index 28332648b5d3f..e19c1788cdb6b 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st
@@ -51,11 +51,14 @@ $if(BytesRef)$
      * returns null. Callers must not release the returned vector as no extra reference is retained by this method.
      */
     OrdinalBytesRefVector asOrdinals();
-$endif$
 
+$endif$
     @Override
     $Type$Vector filter(int... positions);
 
+    @Override
+    $Type$Block keepMask(BooleanVector mask);
+
     @Override
     ReleasableIterator<? extends $Type$Block> lookup(IntBlock positions, ByteSizeValue targetBlockSize);
 
@@ -166,15 +169,15 @@ $if(BytesRef)$
 $else$
             out.write$Type$(get$Type$(0));
 $endif$
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof $Type$ArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof $Type$ArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_ARRAY);
             v.writeArrayVector(positions, out);
 $if(BytesRef)$
-        } else if (version.onOrAfter(TransportVersions.ESQL_ORDINAL_BLOCK) && this instanceof OrdinalBytesRefVector v && v.isDense()) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof OrdinalBytesRefVector v && v.isDense()) {
             out.writeByte(SERIALIZE_VECTOR_ORDINAL);
             v.writeOrdinalVector(out);
 $else$
-        } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof $Type$BigArrayVector v) {
+        } else if (version.onOrAfter(TransportVersions.V_8_14_0) && this instanceof $Type$BigArrayVector v) {
             out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY);
             v.writeArrayVector(positions, out);
 $endif$
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st
index 8f4390e8782c5..d4c6859e64b2a 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st
@@ -44,8 +44,15 @@ $if(BytesRef)$
             return null;
         }
     }
-$endif$
 
+$elseif(boolean)$
+    @Override
+    public ToMask toMask() {
+        vector.incRef();
+        return new ToMask(vector, false);
+    }
+
+$endif$
     @Override
 $if(BytesRef)$
     public BytesRef getBytesRef(int valueIndex, BytesRef dest) {
@@ -71,6 +78,11 @@ $endif$
         return vector.filter(positions).asBlock();
     }
 
+    @Override
+    public $Type$Block keepMask(BooleanVector mask) {
+        return vector.keepMask(mask);
+    }
+
     @Override
     public ReleasableIterator<? extends $Type$Block> lookup(IntBlock positions, ByteSizeValue targetBlockSize) {
         return vector.lookup(positions, targetBlockSize);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/IpBucketedSort.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/IpBucketedSort.java
new file mode 100644
index 0000000000000..0fd38c18d7504
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/IpBucketedSort.java
@@ -0,0 +1,405 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.data.sort;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.BitArray;
+import org.elasticsearch.common.util.ByteArray;
+import org.elasticsearch.common.util.ByteUtils;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.search.sort.BucketedSort;
+import org.elasticsearch.search.sort.SortOrder;
+
+import java.util.Arrays;
+import java.util.stream.IntStream;
+
+/**
+ * Aggregates the top N IP values per bucket.
+ * See {@link BucketedSort} for more information.
+ */
+public class IpBucketedSort implements Releasable {
+    private static final int IP_LENGTH = 16;
+
+    // BytesRefs used in internal methods
+    private final BytesRef scratch1 = new BytesRef();
+    private final BytesRef scratch2 = new BytesRef();
+    /**
+     * Bytes used as temporal storage for scratches
+     */
+    private final byte[] scratchBytes = new byte[IP_LENGTH];
+
+    private final BigArrays bigArrays;
+    private final SortOrder order;
+    private final int bucketSize;
+    /**
+     * {@code true} if the bucket is in heap mode, {@code false} if
+     * it is still gathering.
+     */
+    private final BitArray heapMode;
+    /**
+     * An array containing all the values on all buckets. The structure is as follows:
+     * <p>
+     *     For each bucket, there are bucketSize elements, based on the bucket id (0, 1, 2...).
+     *     Then, for each bucket, it can be in 2 states:
+     * </p>
+     * <ul>
+     *     <li>
+     *         Gather mode: All buckets start in gather mode, and remain here while they have less than bucketSize elements.
+     *         In gather mode, the elements are stored in the array from the highest index to the lowest index.
+     *         The lowest index contains the offset to the next slot to be filled.
+     *         <p>
+     *             This allows us to insert elements in O(1) time.
+     *         </p>
+     *         <p>
+     *             When the bucketSize-th element is collected, the bucket transitions to heap mode, by heapifying its contents.
+     *         </p>
+     *     </li>
+     *     <li>
+     *         Heap mode: The bucket slots are organized as a min heap structure.
+     *         <p>
+     *             The root of the heap is the minimum value in the bucket,
+     *             which allows us to quickly discard new values that are not in the top N.
+     *         </p>
+     *     </li>
+     * </ul>
+     */
+    private ByteArray values;
+
+    public IpBucketedSort(BigArrays bigArrays, SortOrder order, int bucketSize) {
+        this.bigArrays = bigArrays;
+        this.order = order;
+        this.bucketSize = bucketSize;
+        heapMode = new BitArray(0, bigArrays);
+
+        boolean success = false;
+        try {
+            values = bigArrays.newByteArray(0, false);
+            success = true;
+        } finally {
+            if (success == false) {
+                close();
+            }
+        }
+    }
+
+    /**
+     * Collects a {@code value} into a {@code bucket}.
+     * <p>
+     *     It may or may not be inserted in the heap, depending on if it is better than the current root.
+     * </p>
+     */
+    public void collect(BytesRef value, int bucket) {
+        assert value.length == IP_LENGTH;
+        long rootIndex = (long) bucket * bucketSize;
+        if (inHeapMode(bucket)) {
+            if (betterThan(value, get(rootIndex, scratch1))) {
+                set(rootIndex, value);
+                downHeap(rootIndex, 0);
+            }
+            return;
+        }
+        // Gathering mode
+        long requiredSize = (rootIndex + bucketSize) * IP_LENGTH;
+        if (values.size() < requiredSize) {
+            grow(requiredSize);
+        }
+        int next = getNextGatherOffset(rootIndex);
+        assert 0 <= next && next < bucketSize
+            : "Expected next to be in the range of valid buckets [0 <= " + next + " < " + bucketSize + "]";
+        long index = next + rootIndex;
+        set(index, value);
+        if (next == 0) {
+            heapMode.set(bucket);
+            heapify(rootIndex);
+        } else {
+            setNextGatherOffset(rootIndex, next - 1);
+        }
+    }
+
+    /**
+     * The order of the sort.
+     */
+    public SortOrder getOrder() {
+        return order;
+    }
+
+    /**
+     * The number of values to store per bucket.
+     */
+    public int getBucketSize() {
+        return bucketSize;
+    }
+
+    /**
+     * Get the first and last indexes (inclusive, exclusive) of the values for a bucket.
+     * Returns [0, 0] if the bucket has never been collected.
+     */
+    private Tuple<Long, Long> getBucketValuesIndexes(int bucket) {
+        long rootIndex = (long) bucket * bucketSize;
+        if (rootIndex >= values.size() / IP_LENGTH) {
+            // We've never seen this bucket.
+            return Tuple.tuple(0L, 0L);
+        }
+        long start = inHeapMode(bucket) ? rootIndex : (rootIndex + getNextGatherOffset(rootIndex) + 1);
+        long end = rootIndex + bucketSize;
+        return Tuple.tuple(start, end);
+    }
+
+    /**
+     * Merge the values from {@code other}'s {@code otherGroupId} into {@code groupId}.
+     */
+    public void merge(int groupId, IpBucketedSort other, int otherGroupId) {
+        var otherBounds = other.getBucketValuesIndexes(otherGroupId);
+        var scratch = new BytesRef();
+
+        // TODO: This can be improved for heapified buckets by making use of the heap structures
+        for (long i = otherBounds.v1(); i < otherBounds.v2(); i++) {
+            collect(other.get(i, scratch), groupId);
+        }
+    }
+
+    /**
+     * Creates a block with the values from the {@code selected} groups.
+     */
+    public Block toBlock(BlockFactory blockFactory, IntVector selected) {
+        // Check if the selected groups are all empty, to avoid allocating extra memory
+        if (IntStream.range(0, selected.getPositionCount()).map(selected::getInt).noneMatch(bucket -> {
+            var bounds = this.getBucketValuesIndexes(bucket);
+            var size = bounds.v2() - bounds.v1();
+
+            return size > 0;
+        })) {
+            return blockFactory.newConstantNullBlock(selected.getPositionCount());
+        }
+
+        // Used to sort the values in the bucket.
+        var bucketValues = new BytesRef[bucketSize];
+
+        try (var builder = blockFactory.newBytesRefBlockBuilder(selected.getPositionCount())) {
+            for (int s = 0; s < selected.getPositionCount(); s++) {
+                int bucket = selected.getInt(s);
+
+                var bounds = getBucketValuesIndexes(bucket);
+                var size = bounds.v2() - bounds.v1();
+
+                if (size == 0) {
+                    builder.appendNull();
+                    continue;
+                }
+
+                if (size == 1) {
+                    builder.appendBytesRef(get(bounds.v1(), scratch1));
+                    continue;
+                }
+
+                for (int i = 0; i < size; i++) {
+                    bucketValues[i] = get(bounds.v1() + i, new BytesRef());
+                }
+
+                // TODO: Make use of heap structures to faster iterate in order instead of copying and sorting
+                Arrays.sort(bucketValues, 0, (int) size);
+
+                builder.beginPositionEntry();
+                if (order == SortOrder.ASC) {
+                    for (int i = 0; i < size; i++) {
+                        builder.appendBytesRef(bucketValues[i]);
+                    }
+                } else {
+                    for (int i = (int) size - 1; i >= 0; i--) {
+                        builder.appendBytesRef(bucketValues[i]);
+                    }
+                }
+                builder.endPositionEntry();
+            }
+            return builder.build();
+        }
+    }
+
+    /**
+     * Is this bucket a min heap {@code true} or in gathering mode {@code false}?
+     */
+    private boolean inHeapMode(int bucket) {
+        return heapMode.get(bucket);
+    }
+
+    /**
+     * Get the next index that should be "gathered" for a bucket rooted
+     * at {@code rootIndex}.
+     * <p>
+     *     Using the first 4 bytes of the element to store the next gather offset.
+     * </p>
+     */
+    private int getNextGatherOffset(long rootIndex) {
+        values.get(rootIndex * IP_LENGTH, Integer.BYTES, scratch1);
+        assert scratch1.length == Integer.BYTES;
+        return ByteUtils.readIntLE(scratch1.bytes, scratch1.offset);
+    }
+
+    /**
+     * Set the next index that should be "gathered" for a bucket rooted
+     * at {@code rootIndex}.
+     * <p>
+     *     Using the first {@code Integer.BYTES} bytes of the element to store the next gather offset.
+     * </p>
+     */
+    private void setNextGatherOffset(long rootIndex, int offset) {
+        scratch1.bytes = scratchBytes;
+        scratch1.offset = 0;
+        scratch1.length = Integer.BYTES;
+        ByteUtils.writeIntLE(offset, scratch1.bytes, scratch1.offset);
+        values.set(rootIndex * IP_LENGTH, scratch1.bytes, scratch1.offset, scratch1.length);
+    }
+
+    /**
+     * {@code true} if the entry at index {@code lhs} is "better" than
+     * the entry at {@code rhs}. "Better" in this means "lower" for
+     * {@link SortOrder#ASC} and "higher" for {@link SortOrder#DESC}.
+     */
+    private boolean betterThan(BytesRef lhs, BytesRef rhs) {
+        return getOrder().reverseMul() * lhs.compareTo(rhs) < 0;
+    }
+
+    /**
+     * Swap the data at two indices.
+     */
+    private void swap(long lhs, long rhs) {
+        // var tmp = values.get(lhs);
+        values.get(lhs * IP_LENGTH, IP_LENGTH, scratch1);
+        assert scratch1.length == IP_LENGTH;
+        System.arraycopy(scratch1.bytes, scratch1.offset, scratchBytes, 0, scratch1.length);
+
+        // values.set(lhs, values.get(rhs));
+        values.get(rhs * IP_LENGTH, IP_LENGTH, scratch2);
+        assert scratch2.length == IP_LENGTH;
+        values.set(lhs * IP_LENGTH, scratch2.bytes, scratch2.offset, scratch2.length);
+
+        // values.set(rhs, tmp);
+        scratch1.bytes = scratchBytes;
+        scratch1.offset = 0;
+        values.set(rhs * IP_LENGTH, scratch1.bytes, scratch1.offset, scratch1.length);
+    }
+
+    /**
+     * Allocate storage for more buckets and store the "next gather offset"
+     * for those new buckets.
+     */
+    private void grow(long minSize) {
+        long oldMax = values.size() / IP_LENGTH;
+        values = bigArrays.grow(values, minSize);
+        // Set the next gather offsets for all newly allocated buckets.
+        setNextGatherOffsets(oldMax - (oldMax % bucketSize));
+    }
+
+    /**
+     * Maintain the "next gather offsets" for newly allocated buckets.
+     */
+    private void setNextGatherOffsets(long startingAt) {
+        int nextOffset = bucketSize - 1;
+        for (long bucketRoot = startingAt; bucketRoot < values.size() / IP_LENGTH; bucketRoot += bucketSize) {
+            setNextGatherOffset(bucketRoot, nextOffset);
+        }
+    }
+
+    /**
+     * Heapify a bucket whose entries are in random order.
+     * <p>
+     * This works by validating the heap property on each node, iterating
+     * "upwards", pushing any out of order parents "down". Check out the
+     * <a href="https://en.wikipedia.org/w/index.php?title=Binary_heap&oldid=940542991#Building_a_heap">wikipedia</a>
+     * entry on binary heaps for more about this.
+     * </p>
+     * <p>
+     * While this *looks* like it could easily be {@code O(n * log n)}, it is
+     * a fairly well studied algorithm attributed to Floyd. There's
+     * been a bunch of work that puts this at {@code O(n)}, close to 1.88n worst
+     * case.
+     * </p>
+     * <ul>
+     * <li>Hayward, Ryan; McDiarmid, Colin (1991).
+     * <a href="https://web.archive.org/web/20160205023201/http://www.stats.ox.ac.uk/__data/assets/pdf_file/0015/4173/heapbuildjalg.pdf">
+     * Average Case Analysis of Heap Building byRepeated Insertion</a> J. Algorithms.
+     * <li>D.E. Knuth, ”The Art of Computer Programming, Vol. 3, Sorting and Searching”</li>
+     * </ul>
+     * @param rootIndex the index the start of the bucket
+     */
+    private void heapify(long rootIndex) {
+        int maxParent = bucketSize / 2 - 1;
+        for (int parent = maxParent; parent >= 0; parent--) {
+            downHeap(rootIndex, parent);
+        }
+    }
+
+    /**
+     * Correct the heap invariant of a parent and its children. This
+     * runs in {@code O(log n)} time.
+     * @param rootIndex index of the start of the bucket
+     * @param parent Index within the bucket of the parent to check.
+     *               For example, 0 is the "root".
+     */
+    private void downHeap(long rootIndex, int parent) {
+        while (true) {
+            long parentIndex = rootIndex + parent;
+            int worst = parent;
+            long worstIndex = parentIndex;
+            int leftChild = parent * 2 + 1;
+            long leftIndex = rootIndex + leftChild;
+            if (leftChild < bucketSize) {
+                if (betterThan(get(worstIndex, scratch1), get(leftIndex, scratch2))) {
+                    worst = leftChild;
+                    worstIndex = leftIndex;
+                }
+                int rightChild = leftChild + 1;
+                long rightIndex = rootIndex + rightChild;
+                if (rightChild < bucketSize && betterThan(get(worstIndex, scratch1), get(rightIndex, scratch2))) {
+                    worst = rightChild;
+                    worstIndex = rightIndex;
+                }
+            }
+            if (worst == parent) {
+                break;
+            }
+            swap(worstIndex, parentIndex);
+            parent = worst;
+        }
+    }
+
+    /**
+     * Get the IP value at {@code index} and store it in {@code scratch}.
+     * Returns {@code scratch}.
+     * <p>
+     *     {@code index} is an IP index, not a byte index.
+     * </p>
+     */
+    private BytesRef get(long index, BytesRef scratch) {
+        values.get(index * IP_LENGTH, IP_LENGTH, scratch);
+        assert scratch.length == IP_LENGTH;
+        return scratch;
+    }
+
+    /**
+     * Set the IP value at {@code index}.
+     * <p>
+     *     {@code index} is an IP index, not a byte index.
+     * </p>
+     */
+    private void set(long index, BytesRef value) {
+        assert value.length == IP_LENGTH;
+        values.set(index * IP_LENGTH, value.bytes, value.offset, value.length);
+    }
+
+    @Override
+    public final void close() {
+        Releasables.close(values, heapMode);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java
index 11ccfb55a77aa..6f75298e95dd7 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java
@@ -309,7 +309,7 @@ private Status(LuceneOperator operator) {
                 processedQueries = Collections.emptySet();
                 processedShards = Collections.emptySet();
             }
-            processingNanos = in.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS) ? in.readVLong() : 0;
+            processingNanos = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readVLong() : 0;
             sliceIndex = in.readVInt();
             totalSlices = in.readVInt();
             pagesEmitted = in.readVInt();
@@ -325,7 +325,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeCollection(processedQueries, StreamOutput::writeString);
                 out.writeCollection(processedShards, StreamOutput::writeString);
             }
-            if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 out.writeVLong(processingNanos);
             }
             out.writeVInt(sliceIndex);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneQueryExpressionEvaluator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneQueryExpressionEvaluator.java
new file mode 100644
index 0000000000000..dcd13671670d8
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneQueryExpressionEvaluator.java
@@ -0,0 +1,327 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.lucene;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.BulkScorer;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorable;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.data.DocBlock;
+import org.elasticsearch.compute.data.DocVector;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} to run a Lucene {@link Query} during
+ * the compute engine's normal execution, yielding matches/does not match into
+ * a {@link BooleanVector}. It's much faster to push these to the
+ * {@link LuceneSourceOperator} or the like, but sometimes this isn't possible. So
+ * this evaluator is here to save the day.
+ */
+public class LuceneQueryExpressionEvaluator implements EvalOperator.ExpressionEvaluator {
+    public record ShardConfig(Query query, IndexSearcher searcher) {}
+
+    private final BlockFactory blockFactory;
+    private final ShardConfig[] shards;
+    private final int docChannel;
+
+    private ShardState[] perShardState = EMPTY_SHARD_STATES;
+
+    public LuceneQueryExpressionEvaluator(BlockFactory blockFactory, ShardConfig[] shards, int docChannel) {
+        this.blockFactory = blockFactory;
+        this.shards = shards;
+        this.docChannel = docChannel;
+    }
+
+    @Override
+    public Block eval(Page page) {
+        DocVector docs = page.<DocBlock>getBlock(docChannel).asVector();
+        try {
+            if (docs.singleSegmentNonDecreasing()) {
+                return evalSingleSegmentNonDecreasing(docs).asBlock();
+            } else {
+                return evalSlow(docs).asBlock();
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    /**
+     * Evaluate {@link DocVector#singleSegmentNonDecreasing()} documents.
+     * <p>
+     *     ESQL receives documents in DocVector, and they can be in one of two
+     *     states. Either the DocVector contains documents from a single segment
+     *     non-decreasing order, or it doesn't. that first case is much more like
+     *     how Lucene likes to process documents. and it's much more common. So we
+     *     optimize for it.
+     * <p>
+     *     Vectors that are {@link DocVector#singleSegmentNonDecreasing()}
+     *     represent many documents from a single Lucene segment. In Elasticsearch
+     *     terms that's a segment in a single shard. And the document ids are in
+     *     non-decreasing order. Probably just {@code 0, 1, 2, 3, 4, 5...}.
+     *     But maybe something like {@code 0, 5, 6, 10, 10, 10}. Both of those are
+     *     very like how lucene "natively" processes documents and this optimizes
+     *     those accesses.
+     * </p>
+     * <p>
+     *     If the documents are literally {@code 0, 1, ... n} then we use
+     *     {@link BulkScorer#score(LeafCollector, Bits, int, int)} which processes
+     *     a whole range. This should be quite common in the case where we don't
+     *     have deleted documents because that's the order that
+     *     {@link LuceneSourceOperator} produces them.
+     * </p>
+     * <p>
+     *     If there are gaps in the sequence we use {@link Scorer} calls to
+     *     score the sequence. This'll be less fast but isn't going be particularly
+     *     common.
+     * </p>
+     */
+    private BooleanVector evalSingleSegmentNonDecreasing(DocVector docs) throws IOException {
+        ShardState shardState = shardState(docs.shards().getInt(0));
+        SegmentState segmentState = shardState.segmentState(docs.segments().getInt(0));
+        int min = docs.docs().getInt(0);
+        int max = docs.docs().getInt(docs.getPositionCount() - 1);
+        int length = max - min + 1;
+        if (length == docs.getPositionCount() && length > 1) {
+            return segmentState.scoreDense(min, max);
+        }
+        return segmentState.scoreSparse(docs.docs());
+    }
+
+    /**
+     * Evaluate non-{@link DocVector#singleSegmentNonDecreasing()} documents. See
+     * {@link #evalSingleSegmentNonDecreasing} for the meaning of
+     * {@link DocVector#singleSegmentNonDecreasing()} and how we can efficiently
+     * evaluate those segments.
+     * <p>
+     *     This processes the worst case blocks of documents. These can be from any
+     *     number of shards and any number of segments and in any order. We do this
+     *     by iterating the docs in {@code shard ASC, segment ASC, doc ASC} order.
+     *     So, that's segment by segment, docs ascending. We build a boolean block
+     *     out of that. Then we <strong>sort</strong> that to put the booleans in
+     *     the order that the {@link DocVector} came in.
+     * </p>
+     */
+    private BooleanVector evalSlow(DocVector docs) throws IOException {
+        int[] map = docs.shardSegmentDocMapForwards();
+        // Clear any state flags from the previous run
+        int prevShard = -1;
+        int prevSegment = -1;
+        SegmentState segmentState = null;
+        try (BooleanVector.Builder builder = blockFactory.newBooleanVectorFixedBuilder(docs.getPositionCount())) {
+            for (int i = 0; i < docs.getPositionCount(); i++) {
+                int shard = docs.shards().getInt(docs.shards().getInt(map[i]));
+                int segment = docs.segments().getInt(map[i]);
+                if (segmentState == null || prevShard != shard || prevSegment != segment) {
+                    segmentState = shardState(shard).segmentState(segment);
+                    segmentState.initScorer(docs.docs().getInt(map[i]));
+                    prevShard = shard;
+                    prevSegment = segment;
+                }
+                if (segmentState.noMatch) {
+                    builder.appendBoolean(false);
+                } else {
+                    segmentState.scoreSingleDocWithScorer(builder, docs.docs().getInt(map[i]));
+                }
+            }
+            try (BooleanVector outOfOrder = builder.build()) {
+                return outOfOrder.filter(docs.shardSegmentDocMapBackwards());
+            }
+        }
+    }
+
+    @Override
+    public void close() {
+
+    }
+
+    private ShardState shardState(int shard) throws IOException {
+        if (shard >= perShardState.length) {
+            perShardState = ArrayUtil.grow(perShardState, shard + 1);
+        } else if (perShardState[shard] != null) {
+            return perShardState[shard];
+        }
+        perShardState[shard] = new ShardState(shards[shard]);
+        return perShardState[shard];
+    }
+
+    private class ShardState {
+        private final Weight weight;
+        private final IndexSearcher searcher;
+        private SegmentState[] perSegmentState = EMPTY_SEGMENT_STATES;
+
+        ShardState(ShardConfig config) throws IOException {
+            weight = config.searcher.createWeight(config.query, ScoreMode.COMPLETE_NO_SCORES, 0.0f);
+            searcher = config.searcher;
+        }
+
+        SegmentState segmentState(int segment) throws IOException {
+            if (segment >= perSegmentState.length) {
+                perSegmentState = ArrayUtil.grow(perSegmentState, segment + 1);
+            } else if (perSegmentState[segment] != null) {
+                return perSegmentState[segment];
+            }
+            perSegmentState[segment] = new SegmentState(weight, searcher.getLeafContexts().get(segment));
+            return perSegmentState[segment];
+        }
+    }
+
+    private class SegmentState {
+        private final Weight weight;
+        private final LeafReaderContext ctx;
+        /**
+         * Lazily initialed {@link Scorer} for this. {@code null} here means uninitialized
+         * <strong>or</strong> that {@link #noMatch} is true.
+         */
+        private Scorer scorer;
+
+        /**
+         * Lazily initialed {@link BulkScorer} for this. {@code null} here means uninitialized
+         * <strong>or</strong> that {@link #noMatch} is true.
+         */
+        private BulkScorer bulkScorer;
+
+        /**
+         * Set to {@code true} if, in the process of building a {@link Scorer} or {@link BulkScorer},
+         * the {@link Weight} tells us there aren't any matches.
+         */
+        private boolean noMatch;
+
+        private SegmentState(Weight weight, LeafReaderContext ctx) {
+            this.weight = weight;
+            this.ctx = ctx;
+        }
+
+        /**
+         * Score a range using the {@link BulkScorer}. This should be faster
+         * than using {@link #scoreSparse} for dense doc ids.
+         */
+        BooleanVector scoreDense(int min, int max) throws IOException {
+            int length = max - min + 1;
+            if (noMatch) {
+                return blockFactory.newConstantBooleanVector(false, length);
+            }
+            if (bulkScorer == null) {
+                bulkScorer = weight.bulkScorer(ctx);
+                if (bulkScorer == null) {
+                    noMatch = true;
+                    return blockFactory.newConstantBooleanVector(false, length);
+                }
+            }
+            try (DenseCollector collector = new DenseCollector(blockFactory, min, max)) {
+                bulkScorer.score(collector, ctx.reader().getLiveDocs(), min, max + 1);
+                return collector.build();
+            }
+        }
+
+        /**
+         * Score a vector of doc ids using {@link Scorer}. If you have a dense range of
+         * doc ids it'd be faster to use {@link #scoreDense}.
+         */
+        BooleanVector scoreSparse(IntVector docs) throws IOException {
+            initScorer(docs.getInt(0));
+            if (noMatch) {
+                return blockFactory.newConstantBooleanVector(false, docs.getPositionCount());
+            }
+            try (BooleanVector.Builder builder = blockFactory.newBooleanVectorFixedBuilder(docs.getPositionCount())) {
+                for (int i = 0; i < docs.getPositionCount(); i++) {
+                    scoreSingleDocWithScorer(builder, docs.getInt(i));
+                }
+                return builder.build();
+            }
+        }
+
+        private void initScorer(int minDocId) throws IOException {
+            if (noMatch) {
+                return;
+            }
+            if (scorer == null || scorer.iterator().docID() > minDocId) {
+                // The previous block might have been beyond this one, reset the scorer and try again.
+                scorer = weight.scorer(ctx);
+                if (scorer == null) {
+                    noMatch = true;
+                }
+            }
+        }
+
+        private void scoreSingleDocWithScorer(BooleanVector.Builder builder, int doc) throws IOException {
+            if (scorer.iterator().docID() == doc) {
+                builder.appendBoolean(true);
+            } else if (scorer.iterator().docID() > doc) {
+                builder.appendBoolean(false);
+            } else {
+                builder.appendBoolean(scorer.iterator().advance(doc) == doc);
+            }
+        }
+    }
+
+    private static final ShardState[] EMPTY_SHARD_STATES = new ShardState[0];
+    private static final SegmentState[] EMPTY_SEGMENT_STATES = new SegmentState[0];
+
+    /**
+     * Collects matching information for dense range of doc ids. This assumes that
+     * doc ids are sent to {@link LeafCollector#collect(int)} in ascending order
+     * which isn't documented, but @jpountz swears is true.
+     */
+    static class DenseCollector implements LeafCollector, Releasable {
+        private final BooleanVector.FixedBuilder builder;
+        private final int max;
+
+        int next;
+
+        DenseCollector(BlockFactory blockFactory, int min, int max) {
+            this.builder = blockFactory.newBooleanVectorFixedBuilder(max - min + 1);
+            this.max = max;
+            next = min;
+        }
+
+        @Override
+        public void setScorer(Scorable scorable) {}
+
+        @Override
+        public void collect(int doc) {
+            while (next++ < doc) {
+                builder.appendBoolean(false);
+            }
+            builder.appendBoolean(true);
+        }
+
+        public BooleanVector build() {
+            return builder.build();
+        }
+
+        @Override
+        public void finish() {
+            while (next++ <= max) {
+                builder.appendBoolean(false);
+            }
+        }
+
+        @Override
+        public void close() {
+            Releasables.closeExpectNoException(builder);
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/UnsupportedValueSource.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/UnsupportedValueSource.java
deleted file mode 100644
index 3f2632d9a643f..0000000000000
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/UnsupportedValueSource.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.compute.lucene;
-
-import org.apache.lucene.index.LeafReaderContext;
-import org.elasticsearch.common.Rounding;
-import org.elasticsearch.index.fielddata.DocValueBits;
-import org.elasticsearch.index.fielddata.FieldData;
-import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
-
-import java.io.IOException;
-import java.util.function.Function;
-
-public class UnsupportedValueSource extends ValuesSource {
-
-    public static final String UNSUPPORTED_OUTPUT = null;
-    private final ValuesSource originalSource;
-
-    public UnsupportedValueSource(ValuesSource originalSource) {
-        this.originalSource = originalSource;
-    }
-
-    @Override
-    public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException {
-        if (originalSource != null) {
-            try {
-                return originalSource.bytesValues(context);
-            } catch (Exception e) {
-                // ignore and fall back to UNSUPPORTED_OUTPUT
-            }
-        }
-        return FieldData.emptySortedBinary();
-    }
-
-    @Override
-    public DocValueBits docsWithValue(LeafReaderContext context) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    protected Function<Rounding, Rounding.Prepared> roundingPreparer(AggregationContext context) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingOperator.java
index 800b648711f26..05913b7dd5f69 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AbstractPageMappingOperator.java
@@ -112,13 +112,13 @@ public Status(long processNanos, int pagesProcessed) {
         }
 
         protected Status(StreamInput in) throws IOException {
-            processNanos = in.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS) ? in.readVLong() : 0;
+            processNanos = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readVLong() : 0;
             pagesProcessed = in.readVInt();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 out.writeVLong(processNanos);
             }
             out.writeVInt(pagesProcessed);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java
index 20d3f0166f1cb..260b72dc35d9c 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java
@@ -265,7 +265,7 @@ public String toString() {
 
         @Override
         public TransportVersion getMinimalSupportedVersion() {
-            return TransportVersions.ESQL_TIMINGS;
+            return TransportVersions.V_8_14_0;
         }
     }
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java
index 0fed88370a144..2c36b42dee277 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java
@@ -146,7 +146,7 @@ private void checkFailure() {
         Exception e = failureCollector.getFailure();
         if (e != null) {
             discardPages();
-            throw ExceptionsHelper.convertToElastic(e);
+            throw ExceptionsHelper.convertToRuntime(e);
         }
     }
 
@@ -199,7 +199,7 @@ public Page getOutput() {
     }
 
     @Override
-    public SubscribableListener<Void> isBlocked() {
+    public IsBlockedResult isBlocked() {
         // TODO: Add an exchange service between async operation instead?
         if (finished) {
             return Operator.NOT_BLOCKED;
@@ -216,7 +216,7 @@ public SubscribableListener<Void> isBlocked() {
             if (blockedFuture == null) {
                 blockedFuture = new SubscribableListener<>();
             }
-            return blockedFuture;
+            return new IsBlockedResult(blockedFuture, getClass().getSimpleName());
         }
     }
 
@@ -319,7 +319,7 @@ public String toString() {
 
         @Override
         public TransportVersion getMinimalSupportedVersion() {
-            return TransportVersions.ESQL_ENRICH_OPERATOR_STATUS;
+            return TransportVersions.V_8_14_0;
         }
     }
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java
index 17e67335919b1..2578452ad9062 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java
@@ -131,6 +131,14 @@ public void append(BytesRef bytes) {
         append(bytes.bytes, bytes.offset, bytes.length);
     }
 
+    /**
+     * Set the content of the builder to the given bytes.
+     */
+    public void copyBytes(BytesRef newBytes) {
+        clear();
+        append(newBytes);
+    }
+
     /**
      * Reset the builder to an empty bytes array. Doesn't deallocate any memory.
      */
@@ -141,7 +149,7 @@ public void clear() {
     /**
      * Returns a view of the data added as a {@link BytesRef}. Importantly, this does not
      * copy the bytes and any further modification to the {@link BreakingBytesRefBuilder}
-     * will modify the returned {@link BytesRef}. The called must copy the bytes
+     * will modify the returned {@link BytesRef}. The caller must copy the bytes
      * if they wish to keep them.
      */
     public BytesRef bytesRefView() {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java
index 785db826aadd6..acbf8a17b31fd 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java
@@ -127,7 +127,17 @@ public Driver(
         this.statusNanos = statusInterval.nanos();
         this.releasable = releasable;
         this.status = new AtomicReference<>(
-            new DriverStatus(sessionId, startTime, System.currentTimeMillis(), 0, 0, DriverStatus.Status.QUEUED, List.of(), List.of())
+            new DriverStatus(
+                sessionId,
+                startTime,
+                System.currentTimeMillis(),
+                0,
+                0,
+                DriverStatus.Status.QUEUED,
+                List.of(),
+                List.of(),
+                DriverSleeps.empty()
+            )
         );
     }
 
@@ -170,35 +180,36 @@ public DriverContext driverContext() {
      * thread to do other work instead of blocking or busy-spinning on the blocked operator.
      */
     SubscribableListener<Void> run(TimeValue maxTime, int maxIterations, LongSupplier nowSupplier) {
+        updateStatus(0, 0, DriverStatus.Status.RUNNING, "driver running");
         long maxTimeNanos = maxTime.nanos();
         long startTime = nowSupplier.getAsLong();
         long nextStatus = startTime + statusNanos;
         int iter = 0;
         while (true) {
-            SubscribableListener<Void> fut = runSingleLoopIteration();
+            IsBlockedResult isBlocked = runSingleLoopIteration();
             iter++;
-            if (fut.isDone() == false) {
-                updateStatus(nowSupplier.getAsLong() - startTime, iter, DriverStatus.Status.ASYNC);
-                return fut;
+            if (isBlocked.listener().isDone() == false) {
+                updateStatus(nowSupplier.getAsLong() - startTime, iter, DriverStatus.Status.ASYNC, isBlocked.reason());
+                return isBlocked.listener();
             }
             if (isFinished()) {
                 finishNanos = nowSupplier.getAsLong();
-                updateStatus(finishNanos - startTime, iter, DriverStatus.Status.DONE);
+                updateStatus(finishNanos - startTime, iter, DriverStatus.Status.DONE, "driver done");
                 driverContext.finish();
                 Releasables.close(releasable, driverContext.getSnapshot());
-                return Operator.NOT_BLOCKED;
+                return Operator.NOT_BLOCKED.listener();
             }
             long now = nowSupplier.getAsLong();
             if (iter >= maxIterations) {
-                updateStatus(now - startTime, iter, DriverStatus.Status.WAITING);
-                return Operator.NOT_BLOCKED;
+                updateStatus(now - startTime, iter, DriverStatus.Status.WAITING, "driver iterations");
+                return Operator.NOT_BLOCKED.listener();
             }
             if (now - startTime >= maxTimeNanos) {
-                updateStatus(now - startTime, iter, DriverStatus.Status.WAITING);
-                return Operator.NOT_BLOCKED;
+                updateStatus(now - startTime, iter, DriverStatus.Status.WAITING, "driver time");
+                return Operator.NOT_BLOCKED.listener();
             }
             if (now > nextStatus) {
-                updateStatus(now - startTime, iter, DriverStatus.Status.RUNNING);
+                updateStatus(now - startTime, iter, DriverStatus.Status.RUNNING, "driver running");
                 nextStatus = now + statusNanos;
             }
         }
@@ -230,7 +241,7 @@ public void abort(Exception reason, ActionListener<Void> listener) {
         }
     }
 
-    private SubscribableListener<Void> runSingleLoopIteration() {
+    private IsBlockedResult runSingleLoopIteration() {
         ensureNotCancelled();
         boolean movedPage = false;
 
@@ -239,7 +250,7 @@ private SubscribableListener<Void> runSingleLoopIteration() {
             Operator nextOp = activeOperators.get(i + 1);
 
             // skip blocked operator
-            if (op.isBlocked().isDone() == false) {
+            if (op.isBlocked().listener().isDone() == false) {
                 continue;
             }
 
@@ -290,7 +301,10 @@ private SubscribableListener<Void> runSingleLoopIteration() {
 
         if (movedPage == false) {
             return oneOf(
-                activeOperators.stream().map(Operator::isBlocked).filter(laf -> laf.isDone() == false).collect(Collectors.toList())
+                activeOperators.stream()
+                    .map(Operator::isBlocked)
+                    .filter(laf -> laf.listener().isDone() == false)
+                    .collect(Collectors.toList())
             );
         }
         return Operator.NOT_BLOCKED;
@@ -327,7 +341,7 @@ public static void start(
     ) {
         driver.completionListener.addListener(listener);
         if (driver.started.compareAndSet(false, true)) {
-            driver.updateStatus(0, 0, DriverStatus.Status.STARTING);
+            driver.updateStatus(0, 0, DriverStatus.Status.STARTING, "driver starting");
             schedule(DEFAULT_TIME_BEFORE_YIELDING, maxIterations, threadContext, executor, driver, driver.completionListener);
         }
     }
@@ -394,18 +408,23 @@ void onComplete(ActionListener<Void> listener) {
         });
     }
 
-    private static SubscribableListener<Void> oneOf(List<SubscribableListener<Void>> futures) {
-        if (futures.isEmpty()) {
+    private static IsBlockedResult oneOf(List<IsBlockedResult> results) {
+        if (results.isEmpty()) {
             return Operator.NOT_BLOCKED;
         }
-        if (futures.size() == 1) {
-            return futures.get(0);
+        if (results.size() == 1) {
+            return results.get(0);
         }
         SubscribableListener<Void> oneOf = new SubscribableListener<>();
-        for (SubscribableListener<Void> fut : futures) {
-            fut.addListener(oneOf);
+        StringBuilder reason = new StringBuilder();
+        for (IsBlockedResult r : results) {
+            r.listener().addListener(oneOf);
+            if (reason.isEmpty() == false) {
+                reason.append(" OR ");
+            }
+            reason.append(r.reason());
         }
-        return oneOf;
+        return new IsBlockedResult(oneOf, reason.toString());
     }
 
     @Override
@@ -440,7 +459,15 @@ public DriverProfile profile() {
         if (status.status() != DriverStatus.Status.DONE) {
             throw new IllegalStateException("can only get profile from finished driver");
         }
-        return new DriverProfile(finishNanos - startNanos, status.cpuNanos(), status.iterations(), status.completedOperators());
+        return new DriverProfile(
+            status.started(),
+            status.lastUpdated(),
+            finishNanos - startNanos,
+            status.cpuNanos(),
+            status.iterations(),
+            status.completedOperators(),
+            status.sleeps()
+        );
     }
 
     /**
@@ -449,17 +476,44 @@ public DriverProfile profile() {
      * @param extraIterations how many iterations to add to the previous status
      * @param status the status of the overall driver request
      */
-    private void updateStatus(long extraCpuNanos, int extraIterations, DriverStatus.Status status) {
+    private void updateStatus(long extraCpuNanos, int extraIterations, DriverStatus.Status status, String reason) {
         this.status.getAndUpdate(prev -> {
+            long now = System.currentTimeMillis();
+            DriverSleeps sleeps = prev.sleeps();
+
+            // Rebuild the sleeps or bail entirely based on the updated status.
+            // Sorry for the complexity here. If anyone has a nice way to refactor this, be my guest.
+            switch (status) {
+                case ASYNC, WAITING -> sleeps = sleeps.sleep(reason, now);
+                case RUNNING -> {
+                    switch (prev.status()) {
+                        case ASYNC, WAITING -> sleeps = sleeps.wake(now);
+                        case STARTING -> {
+                            if (extraIterations == 0) {
+                                /*
+                                 * 0 extraIterations means we haven't started the loop - we're just
+                                 * signaling that we've woken up. We don't need to signal that when
+                                 * the state is already STARTING because we don't have anything
+                                 * interesting to report. And some tests rely on the status staying
+                                 * in the STARTING state until the first status report.
+                                 */
+                                return prev;
+                            }
+                        }
+                    }
+                }
+            }
+
             return new DriverStatus(
                 sessionId,
                 startTime,
-                System.currentTimeMillis(),
+                now,
                 prev.cpuNanos() + extraCpuNanos,
                 prev.iterations() + extraIterations,
                 status,
                 statusOfCompletedOperators,
-                activeOperators.stream().map(op -> new DriverStatus.OperatorStatus(op.toString(), op.status())).toList()
+                activeOperators.stream().map(op -> new DriverStatus.OperatorStatus(op.toString(), op.status())).toList(),
+                sleeps
             );
         });
     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java
index 00c3771540867..e7b16072f4b66 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverProfile.java
@@ -27,6 +27,16 @@
  * Profile results from a single {@link Driver}.
  */
 public class DriverProfile implements Writeable, ChunkedToXContentObject {
+    /**
+     * Millis since epoch when the driver started.
+     */
+    private final long startMillis;
+
+    /**
+     * Millis since epoch when the driver stopped.
+     */
+    private final long stopMillis;
+
     /**
      * Nanos between creation and completion of the {@link Driver}.
      */
@@ -45,19 +55,39 @@ public class DriverProfile implements Writeable, ChunkedToXContentObject {
     private final long iterations;
 
     /**
-     * Status of each {@link Operator} in the driver when it finishes.
+     * Status of each {@link Operator} in the driver when it finished.
      */
     private final List<DriverStatus.OperatorStatus> operators;
 
-    public DriverProfile(long tookNanos, long cpuNanos, long iterations, List<DriverStatus.OperatorStatus> operators) {
+    private final DriverSleeps sleeps;
+
+    public DriverProfile(
+        long startMillis,
+        long stopMillis,
+        long tookNanos,
+        long cpuNanos,
+        long iterations,
+        List<DriverStatus.OperatorStatus> operators,
+        DriverSleeps sleeps
+    ) {
+        this.startMillis = startMillis;
+        this.stopMillis = stopMillis;
         this.tookNanos = tookNanos;
         this.cpuNanos = cpuNanos;
         this.iterations = iterations;
         this.operators = operators;
+        this.sleeps = sleeps;
     }
 
     public DriverProfile(StreamInput in) throws IOException {
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PROFILE_SLEEPS)) {
+            this.startMillis = in.readVLong();
+            this.stopMillis = in.readVLong();
+        } else {
+            this.startMillis = 0;
+            this.stopMillis = 0;
+        }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             this.tookNanos = in.readVLong();
             this.cpuNanos = in.readVLong();
             this.iterations = in.readVLong();
@@ -67,16 +97,36 @@ public DriverProfile(StreamInput in) throws IOException {
             this.iterations = 0;
         }
         this.operators = in.readCollectionAsImmutableList(DriverStatus.OperatorStatus::new);
+        this.sleeps = DriverSleeps.read(in);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_PROFILE_SLEEPS)) {
+            out.writeVLong(startMillis);
+            out.writeVLong(stopMillis);
+        }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeVLong(tookNanos);
             out.writeVLong(cpuNanos);
             out.writeVLong(iterations);
         }
         out.writeCollection(operators);
+        sleeps.writeTo(out);
+    }
+
+    /**
+     * Millis since epoch when the driver started.
+     */
+    public long startMillis() {
+        return startMillis;
+    }
+
+    /**
+     * Millis since epoch when the driver stopped.
+     */
+    public long stopMillis() {
+        return stopMillis;
     }
 
     /**
@@ -102,13 +152,25 @@ public long iterations() {
         return iterations;
     }
 
+    /**
+     * Status of each {@link Operator} in the driver when it finished.
+     */
     public List<DriverStatus.OperatorStatus> operators() {
         return operators;
     }
 
+    /**
+     * Records of the times the driver has slept.
+     */
+    public DriverSleeps sleeps() {
+        return sleeps;
+    }
+
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         return Iterators.concat(ChunkedToXContentHelper.startObject(), Iterators.single((b, p) -> {
+            b.timeField("start_millis", "start", startMillis);
+            b.timeField("stop_millis", "stop", stopMillis);
             b.field("took_nanos", tookNanos);
             if (b.humanReadable()) {
                 b.field("took_time", TimeValue.timeValueNanos(tookNanos));
@@ -119,7 +181,11 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
             }
             b.field("iterations", iterations);
             return b;
-        }), ChunkedToXContentHelper.array("operators", operators.iterator()), ChunkedToXContentHelper.endObject());
+        }),
+            ChunkedToXContentHelper.array("operators", operators.iterator()),
+            Iterators.single((b, p) -> b.field("sleeps", sleeps)),
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
     @Override
@@ -131,15 +197,18 @@ public boolean equals(Object o) {
             return false;
         }
         DriverProfile that = (DriverProfile) o;
-        return tookNanos == that.tookNanos
+        return startMillis == that.startMillis
+            && stopMillis == that.stopMillis
+            && tookNanos == that.tookNanos
             && cpuNanos == that.cpuNanos
             && iterations == that.iterations
-            && Objects.equals(operators, that.operators);
+            && Objects.equals(operators, that.operators)
+            && sleeps.equals(that.sleeps);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(tookNanos, cpuNanos, iterations, operators);
+        return Objects.hash(startMillis, stopMillis, tookNanos, cpuNanos, iterations, operators, sleeps);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java
new file mode 100644
index 0000000000000..217a0b033bed4
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverSleeps.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Records of the times the driver has slept.
+ * @param counts map from the reason the driver has slept to the number of times it slept for that reason
+ * @param first the first few times the driver slept
+ * @param last the last few times the driver slept
+ */
+public record DriverSleeps(Map<String, Long> counts, List<Sleep> first, List<Sleep> last) implements Writeable, ToXContentObject {
+    /**
+     * A record of a time the driver slept.
+     * @param reason The reason the driver slept
+     * @param sleep Millis since epoch when the driver slept
+     * @param wake Millis since epoch when the driver woke, or 0 if it is currently sleeping
+     */
+    public record Sleep(String reason, long sleep, long wake) implements Writeable, ToXContentObject {
+        Sleep(StreamInput in) throws IOException {
+            this(in.readString(), in.readLong(), in.readLong());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(reason);
+            out.writeLong(sleep);
+            out.writeLong(wake);
+        }
+
+        Sleep wake(long now) {
+            if (isStillSleeping() == false) {
+                throw new IllegalStateException("Already awake.");
+            }
+            return new Sleep(reason, sleep, now);
+        }
+
+        public boolean isStillSleeping() {
+            return wake == 0;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field("reason", reason);
+            builder.timeField("sleep_millis", "sleep", sleep);
+            if (wake > 0) {
+                builder.timeField("wake_millis", "wake", wake);
+            }
+            return builder.endObject();
+        }
+    }
+
+    /**
+     * How many sleeps of the first and last sleeps and wakes to keep.
+     */
+    static final int RECORDS = 10;
+
+    public static DriverSleeps read(StreamInput in) throws IOException {
+        if (in.getTransportVersion().before(TransportVersions.ESQL_PROFILE_SLEEPS)) {
+            return empty();
+        }
+        return new DriverSleeps(
+            in.readImmutableMap(StreamInput::readVLong),
+            in.readCollectionAsList(Sleep::new),
+            in.readCollectionAsList(Sleep::new)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        if (out.getTransportVersion().before(TransportVersions.ESQL_PROFILE_SLEEPS)) {
+            return;
+        }
+        out.writeMap(counts, StreamOutput::writeVLong);
+        out.writeCollection(first);
+        out.writeCollection(last);
+    }
+
+    public static DriverSleeps empty() {
+        return new DriverSleeps(Map.of(), List.of(), List.of());
+    }
+
+    /**
+     * Record a sleep.
+     * @param reason the reason for the sleep
+     * @param now the current time
+     */
+    public DriverSleeps sleep(String reason, long now) {
+        if (last.isEmpty() == false) {
+            Sleep lastLast = last.get(last.size() - 1);
+            if (lastLast.isStillSleeping()) {
+                throw new IllegalStateException("Still sleeping.");
+            }
+        }
+        Map<String, Long> newCounts = new TreeMap<>(counts);
+        newCounts.compute(reason, (k, v) -> v == null ? 1 : v + 1);
+        List<Sleep> newFirst = first.size() < RECORDS ? append(first, reason, now) : first;
+        List<Sleep> newLast = last.size() < RECORDS ? append(last, reason, now) : rollOnto(last, reason, now);
+        return new DriverSleeps(newCounts, newFirst, newLast);
+    }
+
+    /**
+     * Record a wake.
+     * @param now the current time
+     */
+    public DriverSleeps wake(long now) {
+        if (now == 0) {
+            throw new IllegalStateException("Can't wake at epoch. That's used to signal sleeping.");
+        }
+        if (last.isEmpty()) {
+            throw new IllegalStateException("Never slept.");
+        }
+        Sleep lastFirst = first.get(first.size() - 1);
+        List<Sleep> newFirst = lastFirst.wake == 0 ? wake(first, now) : first;
+        return new DriverSleeps(counts, newFirst, wake(last, now));
+    }
+
+    private List<Sleep> append(List<Sleep> old, String reason, long now) {
+        List<Sleep> sleeps = new ArrayList<>(old.size() + 1);
+        sleeps.addAll(old);
+        sleeps.add(new Sleep(reason, now, 0));
+        return Collections.unmodifiableList(sleeps);
+    }
+
+    private List<Sleep> rollOnto(List<Sleep> old, String reason, long now) {
+        List<Sleep> sleeps = new ArrayList<>(old.size());
+        for (int i = 1; i < old.size(); i++) {
+            sleeps.add(old.get(i));
+        }
+        sleeps.add(new Sleep(reason, now, 0));
+        return Collections.unmodifiableList(sleeps);
+    }
+
+    private List<Sleep> wake(List<Sleep> old, long now) {
+        List<Sleep> sleeps = new ArrayList<>(old);
+        sleeps.set(sleeps.size() - 1, old.get(old.size() - 1).wake(now));
+        return Collections.unmodifiableList(sleeps);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.startObject("counts");
+        for (Map.Entry<String, Long> count : counts.entrySet()) {
+            builder.field(count.getKey(), count.getValue());
+        }
+        builder.endObject();
+        toXContent(builder, params, "first", first);
+        toXContent(builder, params, "last", last);
+        return builder.endObject();
+    }
+
+    private static void toXContent(XContentBuilder builder, ToXContent.Params params, String name, List<Sleep> sleeps) throws IOException {
+        builder.startArray(name);
+        for (Sleep sleep : sleeps) {
+            sleep.toXContent(builder, params);
+        }
+        builder.endArray();
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverStatus.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverStatus.java
index f143216303d35..42e3908231206 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverStatus.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverStatus.java
@@ -79,6 +79,8 @@ public class DriverStatus implements Task.Status {
      */
     private final List<OperatorStatus> activeOperators;
 
+    private final DriverSleeps sleeps;
+
     DriverStatus(
         String sessionId,
         long started,
@@ -87,7 +89,8 @@ public class DriverStatus implements Task.Status {
         long iterations,
         Status status,
         List<OperatorStatus> completedOperators,
-        List<OperatorStatus> activeOperators
+        List<OperatorStatus> activeOperators,
+        DriverSleeps sleeps
     ) {
         this.sessionId = sessionId;
         this.started = started;
@@ -97,39 +100,42 @@ public class DriverStatus implements Task.Status {
         this.status = status;
         this.completedOperators = completedOperators;
         this.activeOperators = activeOperators;
+        this.sleeps = sleeps;
     }
 
     public DriverStatus(StreamInput in) throws IOException {
         this.sessionId = in.readString();
-        this.started = in.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS) ? in.readLong() : 0;
+        this.started = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readLong() : 0;
         this.lastUpdated = in.readLong();
-        this.cpuNanos = in.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS) ? in.readVLong() : 0;
-        this.iterations = in.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS) ? in.readVLong() : 0;
-        this.status = Status.valueOf(in.readString());
+        this.cpuNanos = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readVLong() : 0;
+        this.iterations = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readVLong() : 0;
+        this.status = Status.read(in);
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
             this.completedOperators = in.readCollectionAsImmutableList(OperatorStatus::new);
         } else {
             this.completedOperators = List.of();
         }
         this.activeOperators = in.readCollectionAsImmutableList(OperatorStatus::new);
+        this.sleeps = DriverSleeps.read(in);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(sessionId);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeLong(started);
         }
         out.writeLong(lastUpdated);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_TIMINGS)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeVLong(cpuNanos);
             out.writeVLong(iterations);
         }
-        out.writeString(status.toString());
+        status.writeTo(out);
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
             out.writeCollection(completedOperators);
         }
         out.writeCollection(activeOperators);
+        sleeps.writeTo(out);
     }
 
     @Override
@@ -188,6 +194,13 @@ public List<OperatorStatus> completedOperators() {
         return completedOperators;
     }
 
+    /**
+     * Records of the times the driver has slept.
+     */
+    public DriverSleeps sleeps() {
+        return sleeps;
+    }
+
     /**
      * Status of each active {@link Operator} in the driver.
      */
@@ -206,7 +219,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field("cpu_time", TimeValue.timeValueNanos(cpuNanos));
         }
         builder.field("iterations", iterations);
-        builder.field("status", status.toString().toLowerCase(Locale.ROOT));
+        builder.field("status", status, params);
         builder.startArray("completed_operators");
         for (OperatorStatus completed : completedOperators) {
             builder.value(completed);
@@ -217,6 +230,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.value(active);
         }
         builder.endArray();
+        builder.field("sleeps", sleeps, params);
         return builder.endObject();
     }
 
@@ -232,12 +246,13 @@ public boolean equals(Object o) {
             && iterations == that.iterations
             && status == that.status
             && completedOperators.equals(that.completedOperators)
-            && activeOperators.equals(that.activeOperators);
+            && activeOperators.equals(that.activeOperators)
+            && sleeps.equals(that.sleeps);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(sessionId, started, lastUpdated, cpuNanos, iterations, status, completedOperators, activeOperators);
+        return Objects.hash(sessionId, started, lastUpdated, cpuNanos, iterations, status, completedOperators, activeOperators, sleeps);
     }
 
     @Override
@@ -313,7 +328,7 @@ public String toString() {
         }
     }
 
-    public enum Status implements ToXContentFragment {
+    public enum Status implements Writeable, ToXContentFragment {
         QUEUED,
         STARTING,
         RUNNING,
@@ -321,6 +336,15 @@ public enum Status implements ToXContentFragment {
         WAITING,
         DONE;
 
+        public static Status read(StreamInput in) throws IOException {
+            return Status.valueOf(in.readString());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(toString());
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             return builder.value(toString().toLowerCase(Locale.ROOT));
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java
index 99edab038af31..943ba4dc1f4fa 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FailureCollector.java
@@ -46,10 +46,19 @@ public FailureCollector(int maxExceptions) {
         this.maxExceptions = maxExceptions;
     }
 
-    public void unwrapAndCollect(Exception originEx) {
-        final Exception e = originEx instanceof TransportException
-            ? (originEx.getCause() instanceof Exception cause ? cause : new ElasticsearchException(originEx.getCause()))
-            : originEx;
+    private static Exception unwrapTransportException(TransportException te) {
+        final Throwable cause = te.getCause();
+        if (cause == null) {
+            return te;
+        } else if (cause instanceof Exception ex) {
+            return ex;
+        } else {
+            return new ElasticsearchException(cause);
+        }
+    }
+
+    public void unwrapAndCollect(Exception e) {
+        e = e instanceof TransportException te ? unwrapTransportException(te) : e;
         if (ExceptionsHelper.unwrap(e, TaskCancelledException.class) != null) {
             if (cancelledExceptionsCount.incrementAndGet() <= maxExceptions) {
                 cancelledExceptions.add(e);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java
index 07f3ff0be5f14..42bc75a49f4a7 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java
@@ -354,7 +354,7 @@ public String toString() {
 
         @Override
         public TransportVersion getMinimalSupportedVersion() {
-            return TransportVersions.ESQL_TIMINGS;
+            return TransportVersions.V_8_14_0;
         }
     }
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/IsBlockedResult.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/IsBlockedResult.java
new file mode 100644
index 0000000000000..9e9c64dfbfed4
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/IsBlockedResult.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.elasticsearch.action.support.SubscribableListener;
+
+import java.util.Map;
+
+/**
+ * Is this {@link Operator} blocked?
+ * <p>
+ *     If the {@link #listener}'s {@link SubscribableListener#isDone()} method
+ *     returns {@code true} then the {@linkplain Operator} is not blocked.
+ * </p>
+ * <p>
+ *     If the {@linkplain Operator} is blocked then you can
+ *     {@link SubscribableListener#addListener} to the {@link #listener} to be
+ *     notified when the {@linkplain Operator} is unblocked.
+ * </p>
+ * @param listener a listener to check for blocked-ness
+ * @param reason the reason that the {@linkplain Operator} is blocked.
+ *               This is used as a {@link Map} key so this shouldn't
+ *               vary wildly, but it should be descriptive of the reason
+ *               the operator went async.
+ */
+public record IsBlockedResult(SubscribableListener<Void> listener, String reason) {}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Operator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Operator.java
index 1038277c39fe1..663e06756551b 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Operator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Operator.java
@@ -88,11 +88,11 @@ default Status status() {
      * If the operator is not blocked, this method returns {@link #NOT_BLOCKED} which is an already
      * completed future.
      */
-    default SubscribableListener<Void> isBlocked() {
+    default IsBlockedResult isBlocked() {
         return NOT_BLOCKED;
     }
 
-    SubscribableListener<Void> NOT_BLOCKED = SubscribableListener.newSucceeded(null);
+    IsBlockedResult NOT_BLOCKED = new IsBlockedResult(SubscribableListener.newSucceeded(null), "not blocked");
 
     /**
      * A factory for creating intermediate operators.
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/RowInTableLookupOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/RowInTableLookupOperator.java
index 908c973fcad65..0decdc243c4a8 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/RowInTableLookupOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/RowInTableLookupOperator.java
@@ -40,6 +40,12 @@ public String toString() {
      * are never closed, so we need to build them from a non-tracking factory.
      */
     public record Factory(Key[] keys, int[] blockMapping) implements Operator.OperatorFactory {
+        public Factory {
+            if (keys.length < 1) {
+                throw new IllegalArgumentException("expected [keys] to be non-empty");
+            }
+        }
+
         @Override
         public Operator get(DriverContext driverContext) {
             return new RowInTableLookupOperator(driverContext.blockFactory(), keys, blockMapping);
@@ -56,6 +62,9 @@ public String describe() {
     private final int[] blockMapping;
 
     public RowInTableLookupOperator(BlockFactory blockFactory, Key[] keys, int[] blockMapping) {
+        if (keys.length < 1) {
+            throw new IllegalArgumentException("expected [keys] to be non-empty");
+        }
         this.blockMapping = blockMapping;
         this.keys = new ArrayList<>(keys.length);
         Block[] blocks = new Block[keys.length];
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java
index df6c09ea1ff97..ce400ddbdd6f9 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.IsBlockedResult;
 import org.elasticsearch.compute.operator.Operator;
 
 import java.util.Queue;
@@ -83,7 +84,7 @@ private void notifyNotFull() {
         }
     }
 
-    SubscribableListener<Void> waitForWriting() {
+    IsBlockedResult waitForWriting() {
         // maxBufferSize check is not water-tight as more than one sink can pass this check at the same time.
         if (queueSize.get() < maxSize || noMoreInputs) {
             return Operator.NOT_BLOCKED;
@@ -95,11 +96,11 @@ SubscribableListener<Void> waitForWriting() {
             if (notFullFuture == null) {
                 notFullFuture = new SubscribableListener<>();
             }
-            return notFullFuture;
+            return new IsBlockedResult(notFullFuture, "exchange full");
         }
     }
 
-    SubscribableListener<Void> waitForReading() {
+    IsBlockedResult waitForReading() {
         if (size() > 0 || noMoreInputs) {
             return Operator.NOT_BLOCKED;
         }
@@ -110,7 +111,7 @@ SubscribableListener<Void> waitForReading() {
             if (notEmptyFuture == null) {
                 notEmptyFuture = new SubscribableListener<>();
             }
-            return notEmptyFuture;
+            return new IsBlockedResult(notEmptyFuture, "exchange empty");
         }
     }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java
index a365a655370a2..06059944f1310 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java
@@ -115,7 +115,7 @@ public void registerTransportHandler(TransportService transportService) {
      * @throws IllegalStateException if a sink handler for the given id already exists
      */
     public ExchangeSinkHandler createSinkHandler(String exchangeId, int maxBufferSize) {
-        ExchangeSinkHandler sinkHandler = new ExchangeSinkHandler(blockFactory, maxBufferSize, threadPool::relativeTimeInMillis);
+        ExchangeSinkHandler sinkHandler = new ExchangeSinkHandler(blockFactory, maxBufferSize, threadPool.relativeTimeInMillisSupplier());
         if (sinks.putIfAbsent(exchangeId, sinkHandler) != null) {
             throw new IllegalStateException("sink exchanger for id [" + exchangeId + "] already exists");
         }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSink.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSink.java
index 8f0208740b689..e96ca9e39b7e5 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSink.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSink.java
@@ -7,8 +7,8 @@
 
 package org.elasticsearch.compute.operator.exchange;
 
-import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.IsBlockedResult;
 
 /**
  * Sink for exchanging data
@@ -33,5 +33,5 @@ public interface ExchangeSink {
     /**
      * Whether the sink is blocked on adding more pages
      */
-    SubscribableListener<Void> waitForWriting();
+    IsBlockedResult waitForWriting();
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkHandler.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkHandler.java
index ab155d6ee8479..757a3262433c8 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkHandler.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkHandler.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.IsBlockedResult;
 
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -81,7 +82,7 @@ public boolean isFinished() {
         }
 
         @Override
-        public SubscribableListener<Void> waitForWriting() {
+        public IsBlockedResult waitForWriting() {
             return buffer.waitForWriting();
         }
     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkOperator.java
index 01354d681017a..dd89dfe480c36 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkOperator.java
@@ -9,13 +9,13 @@
 
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
-import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.IsBlockedResult;
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.compute.operator.SinkOperator;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -65,13 +65,13 @@ public void finish() {
     }
 
     @Override
-    public SubscribableListener<Void> isBlocked() {
+    public IsBlockedResult isBlocked() {
         return sink.waitForWriting();
     }
 
     @Override
     public boolean needsInput() {
-        return isFinished() == false && isBlocked().isDone();
+        return isFinished() == false && isBlocked().listener().isDone();
     }
 
     @Override
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSource.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSource.java
index 01ed5e3fb6388..aa3374aa26d3f 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSource.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSource.java
@@ -7,8 +7,8 @@
 
 package org.elasticsearch.compute.operator.exchange;
 
-import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.IsBlockedResult;
 
 /**
  * Source for exchanging data
@@ -38,5 +38,5 @@ public interface ExchangeSource {
     /**
      * Allows callers to stop reading from the source when it's blocked
      */
-    SubscribableListener<Void> waitForReading();
+    IsBlockedResult waitForReading();
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java
index 77b535949eb9d..e3fc0e26e34e0 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.FailureCollector;
+import org.elasticsearch.compute.operator.IsBlockedResult;
 import org.elasticsearch.core.Releasable;
 
 import java.util.List;
@@ -53,7 +54,7 @@ private class ExchangeSourceImpl implements ExchangeSource {
         private void checkFailure() {
             Exception e = failure.getFailure();
             if (e != null) {
-                throw ExceptionsHelper.convertToElastic(e);
+                throw ExceptionsHelper.convertToRuntime(e);
             }
         }
 
@@ -70,7 +71,7 @@ public boolean isFinished() {
         }
 
         @Override
-        public SubscribableListener<Void> waitForReading() {
+        public IsBlockedResult waitForReading() {
             return buffer.waitForReading();
         }
 
@@ -178,13 +179,13 @@ void fetchPage() {
                     if (resp.finished()) {
                         onSinkComplete();
                     } else {
-                        SubscribableListener<Void> future = buffer.waitForWriting();
-                        if (future.isDone()) {
+                        IsBlockedResult future = buffer.waitForWriting();
+                        if (future.listener().isDone()) {
                             if (loopControl.tryResume() == false) {
                                 fetchPage();
                             }
                         } else {
-                            future.addListener(ActionListener.wrap(unused -> {
+                            future.listener().addListener(ActionListener.wrap(unused -> {
                                 if (loopControl.tryResume() == false) {
                                     fetchPage();
                                 }
@@ -198,7 +199,7 @@ void fetchPage() {
 
         void onSinkFailed(Exception e) {
             failure.unwrapAndCollect(e);
-            buffer.waitForReading().onResponse(null); // resume the Driver if it is being blocked on reading
+            buffer.waitForReading().listener().onResponse(null); // resume the Driver if it is being blocked on reading
             onSinkComplete();
         }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceOperator.java
index 1efba31bd831b..2d0ce228e81df 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceOperator.java
@@ -9,13 +9,13 @@
 
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
-import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.IsBlockedResult;
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.compute.operator.SourceOperator;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -30,7 +30,7 @@
 public class ExchangeSourceOperator extends SourceOperator {
 
     private final ExchangeSource source;
-    private SubscribableListener<Void> isBlocked = NOT_BLOCKED;
+    private IsBlockedResult isBlocked = NOT_BLOCKED;
     private int pagesEmitted;
 
     public record ExchangeSourceOperatorFactory(Supplier<ExchangeSource> exchangeSources) implements SourceOperatorFactory {
@@ -70,10 +70,10 @@ public void finish() {
     }
 
     @Override
-    public SubscribableListener<Void> isBlocked() {
-        if (isBlocked.isDone()) {
+    public IsBlockedResult isBlocked() {
+        if (isBlocked.listener().isDone()) {
             isBlocked = source.waitForReading();
-            if (isBlocked.isDone()) {
+            if (isBlocked.listener().isDone()) {
                 isBlocked = NOT_BLOCKED;
             }
         }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java
index 8902293ca945f..8b69b5584e65d 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java
@@ -62,6 +62,7 @@
 import org.elasticsearch.compute.operator.PageConsumerOperator;
 import org.elasticsearch.compute.operator.RowInTableLookupOperator;
 import org.elasticsearch.compute.operator.SequenceLongBlockSourceOperator;
+import org.elasticsearch.compute.operator.ShuffleDocsOperator;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
@@ -74,7 +75,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -164,93 +164,51 @@ public void testGroupingWithOrdinals() throws Exception {
             }
             writer.commit();
             Map<BytesRef, Long> actualCounts = new HashMap<>();
-            boolean shuffleDocs = randomBoolean();
-            Operator shuffleDocsOperator = new AbstractPageMappingOperator() {
-                @Override
-                protected Page process(Page page) {
-                    if (shuffleDocs == false) {
-                        return page;
-                    }
-                    DocVector docVector = (DocVector) page.getBlock(0).asVector();
-                    int positionCount = docVector.getPositionCount();
-                    IntVector shards = null;
-                    IntVector segments = null;
-                    IntVector docs = null;
-                    try (
-                        IntVector.Builder shardsBuilder = blockFactory.newIntVectorBuilder(positionCount);
-                        IntVector.Builder segmentsBuilder = blockFactory.newIntVectorBuilder(positionCount);
-                        IntVector.Builder docsBuilder = blockFactory.newIntVectorBuilder(positionCount);
-                    ) {
-                        List<Integer> docIds = new ArrayList<>(positionCount);
-                        for (int i = 0; i < positionCount; i++) {
-                            shardsBuilder.appendInt(docVector.shards().getInt(i));
-                            segmentsBuilder.appendInt(docVector.segments().getInt(i));
-                            docIds.add(docVector.docs().getInt(i));
-                        }
-                        shards = shardsBuilder.build();
-                        segments = segmentsBuilder.build();
-                        Collections.shuffle(docIds, random());
-                        for (Integer d : docIds) {
-                            docsBuilder.appendInt(d);
-                        }
-                        docs = docsBuilder.build();
-                    } finally {
-                        if (docs == null) {
-                            Releasables.closeExpectNoException(docVector, shards, segments);
-                        } else {
-                            Releasables.closeExpectNoException(docVector);
-                        }
-                    }
-                    Block[] blocks = new Block[page.getBlockCount()];
-                    blocks[0] = new DocVector(shards, segments, docs, false).asBlock();
-                    for (int i = 1; i < blocks.length; i++) {
-                        blocks[i] = page.getBlock(i);
-                    }
-                    return new Page(blocks);
-                }
 
-                @Override
-                public String toString() {
-                    return "ShuffleDocs";
+            try (DirectoryReader reader = writer.getReader()) {
+                List<Operator> operators = new ArrayList<>();
+                if (randomBoolean()) {
+                    operators.add(new ShuffleDocsOperator(blockFactory));
                 }
-            };
+                operators.add(new AbstractPageMappingOperator() {
+                    @Override
+                    protected Page process(Page page) {
+                        return page.appendBlock(driverContext.blockFactory().newConstantIntBlockWith(1, page.getPositionCount()));
+                    }
 
-            try (DirectoryReader reader = writer.getReader()) {
+                    @Override
+                    public String toString() {
+                        return "Add(1)";
+                    }
+                });
+                operators.add(
+                    new OrdinalsGroupingOperator(
+                        shardIdx -> new KeywordFieldMapper.KeywordFieldType("g").blockLoader(null),
+                        List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE)),
+                        ElementType.BYTES_REF,
+                        0,
+                        gField,
+                        List.of(CountAggregatorFunction.supplier(List.of(1)).groupingAggregatorFactory(INITIAL)),
+                        randomPageSize(),
+                        driverContext
+                    )
+                );
+                operators.add(
+                    new HashAggregationOperator(
+                        List.of(CountAggregatorFunction.supplier(List.of(1, 2)).groupingAggregatorFactory(FINAL)),
+                        () -> BlockHash.build(
+                            List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF)),
+                            driverContext.blockFactory(),
+                            randomPageSize(),
+                            false
+                        ),
+                        driverContext
+                    )
+                );
                 Driver driver = new Driver(
                     driverContext,
                     luceneOperatorFactory(reader, new MatchAllDocsQuery(), LuceneOperator.NO_LIMIT).get(driverContext),
-                    List.of(shuffleDocsOperator, new AbstractPageMappingOperator() {
-                        @Override
-                        protected Page process(Page page) {
-                            return page.appendBlock(driverContext.blockFactory().newConstantIntBlockWith(1, page.getPositionCount()));
-                        }
-
-                        @Override
-                        public String toString() {
-                            return "Add(1)";
-                        }
-                    },
-                        new OrdinalsGroupingOperator(
-                            shardIdx -> new KeywordFieldMapper.KeywordFieldType("g").blockLoader(null),
-                            List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> SourceLoader.FROM_STORED_SOURCE)),
-                            ElementType.BYTES_REF,
-                            0,
-                            gField,
-                            List.of(CountAggregatorFunction.supplier(List.of(1)).groupingAggregatorFactory(INITIAL)),
-                            randomPageSize(),
-                            driverContext
-                        ),
-                        new HashAggregationOperator(
-                            List.of(CountAggregatorFunction.supplier(List.of(1, 2)).groupingAggregatorFactory(FINAL)),
-                            () -> BlockHash.build(
-                                List.of(new BlockHash.GroupSpec(0, ElementType.BYTES_REF)),
-                                driverContext.blockFactory(),
-                                randomPageSize(),
-                                false
-                            ),
-                            driverContext
-                        )
-                    ),
+                    operators,
                     new PageConsumerOperator(page -> {
                         BytesRefBlock keys = page.getBlock(0);
                         LongBlock counts = page.getBlock(1);
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionTests.java
new file mode 100644
index 0000000000000..adc891a6a977d
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionTests.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.operator.SequenceBytesRefBlockSourceOperator;
+import org.elasticsearch.compute.operator.SourceOperator;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class MaxBytesRefAggregatorFunctionTests extends AggregatorFunctionTestCase {
+    @Override
+    protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
+        return new SequenceBytesRefBlockSourceOperator(
+            blockFactory,
+            IntStream.range(0, size).mapToObj(l -> new BytesRef(randomAlphaOfLengthBetween(0, 100)))
+        );
+    }
+
+    @Override
+    protected AggregatorFunctionSupplier aggregatorFunction(List<Integer> inputChannels) {
+        return new MaxBytesRefAggregatorFunctionSupplier(inputChannels);
+    }
+
+    @Override
+    protected String expectedDescriptionOfAggregator() {
+        return "max of bytes";
+    }
+
+    @Override
+    public void assertSimpleOutput(List<Block> input, Block result) {
+        Optional<BytesRef> max = input.stream().flatMap(b -> allBytesRefs(b)).max(Comparator.naturalOrder());
+        if (max.isEmpty()) {
+            assertThat(result.isNull(0), equalTo(true));
+            return;
+        }
+        assertThat(result.isNull(0), equalTo(false));
+        assertThat(BlockUtils.toJavaObject(result, 0), equalTo(max.get()));
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunctionTests.java
new file mode 100644
index 0000000000000..75a6a839ea62d
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunctionTests.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.LongBytesRefTupleBlockSourceOperator;
+import org.elasticsearch.compute.operator.SourceOperator;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class MaxBytesRefGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase {
+
+    @Override
+    protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
+        return new LongBytesRefTupleBlockSourceOperator(
+            blockFactory,
+            IntStream.range(0, size).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), new BytesRef(randomAlphaOfLengthBetween(0, 100))))
+        );
+    }
+
+    @Override
+    protected DataType acceptedDataType() {
+        return DataType.IP;
+    }
+
+    @Override
+    protected AggregatorFunctionSupplier aggregatorFunction(List<Integer> inputChannels) {
+        return new MaxBytesRefAggregatorFunctionSupplier(inputChannels);
+    }
+
+    @Override
+    protected String expectedDescriptionOfAggregator() {
+        return "max of bytes";
+    }
+
+    @Override
+    protected void assertSimpleGroup(List<Page> input, Block result, int position, Long group) {
+        Optional<BytesRef> max = input.stream().flatMap(p -> allBytesRefs(p, group)).max(Comparator.naturalOrder());
+        if (max.isEmpty()) {
+            assertThat(result.isNull(position), equalTo(true));
+            return;
+        }
+        assertThat(result.isNull(position), equalTo(false));
+        assertThat(BlockUtils.toJavaObject(result, position), equalTo(max.get()));
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionTests.java
new file mode 100644
index 0000000000000..b4383d6b0f56e
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionTests.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.operator.SequenceBytesRefBlockSourceOperator;
+import org.elasticsearch.compute.operator.SourceOperator;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class MinBytesRefAggregatorFunctionTests extends AggregatorFunctionTestCase {
+    @Override
+    protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
+        return new SequenceBytesRefBlockSourceOperator(
+            blockFactory,
+            IntStream.range(0, size).mapToObj(l -> new BytesRef(randomAlphaOfLengthBetween(0, 100)))
+        );
+    }
+
+    @Override
+    protected AggregatorFunctionSupplier aggregatorFunction(List<Integer> inputChannels) {
+        return new MinBytesRefAggregatorFunctionSupplier(inputChannels);
+    }
+
+    @Override
+    protected String expectedDescriptionOfAggregator() {
+        return "min of bytes";
+    }
+
+    @Override
+    public void assertSimpleOutput(List<Block> input, Block result) {
+        Optional<BytesRef> max = input.stream().flatMap(b -> allBytesRefs(b)).min(Comparator.naturalOrder());
+        if (max.isEmpty()) {
+            assertThat(result.isNull(0), equalTo(true));
+            return;
+        }
+        assertThat(result.isNull(0), equalTo(false));
+        assertThat(BlockUtils.toJavaObject(result, 0), equalTo(max.get()));
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunctionTests.java
new file mode 100644
index 0000000000000..d4cfca819f3b7
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunctionTests.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.LongBytesRefTupleBlockSourceOperator;
+import org.elasticsearch.compute.operator.SourceOperator;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class MinBytesRefGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase {
+
+    @Override
+    protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
+        return new LongBytesRefTupleBlockSourceOperator(
+            blockFactory,
+            IntStream.range(0, size).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), new BytesRef(randomAlphaOfLengthBetween(0, 100))))
+        );
+    }
+
+    @Override
+    protected DataType acceptedDataType() {
+        return DataType.IP;
+    }
+
+    @Override
+    protected AggregatorFunctionSupplier aggregatorFunction(List<Integer> inputChannels) {
+        return new MinBytesRefAggregatorFunctionSupplier(inputChannels);
+    }
+
+    @Override
+    protected String expectedDescriptionOfAggregator() {
+        return "min of bytes";
+    }
+
+    @Override
+    protected void assertSimpleGroup(List<Page> input, Block result, int position, Long group) {
+        Optional<BytesRef> max = input.stream().flatMap(p -> allBytesRefs(p, group)).min(Comparator.naturalOrder());
+        if (max.isEmpty()) {
+            assertThat(result.isNull(position), equalTo(true));
+            return;
+        }
+        assertThat(result.isNull(position), equalTo(false));
+        assertThat(BlockUtils.toJavaObject(result, position), equalTo(max.get()));
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionTests.java
new file mode 100644
index 0000000000000..1594f66ed9fe2
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpAggregatorFunctionTests.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.operator.SequenceBytesRefBlockSourceOperator;
+import org.elasticsearch.compute.operator.SourceOperator;
+
+import java.util.List;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.contains;
+
+public class TopIpAggregatorFunctionTests extends AggregatorFunctionTestCase {
+    private static final int LIMIT = 100;
+
+    @Override
+    protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
+        return new SequenceBytesRefBlockSourceOperator(
+            blockFactory,
+            IntStream.range(0, size).mapToObj(l -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean()))))
+        );
+    }
+
+    @Override
+    protected AggregatorFunctionSupplier aggregatorFunction(List<Integer> inputChannels) {
+        return new TopIpAggregatorFunctionSupplier(inputChannels, LIMIT, true);
+    }
+
+    @Override
+    protected String expectedDescriptionOfAggregator() {
+        return "top of ips";
+    }
+
+    @Override
+    public void assertSimpleOutput(List<Block> input, Block result) {
+        Object[] values = input.stream().flatMap(b -> allBytesRefs(b)).sorted().limit(LIMIT).toArray(Object[]::new);
+        assertThat((List<?>) BlockUtils.toJavaObject(result, 0), contains(values));
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunctionTests.java
new file mode 100644
index 0000000000000..da55ff2d7aab3
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunctionTests.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.aggregation;
+
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.LongBytesRefTupleBlockSourceOperator;
+import org.elasticsearch.compute.operator.SourceOperator;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.List;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+
+public class TopIpGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase {
+    private static final int LIMIT = 100;
+
+    @Override
+    protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
+        return new LongBytesRefTupleBlockSourceOperator(
+            blockFactory,
+            IntStream.range(0, size)
+                .mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())))))
+        );
+    }
+
+    @Override
+    protected DataType acceptedDataType() {
+        return DataType.IP;
+    }
+
+    @Override
+    protected AggregatorFunctionSupplier aggregatorFunction(List<Integer> inputChannels) {
+        return new TopIpAggregatorFunctionSupplier(inputChannels, LIMIT, true);
+    }
+
+    @Override
+    protected String expectedDescriptionOfAggregator() {
+        return "top of ips";
+    }
+
+    @Override
+    protected void assertSimpleGroup(List<Page> input, Block result, int position, Long group) {
+        Object[] values = input.stream().flatMap(b -> allBytesRefs(b, group)).sorted().limit(LIMIT).toArray(Object[]::new);
+        if (values.length == 0) {
+            assertThat(result.isNull(position), equalTo(true));
+        } else if (values.length == 1) {
+            assertThat(BlockUtils.toJavaObject(result, position), equalTo(values[0]));
+        } else {
+            assertThat((List<?>) BlockUtils.toJavaObject(result, position), contains(values));
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java
index 311446f184ebf..ad372da47d6b8 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java
@@ -188,6 +188,8 @@ static void assertSingleValueDenseBlock(Block initialBlock) {
 
             initialBlock = block.asVector().asBlock();
         }
+        assertKeepMask(initialBlock);
+        assertKeepMask(initialBlock.asVector());
     }
 
     public void testIntBlock() {
@@ -723,6 +725,7 @@ public void testBytesRefBlockBuilderWithNulls() {
                     assertThat(block.getBytesRef(pos, bytes), equalTo(values[pos]));
                 }
             }
+            assertKeepMask(block);
             releaseAndAssertBreaker(block);
         }
     }
@@ -797,6 +800,12 @@ public void testBooleanBlock() {
             }
             assertLookup(block, positions(blockFactory, positionCount + 1000), singletonList(null));
             assertEmptyLookup(blockFactory, block);
+            try (ToMask mask = block.toMask()) {
+                assertThat(mask.hadMultivaluedFields(), equalTo(false));
+                for (int p = 0; p < positionCount; p++) {
+                    assertThat(mask.mask().getBoolean(p), equalTo(p % 10 == 0));
+                }
+            }
 
             try (BooleanBlock.Builder blockBuilder = blockFactory.newBooleanBlockBuilder(1)) {
                 BooleanBlock copy = blockBuilder.copyFrom(block, 0, block.getPositionCount()).build();
@@ -823,6 +832,7 @@ public void testBooleanBlock() {
             IntStream.range(0, positionCount).mapToObj(ii -> randomBoolean()).forEach(vectorBuilder::appendBoolean);
             BooleanVector vector = vectorBuilder.build();
             assertSingleValueDenseBlock(vector.asBlock());
+            assertToMask(vector);
             releaseAndAssertBreaker(vector.asBlock());
         }
     }
@@ -1355,18 +1365,66 @@ <B extends Block, BB extends Block.Builder, T> void assertNullValues(
         assertTrue(block.isNull(randomNullPosition));
         assertFalse(block.isNull(randomNonNullPosition));
         releaseAndAssertBreaker(block);
+        if (block instanceof BooleanBlock bb) {
+            try (ToMask mask = bb.toMask()) {
+                assertThat(mask.hadMultivaluedFields(), equalTo(false));
+                for (int p = 0; p < positionCount; p++) {
+                    assertThat(mask.mask().getBoolean(p), equalTo(nullsMask.get(p) == false && p % 10 == 0));
+                }
+            }
+        }
+    }
+
+    void assertZeroPositionsAndRelease(BooleanBlock block) {
+        assertToMaskZeroPositions(block);
+        assertZeroPositionsAndRelease((Block) block);
     }
 
     void assertZeroPositionsAndRelease(Block block) {
         assertThat(block.getPositionCount(), is(0));
+        assertKeepMaskEmpty(block);
         releaseAndAssertBreaker(block);
     }
 
+    void assertZeroPositionsAndRelease(BooleanVector vector) {
+        assertToMask(vector);
+        assertZeroPositionsAndRelease((Vector) vector);
+    }
+
     void assertZeroPositionsAndRelease(Vector vector) {
         assertThat(vector.getPositionCount(), is(0));
+        assertKeepMaskEmpty(vector);
         releaseAndAssertBreaker(vector);
     }
 
+    static void assertKeepMaskEmpty(Block block) {
+        try (BooleanVector mask = randomMask(between(0, 1000)); Block masked = block.keepMask(mask)) {
+            if (false == (masked == block || masked.asVector() == block.asVector())) {
+                fail("should return original block or vector");
+            }
+        }
+    }
+
+    static void assertKeepMaskEmpty(Vector vector) {
+        try (BooleanVector mask = randomMask(between(0, 1000)); Block masked = vector.keepMask(mask)) {
+            assertThat(masked.asVector(), sameInstance(vector));
+        }
+    }
+
+    static void assertToMaskZeroPositions(BooleanBlock block) {
+        try (ToMask mask = block.toMask()) {
+            assertThat(mask.mask().getPositionCount(), equalTo(0));
+            assertThat(mask.hadMultivaluedFields(), equalTo(false));
+        }
+    }
+
+    static void assertToMask(BooleanVector vector) {
+        try (ToMask mask = vector.asBlock().toMask()) {
+            assertThat(mask.mask(), sameInstance(vector));
+            assertThat(mask.hadMultivaluedFields(), equalTo(false));
+        }
+    }
+
     void releaseAndAssertBreaker(Block... blocks) {
         assertThat(breaker.getUsed(), greaterThan(0L));
         Page[] pages = Arrays.stream(blocks).map(Page::new).toArray(Page[]::new);
@@ -1743,4 +1801,86 @@ static void assertLookup(Block block, IntBlock positions, List<List<Object>> exp
             assertThat(lookup.hasNext(), equalTo(false));
         }
     }
+
+    static void assertKeepMask(Vector vector) {
+        int maskPositions = vector.getPositionCount();
+        if (randomBoolean()) {
+            maskPositions += between(1, 1000);
+        }
+        try (
+            BooleanVector mask = TestBlockFactory.getNonBreakingInstance().newConstantBooleanVector(true, maskPositions);
+            Block masked = vector.keepMask(mask)
+        ) {
+            assertThat(masked.asVector(), sameInstance(vector));
+        }
+        try (
+            BooleanVector mask = TestBlockFactory.getNonBreakingInstance().newConstantBooleanVector(false, maskPositions);
+            Block masked = vector.keepMask(mask)
+        ) {
+            assertThat(masked.getPositionCount(), equalTo(vector.getPositionCount()));
+            for (int p = 0; p < vector.getPositionCount(); p++) {
+                assertTrue(masked.isNull(p));
+            }
+        }
+        try (BooleanVector mask = randomMask(maskPositions); Block masked = vector.keepMask(mask)) {
+            assertThat(masked.getPositionCount(), equalTo(vector.getPositionCount()));
+            for (int p = 0; p < vector.getPositionCount(); p++) {
+                if (mask.getBoolean(p)) {
+                    assertFalse(masked.isNull(p));
+                    assertEquals(1, masked.getValueCount(p));
+                    assertEquals(BlockUtils.toJavaObject(vector.asBlock(), p), BlockUtils.toJavaObject(masked, p));
+                } else {
+                    assertTrue(masked.isNull(p));
+                }
+            }
+        }
+    }
+
+    static void assertKeepMask(Block block) {
+        int maskPositions = block.getPositionCount();
+        if (randomBoolean()) {
+            maskPositions += between(1, 1000);
+        }
+        try (
+            BooleanVector mask = TestBlockFactory.getNonBreakingInstance().newConstantBooleanVector(true, maskPositions);
+            Block masked = block.keepMask(mask)
+        ) {
+            if (false == (masked == block || masked.asVector() == block.asVector())) {
+                fail("should return original block or vector");
+            }
+        }
+        try (
+            BooleanVector mask = TestBlockFactory.getNonBreakingInstance().newConstantBooleanVector(false, maskPositions);
+            Block masked = block.keepMask(mask)
+        ) {
+            assertThat(masked.getPositionCount(), equalTo(block.getPositionCount()));
+            for (int p = 0; p < block.getPositionCount(); p++) {
+                assertTrue(masked.isNull(p));
+            }
+        }
+        try (BooleanVector mask = randomMask(maskPositions); Block masked = block.keepMask(mask)) {
+            assertThat(masked.getPositionCount(), equalTo(block.getPositionCount()));
+            for (int p = 0; p < block.getPositionCount(); p++) {
+                if (mask.getBoolean(p) && false == block.isNull(p)) {
+                    assertFalse(masked.isNull(p));
+                    assertEquals(block.getValueCount(p), masked.getValueCount(p));
+                    assertEquals(BlockUtils.toJavaObject(block, p), BlockUtils.toJavaObject(masked, p));
+                } else {
+                    assertTrue(masked.isNull(p));
+                }
+            }
+        }
+    }
+
+    /**
+     * Build a random valid "mask" of single valued boolean fields that.
+     */
+    static BooleanVector randomMask(int positions) {
+        try (BooleanVector.Builder builder = TestBlockFactory.getNonBreakingInstance().newBooleanVectorFixedBuilder(positions)) {
+            for (int i = 0; i < positions; i++) {
+                builder.appendBoolean(randomBoolean());
+            }
+            return builder.build();
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayBlockBuilderTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayBlockBuilderTests.java
index 0fd78fd3cb9bf..34d591cd87d84 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayBlockBuilderTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayBlockBuilderTests.java
@@ -12,6 +12,7 @@
 
 import java.io.IOException;
 
+import static org.elasticsearch.compute.data.BasicBlockTests.assertKeepMask;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -40,6 +41,7 @@ public void testLongVector() throws IOException {
                 for (int i = 0; i < numElements; i++) {
                     assertThat(block.getLong(i), equalTo(elements[i]));
                 }
+                assertKeepMask(block);
                 try (LongBlock copy = serializeDeserializeBlock(block)) {
                     assertThat(copy, instanceOf(LongVectorBlock.class));
                     assertThat(block.asVector(), instanceOf(LongArrayVector.class));
@@ -65,6 +67,7 @@ public void testLongVector() throws IOException {
                 for (int i = 0; i < numElements; i++) {
                     assertThat(block.getLong(i), equalTo(elements[i]));
                 }
+                assertKeepMask(block);
                 try (LongBlock copy = serializeDeserializeBlock(block)) {
                     assertThat(copy, instanceOf(LongVectorBlock.class));
                     assertThat(block.asVector(), instanceOf(LongBigArrayVector.class));
@@ -98,6 +101,7 @@ public void testLongBlock() throws IOException {
                 for (int i = 0; i < numElements; i++) {
                     assertThat(block.getLong(i), equalTo(elements[i]));
                 }
+                assertKeepMask(block);
                 try (LongBlock copy = serializeDeserializeBlock(block)) {
                     assertThat(copy, instanceOf(LongArrayBlock.class));
                     assertNull(copy.asVector());
@@ -126,6 +130,7 @@ public void testLongBlock() throws IOException {
                 for (int i = 0; i < numElements; i++) {
                     assertThat(block.getLong(i), equalTo(elements[i]));
                 }
+                assertKeepMask(block);
                 try (LongBlock copy = serializeDeserializeBlock(block)) {
                     assertThat(copy, instanceOf(LongBigArrayBlock.class));
                     assertNull(block.asVector());
@@ -158,6 +163,13 @@ public void testBooleanVector() throws IOException {
                 for (int i = 0; i < numElements; i++) {
                     assertThat(block.getBoolean(i), equalTo(elements[i]));
                 }
+                assertKeepMask(block);
+                try (ToMask mask = block.toMask()) {
+                    assertThat(mask.hadMultivaluedFields(), equalTo(false));
+                    for (int p = 0; p < elements.length; p++) {
+                        assertThat(mask.mask().getBoolean(p), equalTo(elements[p]));
+                    }
+                }
                 try (var copy = serializeDeserializeBlock(block)) {
                     assertThat(copy, instanceOf(BooleanVectorBlock.class));
                     assertThat(block.asVector(), instanceOf(BooleanArrayVector.class));
@@ -183,6 +195,7 @@ public void testBooleanVector() throws IOException {
                 for (int i = 0; i < numElements; i++) {
                     assertThat(block.getBoolean(i), equalTo(elements[i]));
                 }
+                assertKeepMask(block);
                 try (var copy = serializeDeserializeBlock(block)) {
                     assertThat(copy, instanceOf(BooleanVectorBlock.class));
                     assertThat(block.asVector(), instanceOf(BooleanBigArrayVector.class));
@@ -216,6 +229,13 @@ public void testBooleanBlock() throws IOException {
                 for (int i = 0; i < numElements; i++) {
                     assertThat(block.getBoolean(i), equalTo(elements[i]));
                 }
+                assertKeepMask(block);
+                try (ToMask mask = block.toMask()) {
+                    assertThat(mask.hadMultivaluedFields(), equalTo(true));
+                    for (int p = 0; p < elements.length; p++) {
+                        assertThat(mask.mask().getBoolean(p), equalTo(false));
+                    }
+                }
                 try (var copy = serializeDeserializeBlock(block)) {
                     assertThat(copy, instanceOf(BooleanArrayBlock.class));
                     assertNull(copy.asVector());
@@ -244,6 +264,13 @@ public void testBooleanBlock() throws IOException {
                 for (int i = 0; i < numElements; i++) {
                     assertThat(block.getBoolean(i), equalTo(elements[i]));
                 }
+                assertKeepMask(block);
+                try (ToMask mask = block.toMask()) {
+                    assertThat(mask.hadMultivaluedFields(), equalTo(true));
+                    for (int p = 0; p < elements.length; p++) {
+                        assertThat(mask.mask().getBoolean(p), equalTo(false));
+                    }
+                }
                 try (var copy = serializeDeserializeBlock(block)) {
                     assertThat(copy, instanceOf(BooleanBigArrayBlock.class));
                     assertNull(block.asVector());
@@ -257,4 +284,52 @@ public void testBooleanBlock() throws IOException {
         }
         assertThat(blockFactory.breaker().getUsed(), equalTo(0L));
     }
+
+    /**
+     * Tests a block with one value being multivalued and the rest are single valued.
+     */
+    public void testBooleanBlockOneMv() {
+        int mvCount = between(2, 10);
+        int positionCount = randomIntBetween(1000, 5000);
+        blockFactory = new BlockFactory(blockFactory.breaker(), blockFactory.bigArrays(), ByteSizeValue.ofBytes(1));
+        try (var builder = blockFactory.newBooleanBlockBuilder(between(1, mvCount + positionCount))) {
+            boolean[] elements = new boolean[positionCount + mvCount];
+            builder.beginPositionEntry();
+            for (int i = 0; i < mvCount; i++) {
+                elements[i] = randomBoolean();
+                builder.appendBoolean(elements[i]);
+            }
+            builder.endPositionEntry();
+            for (int p = 1; p < positionCount; p++) {
+                elements[mvCount + p] = randomBoolean();
+                builder.appendBoolean(elements[mvCount + p]);
+            }
+            try (var block = builder.build()) {
+                assertThat(block, instanceOf(BooleanBigArrayBlock.class));
+                assertNull(block.asVector());
+                assertThat(block.getPositionCount(), equalTo(positionCount));
+                assertThat(block.getValueCount(0), equalTo(mvCount));
+                for (int i = 0; i < mvCount; i++) {
+                    assertThat(block.getBoolean(block.getFirstValueIndex(0) + i), equalTo(elements[i]));
+                }
+                for (int p = 1; p < positionCount; p++) {
+                    assertThat(block.getValueCount(p), equalTo(1));
+                    assertThat(block.getBoolean(block.getFirstValueIndex(p)), equalTo(elements[mvCount + p]));
+                }
+                assertKeepMask(block);
+                try (ToMask mask = block.toMask()) {
+                    /*
+                     * NOTE: this test is customized to the layout above where we don't make
+                     * any fields with 0 values.
+                     */
+                    assertThat(mask.hadMultivaluedFields(), equalTo(true));
+                    assertThat(mask.mask().getBoolean(0), equalTo(false));
+                    for (int p = 1; p < positionCount; p++) {
+                        assertThat(mask.mask().getBoolean(p), equalTo(elements[mvCount + p]));
+                    }
+                }
+            }
+        }
+        assertThat(blockFactory.breaker().getUsed(), equalTo(0L));
+    }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java
index af4c643a90625..aab8b86f9b795 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java
@@ -72,6 +72,12 @@ public void testBoolean() throws IOException {
             assertEmptyLookup(blockFactory, vector.asBlock());
             assertSerialization(block);
             assertThat(vector.toString(), containsString("BooleanBigArrayVector[positions=" + positionCount));
+            try (ToMask mask = block.toMask()) {
+                assertThat(mask.hadMultivaluedFields(), equalTo(false));
+                for (int p = 0; p < values.length; p++) {
+                    assertThat(mask.mask().getBoolean(p), equalTo(values[p]));
+                }
+            }
         }
     }
 
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockMultiValuedTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockMultiValuedTests.java
index c5e130726844d..e37b2638b56f7 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockMultiValuedTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockMultiValuedTests.java
@@ -31,6 +31,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.nullValue;
 
 public class BlockMultiValuedTests extends ESTestCase {
     @ParametersFactory
@@ -122,6 +123,54 @@ public void testLookupFromSingleManyPages() {
         assertLookup(ByteSizeValue.ofBytes(1), between(1, 32), p -> 1);
     }
 
+    public void testToMask() {
+        if (elementType != ElementType.BOOLEAN) {
+            return;
+        }
+        int positionCount = randomIntBetween(1, 16 * 1024);
+        var b = BasicBlockTests.randomBlock(blockFactory(), elementType, positionCount, nullAllowed, 2, 10, 0, 0);
+        try (ToMask mask = ((BooleanBlock) b.block()).toMask()) {
+            assertThat(mask.hadMultivaluedFields(), equalTo(true));
+            for (int p = 0; p < b.values().size(); p++) {
+                List<Object> v = b.values().get(p);
+                if (v == null) {
+                    assertThat(mask.mask().getBoolean(p), equalTo(false));
+                    continue;
+                }
+                if (v.size() != 1) {
+                    assertThat(mask.mask().getBoolean(p), equalTo(false));
+                    continue;
+                }
+                assertThat(mask.mask().getBoolean(p), equalTo(v.get(0)));
+            }
+        } finally {
+            b.block().close();
+        }
+    }
+
+    public void testMask() {
+        int positionCount = randomIntBetween(1, 16 * 1024);
+        var b = BasicBlockTests.randomBlock(blockFactory(), elementType, positionCount, nullAllowed, 0, 10, 0, 0);
+        try (
+            BooleanVector mask = BasicBlockTests.randomMask(b.values().size() + between(0, 1000));
+            Block masked = b.block().keepMask(mask)
+        ) {
+            for (int p = 0; p < b.values().size(); p++) {
+                List<Object> inputValues = b.values().get(p);
+                List<Object> valuesAtPosition = BasicBlockTests.valuesAtPositions(masked, p, p + 1).get(0);
+                if (inputValues == null || mask.getBoolean(p) == false) {
+                    assertThat(masked.isNull(p), equalTo(true));
+                    assertThat(valuesAtPosition, nullValue());
+                    continue;
+                }
+                assertThat(masked.isNull(p), equalTo(false));
+                assertThat(valuesAtPosition, equalTo(inputValues));
+            }
+        } finally {
+            b.block().close();
+        }
+    }
+
     private void assertFiltered(boolean all, boolean shuffled) {
         int positionCount = randomIntBetween(1, 16 * 1024);
         var b = BasicBlockTests.randomBlock(blockFactory(), elementType, positionCount, nullAllowed, 0, 10, 0, 0);
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/MultiValueBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/MultiValueBlockTests.java
index d3572377912ac..31a7251a03307 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/MultiValueBlockTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/MultiValueBlockTests.java
@@ -15,6 +15,8 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.elasticsearch.compute.data.BasicBlockTests.assertKeepMask;
+import static org.elasticsearch.compute.data.BasicBlockTests.assertKeepMaskEmpty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -54,6 +56,7 @@ public void testIntBlockTrivial1() {
         // cannot get a Vector view
         assertNull(block.asVector());
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(block, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(block);
         block.close();
     }
 
@@ -80,6 +83,7 @@ public void testIntBlockTrivial() {
         assertThat(block.getInt(block.getFirstValueIndex(0)), is(1));
         assertNull(block.asVector());
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(block, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(block);
         block.close();
     }
 
@@ -89,24 +93,35 @@ public void testEmpty() {
             assertThat(intBlock.getPositionCount(), is(0));
             assertThat(intBlock.asVector(), is(notNullValue()));
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+            assertKeepMaskEmpty(intBlock);
             intBlock.close();
 
             LongBlock longBlock = blockFactory.newLongBlockBuilder(initialSize).build();
             assertThat(longBlock.getPositionCount(), is(0));
             assertThat(longBlock.asVector(), is(notNullValue()));
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(longBlock, this::serializeDeserializeBlock, null, Releasable::close);
+            assertKeepMaskEmpty(longBlock);
             longBlock.close();
 
             DoubleBlock doubleBlock = blockFactory.newDoubleBlockBuilder(initialSize).build();
             assertThat(doubleBlock.getPositionCount(), is(0));
             assertThat(doubleBlock.asVector(), is(notNullValue()));
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(doubleBlock, this::serializeDeserializeBlock, null, Releasable::close);
+            assertKeepMaskEmpty(doubleBlock);
             doubleBlock.close();
 
+            FloatBlock floatBlock = blockFactory.newFloatBlockBuilder(initialSize).build();
+            assertThat(floatBlock.getPositionCount(), is(0));
+            assertThat(floatBlock.asVector(), is(notNullValue()));
+            EqualsHashCodeTestUtils.checkEqualsAndHashCode(floatBlock, this::serializeDeserializeBlock, null, Releasable::close);
+            assertKeepMaskEmpty(floatBlock);
+            floatBlock.close();
+
             BytesRefBlock bytesRefBlock = blockFactory.newBytesRefBlockBuilder(initialSize).build();
             assertThat(bytesRefBlock.getPositionCount(), is(0));
             assertThat(bytesRefBlock.asVector(), is(notNullValue()));
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(bytesRefBlock, this::serializeDeserializeBlock, null, Releasable::close);
+            assertKeepMaskEmpty(bytesRefBlock);
             bytesRefBlock.close();
         }
     }
@@ -118,6 +133,7 @@ public void testNullOnly() throws IOException {
             assertThat(intBlock.getValueCount(0), is(0));
             assertNull(intBlock.asVector());
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+            assertKeepMask(intBlock);
             intBlock.close();
 
             LongBlock longBlock = blockFactory.newLongBlockBuilder(initialSize).appendNull().build();
@@ -125,6 +141,7 @@ public void testNullOnly() throws IOException {
             assertThat(longBlock.getValueCount(0), is(0));
             assertNull(longBlock.asVector());
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(longBlock, this::serializeDeserializeBlock, null, Releasable::close);
+            assertKeepMask(longBlock);
             longBlock.close();
 
             DoubleBlock doubleBlock = blockFactory.newDoubleBlockBuilder(initialSize).appendNull().build();
@@ -132,13 +149,23 @@ public void testNullOnly() throws IOException {
             assertThat(doubleBlock.getValueCount(0), is(0));
             assertNull(doubleBlock.asVector());
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(doubleBlock, this::serializeDeserializeBlock, null, Releasable::close);
+            assertKeepMask(doubleBlock);
             doubleBlock.close();
 
+            FloatBlock floatBlock = blockFactory.newFloatBlockBuilder(initialSize).appendNull().build();
+            assertThat(floatBlock.getPositionCount(), is(1));
+            assertThat(floatBlock.getValueCount(0), is(0));
+            assertNull(floatBlock.asVector());
+            EqualsHashCodeTestUtils.checkEqualsAndHashCode(floatBlock, this::serializeDeserializeBlock, null, Releasable::close);
+            assertKeepMask(floatBlock);
+            floatBlock.close();
+
             BytesRefBlock bytesRefBlock = blockFactory.newBytesRefBlockBuilder(initialSize).appendNull().build();
             assertThat(bytesRefBlock.getPositionCount(), is(1));
             assertThat(bytesRefBlock.getValueCount(0), is(0));
             assertNull(bytesRefBlock.asVector());
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(bytesRefBlock, this::serializeDeserializeBlock, null, Releasable::close);
+            assertKeepMask(bytesRefBlock);
             bytesRefBlock.close();
         }
     }
@@ -162,21 +189,31 @@ public void testNullsFollowedByValues() {
         assertThat(intBlock.elementType(), is(equalTo(ElementType.INT)));
         BlockValueAsserter.assertBlockValues(intBlock, blockValues);
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(intBlock);
 
         Block longBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.LONG);
         assertThat(longBlock.elementType(), is(equalTo(ElementType.LONG)));
         BlockValueAsserter.assertBlockValues(longBlock, blockValues);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(longBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(longBlock);
 
         Block doubleBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.DOUBLE);
         assertThat(doubleBlock.elementType(), is(equalTo(ElementType.DOUBLE)));
         BlockValueAsserter.assertBlockValues(doubleBlock, blockValues);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(doubleBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(doubleBlock);
+
+        Block floatBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.DOUBLE);
+        assertThat(floatBlock.elementType(), is(equalTo(ElementType.DOUBLE)));
+        BlockValueAsserter.assertBlockValues(floatBlock, blockValues);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(floatBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(floatBlock);
 
         Block bytesRefBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.BYTES_REF);
         assertThat(bytesRefBlock.elementType(), is(equalTo(ElementType.BYTES_REF)));
         BlockValueAsserter.assertBlockValues(bytesRefBlock, blockValues);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(bytesRefBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(bytesRefBlock);
     }
 
     public void testMultiValuesAndNullsSmall() {
@@ -194,21 +231,31 @@ public void testMultiValuesAndNullsSmall() {
         assertThat(intBlock.elementType(), is(equalTo(ElementType.INT)));
         BlockValueAsserter.assertBlockValues(intBlock, blockValues);
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(intBlock);
 
         Block longBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.LONG);
         assertThat(longBlock.elementType(), is(equalTo(ElementType.LONG)));
         BlockValueAsserter.assertBlockValues(longBlock, blockValues);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(longBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(longBlock);
 
         Block doubleBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.DOUBLE);
         assertThat(doubleBlock.elementType(), is(equalTo(ElementType.DOUBLE)));
         BlockValueAsserter.assertBlockValues(doubleBlock, blockValues);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(doubleBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(doubleBlock);
+
+        Block floatBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.FLOAT);
+        assertThat(floatBlock.elementType(), is(equalTo(ElementType.FLOAT)));
+        BlockValueAsserter.assertBlockValues(floatBlock, blockValues);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(floatBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(floatBlock);
 
         Block bytesRefBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.BYTES_REF);
         assertThat(bytesRefBlock.elementType(), is(equalTo(ElementType.BYTES_REF)));
         BlockValueAsserter.assertBlockValues(bytesRefBlock, blockValues);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(bytesRefBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(bytesRefBlock);
     }
 
     public void testMultiValuesAndNulls() {
@@ -230,21 +277,31 @@ public void testMultiValuesAndNulls() {
         assertThat(intBlock.elementType(), is(equalTo(ElementType.INT)));
         BlockValueAsserter.assertBlockValues(intBlock, blockValues);
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(intBlock);
 
         Block longBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.LONG);
         assertThat(longBlock.elementType(), is(equalTo(ElementType.LONG)));
         BlockValueAsserter.assertBlockValues(longBlock, blockValues);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(longBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(longBlock);
 
         Block doubleBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.DOUBLE);
         assertThat(doubleBlock.elementType(), is(equalTo(ElementType.DOUBLE)));
         BlockValueAsserter.assertBlockValues(doubleBlock, blockValues);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(doubleBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(doubleBlock);
+
+        Block floatBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.FLOAT);
+        assertThat(floatBlock.elementType(), is(equalTo(ElementType.FLOAT)));
+        BlockValueAsserter.assertBlockValues(floatBlock, blockValues);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(floatBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(floatBlock);
 
         Block bytesRefBlock = TestBlockBuilder.blockFromValues(blockValues, ElementType.BYTES_REF);
         assertThat(bytesRefBlock.elementType(), is(equalTo(ElementType.BYTES_REF)));
         BlockValueAsserter.assertBlockValues(bytesRefBlock, blockValues);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(intBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(bytesRefBlock, this::serializeDeserializeBlock, null, Releasable::close);
+        assertKeepMask(bytesRefBlock);
     }
 
     // Tests that the use of Block builder beginPositionEntry (or not) with just a single value,
@@ -265,6 +322,8 @@ public void testSingleNonNullValues() throws IOException {
             TestBlockBuilder.blockFromValues(blockValues.stream().map(List::of).toList(), ElementType.LONG),
             TestBlockBuilder.blockFromSingleValues(blockValues, ElementType.DOUBLE),
             TestBlockBuilder.blockFromValues(blockValues.stream().map(List::of).toList(), ElementType.DOUBLE),
+            TestBlockBuilder.blockFromSingleValues(blockValues, ElementType.FLOAT),
+            TestBlockBuilder.blockFromValues(blockValues.stream().map(List::of).toList(), ElementType.FLOAT),
             TestBlockBuilder.blockFromSingleValues(blockValues, ElementType.BYTES_REF),
             TestBlockBuilder.blockFromValues(blockValues.stream().map(List::of).toList(), ElementType.BYTES_REF)
         );
@@ -273,6 +332,7 @@ public void testSingleNonNullValues() throws IOException {
                 assertThat(block.asVector(), is(notNullValue()));
                 BlockValueAsserter.assertBlockValues(block, blockValues.stream().map(List::of).toList());
                 EqualsHashCodeTestUtils.checkEqualsAndHashCode(block, this::serializeDeserializeBlock, null, Releasable::close);
+                assertKeepMask(block);
             }
         } finally {
             Releasables.close(blocks);
@@ -312,6 +372,8 @@ public void testSingleWithNullValues() {
             TestBlockBuilder.blockFromValues(blockValues.stream().map(MultiValueBlockTests::mapToList).toList(), ElementType.LONG),
             TestBlockBuilder.blockFromSingleValues(blockValues, ElementType.DOUBLE),
             TestBlockBuilder.blockFromValues(blockValues.stream().map(MultiValueBlockTests::mapToList).toList(), ElementType.DOUBLE),
+            TestBlockBuilder.blockFromSingleValues(blockValues, ElementType.FLOAT),
+            TestBlockBuilder.blockFromValues(blockValues.stream().map(MultiValueBlockTests::mapToList).toList(), ElementType.FLOAT),
             TestBlockBuilder.blockFromSingleValues(blockValues, ElementType.BYTES_REF),
             TestBlockBuilder.blockFromValues(blockValues.stream().map(MultiValueBlockTests::mapToList).toList(), ElementType.BYTES_REF)
         );
@@ -319,6 +381,7 @@ public void testSingleWithNullValues() {
             assertThat(block.asVector(), is(nullValue()));
             BlockValueAsserter.assertBlockValues(block, blockValues.stream().map(MultiValueBlockTests::mapToList).toList());
             EqualsHashCodeTestUtils.checkEqualsAndHashCode(block, this::serializeDeserializeBlock, null, Releasable::close);
+            assertKeepMask(block);
         }
     }
 
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/TestBlockBuilder.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/TestBlockBuilder.java
index 33a3531481df9..6472e1fbba4f7 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/TestBlockBuilder.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/TestBlockBuilder.java
@@ -63,6 +63,7 @@ static TestBlockBuilder builderOf(BlockFactory blockFactory, ElementType type) {
             case INT -> new TestIntBlockBuilder(blockFactory, 0);
             case LONG -> new TestLongBlockBuilder(blockFactory, 0);
             case DOUBLE -> new TestDoubleBlockBuilder(blockFactory, 0);
+            case FLOAT -> new TestFloatBlockBuilder(blockFactory, 0);
             case BYTES_REF -> new TestBytesRefBlockBuilder(blockFactory, 0);
             case BOOLEAN -> new TestBooleanBlockBuilder(blockFactory, 0);
             default -> throw new AssertionError(type);
@@ -249,6 +250,66 @@ public void close() {
         }
     }
 
+    private static class TestFloatBlockBuilder extends TestBlockBuilder {
+
+        private final FloatBlock.Builder builder;
+
+        TestFloatBlockBuilder(BlockFactory blockFactory, int estimatedSize) {
+            builder = blockFactory.newFloatBlockBuilder(estimatedSize);
+        }
+
+        @Override
+        public TestBlockBuilder appendObject(Object object) {
+            builder.appendFloat(((Number) object).floatValue());
+            return this;
+        }
+
+        @Override
+        public TestBlockBuilder appendNull() {
+            builder.appendNull();
+            return this;
+        }
+
+        @Override
+        public TestBlockBuilder beginPositionEntry() {
+            builder.beginPositionEntry();
+            return this;
+        }
+
+        @Override
+        public TestBlockBuilder endPositionEntry() {
+            builder.endPositionEntry();
+            return this;
+        }
+
+        @Override
+        public TestBlockBuilder copyFrom(Block block, int beginInclusive, int endExclusive) {
+            builder.copyFrom(block, beginInclusive, endExclusive);
+            return this;
+        }
+
+        @Override
+        public TestBlockBuilder mvOrdering(Block.MvOrdering mvOrdering) {
+            builder.mvOrdering(mvOrdering);
+            return this;
+        }
+
+        @Override
+        public long estimatedBytes() {
+            return builder.estimatedBytes();
+        }
+
+        @Override
+        public FloatBlock build() {
+            return builder.build();
+        }
+
+        @Override
+        public void close() {
+            builder.close();
+        }
+    }
+
     private static class TestBytesRefBlockBuilder extends TestBlockBuilder {
 
         private final BytesRefBlock.Builder builder;
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BooleanBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BooleanBucketedSortTests.java
index 4bead6b4c0c8d..fcece4b6480b8 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BooleanBucketedSortTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BooleanBucketedSortTests.java
@@ -14,27 +14,29 @@
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.search.sort.SortOrder;
 
+import java.util.List;
+
 import static org.hamcrest.Matchers.equalTo;
 
-public class BooleanBucketedSortTests extends BucketedSortTestCase<BooleanBucketedSort> {
+public class BooleanBucketedSortTests extends BucketedSortTestCase<BooleanBucketedSort, Boolean> {
     @Override
     protected BooleanBucketedSort build(SortOrder sortOrder, int bucketSize) {
         return new BooleanBucketedSort(bigArrays(), sortOrder, bucketSize);
     }
 
     @Override
-    protected Object expectedValue(double v) {
-        return toBoolean(v);
+    protected Boolean randomValue() {
+        return randomBoolean();
     }
 
     @Override
-    protected double randomValue() {
-        return randomBoolean() ? 0d : 1d;
+    protected List<Boolean> threeSortedValues() {
+        return List.of(false, true, true);
     }
 
     @Override
-    protected void collect(BooleanBucketedSort sort, double value, int bucket) {
-        sort.collect(toBoolean(value), bucket);
+    protected void collect(BooleanBucketedSort sort, Boolean value, int bucket) {
+        sort.collect(value, bucket);
     }
 
     @Override
@@ -48,15 +50,11 @@ protected Block toBlock(BooleanBucketedSort sort, BlockFactory blockFactory, Int
     }
 
     @Override
-    protected void assertBlockTypeAndValues(Block block, Object... values) {
+    protected void assertBlockTypeAndValues(Block block, List<Boolean> values) {
         assertThat(block.elementType(), equalTo(ElementType.BOOLEAN));
         var typedBlock = (BooleanBlock) block;
-        for (int i = 0; i < values.length; i++) {
-            assertThat(typedBlock.getBoolean(i), equalTo(values[i]));
+        for (int i = 0; i < values.size(); i++) {
+            assertThat(typedBlock.getBoolean(i), equalTo(values.get(i)));
         }
     }
-
-    private boolean toBoolean(double value) {
-        return value > 0;
-    }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java
index 9e1bc145ad4ca..f857f50b2d30f 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java
@@ -25,113 +25,119 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public abstract class BucketedSortTestCase<T extends Releasable> extends ESTestCase {
+public abstract class BucketedSortTestCase<T extends Releasable, V extends Comparable<V>> extends ESTestCase {
     /**
      * Build a {@link T} to test. Sorts built by this method shouldn't need scores.
      */
     protected abstract T build(SortOrder sortOrder, int bucketSize);
 
     /**
-     * Build the expected correctly typed value for a value.
+     * A random value for testing, with the appropriate precision for the type we're testing.
      */
-    protected abstract Object expectedValue(double v);
+    protected abstract V randomValue();
 
     /**
-     * A random value for testing, with the appropriate precision for the type we're testing.
+     * Returns a list of 3 values, in ascending order.
      */
-    protected abstract double randomValue();
+    protected abstract List<V> threeSortedValues();
 
     /**
      * Collect a value into the sort.
      * @param value value to collect, always sent as double just to have
      *        a number to test. Subclasses should cast to their favorite types
      */
-    protected abstract void collect(T sort, double value, int bucket);
+    protected abstract void collect(T sort, V value, int bucket);
 
     protected abstract void merge(T sort, int groupId, T other, int otherGroupId);
 
     protected abstract Block toBlock(T sort, BlockFactory blockFactory, IntVector selected);
 
-    protected abstract void assertBlockTypeAndValues(Block block, Object... values);
+    protected abstract void assertBlockTypeAndValues(Block block, List<V> values);
 
     public final void testNeverCalled() {
         SortOrder order = randomFrom(SortOrder.values());
         try (T sort = build(order, 1)) {
-            assertBlock(sort, randomNonNegativeInt());
+            assertBlock(sort, randomNonNegativeInt(), List.of());
         }
     }
 
     public final void testSingleDoc() {
         try (T sort = build(randomFrom(SortOrder.values()), 1)) {
-            collect(sort, 1, 0);
+            var values = threeSortedValues();
+
+            collect(sort, values.get(0), 0);
 
-            assertBlock(sort, 0, expectedValue(1));
+            assertBlock(sort, 0, List.of(values.get(0)));
         }
     }
 
     public final void testNonCompetitive() {
         try (T sort = build(SortOrder.DESC, 1)) {
-            collect(sort, 2, 0);
-            collect(sort, 1, 0);
+            var values = threeSortedValues();
 
-            assertBlock(sort, 0, expectedValue(2));
+            collect(sort, values.get(1), 0);
+            collect(sort, values.get(0), 0);
+
+            assertBlock(sort, 0, List.of(values.get(1)));
         }
     }
 
     public final void testCompetitive() {
         try (T sort = build(SortOrder.DESC, 1)) {
-            collect(sort, 1, 0);
-            collect(sort, 2, 0);
+            var values = threeSortedValues();
 
-            assertBlock(sort, 0, expectedValue(2));
-        }
-    }
+            collect(sort, values.get(0), 0);
+            collect(sort, values.get(1), 0);
 
-    public final void testNegativeValue() {
-        try (T sort = build(SortOrder.DESC, 1)) {
-            collect(sort, -1, 0);
-            assertBlock(sort, 0, expectedValue(-1));
+            assertBlock(sort, 0, List.of(values.get(1)));
         }
     }
 
     public final void testSomeBuckets() {
         try (T sort = build(SortOrder.DESC, 1)) {
-            collect(sort, 2, 0);
-            collect(sort, 2, 1);
-            collect(sort, 2, 2);
-            collect(sort, 3, 0);
-
-            assertBlock(sort, 0, expectedValue(3));
-            assertBlock(sort, 1, expectedValue(2));
-            assertBlock(sort, 2, expectedValue(2));
-            assertBlock(sort, 3);
+            var values = threeSortedValues();
+
+            collect(sort, values.get(1), 0);
+            collect(sort, values.get(1), 1);
+            collect(sort, values.get(1), 2);
+            collect(sort, values.get(2), 0);
+
+            assertBlock(sort, 0, List.of(values.get(2)));
+            assertBlock(sort, 1, List.of(values.get(1)));
+            assertBlock(sort, 2, List.of(values.get(1)));
+            assertBlock(sort, 3, List.of());
         }
     }
 
     public final void testBucketGaps() {
         try (T sort = build(SortOrder.DESC, 1)) {
-            collect(sort, 2, 0);
-            collect(sort, 2, 2);
+            var values = threeSortedValues();
 
-            assertBlock(sort, 0, expectedValue(2));
-            assertBlock(sort, 1);
-            assertBlock(sort, 2, expectedValue(2));
-            assertBlock(sort, 3);
+            collect(sort, values.get(1), 0);
+            collect(sort, values.get(1), 2);
+
+            assertBlock(sort, 0, List.of(values.get(1)));
+            assertBlock(sort, 1, List.of());
+            assertBlock(sort, 2, List.of(values.get(1)));
+            assertBlock(sort, 3, List.of());
         }
     }
 
     public final void testBucketsOutOfOrder() {
         try (T sort = build(SortOrder.DESC, 1)) {
-            collect(sort, 2, 1);
-            collect(sort, 2, 0);
+            var values = threeSortedValues();
+
+            collect(sort, values.get(1), 1);
+            collect(sort, values.get(1), 0);
 
-            assertBlock(sort, 0, expectedValue(2.0));
-            assertBlock(sort, 1, expectedValue(2.0));
-            assertBlock(sort, 2);
+            assertBlock(sort, 0, List.of(values.get(1)));
+            assertBlock(sort, 1, List.of(values.get(1)));
+            assertBlock(sort, 2, List.of());
         }
     }
 
@@ -143,194 +149,207 @@ public final void testManyBuckets() {
         }
         Collections.shuffle(Arrays.asList(buckets), random());
 
-        double[] maxes = new double[buckets.length];
+        var values = threeSortedValues();
+        List<V> maxes = new ArrayList<V>(Collections.nCopies(buckets.length, null));
 
         try (T sort = build(SortOrder.DESC, 1)) {
             for (int b : buckets) {
-                maxes[b] = 2;
-                collect(sort, 2, b);
+                maxes.set(b, values.get(1));
+                collect(sort, values.get(1), b);
                 if (randomBoolean()) {
-                    maxes[b] = 3;
-                    collect(sort, 3, b);
+                    maxes.set(b, values.get(2));
+                    collect(sort, values.get(2), b);
                 }
                 if (randomBoolean()) {
-                    collect(sort, -1, b);
+                    collect(sort, values.get(0), b);
                 }
             }
             for (int b = 0; b < buckets.length; b++) {
-                assertBlock(sort, b, expectedValue(maxes[b]));
+                assertBlock(sort, b, List.of(maxes.get(b)));
             }
-            assertBlock(sort, buckets.length);
+            assertBlock(sort, buckets.length, List.of());
         }
     }
 
     public final void testTwoHitsDesc() {
         try (T sort = build(SortOrder.DESC, 2)) {
-            collect(sort, 1, 0);
-            collect(sort, 2, 0);
-            collect(sort, 3, 0);
+            var values = threeSortedValues();
 
-            assertBlock(sort, 0, expectedValue(3), expectedValue(2));
+            collect(sort, values.get(0), 0);
+            collect(sort, values.get(1), 0);
+            collect(sort, values.get(2), 0);
+
+            assertBlock(sort, 0, List.of(values.get(2), values.get(1)));
         }
     }
 
     public final void testTwoHitsAsc() {
         try (T sort = build(SortOrder.ASC, 2)) {
-            collect(sort, 1, 0);
-            collect(sort, 2, 0);
-            collect(sort, 3, 0);
+            var values = threeSortedValues();
+
+            collect(sort, values.get(0), 0);
+            collect(sort, values.get(1), 0);
+            collect(sort, values.get(2), 0);
 
-            assertBlock(sort, 0, expectedValue(1), expectedValue(2));
+            assertBlock(sort, 0, List.of(values.get(0), values.get(1)));
         }
     }
 
     public final void testTwoHitsTwoBucket() {
         try (T sort = build(SortOrder.DESC, 2)) {
-            collect(sort, 1, 0);
-            collect(sort, 1, 1);
-            collect(sort, 2, 0);
-            collect(sort, 2, 1);
-            collect(sort, 3, 0);
-            collect(sort, 3, 1);
-            collect(sort, 4, 1);
-
-            assertBlock(sort, 0, expectedValue(3), expectedValue(2));
-            assertBlock(sort, 1, expectedValue(4), expectedValue(3));
+            var values = threeSortedValues();
+
+            collect(sort, values.get(0), 0);
+            collect(sort, values.get(0), 1);
+            collect(sort, values.get(1), 0);
+            collect(sort, values.get(1), 1);
+            collect(sort, values.get(2), 0);
+
+            assertBlock(sort, 0, List.of(values.get(2), values.get(1)));
+            assertBlock(sort, 1, List.of(values.get(1), values.get(0)));
         }
     }
 
     public final void testManyBucketsManyHits() {
         // Set the values in random order
-        double[] values = new double[10000];
-        for (int v = 0; v < values.length; v++) {
-            values[v] = randomValue();
+        List<V> values = new ArrayList<V>();
+        for (int v = 0; v < 10000; v++) {
+            values.add(randomValue());
         }
-        Collections.shuffle(Arrays.asList(values), random());
+        Collections.shuffle(values, random());
 
         int buckets = between(2, 100);
         int bucketSize = between(2, 100);
         try (T sort = build(SortOrder.DESC, bucketSize)) {
             BitArray[] bucketUsed = new BitArray[buckets];
-            Arrays.setAll(bucketUsed, i -> new BitArray(values.length, bigArrays()));
-            for (int doc = 0; doc < values.length; doc++) {
+            Arrays.setAll(bucketUsed, i -> new BitArray(values.size(), bigArrays()));
+            for (int doc = 0; doc < values.size(); doc++) {
                 for (int bucket = 0; bucket < buckets; bucket++) {
                     if (randomBoolean()) {
                         bucketUsed[bucket].set(doc);
-                        collect(sort, values[doc], bucket);
+                        collect(sort, values.get(doc), bucket);
                     }
                 }
             }
             for (int bucket = 0; bucket < buckets; bucket++) {
-                List<Double> bucketValues = new ArrayList<>(values.length);
-                for (int doc = 0; doc < values.length; doc++) {
+                List<V> bucketValues = new ArrayList<>(values.size());
+                for (int doc = 0; doc < values.size(); doc++) {
                     if (bucketUsed[bucket].get(doc)) {
-                        bucketValues.add(values[doc]);
+                        bucketValues.add(values.get(doc));
                     }
                 }
                 bucketUsed[bucket].close();
-                assertBlock(
-                    sort,
-                    bucket,
-                    bucketValues.stream().sorted((lhs, rhs) -> rhs.compareTo(lhs)).limit(bucketSize).map(this::expectedValue).toArray()
-                );
+                assertBlock(sort, bucket, bucketValues.stream().sorted(Comparator.reverseOrder()).limit(bucketSize).toList());
             }
-            assertBlock(sort, buckets);
+            assertBlock(sort, buckets, List.of());
         }
     }
 
     public final void testMergeHeapToHeap() {
         try (T sort = build(SortOrder.ASC, 3)) {
-            collect(sort, 1, 0);
-            collect(sort, 2, 0);
-            collect(sort, 3, 0);
+            var values = threeSortedValues();
+
+            collect(sort, values.get(0), 0);
+            collect(sort, values.get(1), 0);
+            collect(sort, values.get(2), 0);
 
             try (T other = build(SortOrder.ASC, 3)) {
-                collect(other, 1, 0);
-                collect(other, 2, 0);
-                collect(other, 3, 0);
+                collect(other, values.get(0), 0);
+                collect(other, values.get(1), 0);
+                collect(other, values.get(2), 0);
 
                 merge(sort, 0, other, 0);
             }
 
-            assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2));
+            assertBlock(sort, 0, List.of(values.get(0), values.get(0), values.get(1)));
         }
     }
 
     public final void testMergeNoHeapToNoHeap() {
         try (T sort = build(SortOrder.ASC, 3)) {
-            collect(sort, 1, 0);
-            collect(sort, 2, 0);
+            var values = threeSortedValues();
+
+            collect(sort, values.get(0), 0);
+            collect(sort, values.get(1), 0);
 
             try (T other = build(SortOrder.ASC, 3)) {
-                collect(other, 1, 0);
-                collect(other, 2, 0);
+                collect(other, values.get(0), 0);
+                collect(other, values.get(1), 0);
 
                 merge(sort, 0, other, 0);
             }
 
-            assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2));
+            assertBlock(sort, 0, List.of(values.get(0), values.get(0), values.get(1)));
         }
     }
 
     public final void testMergeHeapToNoHeap() {
         try (T sort = build(SortOrder.ASC, 3)) {
-            collect(sort, 1, 0);
-            collect(sort, 2, 0);
+            var values = threeSortedValues();
+
+            collect(sort, values.get(0), 0);
+            collect(sort, values.get(1), 0);
 
             try (T other = build(SortOrder.ASC, 3)) {
-                collect(other, 1, 0);
-                collect(other, 2, 0);
-                collect(other, 3, 0);
+                collect(other, values.get(0), 0);
+                collect(other, values.get(1), 0);
+                collect(other, values.get(2), 0);
 
                 merge(sort, 0, other, 0);
             }
 
-            assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2));
+            assertBlock(sort, 0, List.of(values.get(0), values.get(0), values.get(1)));
         }
     }
 
     public final void testMergeNoHeapToHeap() {
         try (T sort = build(SortOrder.ASC, 3)) {
-            collect(sort, 1, 0);
-            collect(sort, 2, 0);
-            collect(sort, 3, 0);
+            var values = threeSortedValues();
+
+            collect(sort, values.get(0), 0);
+            collect(sort, values.get(1), 0);
+            collect(sort, values.get(2), 0);
 
             try (T other = build(SortOrder.ASC, 3)) {
-                collect(sort, 1, 0);
-                collect(sort, 2, 0);
+                collect(sort, values.get(0), 0);
+                collect(sort, values.get(1), 0);
 
                 merge(sort, 0, other, 0);
             }
 
-            assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2));
+            assertBlock(sort, 0, List.of(values.get(0), values.get(0), values.get(1)));
         }
     }
 
     public final void testMergeHeapToEmpty() {
         try (T sort = build(SortOrder.ASC, 3)) {
+            var values = threeSortedValues();
+
             try (T other = build(SortOrder.ASC, 3)) {
-                collect(other, 1, 0);
-                collect(other, 2, 0);
-                collect(other, 3, 0);
+                collect(other, values.get(0), 0);
+                collect(other, values.get(1), 0);
+                collect(other, values.get(2), 0);
 
                 merge(sort, 0, other, 0);
             }
 
-            assertBlock(sort, 0, expectedValue(1), expectedValue(2), expectedValue(3));
+            assertBlock(sort, 0, List.of(values.get(0), values.get(1), values.get(2)));
         }
     }
 
     public final void testMergeEmptyToHeap() {
         try (T sort = build(SortOrder.ASC, 3)) {
-            collect(sort, 1, 0);
-            collect(sort, 2, 0);
-            collect(sort, 3, 0);
+            var values = threeSortedValues();
+
+            collect(sort, values.get(0), 0);
+            collect(sort, values.get(1), 0);
+            collect(sort, values.get(2), 0);
 
             try (T other = build(SortOrder.ASC, 3)) {
                 merge(sort, 0, other, 0);
             }
 
-            assertBlock(sort, 0, expectedValue(1), expectedValue(2), expectedValue(3));
+            assertBlock(sort, 0, List.of(values.get(0), values.get(1), values.get(2)));
         }
     }
 
@@ -340,20 +359,20 @@ public final void testMergeEmptyToEmpty() {
                 merge(sort, 0, other, randomNonNegativeInt());
             }
 
-            assertBlock(sort, 0);
+            assertBlock(sort, 0, List.of());
         }
     }
 
-    private void assertBlock(T sort, int groupId, Object... values) {
+    protected void assertBlock(T sort, int groupId, List<V> values) {
         var blockFactory = TestBlockFactory.getNonBreakingInstance();
 
         try (var intVector = blockFactory.newConstantIntVector(groupId, 1)) {
             var block = toBlock(sort, blockFactory, intVector);
 
             assertThat(block.getPositionCount(), equalTo(1));
-            assertThat(block.getTotalValueCount(), equalTo(values.length));
+            assertThat(block.getTotalValueCount(), equalTo(values.size()));
 
-            if (values.length == 0) {
+            if (values.isEmpty()) {
                 assertThat(block.elementType(), equalTo(ElementType.NULL));
                 assertThat(block.isNull(0), equalTo(true));
             } else {
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/DoubleBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/DoubleBucketedSortTests.java
index 43b5caa092b9a..e5cc03f7c59cc 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/DoubleBucketedSortTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/DoubleBucketedSortTests.java
@@ -14,26 +14,28 @@
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.search.sort.SortOrder;
 
+import java.util.List;
+
 import static org.hamcrest.Matchers.equalTo;
 
-public class DoubleBucketedSortTests extends BucketedSortTestCase<DoubleBucketedSort> {
+public class DoubleBucketedSortTests extends BucketedSortTestCase<DoubleBucketedSort, Double> {
     @Override
     protected DoubleBucketedSort build(SortOrder sortOrder, int bucketSize) {
         return new DoubleBucketedSort(bigArrays(), sortOrder, bucketSize);
     }
 
     @Override
-    protected Object expectedValue(double v) {
-        return v;
+    protected Double randomValue() {
+        return randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true);
     }
 
     @Override
-    protected double randomValue() {
-        return randomDoubleBetween(Double.MIN_VALUE, Double.MAX_VALUE, true);
+    protected List<Double> threeSortedValues() {
+        return List.of(-Double.MAX_VALUE, randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true), Double.MAX_VALUE);
     }
 
     @Override
-    protected void collect(DoubleBucketedSort sort, double value, int bucket) {
+    protected void collect(DoubleBucketedSort sort, Double value, int bucket) {
         sort.collect(value, bucket);
     }
 
@@ -48,11 +50,11 @@ protected Block toBlock(DoubleBucketedSort sort, BlockFactory blockFactory, IntV
     }
 
     @Override
-    protected void assertBlockTypeAndValues(Block block, Object... values) {
+    protected void assertBlockTypeAndValues(Block block, List<Double> values) {
         assertThat(block.elementType(), equalTo(ElementType.DOUBLE));
         var typedBlock = (DoubleBlock) block;
-        for (int i = 0; i < values.length; i++) {
-            assertThat(typedBlock.getDouble(i), equalTo(values[i]));
+        for (int i = 0; i < values.size(); i++) {
+            assertThat(typedBlock.getDouble(i), equalTo(values.get(i)));
         }
     }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java
index 8b3d288339037..5378e92f688ec 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java
@@ -14,27 +14,29 @@
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.search.sort.SortOrder;
 
+import java.util.List;
+
 import static org.hamcrest.Matchers.equalTo;
 
-public class FloatBucketedSortTests extends BucketedSortTestCase<FloatBucketedSort> {
+public class FloatBucketedSortTests extends BucketedSortTestCase<FloatBucketedSort, Float> {
     @Override
     protected FloatBucketedSort build(SortOrder sortOrder, int bucketSize) {
         return new FloatBucketedSort(bigArrays(), sortOrder, bucketSize);
     }
 
     @Override
-    protected Object expectedValue(double v) {
-        return v;
+    protected Float randomValue() {
+        return randomFloatBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true);
     }
 
     @Override
-    protected double randomValue() {
-        return randomFloatBetween(Float.MIN_VALUE, Float.MAX_VALUE, true);
+    protected List<Float> threeSortedValues() {
+        return List.of(-Float.MAX_VALUE, randomFloatBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true), Float.MAX_VALUE);
     }
 
     @Override
-    protected void collect(FloatBucketedSort sort, double value, int bucket) {
-        sort.collect((float) value, bucket);
+    protected void collect(FloatBucketedSort sort, Float value, int bucket) {
+        sort.collect(value, bucket);
     }
 
     @Override
@@ -48,11 +50,11 @@ protected Block toBlock(FloatBucketedSort sort, BlockFactory blockFactory, IntVe
     }
 
     @Override
-    protected void assertBlockTypeAndValues(Block block, Object... values) {
+    protected void assertBlockTypeAndValues(Block block, List<Float> values) {
         assertThat(block.elementType(), equalTo(ElementType.FLOAT));
         var typedBlock = (FloatBlock) block;
-        for (int i = 0; i < values.length; i++) {
-            assertThat((double) typedBlock.getFloat(i), equalTo(values[i]));
+        for (int i = 0; i < values.size(); i++) {
+            assertThat(typedBlock.getFloat(i), equalTo(values.get(i)));
         }
     }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IntBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IntBucketedSortTests.java
index 70d0a79ea7473..62bf3ea69ceac 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IntBucketedSortTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IntBucketedSortTests.java
@@ -14,27 +14,29 @@
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.search.sort.SortOrder;
 
+import java.util.List;
+
 import static org.hamcrest.Matchers.equalTo;
 
-public class IntBucketedSortTests extends BucketedSortTestCase<IntBucketedSort> {
+public class IntBucketedSortTests extends BucketedSortTestCase<IntBucketedSort, Integer> {
     @Override
     protected IntBucketedSort build(SortOrder sortOrder, int bucketSize) {
         return new IntBucketedSort(bigArrays(), sortOrder, bucketSize);
     }
 
     @Override
-    protected Object expectedValue(double v) {
-        return (int) v;
+    protected Integer randomValue() {
+        return randomInt();
     }
 
     @Override
-    protected double randomValue() {
-        return randomInt();
+    protected List<Integer> threeSortedValues() {
+        return List.of(Integer.MIN_VALUE, randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE), Integer.MAX_VALUE);
     }
 
     @Override
-    protected void collect(IntBucketedSort sort, double value, int bucket) {
-        sort.collect((int) value, bucket);
+    protected void collect(IntBucketedSort sort, Integer value, int bucket) {
+        sort.collect(value, bucket);
     }
 
     @Override
@@ -48,11 +50,11 @@ protected Block toBlock(IntBucketedSort sort, BlockFactory blockFactory, IntVect
     }
 
     @Override
-    protected void assertBlockTypeAndValues(Block block, Object... values) {
+    protected void assertBlockTypeAndValues(Block block, List<Integer> values) {
         assertThat(block.elementType(), equalTo(ElementType.INT));
         var typedBlock = (IntBlock) block;
-        for (int i = 0; i < values.length; i++) {
-            assertThat(typedBlock.getInt(i), equalTo(values[i]));
+        for (int i = 0; i < values.size(); i++) {
+            assertThat(typedBlock.getInt(i), equalTo(values.get(i)));
         }
     }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IpBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IpBucketedSortTests.java
new file mode 100644
index 0000000000000..dad7af26a484d
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IpBucketedSortTests.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.data.sort;
+
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.ElementType;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.search.sort.SortOrder;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class IpBucketedSortTests extends BucketedSortTestCase<IpBucketedSort, BytesRef> {
+    @Override
+    protected IpBucketedSort build(SortOrder sortOrder, int bucketSize) {
+        return new IpBucketedSort(bigArrays(), sortOrder, bucketSize);
+    }
+
+    @Override
+    protected BytesRef randomValue() {
+        return new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())));
+    }
+
+    @Override
+    protected List<BytesRef> threeSortedValues() {
+        return List.of(
+            new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::"))),
+            new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))),
+            new BytesRef(InetAddressPoint.encode(InetAddresses.forString("9999::")))
+        );
+    }
+
+    @Override
+    protected void collect(IpBucketedSort sort, BytesRef value, int bucket) {
+        sort.collect(value, bucket);
+    }
+
+    @Override
+    protected void merge(IpBucketedSort sort, int groupId, IpBucketedSort other, int otherGroupId) {
+        sort.merge(groupId, other, otherGroupId);
+    }
+
+    @Override
+    protected Block toBlock(IpBucketedSort sort, BlockFactory blockFactory, IntVector selected) {
+        return sort.toBlock(blockFactory, selected);
+    }
+
+    @Override
+    protected void assertBlockTypeAndValues(Block block, List<BytesRef> values) {
+        assertThat(block.elementType(), equalTo(ElementType.BYTES_REF));
+        var typedBlock = (BytesRefBlock) block;
+        var scratch = new BytesRef();
+        for (int i = 0; i < values.size(); i++) {
+            assertThat("expected value on block position " + i, typedBlock.getBytesRef(i, scratch), equalTo(values.get(i)));
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/LongBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/LongBucketedSortTests.java
index bceed3b1d95b5..2d60a160c4458 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/LongBucketedSortTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/LongBucketedSortTests.java
@@ -14,28 +14,29 @@
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.search.sort.SortOrder;
 
+import java.util.List;
+
 import static org.hamcrest.Matchers.equalTo;
 
-public class LongBucketedSortTests extends BucketedSortTestCase<LongBucketedSort> {
+public class LongBucketedSortTests extends BucketedSortTestCase<LongBucketedSort, Long> {
     @Override
     protected LongBucketedSort build(SortOrder sortOrder, int bucketSize) {
         return new LongBucketedSort(bigArrays(), sortOrder, bucketSize);
     }
 
     @Override
-    protected Object expectedValue(double v) {
-        return (long) v;
+    protected Long randomValue() {
+        return randomLong();
     }
 
     @Override
-    protected double randomValue() {
-        // 2L^50 fits in the mantisa of a double which the test sort of needs.
-        return randomLongBetween(-2L ^ 50, 2L ^ 50);
+    protected List<Long> threeSortedValues() {
+        return List.of(Long.MIN_VALUE, randomLongBetween(Long.MIN_VALUE, Long.MAX_VALUE), Long.MAX_VALUE);
     }
 
     @Override
-    protected void collect(LongBucketedSort sort, double value, int bucket) {
-        sort.collect((long) value, bucket);
+    protected void collect(LongBucketedSort sort, Long value, int bucket) {
+        sort.collect(value, bucket);
     }
 
     @Override
@@ -49,11 +50,11 @@ protected Block toBlock(LongBucketedSort sort, BlockFactory blockFactory, IntVec
     }
 
     @Override
-    protected void assertBlockTypeAndValues(Block block, Object... values) {
+    protected void assertBlockTypeAndValues(Block block, List<Long> values) {
         assertThat(block.elementType(), equalTo(ElementType.LONG));
         var typedBlock = (LongBlock) block;
-        for (int i = 0; i < values.length; i++) {
-            assertThat(typedBlock.getLong(i), equalTo(values[i]));
+        for (int i = 0; i < values.size(); i++) {
+            assertThat(typedBlock.getLong(i), equalTo(values.get(i)));
         }
     }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryExpressionEvaluatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryExpressionEvaluatorTests.java
new file mode 100644
index 0000000000000..beca522878358
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryExpressionEvaluatorTests.java
@@ -0,0 +1,262 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.lucene;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.KeywordField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.store.BaseDirectoryWrapper;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.OperatorTests;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.ElementType;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.lucene.LuceneQueryExpressionEvaluator.DenseCollector;
+import org.elasticsearch.compute.operator.ComputeTestCase;
+import org.elasticsearch.compute.operator.Driver;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.compute.operator.Operator;
+import org.elasticsearch.compute.operator.OperatorTestCase;
+import org.elasticsearch.compute.operator.ShuffleDocsOperator;
+import org.elasticsearch.compute.operator.TestResultPageSinkOperator;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.index.mapper.BlockDocValuesReader;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.elasticsearch.compute.operator.OperatorTestCase.randomPageSize;
+import static org.hamcrest.Matchers.equalTo;
+
+public class LuceneQueryExpressionEvaluatorTests extends ComputeTestCase {
+    private static final String FIELD = "g";
+
+    public void testDenseCollectorSmall() {
+        try (DenseCollector collector = new DenseCollector(blockFactory(), 0, 2)) {
+            collector.collect(0);
+            collector.collect(1);
+            collector.collect(2);
+            collector.finish();
+            try (BooleanVector result = collector.build()) {
+                for (int i = 0; i <= 2; i++) {
+                    assertThat(result.getBoolean(i), equalTo(true));
+                }
+            }
+        }
+    }
+
+    public void testDenseCollectorSimple() {
+        try (DenseCollector collector = new DenseCollector(blockFactory(), 0, 10)) {
+            collector.collect(2);
+            collector.collect(5);
+            collector.finish();
+            try (BooleanVector result = collector.build()) {
+                for (int i = 0; i < 11; i++) {
+                    assertThat(result.getBoolean(i), equalTo(i == 2 || i == 5));
+                }
+            }
+        }
+    }
+
+    public void testDenseCollector() {
+        int length = between(1, 10_000);
+        int min = between(0, Integer.MAX_VALUE - length - 1);
+        int max = min + length + 1;
+        boolean[] expected = new boolean[length];
+        try (DenseCollector collector = new DenseCollector(blockFactory(), min, max)) {
+            for (int i = 0; i < length; i++) {
+                expected[i] = randomBoolean();
+                if (expected[i]) {
+                    collector.collect(min + i);
+                }
+            }
+            collector.finish();
+            try (BooleanVector result = collector.build()) {
+                for (int i = 0; i < length; i++) {
+                    assertThat(result.getBoolean(i), equalTo(expected[i]));
+                }
+            }
+        }
+    }
+
+    public void testTermQuery() throws IOException {
+        Set<String> values = values();
+        String term = values.iterator().next();
+        List<Page> results = runQuery(values, new TermQuery(new Term(FIELD, term)), false);
+        assertTermQuery(term, results);
+    }
+
+    public void testTermQueryShuffled() throws IOException {
+        Set<String> values = values();
+        String term = values.iterator().next();
+        List<Page> results = runQuery(values, new TermQuery(new Term(FIELD, term)), true);
+        assertTermQuery(term, results);
+    }
+
+    private void assertTermQuery(String term, List<Page> results) {
+        int matchCount = 0;
+        for (Page page : results) {
+            BytesRefVector terms = page.<BytesRefBlock>getBlock(1).asVector();
+            BooleanVector matches = page.<BooleanBlock>getBlock(2).asVector();
+            for (int i = 0; i < page.getPositionCount(); i++) {
+                BytesRef termAtPosition = terms.getBytesRef(i, new BytesRef());
+                assertThat(matches.getBoolean(i), equalTo(termAtPosition.utf8ToString().equals(term)));
+                if (matches.getBoolean(i)) {
+                    matchCount++;
+                }
+            }
+        }
+        assertThat(matchCount, equalTo(1));
+    }
+
+    public void testTermsQuery() throws IOException {
+        testTermsQuery(false);
+    }
+
+    public void testTermsQueryShuffled() throws IOException {
+        testTermsQuery(true);
+    }
+
+    private void testTermsQuery(boolean shuffleDocs) throws IOException {
+        Set<String> values = values();
+        Iterator<String> itr = values.iterator();
+        TreeSet<String> matching = new TreeSet<>();
+        TreeSet<BytesRef> matchingBytes = new TreeSet<>();
+        int expectedMatchCount = between(2, values.size());
+        for (int i = 0; i < expectedMatchCount; i++) {
+            String v = itr.next();
+            matching.add(v);
+            matchingBytes.add(new BytesRef(v));
+        }
+        List<Page> results = runQuery(values, new TermInSetQuery(MultiTermQuery.CONSTANT_SCORE_REWRITE, FIELD, matchingBytes), shuffleDocs);
+        int matchCount = 0;
+        for (Page page : results) {
+            BytesRefVector terms = page.<BytesRefBlock>getBlock(1).asVector();
+            BooleanVector matches = page.<BooleanBlock>getBlock(2).asVector();
+            for (int i = 0; i < page.getPositionCount(); i++) {
+                BytesRef termAtPosition = terms.getBytesRef(i, new BytesRef());
+                assertThat(matches.getBoolean(i), equalTo(matching.contains(termAtPosition.utf8ToString())));
+                if (matches.getBoolean(i)) {
+                    matchCount++;
+                }
+            }
+        }
+        assertThat(matchCount, equalTo(expectedMatchCount));
+    }
+
+    private List<Page> runQuery(Set<String> values, Query query, boolean shuffleDocs) throws IOException {
+        DriverContext driverContext = driverContext();
+        BlockFactory blockFactory = driverContext.blockFactory();
+        return withReader(values, reader -> {
+            IndexSearcher searcher = new IndexSearcher(reader);
+            LuceneQueryExpressionEvaluator.ShardConfig shard = new LuceneQueryExpressionEvaluator.ShardConfig(
+                searcher.rewrite(query),
+                searcher
+            );
+            LuceneQueryExpressionEvaluator luceneQueryEvaluator = new LuceneQueryExpressionEvaluator(
+                blockFactory,
+                new LuceneQueryExpressionEvaluator.ShardConfig[] { shard },
+                0
+            );
+
+            List<Operator> operators = new ArrayList<>();
+            if (shuffleDocs) {
+                operators.add(new ShuffleDocsOperator(blockFactory));
+            }
+            operators.add(
+                new ValuesSourceReaderOperator(
+                    blockFactory,
+                    List.of(
+                        new ValuesSourceReaderOperator.FieldInfo(
+                            FIELD,
+                            ElementType.BYTES_REF,
+                            unused -> new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(FIELD)
+                        )
+                    ),
+                    List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
+                        throw new UnsupportedOperationException();
+                    })),
+                    0
+                )
+            );
+            operators.add(new EvalOperator(blockFactory, luceneQueryEvaluator));
+            List<Page> results = new ArrayList<>();
+            Driver driver = new Driver(
+                driverContext,
+                luceneOperatorFactory(reader, new MatchAllDocsQuery(), LuceneOperator.NO_LIMIT).get(driverContext),
+                operators,
+                new TestResultPageSinkOperator(results::add),
+                () -> {}
+            );
+            OperatorTestCase.runDriver(driver);
+            OperatorTests.assertDriverContext(driverContext);
+            return results;
+        });
+    }
+
+    private <T> T withReader(Set<String> values, CheckedFunction<DirectoryReader, T, IOException> run) throws IOException {
+        try (BaseDirectoryWrapper dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
+            for (String value : values) {
+                writer.addDocument(List.of(new KeywordField(FIELD, value, Field.Store.NO)));
+            }
+            writer.commit();
+            try (DirectoryReader reader = writer.getReader()) {
+                return run.apply(reader);
+            }
+        }
+    }
+
+    private Set<String> values() {
+        int maxNumDocs = between(10, 1_000);
+        int keyLength = randomIntBetween(1, 10);
+        Set<String> values = new HashSet<>();
+        for (int i = 0; i < maxNumDocs; i++) {
+            values.add(randomAlphaOfLength(keyLength));
+        }
+        return values;
+    }
+
+    /**
+     * A {@link DriverContext} with a non-breaking-BigArrays.
+     */
+    private DriverContext driverContext() {
+        BlockFactory blockFactory = blockFactory();
+        return new DriverContext(blockFactory.bigArrays(), blockFactory);
+    }
+
+    static LuceneOperator.Factory luceneOperatorFactory(IndexReader reader, Query query, int limit) {
+        final ShardContext searchContext = new LuceneSourceOperatorTests.MockShardContext(reader, 0);
+        return new LuceneSourceOperator.Factory(
+            List.of(searchContext),
+            ctx -> query,
+            randomFrom(DataPartitioning.values()),
+            randomIntBetween(1, 10),
+            randomPageSize(),
+            limit
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java
index e88f6141049be..ccc3dea78adc8 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java
@@ -1549,7 +1549,11 @@ protected void start(Driver driver, ActionListener<Void> driverListener) {
         PlainActionFuture<Void> future = new PlainActionFuture<>();
         try {
             driverRunner.runToCompletion(drivers, future);
-            future.actionGet(TimeValue.timeValueSeconds(30));
+            /*
+             * We use a 3-minute timer because many of the cases can
+             * take 40 seconds in CI. Locally it's taking 9 seconds.
+             */
+            future.actionGet(TimeValue.timeValueMinutes(3));
         } finally {
             terminate(threadPool);
         }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java
index ae4558d5f8f71..fbcf11cd948c0 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java
@@ -159,49 +159,56 @@ public void doClose() {
         Releasables.close(localBreaker);
     }
 
-    public void testStatus() {
-        BlockFactory blockFactory = blockFactory();
-        DriverContext driverContext = new DriverContext(blockFactory.bigArrays(), blockFactory);
+    class TestOp extends AsyncOperator {
         Map<Page, ActionListener<Page>> handlers = new HashMap<>();
-        AsyncOperator operator = new AsyncOperator(driverContext, 2) {
-            @Override
-            protected void performAsync(Page inputPage, ActionListener<Page> listener) {
-                handlers.put(inputPage, listener);
-            }
 
-            @Override
-            protected void doClose() {
+        TestOp(DriverContext driverContext, int maxOutstandingRequests) {
+            super(driverContext, maxOutstandingRequests);
+        }
 
-            }
-        };
-        assertTrue(operator.isBlocked().isDone());
+        @Override
+        protected void performAsync(Page inputPage, ActionListener<Page> listener) {
+            handlers.put(inputPage, listener);
+        }
+
+        @Override
+        protected void doClose() {
+
+        }
+    }
+
+    public void testStatus() {
+        BlockFactory blockFactory = blockFactory();
+        DriverContext driverContext = new DriverContext(blockFactory.bigArrays(), blockFactory);
+        TestOp operator = new TestOp(driverContext, 2);
+        assertTrue(operator.isBlocked().listener().isDone());
         assertTrue(operator.needsInput());
 
         Page page1 = new Page(driverContext.blockFactory().newConstantNullBlock(1));
         operator.addInput(page1);
-        assertFalse(operator.isBlocked().isDone());
-        SubscribableListener<Void> blocked1 = operator.isBlocked();
+        assertFalse(operator.isBlocked().listener().isDone());
+        SubscribableListener<Void> blocked1 = operator.isBlocked().listener();
         assertTrue(operator.needsInput());
 
         Page page2 = new Page(driverContext.blockFactory().newConstantNullBlock(2));
         operator.addInput(page2);
         assertFalse(operator.needsInput()); // reached the max outstanding requests
-        assertFalse(operator.isBlocked().isDone());
-        assertThat(operator.isBlocked(), equalTo(blocked1));
+        assertFalse(operator.isBlocked().listener().isDone());
+        assertThat(operator.isBlocked(), equalTo(new IsBlockedResult(blocked1, "TestOp")));
 
         Page page3 = new Page(driverContext.blockFactory().newConstantNullBlock(3));
-        handlers.remove(page1).onResponse(page3);
+        operator.handlers.remove(page1).onResponse(page3);
         page1.releaseBlocks();
         assertFalse(operator.needsInput()); // still have 2 outstanding requests
-        assertTrue(operator.isBlocked().isDone());
+        assertTrue(operator.isBlocked().listener().isDone());
         assertTrue(blocked1.isDone());
         assertThat(operator.getOutput(), equalTo(page3));
         page3.releaseBlocks();
 
         assertTrue(operator.needsInput());
-        assertFalse(operator.isBlocked().isDone());
+        assertFalse(operator.isBlocked().listener().isDone());
         Page page4 = new Page(driverContext.blockFactory().newConstantNullBlock(3));
-        handlers.remove(page2).onResponse(page4);
+        operator.handlers.remove(page2).onResponse(page4);
         page2.releaseBlocks();
         assertThat(operator.getOutput(), equalTo(page4));
         page4.releaseBlocks();
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java
index 24f5297a0d6fe..266c17febc5b3 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java
@@ -32,7 +32,7 @@ public void testBreakOnBuild() {
 
     public void testAddByte() {
         testAgainstOracle(() -> new TestIteration() {
-            byte b = randomByte();
+            final byte b = randomByte();
 
             @Override
             public int size() {
@@ -53,7 +53,7 @@ public void applyToOracle(BytesRefBuilder oracle) {
 
     public void testAddBytesRef() {
         testAgainstOracle(() -> new TestIteration() {
-            BytesRef ref = new BytesRef(randomAlphaOfLengthBetween(1, 100));
+            final BytesRef ref = new BytesRef(randomAlphaOfLengthBetween(1, 100));
 
             @Override
             public int size() {
@@ -72,10 +72,23 @@ public void applyToOracle(BytesRefBuilder oracle) {
         });
     }
 
+    public void testCopyBytes() {
+        CircuitBreaker breaker = new MockBigArrays.LimitedBreaker(CircuitBreaker.REQUEST, ByteSizeValue.ofBytes(300));
+        try (BreakingBytesRefBuilder builder = new BreakingBytesRefBuilder(breaker, "test")) {
+            String initialValue = randomAlphaOfLengthBetween(1, 50);
+            builder.copyBytes(new BytesRef(initialValue));
+            assertThat(builder.bytesRefView().utf8ToString(), equalTo(initialValue));
+
+            String newValue = randomAlphaOfLengthBetween(350, 500);
+            Exception e = expectThrows(CircuitBreakingException.class, () -> builder.copyBytes(new BytesRef(newValue)));
+            assertThat(e.getMessage(), equalTo("over test limit"));
+        }
+    }
+
     public void testGrow() {
         testAgainstOracle(() -> new TestIteration() {
-            int length = between(1, 100);
-            byte b = randomByte();
+            final int length = between(1, 100);
+            final byte b = randomByte();
 
             @Override
             public int size() {
@@ -118,10 +131,11 @@ private void testAgainstOracle(Supplier<TestIteration> iterations) {
             assertThat(builder.bytesRefView(), equalTo(oracle.get()));
             while (true) {
                 TestIteration iteration = iterations.get();
-                boolean willResize = builder.length() + iteration.size() >= builder.bytes().length;
+                int targetSize = builder.length() + iteration.size();
+                boolean willResize = targetSize >= builder.bytes().length;
                 if (willResize) {
                     long resizeMemoryUsage = BreakingBytesRefBuilder.SHALLOW_SIZE + ramForArray(builder.bytes().length);
-                    resizeMemoryUsage += ramForArray(ArrayUtil.oversize(builder.length() + iteration.size(), Byte.BYTES));
+                    resizeMemoryUsage += ramForArray(ArrayUtil.oversize(targetSize, Byte.BYTES));
                     if (resizeMemoryUsage > limit) {
                         Exception e = expectThrows(CircuitBreakingException.class, () -> iteration.applyToBuilder(builder));
                         assertThat(e.getMessage(), equalTo("over test limit"));
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverProfileTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverProfileTests.java
index 86655bd3b7f73..27083ea0fcd13 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverProfileTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverProfileTests.java
@@ -20,22 +20,34 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
 
 public class DriverProfileTests extends AbstractWireSerializingTestCase<DriverProfile> {
     public void testToXContent() {
         DriverProfile status = new DriverProfile(
+            123413220000L,
+            123413243214L,
             10012,
             10000,
             12,
             List.of(
                 new DriverStatus.OperatorStatus("LuceneSource", LuceneSourceOperatorStatusTests.simple()),
                 new DriverStatus.OperatorStatus("ValuesSourceReader", ValuesSourceReaderOperatorStatusTests.simple())
+            ),
+            new DriverSleeps(
+                Map.of("driver time", 1L),
+                List.of(new DriverSleeps.Sleep("driver time", 1, 1)),
+                List.of(new DriverSleeps.Sleep("driver time", 1, 1))
             )
         );
         assertThat(Strings.toString(status, true, true), equalTo("""
             {
+              "start" : "1973-11-29T09:27:00.000Z",
+              "start_millis" : 123413220000,
+              "stop" : "1973-11-29T09:27:23.214Z",
+              "stop_millis" : 123413243214,
               "took_nanos" : 10012,
               "took_time" : "10micros",
               "cpu_nanos" : 10000,
@@ -54,7 +66,30 @@ public void testToXContent() {
             """.stripTrailing() + " " + ValuesSourceReaderOperatorStatusTests.simpleToJson().replace("\n", "\n      ") + """
 
                 }
-              ]
+              ],
+              "sleeps" : {
+                "counts" : {
+                  "driver time" : 1
+                },
+                "first" : [
+                  {
+                    "reason" : "driver time",
+                    "sleep" : "1970-01-01T00:00:00.001Z",
+                    "sleep_millis" : 1,
+                    "wake" : "1970-01-01T00:00:00.001Z",
+                    "wake_millis" : 1
+                  }
+                ],
+                "last" : [
+                  {
+                    "reason" : "driver time",
+                    "sleep" : "1970-01-01T00:00:00.001Z",
+                    "sleep_millis" : 1,
+                    "wake" : "1970-01-01T00:00:00.001Z",
+                    "wake_millis" : 1
+                  }
+                ]
+              }
             }"""));
     }
 
@@ -69,24 +104,33 @@ protected DriverProfile createTestInstance() {
             randomNonNegativeLong(),
             randomNonNegativeLong(),
             randomNonNegativeLong(),
-            DriverStatusTests.randomOperatorStatuses()
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            DriverStatusTests.randomOperatorStatuses(),
+            DriverSleepsTests.randomDriverSleeps()
         );
     }
 
     @Override
     protected DriverProfile mutateInstance(DriverProfile instance) throws IOException {
+        long startMillis = instance.startMillis();
+        long stopMillis = instance.stopMillis();
         long tookNanos = instance.tookNanos();
         long cpuNanos = instance.cpuNanos();
         long iterations = instance.iterations();
         var operators = instance.operators();
-        switch (between(0, 3)) {
-            case 0 -> tookNanos = randomValueOtherThan(tookNanos, ESTestCase::randomNonNegativeLong);
-            case 1 -> cpuNanos = randomValueOtherThan(cpuNanos, ESTestCase::randomNonNegativeLong);
-            case 2 -> iterations = randomValueOtherThan(iterations, ESTestCase::randomNonNegativeLong);
-            case 3 -> operators = randomValueOtherThan(operators, DriverStatusTests::randomOperatorStatuses);
+        var sleeps = instance.sleeps();
+        switch (between(0, 6)) {
+            case 0 -> startMillis = randomValueOtherThan(startMillis, ESTestCase::randomNonNegativeLong);
+            case 1 -> stopMillis = randomValueOtherThan(startMillis, ESTestCase::randomNonNegativeLong);
+            case 2 -> tookNanos = randomValueOtherThan(tookNanos, ESTestCase::randomNonNegativeLong);
+            case 3 -> cpuNanos = randomValueOtherThan(cpuNanos, ESTestCase::randomNonNegativeLong);
+            case 4 -> iterations = randomValueOtherThan(iterations, ESTestCase::randomNonNegativeLong);
+            case 5 -> operators = randomValueOtherThan(operators, DriverStatusTests::randomOperatorStatuses);
+            case 6 -> sleeps = randomValueOtherThan(sleeps, DriverSleepsTests::randomDriverSleeps);
             default -> throw new UnsupportedOperationException();
         }
-        return new DriverProfile(tookNanos, cpuNanos, iterations, operators);
+        return new DriverProfile(startMillis, stopMillis, tookNanos, cpuNanos, iterations, operators, sleeps);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverSleepsTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverSleepsTests.java
new file mode 100644
index 0000000000000..a0d956fcd6f6f
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverSleepsTests.java
@@ -0,0 +1,240 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.hamcrest.core.IsEqual.equalTo;
+
+public class DriverSleepsTests extends AbstractWireSerializingTestCase<DriverSleeps> {
+    public static DriverSleeps randomDriverSleeps() {
+        return randomDriverSleeps(between(0, DriverSleeps.RECORDS * 3));
+    }
+
+    private static DriverSleeps randomDriverSleeps(int cycles) {
+        DriverSleeps sleeps = DriverSleeps.empty();
+        long now = 0;
+        for (int i = 0; i < cycles; i++) {
+            now += between(1, 100000);
+            sleeps = sleeps.sleep(randomSleepReason(), now);
+            if (i != cycles - 1 || randomBoolean()) {
+                // Randomly don't wake on the last sleep
+                now += between(1, 100000);
+                sleeps = sleeps.wake(now);
+            }
+        }
+        return sleeps;
+    }
+
+    private static String randomSleepReason() {
+        return randomFrom("driver time", "driver iteration", "exchange empty", "exchange full");
+    }
+
+    public void testEmptyToXContent() {
+        assertThat(Strings.toString(DriverSleeps.empty(), true, true), equalTo("""
+            {
+              "counts" : { },
+              "first" : [ ],
+              "last" : [ ]
+            }"""));
+    }
+
+    public void testSleepingToXContent() {
+        assertThat(Strings.toString(DriverSleeps.empty().sleep("driver iterations", 1723555763000L), true, true), equalTo("""
+            {
+              "counts" : {
+                "driver iterations" : 1
+              },
+              "first" : [
+                {
+                  "reason" : "driver iterations",
+                  "sleep" : "2024-08-13T13:29:23.000Z",
+                  "sleep_millis" : 1723555763000
+                }
+              ],
+              "last" : [
+                {
+                  "reason" : "driver iterations",
+                  "sleep" : "2024-08-13T13:29:23.000Z",
+                  "sleep_millis" : 1723555763000
+                }
+              ]
+            }"""));
+    }
+
+    public void testWakingToXContent() {
+        assertThat(
+            Strings.toString(DriverSleeps.empty().sleep("driver iterations", 1723555763000L).wake(1723555863000L), true, true),
+            equalTo("""
+                {
+                  "counts" : {
+                    "driver iterations" : 1
+                  },
+                  "first" : [
+                    {
+                      "reason" : "driver iterations",
+                      "sleep" : "2024-08-13T13:29:23.000Z",
+                      "sleep_millis" : 1723555763000,
+                      "wake" : "2024-08-13T13:31:03.000Z",
+                      "wake_millis" : 1723555863000
+                    }
+                  ],
+                  "last" : [
+                    {
+                      "reason" : "driver iterations",
+                      "sleep" : "2024-08-13T13:29:23.000Z",
+                      "sleep_millis" : 1723555763000,
+                      "wake" : "2024-08-13T13:31:03.000Z",
+                      "wake_millis" : 1723555863000
+                    }
+                  ]
+                }""")
+        );
+    }
+
+    @Override
+    protected Writeable.Reader<DriverSleeps> instanceReader() {
+        return DriverSleeps::read;
+    }
+
+    @Override
+    protected DriverSleeps createTestInstance() {
+        return randomDriverSleeps();
+    }
+
+    @Override
+    protected DriverSleeps mutateInstance(DriverSleeps instance) throws IOException {
+        if (instance.last().isEmpty()) {
+            return instance.sleep(randomSleepReason(), between(1, 10000));
+        }
+        DriverSleeps.Sleep last = instance.last().get(instance.last().size() - 1);
+        if (last.isStillSleeping()) {
+            return instance.wake(last.sleep() + between(1, 10000));
+        }
+        return instance.sleep(randomSleepReason(), last.wake() + between(1, 10000));
+    }
+
+    public void testTracking() throws IOException {
+        long now = 0;
+        DriverSleeps sleeps = DriverSleeps.empty();
+
+        Map<String, Long> expectedCounts = new TreeMap<>();
+        List<DriverSleeps.Sleep> expectedFirst = new ArrayList<>();
+        assertThat(sleeps, equalTo(new DriverSleeps(expectedCounts, expectedFirst, expectedFirst)));
+
+        /*
+         * Simulate sleeping and waking when the records aren't full.
+         * New sleeps and wakes should show up in both the "first" and "last" fields.
+         */
+        for (int i = 0; i < DriverSleeps.RECORDS; i++) {
+            now++;
+            String reason = randomSleepReason();
+            expectedCounts.compute(reason, (k, v) -> v == null ? 1 : v + 1);
+
+            sleeps = sleeps.sleep(reason, now);
+            expectedFirst.add(new DriverSleeps.Sleep(reason, now, 0));
+            assertThat(sleeps, equalTo(new DriverSleeps(expectedCounts, expectedFirst, expectedFirst)));
+            assertXContent(sleeps, expectedCounts, expectedFirst, expectedFirst);
+
+            now++;
+            sleeps = sleeps.wake(now);
+            expectedFirst.set(expectedFirst.size() - 1, new DriverSleeps.Sleep(reason, now - 1, now));
+            assertThat(sleeps, equalTo(new DriverSleeps(expectedCounts, expectedFirst, expectedFirst)));
+            assertXContent(sleeps, expectedCounts, expectedFirst, expectedFirst);
+        }
+
+        /*
+         * Simulate sleeping and waking when the records are full.
+         * New sleeps and wakes should show up in only the "last" field.
+         */
+        List<DriverSleeps.Sleep> expectedLast = new ArrayList<>(expectedFirst);
+        for (int i = 0; i < 1000; i++) {
+            now++;
+            String reason = randomSleepReason();
+            expectedCounts.compute(reason, (k, v) -> v == null ? 1 : v + 1);
+
+            sleeps = sleeps.sleep(reason, now);
+            expectedLast.remove(0);
+            expectedLast.add(new DriverSleeps.Sleep(reason, now, 0));
+            assertThat(sleeps, equalTo(new DriverSleeps(expectedCounts, expectedFirst, expectedLast)));
+            assertXContent(sleeps, expectedCounts, expectedFirst, expectedLast);
+
+            now++;
+            sleeps = sleeps.wake(now);
+            expectedLast.set(expectedLast.size() - 1, new DriverSleeps.Sleep(reason, now - 1, now));
+            assertThat(sleeps, equalTo(new DriverSleeps(expectedCounts, expectedFirst, expectedLast)));
+            assertXContent(sleeps, expectedCounts, expectedFirst, expectedLast);
+        }
+    }
+
+    public void assertXContent(
+        DriverSleeps sleeps,
+        Map<String, Long> expectedCounts,
+        List<DriverSleeps.Sleep> expectedFirst,
+        List<DriverSleeps.Sleep> expectedLast
+    ) throws IOException {
+        try (BytesStreamOutput expected = new BytesStreamOutput()) {
+            try (XContentBuilder b = new XContentBuilder(XContentType.JSON.xContent(), expected).prettyPrint().humanReadable(true)) {
+                b.startObject();
+                b.startObject("counts");
+                {
+                    for (Map.Entry<String, Long> e : expectedCounts.entrySet()) {
+                        b.field(e.getKey(), e.getValue());
+                    }
+                }
+                b.endObject();
+                {
+                    b.startArray("first");
+                    for (DriverSleeps.Sleep sleep : expectedFirst) {
+                        sleep.toXContent(b, ToXContent.EMPTY_PARAMS);
+                    }
+                    b.endArray();
+                }
+                {
+                    b.startArray("last");
+                    for (DriverSleeps.Sleep sleep : expectedLast) {
+                        sleep.toXContent(b, ToXContent.EMPTY_PARAMS);
+                    }
+                    b.endArray();
+                }
+                b.endObject();
+            }
+            assertThat(Strings.toString(sleeps, true, true), equalTo(expected.bytes().utf8ToString()));
+        }
+    }
+
+    public void testWakeNeverSlept() {
+        Exception e = expectThrows(IllegalStateException.class, () -> DriverSleeps.empty().wake(1));
+        assertThat(e.getMessage(), equalTo("Never slept."));
+    }
+
+    public void testWakeWhileAwake() {
+        Exception e = expectThrows(IllegalStateException.class, () -> DriverSleeps.empty().sleep(randomSleepReason(), 1).wake(2).wake(3));
+        assertThat(e.getMessage(), equalTo("Already awake."));
+    }
+
+    public void testSleepWhileSleeping() {
+        Exception e = expectThrows(
+            IllegalStateException.class,
+            () -> DriverSleeps.empty().sleep(randomSleepReason(), 1).sleep(randomSleepReason(), 2)
+        );
+        assertThat(e.getMessage(), equalTo("Still sleeping."));
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverStatusTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverStatusTests.java
index e82cbb831cff2..b46d9f3f4add7 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverStatusTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverStatusTests.java
@@ -23,6 +23,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -40,7 +41,12 @@ public void testToXContent() {
                 new DriverStatus.OperatorStatus("LuceneSource", LuceneSourceOperatorStatusTests.simple()),
                 new DriverStatus.OperatorStatus("ValuesSourceReader", ValuesSourceReaderOperatorStatusTests.simple())
             ),
-            List.of(new DriverStatus.OperatorStatus("ExchangeSink", ExchangeSinkOperatorStatusTests.simple()))
+            List.of(new DriverStatus.OperatorStatus("ExchangeSink", ExchangeSinkOperatorStatusTests.simple())),
+            new DriverSleeps(
+                Map.of("driver time", 1L),
+                List.of(new DriverSleeps.Sleep("driver time", 1, 1)),
+                List.of(new DriverSleeps.Sleep("driver time", 1, 1))
+            )
         );
         assertThat(Strings.toString(status, true, true), equalTo("""
             {
@@ -72,7 +78,30 @@ public void testToXContent() {
             """.stripTrailing() + " " + ExchangeSinkOperatorStatusTests.simpleToJson().replace("\n", "\n      ") + """
 
                 }
-              ]
+              ],
+              "sleeps" : {
+                "counts" : {
+                  "driver time" : 1
+                },
+                "first" : [
+                  {
+                    "reason" : "driver time",
+                    "sleep" : "1970-01-01T00:00:00.001Z",
+                    "sleep_millis" : 1,
+                    "wake" : "1970-01-01T00:00:00.001Z",
+                    "wake_millis" : 1
+                  }
+                ],
+                "last" : [
+                  {
+                    "reason" : "driver time",
+                    "sleep" : "1970-01-01T00:00:00.001Z",
+                    "sleep_millis" : 1,
+                    "wake" : "1970-01-01T00:00:00.001Z",
+                    "wake_millis" : 1
+                  }
+                ]
+              }
             }"""));
     }
 
@@ -91,7 +120,8 @@ protected DriverStatus createTestInstance() {
             randomNonNegativeLong(),
             randomStatus(),
             randomOperatorStatuses(),
-            randomOperatorStatuses()
+            randomOperatorStatuses(),
+            DriverSleepsTests.randomDriverSleeps()
         );
     }
 
@@ -127,7 +157,8 @@ protected DriverStatus mutateInstance(DriverStatus instance) throws IOException
         var status = instance.status();
         var completedOperators = instance.completedOperators();
         var activeOperators = instance.activeOperators();
-        switch (between(0, 7)) {
+        var sleeps = instance.sleeps();
+        switch (between(0, 8)) {
             case 0 -> sessionId = randomValueOtherThan(sessionId, this::randomSessionId);
             case 1 -> started = randomValueOtherThan(started, ESTestCase::randomNonNegativeLong);
             case 2 -> lastUpdated = randomValueOtherThan(lastUpdated, ESTestCase::randomNonNegativeLong);
@@ -136,9 +167,10 @@ protected DriverStatus mutateInstance(DriverStatus instance) throws IOException
             case 5 -> status = randomValueOtherThan(status, this::randomStatus);
             case 6 -> completedOperators = randomValueOtherThan(completedOperators, DriverStatusTests::randomOperatorStatuses);
             case 7 -> activeOperators = randomValueOtherThan(activeOperators, DriverStatusTests::randomOperatorStatuses);
+            case 8 -> sleeps = randomValueOtherThan(sleeps, DriverSleepsTests::randomDriverSleeps);
             default -> throw new UnsupportedOperationException();
         }
-        return new DriverStatus(sessionId, started, lastUpdated, cpuNanos, iterations, status, completedOperators, activeOperators);
+        return new DriverStatus(sessionId, started, lastUpdated, cpuNanos, iterations, status, completedOperators, activeOperators, sleeps);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java
index d5fa0a1eaecc9..637cbe8892b3e 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FailureCollectorTests.java
@@ -7,12 +7,15 @@
 
 package org.elasticsearch.compute.operator;
 
+import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.NodeDisconnectedException;
 import org.elasticsearch.transport.RemoteTransportException;
+import org.elasticsearch.transport.TransportException;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -25,6 +28,9 @@
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
+import static org.hamcrest.Matchers.arrayWithSize;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThan;
 
 public class FailureCollectorTests extends ESTestCase {
@@ -87,4 +93,17 @@ public void testEmpty() {
         assertFalse(collector.hasFailure());
         assertNull(collector.getFailure());
     }
+
+    public void testTransportExceptions() {
+        FailureCollector collector = new FailureCollector(5);
+        collector.unwrapAndCollect(new NodeDisconnectedException(DiscoveryNodeUtils.builder("node-1").build(), "/field_caps"));
+        collector.unwrapAndCollect(new TransportException(new CircuitBreakingException("too large", CircuitBreaker.Durability.TRANSIENT)));
+        Exception failure = collector.getFailure();
+        assertNotNull(failure);
+        assertThat(failure, instanceOf(NodeDisconnectedException.class));
+        assertThat(failure.getMessage(), equalTo("[][0.0.0.0:1][/field_caps] disconnected"));
+        Throwable[] suppressed = failure.getSuppressed();
+        assertThat(suppressed, arrayWithSize(1));
+        assertThat(suppressed[0], instanceOf(CircuitBreakingException.class));
+    }
 }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java
index fa72721545ab9..c0396fdc469aa 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java
@@ -204,7 +204,11 @@ protected void start(Driver driver, ActionListener<Void> listener) {
     List<Driver> createDriversForInput(List<Page> input, List<Page> results, boolean throwingOp) {
         Collection<List<Page>> splitInput = randomSplits(input, randomIntBetween(2, 4));
         BlockFactory factory = blockFactory();
-        ExchangeSinkHandler sinkExchanger = new ExchangeSinkHandler(factory, randomIntBetween(2, 10), threadPool::relativeTimeInMillis);
+        ExchangeSinkHandler sinkExchanger = new ExchangeSinkHandler(
+            factory,
+            randomIntBetween(2, 10),
+            threadPool.relativeTimeInMillisSupplier()
+        );
         ExchangeSourceHandler sourceExchanger = new ExchangeSourceHandler(randomIntBetween(1, 4), threadPool.executor(ESQL_TEST_EXECUTOR));
         sourceExchanger.addRemoteSink(sinkExchanger::fetchPageAsync, 1);
 
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ShuffleDocsOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ShuffleDocsOperator.java
new file mode 100644
index 0000000000000..955d0237c65f7
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ShuffleDocsOperator.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.DocVector;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.core.Releasables;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.lucene.tests.util.LuceneTestCase.random;
+
+public class ShuffleDocsOperator extends AbstractPageMappingOperator {
+    private final BlockFactory blockFactory;
+
+    public ShuffleDocsOperator(BlockFactory blockFactory) {
+        this.blockFactory = blockFactory;
+    }
+
+    @Override
+    protected Page process(Page page) {
+        DocVector docVector = (DocVector) page.getBlock(0).asVector();
+        int positionCount = docVector.getPositionCount();
+        IntVector shards = null;
+        IntVector segments = null;
+        IntVector docs = null;
+        try (
+            IntVector.Builder shardsBuilder = blockFactory.newIntVectorBuilder(positionCount);
+            IntVector.Builder segmentsBuilder = blockFactory.newIntVectorBuilder(positionCount);
+            IntVector.Builder docsBuilder = blockFactory.newIntVectorBuilder(positionCount);
+        ) {
+            List<Integer> docIds = new ArrayList<>(positionCount);
+            for (int i = 0; i < positionCount; i++) {
+                shardsBuilder.appendInt(docVector.shards().getInt(i));
+                segmentsBuilder.appendInt(docVector.segments().getInt(i));
+                docIds.add(docVector.docs().getInt(i));
+            }
+            shards = shardsBuilder.build();
+            segments = segmentsBuilder.build();
+            Collections.shuffle(docIds, random());
+            for (Integer d : docIds) {
+                docsBuilder.appendInt(d);
+            }
+            docs = docsBuilder.build();
+        } finally {
+            if (docs == null) {
+                Releasables.closeExpectNoException(docVector, shards, segments);
+            } else {
+                Releasables.closeExpectNoException(docVector);
+            }
+        }
+        Block[] blocks = new Block[page.getBlockCount()];
+        blocks[0] = new DocVector(shards, segments, docs, false).asBlock();
+        for (int i = 1; i < blocks.length; i++) {
+            blocks[i] = page.getBlock(i);
+        }
+        return new Page(blocks);
+    }
+
+    @Override
+    public String toString() {
+        return "ShuffleDocs";
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java
index 51332b3c8997a..ab785e739d080 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java
@@ -91,7 +91,7 @@ public void testBasic() throws Exception {
         for (int i = 0; i < pages.length; i++) {
             pages[i] = new Page(blockFactory.newConstantIntBlockWith(i, 2));
         }
-        ExchangeSinkHandler sinkExchanger = new ExchangeSinkHandler(blockFactory, 2, threadPool::relativeTimeInMillis);
+        ExchangeSinkHandler sinkExchanger = new ExchangeSinkHandler(blockFactory, 2, threadPool.relativeTimeInMillisSupplier());
         ExchangeSink sink1 = sinkExchanger.createExchangeSink();
         ExchangeSink sink2 = sinkExchanger.createExchangeSink();
         ExchangeSourceHandler sourceExchanger = new ExchangeSourceHandler(3, threadPool.executor(ESQL_TEST_EXECUTOR));
@@ -99,40 +99,40 @@ public void testBasic() throws Exception {
         sourceExchanger.addCompletionListener(sourceCompletion);
         ExchangeSource source = sourceExchanger.createExchangeSource();
         sourceExchanger.addRemoteSink(sinkExchanger::fetchPageAsync, 1);
-        SubscribableListener<Void> waitForReading = source.waitForReading();
+        SubscribableListener<Void> waitForReading = source.waitForReading().listener();
         assertFalse(waitForReading.isDone());
         assertNull(source.pollPage());
-        assertTrue(sink1.waitForWriting().isDone());
+        assertTrue(sink1.waitForWriting().listener().isDone());
         randomFrom(sink1, sink2).addPage(pages[0]);
         randomFrom(sink1, sink2).addPage(pages[1]);
         // source and sink buffers can store 5 pages
         for (Page p : List.of(pages[2], pages[3], pages[4])) {
             ExchangeSink sink = randomFrom(sink1, sink2);
-            assertBusy(() -> assertTrue(sink.waitForWriting().isDone()));
+            assertBusy(() -> assertTrue(sink.waitForWriting().listener().isDone()));
             sink.addPage(p);
         }
         // sink buffer is full
-        assertFalse(randomFrom(sink1, sink2).waitForWriting().isDone());
-        assertBusy(() -> assertTrue(source.waitForReading().isDone()));
+        assertFalse(randomFrom(sink1, sink2).waitForWriting().listener().isDone());
+        assertBusy(() -> assertTrue(source.waitForReading().listener().isDone()));
         assertEquals(pages[0], source.pollPage());
-        assertBusy(() -> assertTrue(source.waitForReading().isDone()));
+        assertBusy(() -> assertTrue(source.waitForReading().listener().isDone()));
         assertEquals(pages[1], source.pollPage());
         // sink can write again
-        assertBusy(() -> assertTrue(randomFrom(sink1, sink2).waitForWriting().isDone()));
+        assertBusy(() -> assertTrue(randomFrom(sink1, sink2).waitForWriting().listener().isDone()));
         randomFrom(sink1, sink2).addPage(pages[5]);
-        assertBusy(() -> assertTrue(randomFrom(sink1, sink2).waitForWriting().isDone()));
+        assertBusy(() -> assertTrue(randomFrom(sink1, sink2).waitForWriting().listener().isDone()));
         randomFrom(sink1, sink2).addPage(pages[6]);
         // sink buffer is full
-        assertFalse(randomFrom(sink1, sink2).waitForWriting().isDone());
+        assertFalse(randomFrom(sink1, sink2).waitForWriting().listener().isDone());
         sink1.finish();
         assertTrue(sink1.isFinished());
         for (int i = 0; i < 5; i++) {
-            assertBusy(() -> assertTrue(source.waitForReading().isDone()));
+            assertBusy(() -> assertTrue(source.waitForReading().listener().isDone()));
             assertEquals(pages[2 + i], source.pollPage());
         }
         // source buffer is empty
-        assertFalse(source.waitForReading().isDone());
-        assertBusy(() -> assertTrue(sink2.waitForWriting().isDone()));
+        assertFalse(source.waitForReading().listener().isDone());
+        assertBusy(() -> assertTrue(sink2.waitForWriting().listener().isDone()));
         sink2.finish();
         assertTrue(sink2.isFinished());
         assertTrue(source.isFinished());
@@ -334,7 +334,7 @@ public void testConcurrentWithHandlers() {
             if (sinkHandlers.isEmpty() == false && randomBoolean()) {
                 sinkHandler = randomFrom(sinkHandlers);
             } else {
-                sinkHandler = new ExchangeSinkHandler(blockFactory, randomExchangeBuffer(), threadPool::relativeTimeInMillis);
+                sinkHandler = new ExchangeSinkHandler(blockFactory, randomExchangeBuffer(), threadPool.relativeTimeInMillisSupplier());
                 sourceExchanger.addRemoteSink(sinkHandler::fetchPageAsync, randomIntBetween(1, 3));
                 sinkHandlers.add(sinkHandler);
             }
@@ -352,17 +352,17 @@ public void testEarlyTerminate() {
         IntBlock block2 = blockFactory.newConstantIntBlockWith(1, 2);
         Page p1 = new Page(block1);
         Page p2 = new Page(block2);
-        ExchangeSinkHandler sinkExchanger = new ExchangeSinkHandler(blockFactory, 2, threadPool::relativeTimeInMillis);
+        ExchangeSinkHandler sinkExchanger = new ExchangeSinkHandler(blockFactory, 2, threadPool.relativeTimeInMillisSupplier());
         ExchangeSink sink = sinkExchanger.createExchangeSink();
         sink.addPage(p1);
         sink.addPage(p2);
-        assertFalse(sink.waitForWriting().isDone());
+        assertFalse(sink.waitForWriting().listener().isDone());
         PlainActionFuture<ExchangeResponse> future = new PlainActionFuture<>();
         sinkExchanger.fetchPageAsync(true, future);
         ExchangeResponse resp = future.actionGet();
         assertTrue(resp.finished());
         assertNull(resp.takePage());
-        assertTrue(sink.waitForWriting().isDone());
+        assertTrue(sink.waitForWriting().listener().isDone());
         assertTrue(sink.isFinished());
     }
 
diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java
index 443813442165a..f2633dfffb0fe 100644
--- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java
+++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
+import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -22,9 +23,12 @@
 import java.util.Locale;
 
 import static org.elasticsearch.core.TimeValue.timeValueNanos;
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class EsqlAsyncSecurityIT extends EsqlSecurityIT {
 
@@ -45,6 +49,11 @@ protected Response runESQLCommand(String user, String command) throws IOExceptio
         return getResponse;
     }
 
+    @Override
+    protected MapMatcher responseMatcher() {
+        return super.responseMatcher().entry("is_running", equalTo(false)).entry("id", allOf(notNullValue(), instanceOf(String.class)));
+    }
+
     @Override
     public void testUnauthorizedIndices() throws IOException {
         super.testUnauthorizedIndices();
@@ -58,7 +67,7 @@ public void testUnauthorizedIndices() throws IOException {
             var getResponse = runAsyncGet("user1", id); // sanity
             assertOK(getResponse);
             ResponseException error;
-            error = expectThrows(ResponseException.class, () -> runAsyncGet("user2", id));
+            error = expectThrows(ResponseException.class, () -> runAsyncGet("user2", id, true));
             // resource not found exception if the authenticated user is not the creator of the original task
             assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(404));
 
@@ -76,7 +85,7 @@ public void testUnauthorizedIndices() throws IOException {
             var getResponse = runAsyncGet("user2", id); // sanity
             assertOK(getResponse);
             ResponseException error;
-            error = expectThrows(ResponseException.class, () -> runAsyncGet("user1", id));
+            error = expectThrows(ResponseException.class, () -> runAsyncGet("user1", id, true));
             assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(404));
 
             error = expectThrows(ResponseException.class, () -> runAsyncDelete("user1", id));
@@ -108,6 +117,10 @@ private Response runAsync(String user, String command) throws IOException {
     }
 
     private Response runAsyncGet(String user, String id) throws IOException {
+        return runAsyncGet(user, id, false);
+    }
+
+    private Response runAsyncGet(String user, String id, boolean isAsyncIdNotFound_Expected) throws IOException {
         int tries = 0;
         while (tries < 10) {
             // Sometimes we get 404s fetching the task status.
@@ -120,22 +133,32 @@ private Response runAsyncGet(String user, String id) throws IOException {
                 logResponse(response);
                 return response;
             } catch (ResponseException e) {
-                if (e.getResponse().getStatusLine().getStatusCode() == 404
-                    && EntityUtils.toString(e.getResponse().getEntity()).contains("no such index [.async-search]")) {
-                    /*
-                     * Work around https://github.com/elastic/elasticsearch/issues/110304 - the .async-search
-                     * index may not exist when we try the fetch, but it should exist on next attempt.
-                     */
+                var statusCode = e.getResponse().getStatusLine().getStatusCode();
+                var message = EntityUtils.toString(e.getResponse().getEntity());
+
+                if (statusCode == 404 && message.contains("no such index [.async-search]")) {
+                    // Work around https://github.com/elastic/elasticsearch/issues/110304 - the .async-search
+                    // index may not exist when we try the fetch, but it should exist on next attempt.
                     logger.warn("async-search index does not exist", e);
                     try {
                         Thread.sleep(1000);
                     } catch (InterruptedException ex) {
                         throw new RuntimeException(ex);
                     }
+                } else if (statusCode == 404 && false == isAsyncIdNotFound_Expected && message.contains("resource_not_found_exception")) {
+                    // Work around for https://github.com/elastic/elasticsearch/issues/112110
+                    // The async id is not indexed quickly enough in .async-search index for us to retrieve it.
+                    logger.warn("async id not found", e);
+                    try {
+                        Thread.sleep(500);
+                    } catch (InterruptedException ex) {
+                        throw new RuntimeException(ex);
+                    }
                 } else {
                     throw e;
                 }
                 tries++;
+                logger.warn("retry [" + tries + "] for GET /_query/async/" + id);
             }
         }
         throw new IllegalStateException("couldn't find task status");
diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
index ca477d4256527..2b162b4f18ead 100644
--- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
+++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
@@ -18,6 +18,8 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.util.resource.Resource;
@@ -43,7 +45,6 @@ public class EsqlSecurityIT extends ESRestTestCase {
         .distribution(DistributionType.DEFAULT)
         .setting("xpack.license.self_generated.type", "trial")
         .setting("xpack.security.enabled", "true")
-        .setting("xpack.ml.enabled", "false")
         .rolesFile(Resource.fromClasspath("roles.yml"))
         .user("test-admin", "x-pack-test-password", "test-admin", true)
         .user("user1", "x-pack-test-password", "user1", false)
@@ -53,6 +54,8 @@ public class EsqlSecurityIT extends ESRestTestCase {
         .user("user5", "x-pack-test-password", "user5", false)
         .user("fls_user", "x-pack-test-password", "fls_user", false)
         .user("metadata1_read2", "x-pack-test-password", "metadata1_read2", false)
+        .user("alias_user1", "x-pack-test-password", "alias_user1", false)
+        .user("alias_user2", "x-pack-test-password", "alias_user2", false)
         .build();
 
     @Override
@@ -100,6 +103,38 @@ public void indexDocuments() throws IOException {
         indexDocument("indexpartial", 1, 32.0, "marketing");
         indexDocument("indexpartial", 2, 40.0, "sales");
         refresh("indexpartial");
+
+        if (aliasExists("second-alias") == false) {
+            Request aliasRequest = new Request("POST", "_aliases");
+            aliasRequest.setJsonEntity("""
+                {
+                    "actions": [
+                        {
+                          "add": {
+                            "alias": "first-alias",
+                            "index": "index-user1",
+                            "filter": {
+                                "term": {
+                                    "org": "sales"
+                                }
+                            }
+                          }
+                        },
+                        {
+                          "add": {
+                            "alias": "second-alias",
+                            "index": "index-user2"
+                          }
+                        }
+                    ]
+                }
+                """);
+            assertOK(client().performRequest(aliasRequest));
+        }
+    }
+
+    protected MapMatcher responseMatcher() {
+        return matchesMap();
     }
 
     public void testAllowedIndices() throws Exception {
@@ -108,23 +143,61 @@ public void testAllowedIndices() throws Exception {
             assertOK(resp);
             Map<String, Object> respMap = entityAsMap(resp);
             assertThat(respMap.get("columns"), equalTo(List.of(Map.of("name", "sum", "type", "double"))));
-            assertThat(respMap.get("values"), equalTo(List.of(List.of(30.0))));
+            assertThat(respMap.get("values"), equalTo(List.of(List.of(30.0d))));
         }
 
         for (String user : List.of("test-admin", "user1")) {
             Response resp = runESQLCommand(user, "from index-user1 | stats sum=sum(value)");
             assertOK(resp);
-            Map<String, Object> respMap = entityAsMap(resp);
-            assertThat(respMap.get("columns"), equalTo(List.of(Map.of("name", "sum", "type", "double"))));
-            assertThat(respMap.get("values"), equalTo(List.of(List.of(43.0))));
+            MapMatcher matcher = responseMatcher().entry("columns", List.of(Map.of("name", "sum", "type", "double")))
+                .entry("values", List.of(List.of(43.0d)));
+            assertMap(entityAsMap(resp), matcher);
         }
 
         for (String user : List.of("test-admin", "user2")) {
             Response resp = runESQLCommand(user, "from index-user2 | stats sum=sum(value)");
             assertOK(resp);
-            Map<String, Object> respMap = entityAsMap(resp);
-            assertThat(respMap.get("columns"), equalTo(List.of(Map.of("name", "sum", "type", "double"))));
-            assertThat(respMap.get("values"), equalTo(List.of(List.of(72.0))));
+            MapMatcher matcher = responseMatcher().entry("columns", List.of(Map.of("name", "sum", "type", "double")))
+                .entry("values", List.of(List.of(72.0d)));
+            assertMap(entityAsMap(resp), matcher);
+        }
+        for (var index : List.of("index-user2", "index-user*", "index*")) {
+            Response resp = runESQLCommand("metadata1_read2", "from " + index + " | stats sum=sum(value)");
+            assertOK(resp);
+            MapMatcher matcher = responseMatcher().entry("columns", List.of(Map.of("name", "sum", "type", "double")))
+                .entry("values", List.of(List.of(72.0d)));
+            assertMap(entityAsMap(resp), matcher);
+        }
+    }
+
+    public void testAliases() throws Exception {
+        for (var index : List.of("second-alias", "second-*", "second-*,index*")) {
+            Response resp = runESQLCommand(
+                "alias_user2",
+                "from " + index + " METADATA _index" + "| stats sum=sum(value), index=VALUES(_index)"
+            );
+            assertOK(resp);
+            MapMatcher matcher = responseMatcher().entry(
+                "columns",
+                List.of(Map.of("name", "sum", "type", "double"), Map.of("name", "index", "type", "keyword"))
+            ).entry("values", List.of(List.of(72.0d, "index-user2")));
+            assertMap(entityAsMap(resp), matcher);
+        }
+    }
+
+    public void testAliasFilter() throws Exception {
+        for (var index : List.of("first-alias", "first-alias,index-*", "first-*,index-*")) {
+            Response resp = runESQLCommand("alias_user1", "from " + index + " METADATA _index" + "| KEEP _index, org, value | LIMIT 10");
+            assertOK(resp);
+            MapMatcher matcher = responseMatcher().entry(
+                "columns",
+                List.of(
+                    Map.of("name", "_index", "type", "keyword"),
+                    Map.of("name", "org", "type", "keyword"),
+                    Map.of("name", "value", "type", "double")
+                )
+            ).entry("values", List.of(List.of("index-user1", "sales", 31.0d)));
+            assertMap(entityAsMap(resp), matcher);
         }
     }
 
@@ -135,6 +208,12 @@ public void testUnauthorizedIndices() throws IOException {
 
         error = expectThrows(ResponseException.class, () -> runESQLCommand("user2", "from index-user1 | stats sum(value)"));
         assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));
+
+        error = expectThrows(ResponseException.class, () -> runESQLCommand("alias_user2", "from index-user2 | stats sum(value)"));
+        assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));
+
+        error = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", "from index-user1 | stats sum(value)"));
+        assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));
     }
 
     public void testInsufficientPrivilege() {
@@ -143,19 +222,97 @@ public void testInsufficientPrivilege() {
         assertThat(error.getMessage(), containsString("Unknown index [index-user1]"));
     }
 
+    public void testIndexPatternErrorMessageComparison_ESQL_SearchDSL() throws Exception {
+        // _search match_all query on the index-user1,index-user2 index pattern
+        XContentBuilder json = JsonXContent.contentBuilder();
+        json.startObject();
+        json.field("query", QueryBuilders.matchAllQuery());
+        json.endObject();
+        Request searchRequest = new Request("GET", "/index-user1,index-user2/_search");
+        searchRequest.setJsonEntity(Strings.toString(json));
+        searchRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", "metadata1_read2"));
+
+        // ES|QL query on the same index pattern
+        var esqlResp = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2"));
+        var srchResp = expectThrows(ResponseException.class, () -> client().performRequest(searchRequest));
+
+        for (ResponseException r : List.of(esqlResp, srchResp)) {
+            assertThat(
+                EntityUtils.toString(r.getResponse().getEntity()),
+                containsString(
+                    "unauthorized for user [test-admin] run as [metadata1_read2] with effective roles [metadata1_read2] on indices [index-user1]"
+                )
+            );
+        }
+        assertThat(esqlResp.getResponse().getStatusLine().getStatusCode(), equalTo(srchResp.getResponse().getStatusLine().getStatusCode()));
+    }
+
     public void testLimitedPrivilege() throws Exception {
-        Response resp = runESQLCommand("metadata1_read2", """
-            FROM index-user1,index-user2 METADATA _index
-            | STATS sum=sum(value), index=VALUES(_index)
-            """);
-        assertOK(resp);
-        Map<String, Object> respMap = entityAsMap(resp);
+        ResponseException resp = expectThrows(
+            ResponseException.class,
+            () -> runESQLCommand(
+                "metadata1_read2",
+                "FROM index-user1,index-user2 METADATA _index | STATS sum=sum(value), index=VALUES(_index)"
+            )
+        );
         assertThat(
-            respMap.get("columns"),
-            equalTo(List.of(Map.of("name", "sum", "type", "double"), Map.of("name", "index", "type", "keyword")))
+            EntityUtils.toString(resp.getResponse().getEntity()),
+            containsString(
+                "unauthorized for user [test-admin] run as [metadata1_read2] with effective roles [metadata1_read2] on indices [index-user1]"
+            )
+        );
+        assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN));
+
+        resp = expectThrows(
+            ResponseException.class,
+            () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 METADATA _index | STATS index=VALUES(_index)")
+        );
+        assertThat(
+            EntityUtils.toString(resp.getResponse().getEntity()),
+            containsString(
+                "unauthorized for user [test-admin] run as [metadata1_read2] with effective roles [metadata1_read2] on indices [index-user1]"
+            )
         );
-        assertThat(respMap.get("values"), equalTo(List.of(List.of(72.0, "index-user2"))));
+        assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN));
 
+        resp = expectThrows(
+            ResponseException.class,
+            () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 | STATS sum=sum(value)")
+        );
+        assertThat(
+            EntityUtils.toString(resp.getResponse().getEntity()),
+            containsString(
+                "unauthorized for user [test-admin] run as [metadata1_read2] with effective roles [metadata1_read2] on indices [index-user1]"
+            )
+        );
+        assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN));
+
+        resp = expectThrows(
+            ResponseException.class,
+            () -> runESQLCommand("alias_user1", "FROM first-alias,index-user1 METADATA _index | KEEP _index, org, value | LIMIT 10")
+        );
+        assertThat(
+            EntityUtils.toString(resp.getResponse().getEntity()),
+            containsString(
+                "unauthorized for user [test-admin] run as [alias_user1] with effective roles [alias_user1] on indices [index-user1]"
+            )
+        );
+        assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN));
+
+        resp = expectThrows(
+            ResponseException.class,
+            () -> runESQLCommand(
+                "alias_user2",
+                "from second-alias,index-user2 METADATA _index | stats sum=sum(value), index=VALUES(_index)"
+            )
+        );
+        assertThat(
+            EntityUtils.toString(resp.getResponse().getEntity()),
+            containsString(
+                "unauthorized for user [test-admin] run as [alias_user2] with effective roles [alias_user2] on indices [index-user2]"
+            )
+        );
+        assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN));
     }
 
     public void testDocumentLevelSecurity() throws Exception {
diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml b/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml
index 6225711918608..5c0164782d181 100644
--- a/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml
+++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml
@@ -40,6 +40,20 @@ metadata1_read2:
     - names: [ 'index-user2' ]
       privileges: [ 'read' ]
 
+alias_user1:
+  cluster: []
+  indices:
+    - names: [ 'first-alias' ]
+      privileges:
+        - read
+
+alias_user2:
+  cluster: []
+  indices:
+    - names: [ 'second-alias' ]
+      privileges:
+        - read
+
 user3:
   cluster: []
   indices:
diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java
index 5676a8bce3ede..d0d6d5fa49c42 100644
--- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java
+++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.TestFeatureService;
-import org.elasticsearch.xpack.esql.core.CsvSpecReader.CsvTestCase;
+import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
 import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase;
 import org.junit.AfterClass;
 import org.junit.Before;
diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/Clusters.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/Clusters.java
index f20d758132cbb..fa8cb49c59aed 100644
--- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/Clusters.java
+++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/Clusters.java
@@ -12,9 +12,13 @@
 import org.elasticsearch.test.cluster.util.Version;
 
 public class Clusters {
+
+    static final String REMOTE_CLUSTER_NAME = "remote_cluster";
+    static final String LOCAL_CLUSTER_NAME = "local_cluster";
+
     public static ElasticsearchCluster remoteCluster() {
         return ElasticsearchCluster.local()
-            .name("remote_cluster")
+            .name(REMOTE_CLUSTER_NAME)
             .distribution(DistributionType.DEFAULT)
             .version(Version.fromString(System.getProperty("tests.old_cluster_version")))
             .nodes(2)
@@ -28,7 +32,7 @@ public static ElasticsearchCluster remoteCluster() {
 
     public static ElasticsearchCluster localCluster(ElasticsearchCluster remoteCluster) {
         return ElasticsearchCluster.local()
-            .name("local_cluster")
+            .name(LOCAL_CLUSTER_NAME)
             .distribution(DistributionType.DEFAULT)
             .version(Version.CURRENT)
             .nodes(2)
diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/EsqlRestValidationIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/EsqlRestValidationIT.java
new file mode 100644
index 0000000000000..21307c5362417
--- /dev/null
+++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/EsqlRestValidationIT.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.ccq;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.apache.http.HttpHost;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.test.TestClustersThreadFilter;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xpack.esql.qa.rest.EsqlRestValidationTestCase;
+import org.junit.AfterClass;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+import java.util.StringJoiner;
+
+import static org.elasticsearch.xpack.esql.ccq.Clusters.REMOTE_CLUSTER_NAME;
+
+@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
+public class EsqlRestValidationIT extends EsqlRestValidationTestCase {
+    static ElasticsearchCluster remoteCluster = Clusters.remoteCluster();
+    static ElasticsearchCluster localCluster = Clusters.localCluster(remoteCluster);
+
+    @ClassRule
+    public static TestRule clusterRule = RuleChain.outerRule(remoteCluster).around(localCluster);
+    private static RestClient remoteClient;
+
+    @Override
+    protected String getTestRestCluster() {
+        return localCluster.getHttpAddresses();
+    }
+
+    @AfterClass
+    public static void closeRemoteClients() throws IOException {
+        try {
+            IOUtils.close(remoteClient);
+        } finally {
+            remoteClient = null;
+        }
+    }
+
+    @Override
+    protected String clusterSpecificIndexName(String pattern) {
+        StringJoiner sj = new StringJoiner(",");
+        for (String index : pattern.split(",")) {
+            sj.add(remoteClusterIndex(index));
+        }
+        return sj.toString();
+    }
+
+    private static String remoteClusterIndex(String indexName) {
+        return REMOTE_CLUSTER_NAME + ":" + indexName;
+    }
+
+    @Override
+    protected RestClient provisioningClient() throws IOException {
+        return remoteClusterClient();
+    }
+
+    @Override
+    protected RestClient provisioningAdminClient() throws IOException {
+        return remoteClusterClient();
+    }
+
+    private RestClient remoteClusterClient() throws IOException {
+        if (remoteClient == null) {
+            var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses());
+            remoteClient = buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0]));
+        }
+        return remoteClient;
+    }
+}
diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java
index 6143c2b4148f7..3e799730f7269 100644
--- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java
+++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java
@@ -21,9 +21,9 @@
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.TestFeatureService;
-import org.elasticsearch.xpack.esql.core.CsvSpecReader;
-import org.elasticsearch.xpack.esql.core.CsvSpecReader.CsvTestCase;
-import org.elasticsearch.xpack.esql.core.SpecReader;
+import org.elasticsearch.xpack.esql.CsvSpecReader;
+import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
+import org.elasticsearch.xpack.esql.SpecReader;
 import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase;
 import org.junit.AfterClass;
 import org.junit.ClassRule;
@@ -42,10 +42,10 @@
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.esql.CsvSpecReader.specParser;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled;
 import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.ENRICH_SOURCE_INDICES;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
-import static org.elasticsearch.xpack.esql.core.CsvSpecReader.specParser;
 import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
@@ -110,6 +110,8 @@ protected void shouldSkipTest(String testName) throws IOException {
             "Test " + testName + " is skipped on " + Clusters.oldVersion(),
             isEnabled(testName, instructions, Clusters.oldVersion())
         );
+        assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains("inlinestats"));
+        assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains("inlinestats_v2"));
     }
 
     private TestFeatureService remoteFeaturesService() throws IOException {
diff --git a/x-pack/plugin/esql/qa/server/multi-node/build.gradle b/x-pack/plugin/esql/qa/server/multi-node/build.gradle
index 6bba58b721a94..9f8ca78aba81e 100644
--- a/x-pack/plugin/esql/qa/server/multi-node/build.gradle
+++ b/x-pack/plugin/esql/qa/server/multi-node/build.gradle
@@ -8,6 +8,9 @@ dependencies {
   javaRestTestImplementation project(xpackModule('esql:qa:testFixtures'))
   javaRestTestImplementation project(xpackModule('esql:qa:server'))
   yamlRestTestImplementation project(xpackModule('esql:qa:server'))
+
+  clusterPlugins project(':plugins:mapper-size')
+  clusterPlugins project(':plugins:mapper-murmur3')
 }
 
 GradleUtils.extendSourceSet(project, "javaRestTest", "yamlRestTest")
diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/Clusters.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/Clusters.java
index 4aa17801fa217..3a68aee9fd205 100644
--- a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/Clusters.java
+++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/Clusters.java
@@ -8,15 +8,17 @@
 package org.elasticsearch.xpack.esql.qa.multi_node;
 
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 
 public class Clusters {
-    public static ElasticsearchCluster testCluster() {
+    public static ElasticsearchCluster testCluster(LocalClusterConfigProvider configProvider) {
         return ElasticsearchCluster.local()
             .distribution(DistributionType.DEFAULT)
             .nodes(2)
             .setting("xpack.security.enabled", "false")
             .setting("xpack.license.self_generated.type", "trial")
+            .apply(() -> configProvider)
             .build();
     }
 }
diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlRestValidationIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlRestValidationIT.java
new file mode 100644
index 0000000000000..0187bafe19fce
--- /dev/null
+++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlRestValidationIT.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.multi_node;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.elasticsearch.test.TestClustersThreadFilter;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xpack.esql.qa.rest.EsqlRestValidationTestCase;
+import org.junit.ClassRule;
+
+@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
+public class EsqlRestValidationIT extends EsqlRestValidationTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> {});
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}
diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java
index 93385ec9efd89..bda10709ed947 100644
--- a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java
+++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java
@@ -8,13 +8,13 @@
 package org.elasticsearch.xpack.esql.qa.multi_node;
 
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.xpack.esql.core.CsvSpecReader.CsvTestCase;
+import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
 import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase;
 import org.junit.ClassRule;
 
 public class EsqlSpecIT extends EsqlSpecTestCase {
     @ClassRule
-    public static ElasticsearchCluster cluster = Clusters.testCluster();
+    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> {});
 
     @Override
     protected String getTestRestCluster() {
diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/FieldExtractorIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/FieldExtractorIT.java
index bcb83a31f7641..9ec454db2d325 100644
--- a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/FieldExtractorIT.java
+++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/FieldExtractorIT.java
@@ -17,7 +17,7 @@
 @ThreadLeakFilters(filters = TestClustersThreadFilter.class)
 public class FieldExtractorIT extends FieldExtractorTestCase {
     @ClassRule
-    public static ElasticsearchCluster cluster = Clusters.testCluster();
+    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> {});
 
     @Override
     protected String getTestRestCluster() {
diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/RestEsqlIT.java
new file mode 100644
index 0000000000000..7e98d486d6c2e
--- /dev/null
+++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/RestEsqlIT.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.multi_node;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase;
+import org.junit.ClassRule;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class RestEsqlIT extends RestEsqlTestCase {
+    @ClassRule
+    public static ElasticsearchCluster cluster = Clusters.testCluster(
+        specBuilder -> specBuilder.plugin("mapper-size").plugin("mapper-murmur3")
+    );
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @ParametersFactory(argumentFormatting = "%1s")
+    public static List<Object[]> modes() {
+        return Arrays.stream(Mode.values()).map(m -> new Object[] { m }).toList();
+    }
+
+    public RestEsqlIT(Mode mode) {
+        super(mode);
+    }
+}
diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlClientYamlIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlClientYamlIT.java
index d3ddae16e8af1..62e83c9280087 100644
--- a/x-pack/plugin/esql/qa/server/multi-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlClientYamlIT.java
+++ b/x-pack/plugin/esql/qa/server/multi-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlClientYamlIT.java
@@ -19,7 +19,7 @@
 
 public class EsqlClientYamlIT extends ESClientYamlSuiteTestCase {
     @ClassRule
-    public static ElasticsearchCluster cluster = Clusters.testCluster();
+    public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> {});
 
     @Override
     protected String getTestRestCluster() {
diff --git a/x-pack/plugin/esql/qa/server/single-node/build.gradle b/x-pack/plugin/esql/qa/server/single-node/build.gradle
index 865d7cf5f5e6c..ab8e3d4b32d9a 100644
--- a/x-pack/plugin/esql/qa/server/single-node/build.gradle
+++ b/x-pack/plugin/esql/qa/server/single-node/build.gradle
@@ -20,10 +20,8 @@ dependencies {
   javaRestTestImplementation("org.slf4j:slf4j-nop:${versions.slf4j}")
   javaRestTestImplementation('org.apache.arrow:arrow-memory-unsafe:16.1.0')
 
-  dependencies {
-    clusterPlugins project(':plugins:mapper-size')
-    clusterPlugins project(':plugins:mapper-murmur3')
-  }
+  clusterPlugins project(':plugins:mapper-size')
+  clusterPlugins project(':plugins:mapper-murmur3')
 }
 
 restResources {
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlRestValidationIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlRestValidationIT.java
new file mode 100644
index 0000000000000..5a31fc722eec1
--- /dev/null
+++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlRestValidationIT.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.single_node;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.elasticsearch.test.TestClustersThreadFilter;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xpack.esql.qa.rest.EsqlRestValidationTestCase;
+import org.junit.ClassRule;
+
+@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
+public class EsqlRestValidationIT extends EsqlRestValidationTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = Clusters.testCluster();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java
index 15d55e0258110..676fffd553ca8 100644
--- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java
+++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java
@@ -11,7 +11,7 @@
 
 import org.elasticsearch.test.TestClustersThreadFilter;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.xpack.esql.core.CsvSpecReader.CsvTestCase;
+import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
 import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase;
 import org.junit.ClassRule;
 
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java
index bf54dcbfa96f6..44550c62bd7c5 100644
--- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java
+++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java
@@ -10,15 +10,19 @@
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 
 import org.apache.http.util.EntityUtils;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.elasticsearch.Build;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ListMatcher;
+import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.TestClustersThreadFilter;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.LogType;
+import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase;
 import org.hamcrest.Matchers;
 import org.junit.Assert;
@@ -27,14 +31,22 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
+import static org.elasticsearch.test.ListMatcher.matchesList;
+import static org.elasticsearch.test.MapMatcher.assertMap;
+import static org.elasticsearch.test.MapMatcher.matchesMap;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.startsWith;
 import static org.hamcrest.core.Is.is;
 
 @ThreadLeakFilters(filters = TestClustersThreadFilter.class)
@@ -49,7 +61,7 @@ protected String getTestRestCluster() {
         return cluster.getHttpAddresses();
     }
 
-    @ParametersFactory
+    @ParametersFactory(argumentFormatting = "%1s")
     public static List<Object[]> modes() {
         return Arrays.stream(Mode.values()).map(m -> new Object[] { m }).toList();
     }
@@ -59,19 +71,7 @@ public RestEsqlIT(Mode mode) {
     }
 
     public void testBasicEsql() throws IOException {
-        StringBuilder b = new StringBuilder();
-        for (int i = 0; i < 1000; i++) {
-            b.append(String.format(Locale.ROOT, """
-                {"create":{"_index":"%s"}}
-                {"@timestamp":"2020-12-12","test":"value%s","value":%d}
-                """, testIndexName(), i, i));
-        }
-        Request bulk = new Request("POST", "/_bulk");
-        bulk.addParameter("refresh", "true");
-        bulk.addParameter("filter_path", "errors");
-        bulk.setJsonEntity(b.toString());
-        Response response = client().performRequest(bulk);
-        Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
+        indexTimestampData(1);
 
         RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | stats avg(value)");
         if (Build.current().isSnapshot()) {
@@ -273,6 +273,291 @@ public void testTableDuplicateNames() throws IOException {
         assertThat(re.getMessage(), containsString("[6:10] Duplicate field 'a'"));
     }
 
+    public void testProfile() throws IOException {
+        indexTimestampData(1);
+
+        RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | STATS AVG(value)");
+        builder.profile(true);
+        if (Build.current().isSnapshot()) {
+            // Lock to shard level partitioning, so we get consistent profile output
+            builder.pragmas(Settings.builder().put("data_partitioning", "shard").build());
+        }
+        Map<String, Object> result = runEsql(builder);
+        assertMap(
+            result,
+            matchesMap().entry("columns", matchesList().item(matchesMap().entry("name", "AVG(value)").entry("type", "double")))
+                .entry("values", List.of(List.of(499.5d)))
+                .entry("profile", matchesMap().entry("drivers", instanceOf(List.class)))
+        );
+
+        List<List<String>> signatures = new ArrayList<>();
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> profiles = (List<Map<String, Object>>) ((Map<String, Object>) result.get("profile")).get("drivers");
+        for (Map<String, Object> p : profiles) {
+            fixTypesOnProfile(p);
+            assertThat(p, commonProfile());
+            List<String> sig = new ArrayList<>();
+            @SuppressWarnings("unchecked")
+            List<Map<String, Object>> operators = (List<Map<String, Object>>) p.get("operators");
+            for (Map<String, Object> o : operators) {
+                sig.add(checkOperatorProfile(o));
+            }
+            signatures.add(sig);
+        }
+        assertThat(
+            signatures,
+            containsInAnyOrder(
+                matchesList().item("LuceneSourceOperator")
+                    .item("ValuesSourceReaderOperator")
+                    .item("AggregationOperator")
+                    .item("ExchangeSinkOperator"),
+                matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator"),
+                matchesList().item("ExchangeSourceOperator")
+                    .item("AggregationOperator")
+                    .item("ProjectOperator")
+                    .item("LimitOperator")
+                    .item("EvalOperator")
+                    .item("ProjectOperator")
+                    .item("OutputOperator")
+            )
+        );
+    }
+
+    public void testInlineStatsProfile() throws IOException {
+        assumeTrue("INLINESTATS only available on snapshots", Build.current().isSnapshot());
+        indexTimestampData(1);
+
+        RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | INLINESTATS AVG(value) | SORT value ASC");
+        builder.profile(true);
+        if (Build.current().isSnapshot()) {
+            // Lock to shard level partitioning, so we get consistent profile output
+            builder.pragmas(Settings.builder().put("data_partitioning", "shard").build());
+        }
+        Map<String, Object> result = runEsql(builder);
+        ListMatcher values = matchesList();
+        for (int i = 0; i < 1000; i++) {
+            values = values.item(matchesList().item("2020-12-12T00:00:00.000Z").item("value" + i).item("value" + i).item(i).item(499.5));
+        }
+        assertMap(
+            result,
+            matchesMap().entry(
+                "columns",
+                matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date"))
+                    .item(matchesMap().entry("name", "test").entry("type", "text"))
+                    .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword"))
+                    .item(matchesMap().entry("name", "value").entry("type", "long"))
+                    .item(matchesMap().entry("name", "AVG(value)").entry("type", "double"))
+            ).entry("values", values).entry("profile", matchesMap().entry("drivers", instanceOf(List.class)))
+        );
+
+        List<List<String>> signatures = new ArrayList<>();
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> profiles = (List<Map<String, Object>>) ((Map<String, Object>) result.get("profile")).get("drivers");
+        for (Map<String, Object> p : profiles) {
+            fixTypesOnProfile(p);
+            assertThat(p, commonProfile());
+            List<String> sig = new ArrayList<>();
+            @SuppressWarnings("unchecked")
+            List<Map<String, Object>> operators = (List<Map<String, Object>>) p.get("operators");
+            for (Map<String, Object> o : operators) {
+                sig.add(checkOperatorProfile(o));
+            }
+            signatures.add(sig);
+        }
+        assertThat(
+            signatures,
+            containsInAnyOrder(
+                // First pass read and start agg
+                matchesList().item("LuceneSourceOperator")
+                    .item("ValuesSourceReaderOperator")
+                    .item("AggregationOperator")
+                    .item("ExchangeSinkOperator"),
+                // First pass node level reduce
+                matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator"),
+                // First pass finish agg
+                matchesList().item("ExchangeSourceOperator")
+                    .item("AggregationOperator")
+                    .item("ProjectOperator")
+                    .item("EvalOperator")
+                    .item("ProjectOperator")
+                    .item("OutputOperator"),
+                // Second pass read and join via eval
+                matchesList().item("LuceneSourceOperator")
+                    .item("EvalOperator")
+                    .item("ValuesSourceReaderOperator")
+                    .item("TopNOperator")
+                    .item("ValuesSourceReaderOperator")
+                    .item("ProjectOperator")
+                    .item("ExchangeSinkOperator"),
+                // Second pass node level reduce
+                matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator"),
+                // Second pass finish
+                matchesList().item("ExchangeSourceOperator").item("TopNOperator").item("OutputOperator")
+            )
+        );
+    }
+
+    public void testForceSleepsProfile() throws IOException {
+        assumeTrue("requires pragmas", Build.current().isSnapshot());
+
+        Request createIndex = new Request("PUT", testIndexName());
+        createIndex.setJsonEntity("""
+            {
+              "settings": {
+                "index": {
+                  "number_of_shards": 1
+                }
+              }
+            }""");
+        Response response = client().performRequest(createIndex);
+        assertThat(
+            entityToMap(response.getEntity(), XContentType.JSON),
+            matchesMap().entry("shards_acknowledged", true).entry("index", testIndexName()).entry("acknowledged", true)
+        );
+
+        int groupCount = 300;
+        for (int group1 = 0; group1 < groupCount; group1++) {
+            StringBuilder b = new StringBuilder();
+            for (int group2 = 0; group2 < groupCount; group2++) {
+                b.append(String.format(Locale.ROOT, """
+                    {"create":{"_index":"%s"}}
+                    {"@timestamp":"2020-12-12","value":1,"group1":%d,"group2":%d}
+                    """, testIndexName(), group1, group2));
+            }
+            Request bulk = new Request("POST", "/_bulk");
+            bulk.addParameter("refresh", "true");
+            bulk.addParameter("filter_path", "errors");
+            bulk.setJsonEntity(b.toString());
+            response = client().performRequest(bulk);
+            Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
+        }
+
+        RequestObjectBuilder builder = requestObjectBuilder().query(
+            fromIndex() + " | STATS AVG(value), MAX(value), MIN(value) BY group1, group2 | SORT group1, group2 ASC | LIMIT 10"
+        );
+        // Lock to shard level partitioning, so we get consistent profile output
+        builder.pragmas(Settings.builder().put("data_partitioning", "shard").put("page_size", 10).build());
+        builder.profile(true);
+        Map<String, Object> result = runEsql(builder);
+        List<List<?>> expectedValues = new ArrayList<>();
+        for (int group2 = 0; group2 < 10; group2++) {
+            expectedValues.add(List.of(1.0, 1, 1, 0, group2));
+        }
+        assertMap(
+            result,
+            matchesMap().entry(
+                "columns",
+                matchesList().item(matchesMap().entry("name", "AVG(value)").entry("type", "double"))
+                    .item(matchesMap().entry("name", "MAX(value)").entry("type", "long"))
+                    .item(matchesMap().entry("name", "MIN(value)").entry("type", "long"))
+                    .item(matchesMap().entry("name", "group1").entry("type", "long"))
+                    .item(matchesMap().entry("name", "group2").entry("type", "long"))
+            ).entry("values", expectedValues).entry("profile", matchesMap().entry("drivers", instanceOf(List.class)))
+        );
+
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> profiles = (List<Map<String, Object>>) ((Map<String, Object>) result.get("profile")).get("drivers");
+
+        for (Map<String, Object> p : profiles) {
+            fixTypesOnProfile(p);
+            assertMap(p, commonProfile());
+            @SuppressWarnings("unchecked")
+            Map<String, Object> sleeps = (Map<String, Object>) p.get("sleeps");
+            String operators = p.get("operators").toString();
+            MapMatcher sleepMatcher = matchesMap().entry("reason", "exchange empty")
+                .entry("sleep_millis", greaterThan(0L))
+                .entry("wake_millis", greaterThan(0L));
+            if (operators.contains("LuceneSourceOperator")) {
+                assertMap(sleeps, matchesMap().entry("counts", Map.of()).entry("first", List.of()).entry("last", List.of()));
+            } else if (operators.contains("ExchangeSourceOperator")) {
+                if (operators.contains("ExchangeSinkOperator")) {
+                    assertMap(sleeps, matchesMap().entry("counts", matchesMap().entry("exchange empty", greaterThan(0))).extraOk());
+                    @SuppressWarnings("unchecked")
+                    List<Map<String, Object>> first = (List<Map<String, Object>>) sleeps.get("first");
+                    for (Map<String, Object> s : first) {
+                        assertMap(s, sleepMatcher);
+                    }
+                    @SuppressWarnings("unchecked")
+                    List<Map<String, Object>> last = (List<Map<String, Object>>) sleeps.get("last");
+                    for (Map<String, Object> s : last) {
+                        assertMap(s, sleepMatcher);
+                    }
+
+                } else {
+                    assertMap(
+                        sleeps,
+                        matchesMap().entry("counts", matchesMap().entry("exchange empty", 1))
+                            .entry("first", List.of(sleepMatcher))
+                            .entry("last", List.of(sleepMatcher))
+                    );
+                }
+            } else {
+                fail("unknown signature: " + operators);
+            }
+        }
+    }
+
+    private MapMatcher commonProfile() {
+        return matchesMap().entry("start_millis", greaterThan(0L))
+            .entry("stop_millis", greaterThan(0L))
+            .entry("iterations", greaterThan(0L))
+            .entry("cpu_nanos", greaterThan(0L))
+            .entry("took_nanos", greaterThan(0L))
+            .entry("operators", instanceOf(List.class))
+            .entry("sleeps", matchesMap().extraOk());
+    }
+
+    /**
+     * Fix some of the types on the profile results. Sometimes they
+     * come back as integers and sometimes longs. This just promotes
+     * them to long every time.
+     */
+    private void fixTypesOnProfile(Map<String, Object> profile) {
+        profile.put("iterations", ((Number) profile.get("iterations")).longValue());
+        profile.put("cpu_nanos", ((Number) profile.get("cpu_nanos")).longValue());
+        profile.put("took_nanos", ((Number) profile.get("took_nanos")).longValue());
+    }
+
+    private String checkOperatorProfile(Map<String, Object> o) {
+        String name = (String) o.get("operator");
+        name = name.replaceAll("\\[.+", "");
+        MapMatcher status = switch (name) {
+            case "LuceneSourceOperator" -> matchesMap().entry("processed_slices", greaterThan(0))
+                .entry("processed_shards", List.of(testIndexName() + ":0"))
+                .entry("total_slices", greaterThan(0))
+                .entry("slice_index", 0)
+                .entry("slice_max", 0)
+                .entry("slice_min", 0)
+                .entry("current", DocIdSetIterator.NO_MORE_DOCS)
+                .entry("pages_emitted", greaterThan(0))
+                .entry("processing_nanos", greaterThan(0))
+                .entry("processed_queries", List.of("*:*"));
+            case "ValuesSourceReaderOperator" -> basicProfile().entry("readers_built", matchesMap().extraOk());
+            case "AggregationOperator" -> matchesMap().entry("pages_processed", greaterThan(0)).entry("aggregation_nanos", greaterThan(0));
+            case "ExchangeSinkOperator" -> matchesMap().entry("pages_accepted", greaterThan(0));
+            case "ExchangeSourceOperator" -> matchesMap().entry("pages_emitted", greaterThan(0)).entry("pages_waiting", 0);
+            case "ProjectOperator", "EvalOperator" -> basicProfile();
+            case "LimitOperator" -> matchesMap().entry("pages_processed", greaterThan(0))
+                .entry("limit", 1000)
+                .entry("limit_remaining", 999);
+            case "OutputOperator" -> null;
+            case "TopNOperator" -> matchesMap().entry("occupied_rows", 0)
+                .entry("ram_used", instanceOf(String.class))
+                .entry("ram_bytes_used", greaterThan(0));
+            default -> throw new AssertionError("unexpected status: " + o);
+        };
+        MapMatcher expectedOp = matchesMap().entry("operator", startsWith(name));
+        if (status != null) {
+            expectedOp = expectedOp.entry("status", status);
+        }
+        assertMap(o, expectedOp);
+        return name;
+    }
+
+    private MapMatcher basicProfile() {
+        return matchesMap().entry("pages_processed", greaterThan(0)).entry("process_nanos", greaterThan(0));
+    }
+
     private void assertException(String query, String... errorMessages) throws IOException {
         ResponseException re = expectThrows(ResponseException.class, () -> runEsqlSync(requestObjectBuilder().query(query)));
         assertThat(re.getResponse().getStatusLine().getStatusCode(), equalTo(400));
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlRestValidationTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlRestValidationTestCase.java
new file mode 100644
index 0000000000000..9ec4f60f4c843
--- /dev/null
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlRestValidationTestCase.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.rest;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.WarningsHandler;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public abstract class EsqlRestValidationTestCase extends ESRestTestCase {
+
+    private static final String indexName = "test_esql";
+    private static final String aliasName = "alias-test_esql";
+    protected static final String[] existentIndexWithWildcard = new String[] {
+        indexName + ",inexistent*",
+        indexName + "*,inexistent*",
+        "inexistent*," + indexName };
+    private static final String[] existentIndexWithoutWildcard = new String[] { indexName + ",inexistent", "inexistent," + indexName };
+    protected static final String[] existentAliasWithWildcard = new String[] {
+        aliasName + ",inexistent*",
+        aliasName + "*,inexistent*",
+        "inexistent*," + aliasName };
+    private static final String[] existentAliasWithoutWildcard = new String[] { aliasName + ",inexistent", "inexistent," + aliasName };
+    private static final String[] inexistentIndexNameWithWildcard = new String[] { "inexistent*", "inexistent1*,inexistent2*" };
+    private static final String[] inexistentIndexNameWithoutWildcard = new String[] { "inexistent", "inexistent1,inexistent2" };
+    private static final String createAlias = "{\"actions\":[{\"add\":{\"index\":\"" + indexName + "\",\"alias\":\"" + aliasName + "\"}}]}";
+    private static final String removeAlias = "{\"actions\":[{\"remove\":{\"index\":\""
+        + indexName
+        + "\",\"alias\":\""
+        + aliasName
+        + "\"}}]}";
+
+    @Before
+    @After
+    public void assertRequestBreakerEmpty() throws Exception {
+        EsqlSpecTestCase.assertRequestBreakerEmpty();
+    }
+
+    @Before
+    public void prepareIndices() throws IOException {
+        if (provisioningClient().performRequest(new Request("HEAD", "/" + indexName)).getStatusLine().getStatusCode() == 404) {
+            var request = new Request("PUT", "/" + indexName);
+            request.setJsonEntity("{\"mappings\": {\"properties\": {\"foo\":{\"type\":\"keyword\"}}}}");
+            provisioningClient().performRequest(request);
+        }
+        assertOK(provisioningAdminClient().performRequest(new Request("POST", "/" + indexName + "/_refresh")));
+    }
+
+    @After
+    public void wipeTestData() throws IOException {
+        try {
+            var response = provisioningAdminClient().performRequest(new Request("DELETE", "/" + indexName));
+            assertEquals(200, response.getStatusLine().getStatusCode());
+        } catch (ResponseException re) {
+            assertEquals(404, re.getResponse().getStatusLine().getStatusCode());
+        }
+    }
+
+    private String getInexistentIndexErrorMessage() {
+        return "\"reason\" : \"Found 1 problem\\nline 1:1: Unknown index ";
+    }
+
+    public void testInexistentIndexNameWithWildcard() throws IOException {
+        assertErrorMessages(inexistentIndexNameWithWildcard, getInexistentIndexErrorMessage(), 400);
+    }
+
+    public void testInexistentIndexNameWithoutWildcard() throws IOException {
+        assertErrorMessages(inexistentIndexNameWithoutWildcard, getInexistentIndexErrorMessage(), 400);
+    }
+
+    public void testExistentIndexWithoutWildcard() throws IOException {
+        for (String indexName : existentIndexWithoutWildcard) {
+            assertErrorMessage(indexName, "\"reason\" : \"no such index [inexistent]\"", 404);
+        }
+    }
+
+    public void testExistentIndexWithWildcard() throws IOException {
+        assertValidRequestOnIndices(existentIndexWithWildcard);
+    }
+
+    public void testAlias() throws IOException {
+        createAlias();
+
+        for (String indexName : existentAliasWithoutWildcard) {
+            assertErrorMessage(indexName, "\"reason\" : \"no such index [inexistent]\"", 404);
+        }
+        assertValidRequestOnIndices(existentAliasWithWildcard);
+
+        deleteAlias();
+    }
+
+    private void assertErrorMessages(String[] indices, String errorMessage, int statusCode) throws IOException {
+        for (String indexName : indices) {
+            assertErrorMessage(indexName, errorMessage + "[" + clusterSpecificIndexName(indexName) + "]", statusCode);
+        }
+    }
+
+    protected String clusterSpecificIndexName(String indexName) {
+        return indexName;
+    }
+
+    private void assertErrorMessage(String indexName, String errorMessage, int statusCode) throws IOException {
+        var specificName = clusterSpecificIndexName(indexName);
+        final var request = createRequest(specificName);
+        ResponseException exc = expectThrows(ResponseException.class, () -> client().performRequest(request));
+
+        assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(statusCode));
+        assertThat(exc.getMessage(), containsString(errorMessage));
+    }
+
+    private Request createRequest(String indexName) throws IOException {
+        final var request = new Request("POST", "/_query");
+        request.addParameter("error_trace", "true");
+        request.addParameter("pretty", "true");
+        request.setJsonEntity(
+            Strings.toString(JsonXContent.contentBuilder().startObject().field("query", "from " + indexName).endObject())
+        );
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        options.setWarningsHandler(WarningsHandler.PERMISSIVE);
+        request.setOptions(options);
+        return request;
+    }
+
+    private void assertValidRequestOnIndices(String[] indices) throws IOException {
+        for (String indexName : indices) {
+            final var request = createRequest(clusterSpecificIndexName(indexName));
+            Response response = client().performRequest(request);
+            assertOK(response);
+        }
+    }
+
+    // Returned client is used to load the test data, either in the local cluster or a remote one (for
+    // multi-clusters). The client()/adminClient() will always connect to the local cluster
+    protected RestClient provisioningClient() throws IOException {
+        return client();
+    }
+
+    protected RestClient provisioningAdminClient() throws IOException {
+        return adminClient();
+    }
+
+    private void createAlias() throws IOException {
+        var r = new Request("POST", "_aliases");
+        r.setJsonEntity(createAlias);
+        assertOK(provisioningClient().performRequest(r));
+    }
+
+    private void deleteAlias() throws IOException {
+        var r = new Request("POST", "/_aliases/");
+        r.setJsonEntity(removeAlias);
+        assertOK(provisioningAdminClient().performRequest(r));
+    }
+}
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
index a9074073f1f19..1543ba039dc6d 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
@@ -26,10 +26,10 @@
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.TestFeatureService;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
 import org.elasticsearch.xpack.esql.CsvTestUtils;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
-import org.elasticsearch.xpack.esql.core.CsvSpecReader.CsvTestCase;
-import org.elasticsearch.xpack.esql.core.SpecReader;
+import org.elasticsearch.xpack.esql.SpecReader;
 import org.elasticsearch.xpack.esql.plugin.EsqlFeatures;
 import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.RequestObjectBuilder;
 import org.junit.After;
@@ -61,13 +61,13 @@
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.elasticsearch.xpack.esql.CsvAssert.assertData;
 import static org.elasticsearch.xpack.esql.CsvAssert.assertMetadata;
+import static org.elasticsearch.xpack.esql.CsvSpecReader.specParser;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.ExpectedResults;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.loadCsvSpecValues;
 import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.CSV_DATASET_MAP;
 import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadDataSetIntoEs;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
-import static org.elasticsearch.xpack.esql.core.CsvSpecReader.specParser;
 
 // This test can run very long in serverless configurations
 @TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java
index 085bc7a22f185..6fec6e2fca281 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java
@@ -231,7 +231,8 @@ public void testScaledFloat() throws IOException {
     private Matcher<Double> scaledFloatMatcher(double scalingFactor, double d) {
         long encoded = Math.round(d * scalingFactor);
         double decoded = encoded / scalingFactor;
-        return closeTo(decoded, Math.ulp(decoded));
+        // We can lose a little more the ulp in the round trip.
+        return closeTo(decoded, Math.ulp(decoded) * 2);
     }
 
     public void testBoolean() throws IOException {
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
index a672e4a50612c..6f5297bbeef4d 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
@@ -11,6 +11,7 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.nio.entity.NByteArrayEntity;
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Build;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
@@ -20,16 +21,19 @@
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
+import org.elasticsearch.test.ListMatcher;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 
 import java.io.ByteArrayOutputStream;
@@ -52,11 +56,14 @@
 
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
+import static org.elasticsearch.test.ListMatcher.matchesList;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
 import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.Mode.ASYNC;
 import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.Mode.SYNC;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
+import static org.hamcrest.Matchers.any;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.emptyOrNullString;
@@ -119,6 +126,10 @@ public static class RequestObjectBuilder {
 
         private Boolean keepOnCompletion = null;
 
+        private Boolean profile = null;
+
+        private CheckedConsumer<XContentBuilder, IOException> filter;
+
         public RequestObjectBuilder() throws IOException {
             this(randomFrom(XContentType.values()));
         }
@@ -180,6 +191,16 @@ public RequestObjectBuilder pragmas(Settings pragmas) throws IOException {
             return this;
         }
 
+        public RequestObjectBuilder profile(boolean profile) {
+            this.profile = profile;
+            return this;
+        }
+
+        public RequestObjectBuilder filter(CheckedConsumer<XContentBuilder, IOException> filter) {
+            this.filter = filter;
+            return this;
+        }
+
         public RequestObjectBuilder build() throws IOException {
             if (isBuilt == false) {
                 if (tables != null) {
@@ -195,6 +216,14 @@ public RequestObjectBuilder build() throws IOException {
                     }
                     builder.endObject();
                 }
+                if (profile != null) {
+                    builder.field("profile", profile);
+                }
+                if (filter != null) {
+                    builder.startObject("filter");
+                    filter.accept(builder);
+                    builder.endObject();
+                }
                 builder.endObject();
                 isBuilt = true;
             }
@@ -412,6 +441,65 @@ public void testOutOfRangeComparisons() throws IOException {
         }
     }
 
+    // Test the Range created in PushFiltersToSource for qualified pushable filters on the same field
+    public void testInternalRange() throws IOException {
+        final int NUM_SINGLE_VALUE_ROWS = 100;
+        bulkLoadTestData(NUM_SINGLE_VALUE_ROWS);
+        bulkLoadTestData(10, NUM_SINGLE_VALUE_ROWS, false, RestEsqlTestCase::createDocumentWithMVs);
+        bulkLoadTestData(5, NUM_SINGLE_VALUE_ROWS + 10, false, RestEsqlTestCase::createDocumentWithNulls);
+
+        String upperBound = randomFrom(" < ", " <= ");
+        String lowerBound = randomFrom(" > ", " >= ");
+
+        String predicate = "{}" + upperBound + "{} and {}" + lowerBound + "{} and {} != {}";
+        int half = NUM_SINGLE_VALUE_ROWS / 2;
+        int halfPlusThree = half + 3;
+        List<String> predicates = List.of(
+            format(null, predicate, "integer", half, "integer", -1, "integer", half),
+            format(null, predicate, "short", half, "short", -1, "short", half),
+            format(null, predicate, "byte", half, "byte", -1, "byte", half),
+            format(null, predicate, "long", half, "long", -1, "long", half),
+            format(null, predicate, "double", half, "double", -1.0, "double", half),
+            format(null, predicate, "float", half, "float", -1.0, "float", half),
+            format(null, predicate, "half_float", half, "half_float", -1.0, "half_float", half),
+            format(null, predicate, "scaled_float", half, "scaled_float", -1.0, "scaled_float", half),
+            format(
+                null,
+                predicate,
+                "date",
+                "\"" + dateTimeToString(half) + "\"",
+                "date",
+                "\"1001-01-01\"",
+                "date",
+                "\"" + dateTimeToString(half) + "\""
+            ),
+            // keyword6-9 is greater than keyword53, [54,99] + [6, 9], 50 items in total
+            format(
+                null,
+                predicate,
+                "keyword",
+                "\"keyword999\"",
+                "keyword",
+                "\"keyword" + halfPlusThree + "\"",
+                "keyword",
+                "\"keyword" + halfPlusThree + "\""
+            ),
+            format(null, predicate, "ip", "\"127.0.0." + half + "\"", "ip", "\"126.0.0.0\"", "ip", "\"127.0.0." + half + "\""),
+            format(null, predicate, "version", "\"1.2." + half + "\"", "version", "\"1.2\"", "version", "\"1.2." + half + "\"")
+        );
+
+        for (String p : predicates) {
+            var query = requestObjectBuilder().query(format(null, "from {} | where {}", testIndexName(), p));
+            var result = runEsql(query, List.of(), NO_WARNINGS_REGEX, mode);
+            var values = as(result.get("values"), ArrayList.class);
+            assertThat(
+                format(null, "Comparison [{}] should return all rows with single values.", p),
+                values.size(),
+                is(NUM_SINGLE_VALUE_ROWS / 2)
+            );
+        }
+    }
+
     public void testWarningHeadersOnFailedConversions() throws IOException {
         int count = randomFrom(10, 40, 60);
         bulkLoadTestData(count);
@@ -488,16 +576,18 @@ public void testErrorMessageForInvalidParams() throws IOException {
             () -> runEsqlSync(
                 requestObjectBuilder().query("row a = 1 | eval x = ?, y = ?")
                     .params(
-                        "[{\"1\": \"v1\"}, {\"1-\": \"v1\"}, {\"_a\": \"v1\"}, {\"@-#\": \"v1\"}, true, 123, "
-                            + "{\"type\": \"byte\", \"value\": 5}]"
+                        "[{\"1\": \"v1\"}, {\"1-\": \"v1\"}, {\"-a\": \"v1\"}, {\"@-#\": \"v1\"}, true, 123, "
+                            + "{\"type\": \"byte\", \"value\": 5}, {\"_1\": \"v1\"}, {\"_a\": \"v1\"}]"
                     )
             )
         );
         String error = EntityUtils.toString(re.getResponse().getEntity()).replaceAll("\\\\\n\s+\\\\", "");
         assertThat(error, containsString("[1] is not a valid parameter name"));
         assertThat(error, containsString("[1-] is not a valid parameter name"));
-        assertThat(error, containsString("[_a] is not a valid parameter name"));
+        assertThat(error, containsString("[-a] is not a valid parameter name"));
         assertThat(error, containsString("[@-#] is not a valid parameter name"));
+        assertThat(error, not(containsString("[_a] is not a valid parameter name")));
+        assertThat(error, not(containsString("[_1] is not a valid parameter name")));
         assertThat(error, containsString("Params cannot contain both named and unnamed parameters"));
         assertThat(error, containsString("Cannot parse more than one key:value pair as parameter"));
         re = expectThrows(
@@ -512,7 +602,6 @@ public void testErrorMessageForInvalidParams() throws IOException {
             EntityUtils.toString(re.getResponse().getEntity()),
             containsString("No parameter is defined for position 2, did you mean position 1")
         );
-
         re = expectThrows(
             ResponseException.class,
             () -> runEsqlSync(requestObjectBuilder().query("row a = ?n0").params("[{\"n1\": \"v1\"}]"))
@@ -567,6 +656,141 @@ public void testErrorMessageForArrayValuesInParams() throws IOException {
         assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("n1=[5, 6, 7] is not supported as a parameter"));
     }
 
+    public void testComplexFieldNames() throws IOException {
+        bulkLoadTestData(1);
+        // catch verification exception, field names not found
+        int fieldNumber = 5000;
+        String q1 = fromIndex() + queryWithComplexFieldNames(fieldNumber);
+        ResponseException e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query(q1)));
+        assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
+        assertThat(e.getMessage(), containsString("verification_exception"));
+
+        // catch automaton's TooComplexToDeterminizeException
+        fieldNumber = 6000;
+        final String q2 = fromIndex() + queryWithComplexFieldNames(fieldNumber);
+        e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query(q2)));
+        assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
+        assertThat(e.getMessage(), containsString("The field names are too complex to process"));
+    }
+
+    /**
+     * INLINESTATS <strong>can</strong> group on {@code NOW()}. It's a little silly, but
+     * doing something like {@code DATE_TRUNC(1 YEAR, NOW() - 1970-01-01T00:00:00Z)} is
+     * much more sensible. But just grouping on {@code NOW()} is enough to test this.
+     * <p>
+     *     This works because {@code NOW()} locks it's value at the start of the entire
+     *     query. It's part of the "configuration" of the query.
+     * </p>
+     */
+    public void testInlineStatsNow() throws IOException {
+        assumeTrue("INLINESTATS only available on snapshots", Build.current().isSnapshot());
+        indexTimestampData(1);
+
+        RequestObjectBuilder builder = requestObjectBuilder().query(
+            fromIndex() + " | EVAL now=NOW() | INLINESTATS AVG(value) BY now | SORT value ASC"
+        );
+        Map<String, Object> result = runEsql(builder);
+        ListMatcher values = matchesList();
+        for (int i = 0; i < 1000; i++) {
+            values = values.item(
+                matchesList().item("2020-12-12T00:00:00.000Z")
+                    .item("value" + i)
+                    .item("value" + i)
+                    .item(i)
+                    .item(any(String.class))
+                    .item(499.5)
+            );
+        }
+        assertMap(
+            result,
+            matchesMap().entry(
+                "columns",
+                matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date"))
+                    .item(matchesMap().entry("name", "test").entry("type", "text"))
+                    .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword"))
+                    .item(matchesMap().entry("name", "value").entry("type", "long"))
+                    .item(matchesMap().entry("name", "now").entry("type", "date"))
+                    .item(matchesMap().entry("name", "AVG(value)").entry("type", "double"))
+            ).entry("values", values)
+        );
+    }
+
+    public void testTopLevelFilter() throws IOException {
+        indexTimestampData(3); // Multiple shards has caused a bug in the past with the merging case below
+
+        RequestObjectBuilder builder = requestObjectBuilder().filter(b -> {
+            b.startObject("range");
+            {
+                b.startObject("@timestamp").field("gte", "2020-12-12").endObject();
+            }
+            b.endObject();
+        }).query(fromIndex() + " | STATS SUM(value)");
+        assertMap(
+            runEsql(builder),
+            matchesMap().entry("columns", matchesList().item(matchesMap().entry("name", "SUM(value)").entry("type", "long")))
+                .entry("values", List.of(List.of(499500)))
+        );
+    }
+
+    public void testTopLevelFilterMerged() throws IOException {
+        indexTimestampData(3); // Multiple shards has caused a bug in the past with the merging case below
+
+        RequestObjectBuilder builder = requestObjectBuilder().filter(b -> {
+            b.startObject("range");
+            {
+                b.startObject("@timestamp").field("gte", "2020-12-12").endObject();
+            }
+            b.endObject();
+        }).query(fromIndex() + " | WHERE value == 12 | STATS SUM(value)");
+        assertMap(
+            runEsql(builder),
+            matchesMap().entry("columns", matchesList().item(matchesMap().entry("name", "SUM(value)").entry("type", "long")))
+                .entry("values", List.of(List.of(12)))
+        );
+    }
+
+    public void testTopLevelFilterBoolMerged() throws IOException {
+        indexTimestampData(3); // Multiple shards has caused a bug in the past
+
+        for (int i = 0; i < 100; i++) {
+            // Run the query many times so we're more likely to bump into any sort of modification problems
+            RequestObjectBuilder builder = requestObjectBuilder().filter(b -> {
+                b.startObject("bool");
+                {
+                    b.startArray("filter");
+                    {
+                        b.startObject().startObject("range");
+                        {
+                            b.startObject("@timestamp").field("gte", "2020-12-12").endObject();
+                        }
+                        b.endObject().endObject();
+                        b.startObject().startObject("match");
+                        {
+                            b.field("test", "value12");
+                        }
+                        b.endObject().endObject();
+                    }
+                    b.endArray();
+                }
+                b.endObject();
+            }).query(fromIndex() + " | WHERE @timestamp > \"2010-01-01\" | STATS SUM(value)");
+            assertMap(
+                runEsql(builder),
+                matchesMap().entry("columns", matchesList().item(matchesMap().entry("name", "SUM(value)").entry("type", "long")))
+                    .entry("values", List.of(List.of(12)))
+            );
+        }
+    }
+
+    private static String queryWithComplexFieldNames(int field) {
+        StringBuilder query = new StringBuilder();
+        query.append(" | keep ").append(randomAlphaOfLength(10)).append(1);
+        for (int i = 2; i <= field; i++) {
+            query.append(", ").append(randomAlphaOfLength(10)).append(i);
+        }
+        return query.toString();
+    }
+
     private static String expectedTextBody(String format, int count, @Nullable Character csvDelimiter) {
         StringBuilder sb = new StringBuilder();
         switch (format) {
@@ -680,17 +904,24 @@ public static Map<String, Object> runEsqlAsync(
         checkKeepOnCompletion(requestObject, json);
         String id = (String) json.get("id");
 
+        var supportsAsyncHeaders = clusterHasCapability("POST", "/_query", List.of(), List.of("async_query_status_headers")).orElse(false);
+
         if (id == null) {
             // no id returned from an async call, must have completed immediately and without keep_on_completion
             assertThat(requestObject.keepOnCompletion(), either(nullValue()).or(is(false)));
             assertThat((boolean) json.get("is_running"), is(false));
+            if (supportsAsyncHeaders) {
+                assertThat(response.getHeader("X-Elasticsearch-Async-Id"), nullValue());
+                assertThat(response.getHeader("X-Elasticsearch-Async-Is-Running"), is("?0"));
+            }
             assertWarnings(response, expectedWarnings, expectedWarningsRegex);
             json.remove("is_running"); // remove this to not mess up later map assertions
             return Collections.unmodifiableMap(json);
         } else {
             // async may not return results immediately, so may need an async get
             assertThat(id, is(not(emptyOrNullString())));
-            if ((boolean) json.get("is_running") == false) {
+            boolean isRunning = (boolean) json.get("is_running");
+            if (isRunning == false) {
                 // must have completed immediately so keep_on_completion must be true
                 assertThat(requestObject.keepOnCompletion(), is(true));
                 assertWarnings(response, expectedWarnings, expectedWarningsRegex);
@@ -702,6 +933,12 @@ public static Map<String, Object> runEsqlAsync(
                 assertThat(json.get("columns"), is(equalTo(List.<Map<String, String>>of()))); // no partial results
                 assertThat(json.get("pages"), nullValue());
             }
+
+            if (supportsAsyncHeaders) {
+                assertThat(response.getHeader("X-Elasticsearch-Async-Id"), is(id));
+                assertThat(response.getHeader("X-Elasticsearch-Async-Is-Running"), is(isRunning ? "?1" : "?0"));
+            }
+
             // issue a second request to "async get" the results
             Request getRequest = prepareAsyncGetRequest(id);
             getRequest.setOptions(options);
@@ -730,7 +967,7 @@ static Map<String, Object> removeAsyncProperties(Map<String, Object> map) {
         return Collections.unmodifiableMap(copy);
     }
 
-    static Map<String, Object> entityToMap(HttpEntity entity, XContentType expectedContentType) throws IOException {
+    protected static Map<String, Object> entityToMap(HttpEntity entity, XContentType expectedContentType) throws IOException {
         try (InputStream content = entity.getContent()) {
             XContentType xContentType = XContentType.fromMediaType(entity.getContentType().getValue());
             assertEquals(expectedContentType, xContentType);
@@ -973,4 +1210,35 @@ protected boolean preserveClusterUponCompletion() {
     public void assertRequestBreakerEmpty() throws Exception {
         EsqlSpecTestCase.assertRequestBreakerEmpty();
     }
+
+    protected void indexTimestampData(int shards) throws IOException {
+        Request createIndex = new Request("PUT", testIndexName());
+        createIndex.setJsonEntity("""
+            {
+              "settings": {
+                "index": {
+                  "number_of_shards": %shards%
+                }
+              }
+            }""".replace("%shards%", Integer.toString(shards)));
+        Response response = client().performRequest(createIndex);
+        assertThat(
+            entityToMap(response.getEntity(), XContentType.JSON),
+            matchesMap().entry("shards_acknowledged", true).entry("index", testIndexName()).entry("acknowledged", true)
+        );
+
+        StringBuilder b = new StringBuilder();
+        for (int i = 0; i < 1000; i++) {
+            b.append(String.format(Locale.ROOT, """
+                {"create":{"_index":"%s"}}
+                {"@timestamp":"2020-12-12","test":"value%s","value":%d}
+                """, testIndexName(), i, i));
+        }
+        Request bulk = new Request("POST", "/_bulk");
+        bulk.addParameter("refresh", "true");
+        bulk.addParameter("filter_path", "errors");
+        bulk.setJsonEntity(b.toString());
+        response = client().performRequest(bulk);
+        Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
+    }
 }
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/ConfigurationTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/ConfigurationTestUtils.java
new file mode 100644
index 0000000000000..fce88be2a3750
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/ConfigurationTestUtils.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql;
+
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.data.ElementType;
+import org.elasticsearch.compute.lucene.DataPartitioning;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.action.ParseTables;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.planner.PlannerUtils;
+import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
+import org.elasticsearch.xpack.esql.session.Configuration;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.lucene.tests.util.LuceneTestCase.random;
+import static org.apache.lucene.tests.util.LuceneTestCase.randomLocale;
+import static org.elasticsearch.test.ESTestCase.between;
+import static org.elasticsearch.test.ESTestCase.frequently;
+import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
+import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween;
+import static org.elasticsearch.test.ESTestCase.randomBoolean;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
+import static org.elasticsearch.test.ESTestCase.randomIntBetween;
+import static org.elasticsearch.test.ESTestCase.randomNonNegativeInt;
+import static org.elasticsearch.test.ESTestCase.randomRealisticUnicodeOfLength;
+import static org.elasticsearch.test.ESTestCase.randomZone;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
+import static org.elasticsearch.xpack.esql.session.Configuration.QUERY_COMPRESS_THRESHOLD_CHARS;
+
+/**
+ * Test utilities for building random {@link Configuration}s.
+ */
+public class ConfigurationTestUtils {
+    public static Configuration randomConfiguration() {
+        int len = randomIntBetween(1, 300) + (frequently() ? 0 : QUERY_COMPRESS_THRESHOLD_CHARS);
+        return randomConfiguration(randomRealisticUnicodeOfLength(len), randomTables());
+    }
+
+    public static Configuration randomConfiguration(String query, Map<String, Map<String, Column>> tables) {
+        var zoneId = randomZone();
+        var locale = randomLocale(random());
+        var username = randomAlphaOfLengthBetween(1, 10);
+        var clusterName = randomAlphaOfLengthBetween(3, 10);
+        var truncation = randomNonNegativeInt();
+        var defaultTruncation = randomNonNegativeInt();
+        boolean profile = randomBoolean();
+
+        return new Configuration(
+            zoneId,
+            locale,
+            username,
+            clusterName,
+            randomQueryPragmas(),
+            truncation,
+            defaultTruncation,
+            query,
+            profile,
+            tables
+        );
+    }
+
+    private static QueryPragmas randomQueryPragmas() {
+        return new QueryPragmas(
+            Settings.builder().put(QueryPragmas.DATA_PARTITIONING.getKey(), randomFrom(DataPartitioning.values())).build()
+        );
+    }
+
+    /**
+     * Build random "tables" to use for {@link Configuration#tables()}.
+     */
+    public static Map<String, Map<String, Column>> randomTables() {
+        if (randomBoolean()) {
+            return Map.of();
+        }
+        int count = between(1, 10);
+        Map<String, Map<String, Column>> tables = new HashMap<>(count);
+        try {
+            for (int i = 0; i < 10; i++) {
+                tables.put(randomAlphaOfLength(i + 1), randomColumns());
+            }
+            return tables;
+        } finally {
+            if (tables.size() != count) {
+                Releasables.close(
+                    Releasables.wrap(
+                        Iterators.flatMap(tables.values().iterator(), columns -> Iterators.map(columns.values().iterator(), Column::values))
+                    )
+                );
+            }
+        }
+    }
+
+    static Map<String, Column> randomColumns() {
+        int count = between(1, 10);
+        Map<String, Column> columns = new HashMap<>(count);
+        int positions = between(1, 10_000);
+        try {
+            for (int i = 0; i < count; i++) {
+                String name = randomAlphaOfLength(i + 1);
+                DataType dataType = randomFrom(ParseTables.SUPPORTED_TYPES);
+                ElementType type = PlannerUtils.toElementType(dataType);
+                try (
+                    Block.Builder builder = type.newBlockBuilder(
+                        positions,
+                        new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE)
+                    )
+                ) {
+                    for (int p = 0; p < positions; p++) {
+                        BlockUtils.appendValue(builder, randomLiteral(dataType).value(), type);
+                    }
+                    columns.put(name, new Column(dataType, builder.build()));
+                }
+            }
+            return columns;
+        } finally {
+            if (columns.size() != count) {
+                Releasables.close(Releasables.wrap(Iterators.map(columns.values().iterator(), Column::values)));
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java
index 875058ba6e0e4..36d785c24ab23 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.search.DocValueFormat;
@@ -127,6 +128,7 @@ private static void assertMetadata(
 
                 if (blockType == Type.LONG
                     && (expectedType == Type.DATETIME
+                        || expectedType == Type.DATE_NANOS
                         || expectedType == Type.GEO_POINT
                         || expectedType == Type.CARTESIAN_POINT
                         || expectedType == UNSIGNED_LONG)) {
@@ -214,6 +216,12 @@ public static void assertData(
                         // convert the long from CSV back to its STRING form
                         if (expectedType == Type.DATETIME) {
                             expectedValue = rebuildExpected(expectedValue, Long.class, x -> UTC_DATE_TIME_FORMATTER.formatMillis((long) x));
+                        } else if (expectedType == Type.DATE_NANOS) {
+                            expectedValue = rebuildExpected(
+                                expectedValue,
+                                Long.class,
+                                x -> DateFormatter.forPattern("strict_date_optional_time_nanos").formatNanos((long) x)
+                            );
                         } else if (expectedType == Type.GEO_POINT) {
                             expectedValue = rebuildExpected(expectedValue, BytesRef.class, x -> GEO.wkbToWkt((BytesRef) x));
                         } else if (expectedType == Type.CARTESIAN_POINT) {
diff --git a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/CsvSpecReader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java
similarity index 99%
rename from x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/CsvSpecReader.java
rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java
index 8e5a228af00d6..781ae5531c6f0 100644
--- a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/CsvSpecReader.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core;
+package org.elasticsearch.xpack.esql;
 
 import java.util.ArrayList;
 import java.util.List;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
index bfa9c11cbcd53..c934a8926ee7e 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
@@ -37,6 +37,7 @@
 import java.io.StringReader;
 import java.math.BigDecimal;
 import java.net.URL;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -52,8 +53,9 @@
 import static org.elasticsearch.common.Strings.delimitedListToStringArray;
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.reader;
-import static org.elasticsearch.xpack.esql.core.SpecReader.shouldSkipLine;
+import static org.elasticsearch.xpack.esql.SpecReader.shouldSkipLine;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToUnsignedLong;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.ISO_DATE_WITH_NANOS;
 import static org.elasticsearch.xpack.esql.core.util.DateUtils.UTC_DATE_TIME_FORMATTER;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
@@ -444,6 +446,13 @@ public enum Type {
             (l, r) -> l instanceof Long maybeIP ? maybeIP.compareTo((Long) r) : l.toString().compareTo(r.toString()),
             Long.class
         ),
+        DATE_NANOS(x -> {
+            if (x == null) {
+                return null;
+            }
+            Instant parsed = DateFormatters.from(ISO_DATE_WITH_NANOS.parse(x)).toInstant();
+            return parsed.getEpochSecond() * 1_000_000_000 + parsed.getNano();
+        }, (l, r) -> l instanceof Long maybeIP ? maybeIP.compareTo((Long) r) : l.toString().compareTo(r.toString()), Long.class),
         BOOLEAN(Booleans::parseBoolean, Boolean.class),
         GEO_POINT(x -> x == null ? null : GEO.wktToWkb(x), BytesRef.class),
         CARTESIAN_POINT(x -> x == null ? null : CARTESIAN.wktToWkb(x), BytesRef.class),
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
index f9b768d67d574..9ee22113a4244 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
@@ -55,6 +55,7 @@ public class CsvTestsDataLoader {
     private static final TestsDataset HOSTS = new TestsDataset("hosts", "mapping-hosts.json", "hosts.csv");
     private static final TestsDataset APPS = new TestsDataset("apps", "mapping-apps.json", "apps.csv");
     private static final TestsDataset LANGUAGES = new TestsDataset("languages", "mapping-languages.json", "languages.csv");
+    private static final TestsDataset ALERTS = new TestsDataset("alerts", "mapping-alerts.json", "alerts.csv");
     private static final TestsDataset UL_LOGS = new TestsDataset("ul_logs", "mapping-ul_logs.json", "ul_logs.csv");
     private static final TestsDataset SAMPLE_DATA = new TestsDataset("sample_data", "mapping-sample_data.json", "sample_data.csv");
     private static final TestsDataset SAMPLE_DATA_STR = new TestsDataset(
@@ -67,6 +68,11 @@ public class CsvTestsDataLoader {
         "mapping-sample_data_ts_long.json",
         "sample_data_ts_long.csv"
     );
+    private static final TestsDataset MISSING_IP_SAMPLE_DATA = new TestsDataset(
+        "missing_ip_sample_data",
+        "mapping-missing_ip_sample_data.json",
+        "missing_ip_sample_data.csv"
+    );
     private static final TestsDataset CLIENT_IPS = new TestsDataset("clientips", "mapping-clientips.json", "clientips.csv");
     private static final TestsDataset CLIENT_CIDR = new TestsDataset("client_cidr", "mapping-client_cidr.json", "client_cidr.csv");
     private static final TestsDataset AGES = new TestsDataset("ages", "mapping-ages.json", "ages.csv");
@@ -75,6 +81,7 @@ public class CsvTestsDataLoader {
     private static final TestsDataset AIRPORTS = new TestsDataset("airports", "mapping-airports.json", "airports.csv");
     private static final TestsDataset AIRPORTS_MP = new TestsDataset("airports_mp", "mapping-airports.json", "airports_mp.csv");
     private static final TestsDataset AIRPORTS_WEB = new TestsDataset("airports_web", "mapping-airports_web.json", "airports_web.csv");
+    private static final TestsDataset DATE_NANOS = new TestsDataset("date_nanos", "mapping-date_nanos.json", "date_nanos.csv");
     private static final TestsDataset COUNTRIES_BBOX = new TestsDataset(
         "countries_bbox",
         "mapping-countries_bbox.json",
@@ -98,6 +105,7 @@ public class CsvTestsDataLoader {
     private static final TestsDataset DISTANCES = new TestsDataset("distances", "mapping-distances.json", "distances.csv");
     private static final TestsDataset K8S = new TestsDataset("k8s", "k8s-mappings.json", "k8s.csv", "k8s-settings.json", true);
     private static final TestsDataset ADDRESSES = new TestsDataset("addresses", "mapping-addresses.json", "addresses.csv", null, true);
+    private static final TestsDataset BOOKS = new TestsDataset("books", "mapping-books.json", "books.csv", null, true);
 
     public static final Map<String, TestsDataset> CSV_DATASET_MAP = Map.ofEntries(
         Map.entry(EMPLOYEES.indexName, EMPLOYEES),
@@ -106,8 +114,10 @@ public class CsvTestsDataLoader {
         Map.entry(LANGUAGES.indexName, LANGUAGES),
         Map.entry(UL_LOGS.indexName, UL_LOGS),
         Map.entry(SAMPLE_DATA.indexName, SAMPLE_DATA),
+        Map.entry(ALERTS.indexName, ALERTS),
         Map.entry(SAMPLE_DATA_STR.indexName, SAMPLE_DATA_STR),
         Map.entry(SAMPLE_DATA_TS_LONG.indexName, SAMPLE_DATA_TS_LONG),
+        Map.entry(MISSING_IP_SAMPLE_DATA.indexName, MISSING_IP_SAMPLE_DATA),
         Map.entry(CLIENT_IPS.indexName, CLIENT_IPS),
         Map.entry(CLIENT_CIDR.indexName, CLIENT_CIDR),
         Map.entry(AGES.indexName, AGES),
@@ -120,9 +130,11 @@ public class CsvTestsDataLoader {
         Map.entry(COUNTRIES_BBOX_WEB.indexName, COUNTRIES_BBOX_WEB),
         Map.entry(AIRPORT_CITY_BOUNDARIES.indexName, AIRPORT_CITY_BOUNDARIES),
         Map.entry(CARTESIAN_MULTIPOLYGONS.indexName, CARTESIAN_MULTIPOLYGONS),
+        Map.entry(DATE_NANOS.indexName, DATE_NANOS),
         Map.entry(K8S.indexName, K8S),
         Map.entry(DISTANCES.indexName, DISTANCES),
-        Map.entry(ADDRESSES.indexName, ADDRESSES)
+        Map.entry(ADDRESSES.indexName, ADDRESSES),
+        Map.entry(BOOKS.indexName, BOOKS)
     );
 
     private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json");
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
index 2bf3baf845010..1774f4a6d8989 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
@@ -7,9 +7,12 @@
 
 package org.elasticsearch.xpack.esql;
 
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.sandbox.document.HalfFloatPoint;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
@@ -22,8 +25,11 @@
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geo.ShapeTestUtils;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
 import org.elasticsearch.xpack.esql.analysis.Verifier;
@@ -31,14 +37,12 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DateUtils;
 import org.elasticsearch.xpack.esql.core.type.EsField;
-import org.elasticsearch.xpack.esql.core.type.TypesTests;
+import org.elasticsearch.xpack.esql.core.util.DateUtils;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
@@ -46,6 +50,7 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
+import org.elasticsearch.xpack.esql.index.EsIndex;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
@@ -53,16 +58,17 @@
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.stats.Metrics;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
+import org.elasticsearch.xpack.versionfield.Version;
 import org.junit.Assert;
 
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
 import java.net.URL;
 import java.net.URLConnection;
 import java.nio.charset.StandardCharsets;
@@ -72,6 +78,8 @@
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.time.Duration;
+import java.time.Period;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.Iterator;
@@ -88,24 +96,38 @@
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
+import static org.elasticsearch.test.ESTestCase.between;
 import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
 import static org.elasticsearch.test.ESTestCase.randomBoolean;
+import static org.elasticsearch.test.ESTestCase.randomByte;
+import static org.elasticsearch.test.ESTestCase.randomDouble;
+import static org.elasticsearch.test.ESTestCase.randomFloat;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
+import static org.elasticsearch.test.ESTestCase.randomInt;
+import static org.elasticsearch.test.ESTestCase.randomIntBetween;
+import static org.elasticsearch.test.ESTestCase.randomIp;
+import static org.elasticsearch.test.ESTestCase.randomLong;
+import static org.elasticsearch.test.ESTestCase.randomLongBetween;
+import static org.elasticsearch.test.ESTestCase.randomMillisUpToYear9999;
+import static org.elasticsearch.test.ESTestCase.randomShort;
 import static org.elasticsearch.test.ESTestCase.randomZone;
 import static org.elasticsearch.test.ListMatcher.matchesList;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
+import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertTrue;
 
 public final class EsqlTestUtils {
+
     public static final Literal ONE = new Literal(Source.EMPTY, 1, DataType.INTEGER);
     public static final Literal TWO = new Literal(Source.EMPTY, 2, DataType.INTEGER);
     public static final Literal THREE = new Literal(Source.EMPTY, 3, DataType.INTEGER);
     public static final Literal FOUR = new Literal(Source.EMPTY, 4, DataType.INTEGER);
     public static final Literal FIVE = new Literal(Source.EMPTY, 5, DataType.INTEGER);
-    private static final Literal SIX = new Literal(Source.EMPTY, 6, DataType.INTEGER);
+    public static final Literal SIX = new Literal(Source.EMPTY, 6, DataType.INTEGER);
 
     public static Equals equalsOf(Expression left, Expression right) {
         return new Equals(EMPTY, left, right, null);
@@ -155,10 +177,6 @@ public static Literal of(Object value) {
         return of(Source.EMPTY, value);
     }
 
-    public static Configuration randomConfiguration() {
-        return new Configuration(randomZone(), randomAlphaOfLength(10), randomAlphaOfLength(10));
-    }
-
     /**
      * Utility method for creating 'in-line' Literals (out of values instead of expressions).
      */
@@ -169,6 +187,10 @@ public static Literal of(Source source, Object value) {
         return new Literal(source, value, DataType.fromJava(value));
     }
 
+    public static ReferenceAttribute referenceAttribute(String name, DataType type) {
+        return new ReferenceAttribute(EMPTY, name, type);
+    }
+
     public static Range rangeOf(Expression value, Expression lower, boolean includeLower, Expression upper, boolean includeUpper) {
         return new Range(EMPTY, value, lower, includeLower, upper, includeUpper, randomZone());
     }
@@ -227,14 +249,14 @@ public boolean isIndexed(String field) {
 
     private static final Map<String, Map<String, Column>> TABLES = tables();
 
-    public static final EsqlConfiguration TEST_CFG = configuration(new QueryPragmas(Settings.EMPTY));
+    public static final Configuration TEST_CFG = configuration(new QueryPragmas(Settings.EMPTY));
 
     public static final Verifier TEST_VERIFIER = new Verifier(new Metrics());
 
     private EsqlTestUtils() {}
 
-    public static EsqlConfiguration configuration(QueryPragmas pragmas, String query) {
-        return new EsqlConfiguration(
+    public static Configuration configuration(QueryPragmas pragmas, String query) {
+        return new Configuration(
             DateUtils.UTC,
             Locale.US,
             null,
@@ -248,11 +270,11 @@ public static EsqlConfiguration configuration(QueryPragmas pragmas, String query
         );
     }
 
-    public static EsqlConfiguration configuration(QueryPragmas pragmas) {
+    public static Configuration configuration(QueryPragmas pragmas) {
         return configuration(pragmas, StringUtils.EMPTY);
     }
 
-    public static EsqlConfiguration configuration(String query) {
+    public static Configuration configuration(String query) {
         return configuration(new QueryPragmas(Settings.EMPTY), query);
     }
 
@@ -274,7 +296,7 @@ public static <T> T as(Object node, Class<T> type) {
     }
 
     public static Map<String, EsField> loadMapping(String name) {
-        return TypesTests.loadMapping(EsqlDataTypeRegistry.INSTANCE, name, true);
+        return LoadMapping.loadMapping(name);
     }
 
     public static String loadUtf8TextFile(String name) {
@@ -596,4 +618,54 @@ public static Tuple<String, String> pathAndName(String string) {
         return new Tuple<>(folder, file);
     }
 
+    /**
+     * Generate a random value of the appropriate type to fit into blocks of {@code e}.
+     */
+    public static Literal randomLiteral(DataType type) {
+        return new Literal(Source.EMPTY, switch (type) {
+            case BOOLEAN -> randomBoolean();
+            case BYTE -> randomByte();
+            case SHORT -> randomShort();
+            case INTEGER, COUNTER_INTEGER -> randomInt();
+            case UNSIGNED_LONG, LONG, COUNTER_LONG -> randomLong();
+            case DATE_PERIOD -> Period.of(randomIntBetween(-1000, 1000), randomIntBetween(-13, 13), randomIntBetween(-32, 32));
+            case DATETIME -> randomMillisUpToYear9999();
+            case DATE_NANOS -> randomLong();
+            case DOUBLE, SCALED_FLOAT, COUNTER_DOUBLE -> randomDouble();
+            case FLOAT -> randomFloat();
+            case HALF_FLOAT -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat()));
+            case KEYWORD -> new BytesRef(randomAlphaOfLength(5));
+            case IP -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())));
+            case TIME_DURATION -> Duration.ofMillis(randomLongBetween(-604800000L, 604800000L)); // plus/minus 7 days
+            case TEXT -> new BytesRef(randomAlphaOfLength(50));
+            case VERSION -> randomVersion().toBytesRef();
+            case GEO_POINT -> GEO.asWkb(GeometryTestUtils.randomPoint());
+            case CARTESIAN_POINT -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint());
+            case GEO_SHAPE -> GEO.asWkb(GeometryTestUtils.randomGeometry(randomBoolean()));
+            case CARTESIAN_SHAPE -> CARTESIAN.asWkb(ShapeTestUtils.randomGeometry(randomBoolean()));
+            case NULL -> null;
+            case SOURCE -> {
+                try {
+                    yield BytesReference.bytes(
+                        JsonXContent.contentBuilder().startObject().field(randomAlphaOfLength(3), randomAlphaOfLength(10)).endObject()
+                    ).toBytesRef();
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+            case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException(
+                "can't make random values for [" + type.typeName() + "]"
+            );
+        }, type);
+    }
+
+    static Version randomVersion() {
+        // TODO degenerate versions and stuff
+        return switch (between(0, 2)) {
+            case 0 -> new Version(Integer.toString(between(0, 100)));
+            case 1 -> new Version(between(0, 100) + "." + between(0, 100));
+            case 2 -> new Version(between(0, 100) + "." + between(0, 100) + "." + between(0, 100));
+            default -> throw new IllegalArgumentException();
+        };
+    }
 }
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java
new file mode 100644
index 0000000000000..e750502cad198
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.index.mapper.TimeSeriesParams;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DateEsField;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.type.KeywordEsField;
+import org.elasticsearch.xpack.esql.core.type.TextEsField;
+import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
+import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
+import static org.junit.Assert.assertNotNull;
+
+public class LoadMapping {
+    public static Map<String, EsField> loadMapping(String name) {
+        InputStream stream = LoadMapping.class.getResourceAsStream("/" + name);
+        assertNotNull("Could not find mapping resource:" + name, stream);
+        return loadMapping(stream);
+    }
+
+    private static Map<String, EsField> loadMapping(InputStream stream) {
+        try (InputStream in = stream) {
+            Map<String, Object> map = XContentHelper.convertToMap(JsonXContent.jsonXContent, in, true);
+            return fromEs(map);
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<String, EsField> fromEs(Map<String, Object> asMap) {
+        Map<String, Object> props = null;
+        if (asMap != null && asMap.isEmpty() == false) {
+            props = (Map<String, Object>) asMap.get("properties");
+        }
+        return props == null || props.isEmpty() ? emptyMap() : startWalking(props);
+    }
+
+    private static Map<String, EsField> startWalking(Map<String, Object> mapping) {
+        Map<String, EsField> types = new LinkedHashMap<>();
+
+        if (mapping == null) {
+            return emptyMap();
+        }
+        for (Map.Entry<String, Object> entry : mapping.entrySet()) {
+            walkMapping(entry.getKey(), entry.getValue(), types);
+        }
+
+        return types;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void walkMapping(String name, Object value, Map<String, EsField> mapping) {
+        // object type - only root or nested docs supported
+        if (value instanceof Map) {
+            Map<String, Object> content = (Map<String, Object>) value;
+
+            if ("nested".equals(content.get("type"))) {
+                // Nested fields are entirely removed by IndexResolver so we mimic it.
+                return;
+            }
+            // extract field type
+            DataType esDataType = getType(content);
+            final Map<String, EsField> properties;
+            if (esDataType == OBJECT) {
+                properties = fromEs(content);
+            } else if (content.containsKey("fields")) {
+                // Check for multifields
+                Object fields = content.get("fields");
+                if (fields instanceof Map) {
+                    properties = startWalking((Map<String, Object>) fields);
+                } else {
+                    properties = Collections.emptyMap();
+                }
+            } else {
+                properties = fromEs(content);
+            }
+            boolean docValues = boolSetting(content.get("doc_values"), esDataType.hasDocValues());
+            final EsField field;
+            if (esDataType == TEXT) {
+                field = new TextEsField(name, properties, docValues);
+            } else if (esDataType == KEYWORD) {
+                int length = intSetting(content.get("ignore_above"), Short.MAX_VALUE);
+                boolean normalized = Strings.hasText(textSetting(content.get("normalizer"), null));
+                field = new KeywordEsField(name, properties, docValues, length, normalized);
+            } else if (esDataType == DATETIME) {
+                field = DateEsField.dateEsField(name, properties, docValues);
+            } else if (esDataType == UNSUPPORTED) {
+                String type = content.get("type").toString();
+                field = new UnsupportedEsField(name, type, null, properties);
+                propagateUnsupportedType(name, type, properties);
+            } else {
+                field = new EsField(name, esDataType, properties, docValues);
+            }
+            mapping.put(name, field);
+        } else {
+            throw new IllegalArgumentException("Unrecognized mapping " + value);
+        }
+    }
+
+    private static DataType getType(Map<String, Object> content) {
+        if (content.containsKey("type")) {
+            String typeName = content.get("type").toString();
+            if ("constant_keyword".equals(typeName) || "wildcard".equals(typeName)) {
+                return KEYWORD;
+            }
+            final Object metricsTypeParameter = content.get(TimeSeriesParams.TIME_SERIES_METRIC_PARAM);
+            final TimeSeriesParams.MetricType metricType;
+            if (metricsTypeParameter instanceof String str) {
+                metricType = TimeSeriesParams.MetricType.fromString(str);
+            } else {
+                metricType = (TimeSeriesParams.MetricType) metricsTypeParameter;
+            }
+            try {
+                return EsqlDataTypeRegistry.INSTANCE.fromEs(typeName, metricType);
+            } catch (IllegalArgumentException ex) {
+                return UNSUPPORTED;
+            }
+        } else if (content.containsKey("properties")) {
+            return OBJECT;
+        } else {
+            return UNSUPPORTED;
+        }
+    }
+
+    private static String textSetting(Object value, String defaultValue) {
+        return value == null ? defaultValue : value.toString();
+    }
+
+    private static boolean boolSetting(Object value, boolean defaultValue) {
+        return value == null ? defaultValue : Booleans.parseBoolean(value.toString(), defaultValue);
+    }
+
+    private static int intSetting(Object value, int defaultValue) {
+        return value == null ? defaultValue : Integer.parseInt(value.toString());
+    }
+
+    public static void propagateUnsupportedType(String inherited, String originalType, Map<String, EsField> properties) {
+        if (properties != null && properties.isEmpty() == false) {
+            for (Map.Entry<String, EsField> entry : properties.entrySet()) {
+                EsField field = entry.getValue();
+                UnsupportedEsField u;
+                if (field instanceof UnsupportedEsField) {
+                    u = (UnsupportedEsField) field;
+                    u = new UnsupportedEsField(u.getName(), originalType, inherited, u.getProperties());
+                } else {
+                    u = new UnsupportedEsField(field.getName(), originalType, inherited, field.getProperties());
+                }
+                entry.setValue(u);
+                propagateUnsupportedType(inherited, originalType, u.getProperties());
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/SpecReader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/SpecReader.java
similarity index 94%
rename from x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/SpecReader.java
rename to x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/SpecReader.java
index c96f360cc95f0..793268f18184d 100644
--- a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/SpecReader.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/SpecReader.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core;
+package org.elasticsearch.xpack.esql;
 
 import org.elasticsearch.common.Strings;
 
@@ -18,7 +18,6 @@
 import java.util.Objects;
 
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.xpack.esql.core.TestUtils.pathAndName;
 import static org.junit.Assert.assertNull;
 
 public final class SpecReader {
@@ -45,14 +44,14 @@ public static List<Object[]> readScriptSpec(List<URL> urls, Parser parser) throw
     }
 
     public static List<Object[]> readURLSpec(URL source, Parser parser) throws Exception {
-        String fileName = pathAndName(source.getFile()).v2();
+        String fileName = EsqlTestUtils.pathAndName(source.getFile()).v2();
         String groupName = fileName.substring(0, fileName.lastIndexOf('.'));
 
         Map<String, Integer> testNames = new LinkedHashMap<>();
         List<Object[]> testCases = new ArrayList<>();
 
         String testName = null;
-        try (BufferedReader reader = TestUtils.reader(source)) {
+        try (BufferedReader reader = EsqlTestUtils.reader(source)) {
             String line;
             int lineNumber = 1;
             while ((line = reader.readLine()) != null) {
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/alerts.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/alerts.csv
new file mode 100644
index 0000000000000..dbf13fd89176c
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/alerts.csv
@@ -0,0 +1,11 @@
+host.name:keyword,kibana.alert.risk_score:double
+test-host-1,21.0
+test-host-2,17.0
+test-host-2,23.0
+test-host-1,45.0
+test-host-2,12.0
+test-host-2,16.0
+test-host-1,21.0
+test-host-1,70.0
+test-host-1,21.0
+test-host-2,5.0
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/books.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/books.csv
new file mode 100644
index 0000000000000..1deefaa3c6475
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/books.csv
@@ -0,0 +1,80 @@
+book_no:keyword,title:text,author:text,year:integer,publisher:text,ratings:float,description:text
+2924,A Gentle Creature and Other Stories: White Nights\, A Gentle Creature\, and The Dream of a Ridiculous Man (The World's Classics),[Fyodor Dostoevsky, Alan Myers, W. J. Leatherbarrow],2009,Oxford Paperbacks,4.00,In these stories Dostoevsky explores both the figure of the dreamer divorced from reality and also his own ambiguous attitude to utopianism\, themes central to many of his great novels. This new translation captures the power and lyricism of Dostoevsky's writing\, while the introduction examines the stories in relation to one another and to his novels.
+7670,A Middle English Reader and Vocabulary,[Kenneth Sisam, J. R. R. Tolkien],2011,Courier Corporation,4.33,This highly respected anthology of medieval English literature features poetry\, prose and popular tales from Arthurian legend and classical mythology. Includes notes on each extract\, appendices\, and an extensive glossary by J. R. R. Tolkien.
+7381,A Psychic in the Heartland: The Extraordinary Experiences of a Small Town Doctor,Bettilu Stein Faulkner,2003,Red Wheel/Weiser,4.50,The true story of a small-town doctor destined to live his life along two paths: one as a successful physician\, the other as a psychic with ever more interesting adventures. Experiencing a wide range of spiritual phenomena\, Dr. Riblet Hout learned about the connection between the healer and the healed\, our individual missions on earth\, free will\, and our relationship with God. He also paints a vivid picture of life on the other side as well as the moment of transition from physical life to afterlife.
+2883,A Summer of Faulkner: As I Lay Dying/The Sound and the Fury/Light in August (Oprah's Book Club),William Faulkner,2005,Vintage Books,3.89,Presents three novels\, including As I Lay Dying\, in which the Bundren family journeys across Mississippi to bury their mother\, The Sound and the Fury\, in which Caddy Compson's story is narrated by her three brothers\, and Light in August\, in which th
+4023,A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings,[Walter Scheps, Agnes Perkins, Charles Adolph Huttar, John Ronald Reuel Tolkien],1975,Open Court Publishing,4.67,The structure\, content\, and character of Tolkien's The Hobbit and The Lord of the Rings are dealt with in ten critical essays.
+2382,A Wizard of Earthsea (Earthsea Trilogy Ser.),Ursula K. Le Guin,1991,Atheneum Books for Young Readers,4.01,A boy grows to manhood while attempting to subdue the evil he unleashed on the world as an apprentice to the Master Wizard.
+7541,A Writer's Diary (Volume 1: 1873-1876),Fyodor Dostoevsky,1997,Northwestern University Press,4.50,Winner of the AATSEEL Outstanding Translation Award This is the first paperback edition of the complete collection of writings that has been called Dostoevsky's boldest experiment with literary form\, it is a uniquely encyclopedic forum of fictional and nonfictional genres. The Diary's radical format was matched by the extreme range of its contents. In a single frame it incorporated an astonishing variety of material: short stories\, humorous sketches\, reports on sensational crimes\, historical predictions\, portraits of famous people\, autobiographical pieces\, and plans for stories\, some of which were never written while others appeared in the Diary itself.
+7400,Anna Karenina: Television Tie-In Edition (Signet classics),[Leo Tolstoy, SBP Editors],2019,Samaira Book Publishers,4.45,The Russian novelist and moral philosopher Leo Tolstoy (1828-1910) ranks as one of the world s great writers\, and his 'War and Peace' has been called the greatest novel ever written. But during his long lifetime\, Tolstoy also wrote enough shorter works to fill many volumes. The message in all his stories is presented with such humour that the reader hardly realises that it is strongly didactic. These stories give a snapshot of Russia and its people in the late nineteenth century.
+4917,Autumn of the Patriarch,Gabriel Garcia Marquez,2014,Penguin UK,4.33,Gabriel Garcia Marquez\, winner of the 1982 Nobel Prize for Literature and author of One Hundred Years of Solitude\, explores the loneliness of power in Autumn of the Patriarch. 'Over the weekend the vultures got into the presidential palace by pecking through the screens on the balcony windows and the flapping of their wings stirred up the stagnant time inside' As the citizens of an unnamed Caribbean nation creep through dusty corridors in search of their tyrannical leader\, they cannot comprehend that the frail and withered man lying dead on the floor can be the self-styled General of the Universe. Their arrogant\, manically violent leader\, known for serving up traitors to dinner guests and drowning young children at sea\, can surely not die the humiliating death of a mere mortal? Tracing the demands of a man whose egocentric excesses mask the loneliness of isolation and whose lies have become so ingrained that they are indistinguishable from truth\, Marquez has created a fantastical portrait of despotism that rings with an air of reality. 'Delights with its quirky humanity and black humour and impresses by its total originality' Vogue 'Captures perfectly the moral squalor and political paralysis that enshrouds a society awaiting the death of a long-term dictator' Guardian 'Marquez writes in this lyrical\, magical language that no-one else can do' Salman Rushdie
+9896,Barn burning (A tale blazer book),William Faulkner,1979,Perfection Learning,3.50,Reprinted from Collected Stories of William Faulkner\, by permission of Random House\, Inc.
+9607,Beowolf: The monsters and the critics,John Ronald Reuel Tolkien,1997,HarperCollins UK,4.12,A collection of seven essays by J.R.R. Tolkien arising out of Tolkien's work in medieval literature
+1985,Brothers Karamazov,Fyodor Dostoevsky,2015,First Avenue Editions,5.00,Four brothers reunite in their hometown in Russia. The murder of their father forces the brothers to question their beliefs about each other\, religion\, and morality.
+2713,Collected Stories of William Faulkner,William Faulkner,1995,Vintage,4.53,A collection of short stories focuses on the people of rural Mississippi
+2464,Conversations with Kurt Vonnegut (Literary Conversations),Kurt Vonnegut,1988,Univ. Press of Mississippi,4.40,Gathers interviews with Vonnegut from each period of his career and offers a brief profile of his life and accomplishments
+8534,Crime and Punishment (Oxford World's Classics),Fyodor Dostoevsky,2017,Oxford University Press,4.38,'One death\, in exchange for thousands of lives - it's simple arithmetic!' A new translation of Dostoevsky's epic masterpiece\, Crime and Punishment (1866). The impoverished student Raskolnikov decides to free himself from debt by killing an old moneylender\, an act he sees as elevating himself above conventional morality. Like Napoleon he will assert his will and his crime will be justified by its elimination of 'vermin' for the sake of the greater good. But Raskolnikov is torn apart by fear\, guilt\, and a growing conscience under the influence of his love for Sonya. Meanwhile the police detective Porfiry is on his trial. It is a powerfully psychological novel\, in which the St Petersburg setting\, Dostoevsky's own circumstances\, and contemporary social problems all play their part.
+8605,Dead Souls,Nikolai Gogol,1997,Vintage,4.28,Chichikov\, an amusing and often confused schemer\, buys deceased serfs' names from landholders' poll tax lists hoping to mortgage them for profit
+6970,Domestic Goddesses,Edith Vonnegut,1998,Pomegranate,4.67,In this immensely charming and insightful book\, artist Edith Vonnegut takes issue with traditional art imagery in which women are shown as weak and helpless. Through twenty-seven of her own paintings interspersed with her text\, she poignantly -- and humorously -- illustrates her maxim that the lives of mothers and homemakers are filled with endless challenges and vital decisions that should be portrayed with the dignity they deserve. In Vonnegut's paintings\, one woman bravely blocks the sun from harming a child (Sun Block) while another vacuums the stairs with angelic figures singing her praises (Electrolux). In contrasting her own Domestic Goddesses with the diaphanous women of classical art (seven paintings by masters such as Titian and Botticelli are included)\, she 'expresses the importance of traditional roles of women so cleverly and with such joy that her message and images will be forever emblazoned on our collective psyche.
+4814,El Coronel No Tiene Quien Le Escriba / No One Writes to the Colonel (Spanish Edition),Gabriel Garcia Marquez,2005,Harper Collins,4.45,Written with compassionate realism and wit\, the stories in this mesmerizing collection depict the disparities of town and village life in South America\, of the frightfully poor and outrageously rich\, of memories and illusions\, and of lost opportunities and present joys.
+4636,FINAL WITNESS,Simon Tolkien,2004,Random House Digital\, Inc.,3.94,The murder of Lady Anne Robinson by two intruders causes a schism in the victim's family when her son convinces police that his father's beautiful personal assistant hired the killers\, while his father\, the British minister of defense\, refuses to believe his son and marries the accused. A first novel. Reprint.
+2936,Fellowship of the Ring 2ND Edition,John Ronald Reuel Tolkien,2008,HarperCollins UK,4.43,Sauron\, the Dark Lord\, has gathered to him all the Rings of Power - the means by which he intends to rule Middle-earth. All he lacks in his plans for dominion is the One Ring - the ring that rules them all - which has fallen into the hands of the hobbit\, Bilbo Baggins. In a sleepy village in the Shire\, young Frodo Baggins finds himself faced with an immense task\, as his elderly cousin Bilbo entrusts the Ring to his care. Frodo must leave his home and make a perilous journey across Middle-earth to the Cracks of Doom\, there to destroy the Ring and foil the Dark Lord in his evil purpose. JRR Tolkien's great work of imaginative fiction has been labelled both a heroic romance and a classic fantasy fiction. By turns comic and homely\, epic and diabolic\, the narrative moves through countless changes of scene and character in an imaginary world which is totally convincing in its detail.
+8956,GOD BLESS YOU MR. ROSEWATER : Or Pearls Before Swine,Kurt Vonnegut,1970,New York : Dell,4.00,A lawyer schemes to gain control of a large fortune by having the present claimant declared insane.
+6818,Hadji Murad,Leo Tolstoy,2022,Hachette UK,3.88,'How truth thickens and deepens when it migrates from didactic fable to the raw experience of a visceral awakening is one of the thrills of Tolstoy's stories' Sharon Cameron in her preface to Hadji Murad and Other Stories This\, the third volume of Tolstoy's shorter fiction concentrates on his later stories\, including one of his greatest\, 'Hadji Murad'. In the stark form of homily that shapes these later works\, life considered as one's own has no rational meaning. From the chain of events that follows in the wake of two schoolboys' deception in 'The Forged Coupon' to the disillusionment of the narrator in 'After the Ball' we see\, in Virginia Woolf's observation\, that Tolstoy puts at the centre of his writing one 'who gathers into himself all experience\, turns the world round between his fingers\, and never ceases to ask\, even as he enjoys it\, what is the meaning of it'. The riverrun edition reissues the translation of Louise and Aylmer Maude\, whose influential versions of Tolstoy first brought his work to a wide readership in English.
+3950,Hocus,Kurt Vonnegut,1997,Penguin,4.67,Tarkington College\, a small\, exclusive college in upstate New York\, is turned upside down when ten thousand prisoners from the maximum security prison across Lake Mohiga break out and head for the college
+5404,Intruder in the dust,William Faulkner,2011,Vintage,3.18,A classic Faulkner novel which explores the lives of a family of characters in the South. An aging black who has long refused to adopt the black's traditionally servile attitude is wrongfully accused of murdering a white man.
+5578,Intruder in the dust: A novel,William Faulkner,1991,Vintage,3.18,Dramatizes the events that surround the murder of a white man in a volatile Southern community
+6380,La hojarasca (Spanish Edition),Gabriel Garcia Marquez,1979,Harper Collins,3.75,Translated from the Spanish by Gregory Rabassa
+5335,Letters of J R R Tolkien,J.R.R. Tolkien,2014,HarperCollins,4.70,This collection will entertain all who appreciate the art of masterful letter writing. The Letters of J.R.R Tolkien sheds much light on Tolkien's creative genius and grand design for the creation of a whole new world: Middle-earth. Featuring a radically expanded index\, this volume provides a valuable research tool for all fans wishing to trace the evolution of THE HOBBIT and THE LORD OF THE RINGS.
+3870,My First 100 Words in Spanish/English (My First 100 Words Pull-Tab Book),Keith Faulkner,1998,Libros Para Ninos,4.50,Learning a foreign language has never been this much fun! Just pull the sturdy tabs and change the words under the pictures from English to Spanish and back again to English!
+4502,O'Brian's Bride,Colleen Faulkner,1995,Zebra Books,5.00,Abandoning her pampered English life to marry a man in the American colonies\, Elizabeth finds her new world shattered when her husband is killed in an accident\, leaving her in charge of a business on the untamed frontier. Original.
+7635,Oliphaunt (Beastly Verse),J. R. R. Tolkien,1989,Contemporary Books,2.50,A poem in which an elephant describes himself and his way of life. On board pages.
+3254,Pearl and Sir Orfeo,[John Ronald Reuel Tolkien, Christopher Tolkien],1995,Harpercollins Pub Limited,5.00,Three epic poems from 14th century England speak of life during the age of chivalry. Translated from medieval English.
+3677,Planet of Exile,Ursula K. Le Guin,1979,Orion,4.20,PLAYAWAY: An alliance between the powerful Tevars and the brown-skinned\, clairvoyant Farbons must take place if the two colonies are to withstand the fierce attack of the nomadic tribes from the north of the planet Eltanin.
+4289,Poems from the Hobbit,J R R Tolkien,1999,HarperCollins Publishers,4.00,A collection of J.R.R. Tolkien's Hobbit poems in a miniature hardback volume complete with illustrations by Tolkien himself. Far over misty mountains cold To dungeons deep and caverns old We must away ere break of day To seek the pale enchanted gold. J.R.R. Tolkien's acclaimed The Hobbit contains 12 poems which are themselves masterpieces of writing. This miniature book\, illustrated with 30 of Tolkien's own paintings and drawings from the book -- some quite rare and all in full colour -- includes all the poems\, plus Gollum's eight riddles in verse\, and will be a perfect keepsake for lovers of The Hobbit and of accomplished poetry.
+6151,Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks),[Keith Faulkner, Rory Tyger],2003,Dutton Childrens Books,5.00,Toby the turtle goes from in-line skates to a motorcycle to a rocketship with a handful of balloons that pop\, one by one\, along the way.
+3535,Rainbow's End: A Magical Story and Moneybox,[Keith Faulkner, Beverlie Manson],2003,Barrons Juveniles,4.00,In this combination picture storybook and coin bank\, the unusual front cover shows an illustration from the story that's embellished with five transparent plastic windows. Opening the book\, children will find a story about a poor little ballerina who is crying because her dancing shoes are worn and she has no money to replace them. Full color. Consumable.
+8423,Raising Faithful Kids in a Fast-Paced World,Paul Faulkner,1995,Howard Publishing Company,5.00,To find help for struggling parents\, Dr. Paul Faulkner--renowned family counselor and popular speaker--interviewed 30 successful families who have managed to raise faithful kids while also maintaining demanding careers. The invaluable strategies and methods he gleaned are now available in this powerful book delivered in Dr. Faulkner's warm\, humorous style.
+1463,Realms of Tolkien: Images of Middle-earth,J. R. R. Tolkien,1997,HarperCollins Publishers,4.00,Twenty new and familiar Tolkien artists are represented in this fabulous volume\, breathing an extraordinary variety of life into 58 different scenes\, each of which is accompanied by appropriate passage from The Hobbit and The Lord of the Rings and The Silmarillion
+6323,Resurrection (The Penguin classics),Leo Tolstoy,2009,Penguin,3.25,Leo Tolstoy's last completed novel\, Resurrection is an intimate\, psychological tale of guilt\, anger and forgiveness Serving on the jury at a murder trial\, Prince Dmitri Nekhlyudov is devastated when he sees the prisoner - Katyusha\, a young maid he seduced and abandoned years before. As Dmitri faces the consequences of his actions\, he decides to give up his life of wealth and luxury to devote himself to rescuing Katyusha\, even if it means following her into exile in Siberia. But can a man truly find redemption by saving another person? Tolstoy's most controversial novel\, Resurrection (1899) is a scathing indictment of injustice\, corruption and hypocrisy at all levels of society. Creating a vast panorama of Russian life\, from peasants to aristocrats\, bureaucrats to convicts\, it reveals Tolstoy's magnificent storytelling powers. Anthony Briggs' superb new translation preserves Tolstoy's gripping realism and satirical humour. In his introduction\, Briggs discusses the true story behind Resurrection\, Tolstoy's political and religious reasons for writing the novel\, his gift for characterization and the compelling psychological portrait of Dmitri. This edition also includes a chronology\, notes and a summary of chapters. For more than seventy years\, Penguin has been the leading publisher of classic literature in the English-speaking world. With more than 1\,700 titles\, Penguin Classics represents a global bookshelf of the best works throughout history and across genres and disciplines. Readers trust the series to provide authoritative texts enhanced by introductions and notes by distinguished scholars and contemporary authors\, as well as up-to-date translations by award-winning translators.
+2714,Return of the King Being the Third Part of The Lord of the Rings,J. R. R. Tolkien,2012,HarperCollins,4.60,Concluding the story begun in The Hobbit\, this is the final part of Tolkien s epic masterpiece\, The Lord of the Rings\, featuring an exclusive cover image from the film\, the definitive text\, and a detailed map of Middle-earth. The armies of the Dark Lord Sauron are massing as his evil shadow spreads ever wider. Men\, Dwarves\, Elves and Ents unite forces to do battle agains the Dark. Meanwhile\, Frodo and Sam struggle further into Mordor in their heroic quest to destroy the One Ring. The devastating conclusion of J.R.R. Tolkien s classic tale of magic and adventure\, begun in The Fellowship of the Ring and The Two Towers\, features the definitive edition of the text and includes the Appendices and a revised Index in full. To celebrate the release of the first of Peter Jackson s two-part film adaptation of The Hobbit\, THE HOBBIT: AN UNEXPECTED JOURNEY\, this third part of The Lord of the Rings is available for a limited time with an exclusive cover image from Peter Jackson s award-winning trilogy.
+7350,Return of the Shadow,[John Ronald Reuel Tolkien, Christopher Tolkien],2000,Mariner Books,5.00,In this sixth volume of The History of Middle-earth the story reaches The Lord of the Rings. In The Return of the Shadow (an abandoned title for the first volume) Christopher Tolkien describes\, with full citation of the earliest notes\, outline plans\, and narrative drafts\, the intricate evolution of The Fellowship of the Ring and the gradual emergence of the conceptions that transformed what J.R.R. Tolkien for long believed would be a far shorter book\, 'a sequel to The Hobbit'. The enlargement of Bilbo's 'magic ring' into the supremely potent and dangerous Ruling Ring of the Dark Lord is traced and the precise moment is seen when\, in an astonishing and unforeseen leap in the earliest narrative\, a Black Rider first rode into the Shire\, his significance still unknown. The character of the hobbit called Trotter (afterwards Strider or Aragorn) is developed while his indentity remains an absolute puzzle\, and the suspicion only very slowly becomes certainty that he must after all be a Man. The hobbits\, Frodo's companions\, undergo intricate permutations of name and personality\, and other major figures appear in strange modes: a sinister Treebeard\, in league with the Enemy\, a ferocious and malevolent Farmer Maggot. The story in this book ends at the point where J.R.R. Tolkien halted in the story for a long time\, as the Company of the Ring\, still lacking Legolas and Gimli\, stood before the tomb of Balin in the Mines of Moria. The Return of the Shadow is illustrated with reproductions of the first maps and notable pages from the earliest manuscripts.
+6760,Roverandom,J. R. R. Tolkien,1999,Mariner Books,4.38,Rover\, a dog who has been turned into a toy dog encounters rival wizards and experiences various adventures on the moon with giant spiders\, dragon moths\, and the Great White Dragon. By the author of The Hobbit. Reprint.
+8873,Searoad: Chronicles of Klatsand,Ursula K. Le Guin,2004,Shambhala Publications,5.00,A series of interlinking tales and a novella by the author of the Earthsea trilogy portrays the triumphs and struggles of several generations of women who independently control Klatsand\, a small resort town on the Oregon coast. Reprint.
+2378,Selected Letters of Lucretia Coffin Mott (Women in American History),[Lucretia Mott, Holly Byers Ochoa, Carol Faulkner],2002,University of Illinois Press,5.00,Dedicated to reform of almost every kind - temperance\, peace\, equal rights\, woman suffrage\, nonresistance\, and the abolition of slavery - Mott viewed women's rights as only one element of a broad-based reform agenda for American society.
+1502,Selected Passages from Correspondence with Friends,Nikolai Vasilevich Gogol,2009,Vanderbilt University Press,4.00,Nikolai Gogol wrote some letters to his friends\, none of which were a nose of high rank. Many are reproduced here (the letters\, not noses).
+5996,Smith of Wooten Manor & Farmer Giles of Ham,John Ronald Reuel Tolkien,1969,Del Rey,4.91,Two bewitching fantasies by J.R.R. Tolkien\, beloved author of THE HOBBIT. In SMITH OF WOOTTON MAJOR\, Tolkien explores the gift of fantasy\, and what it means to the life and character of the man who receives it. And FARMER GILES OF HAM tells a delightfully ribald mock-heroic tale\, where a dragon who invades a town refuses to fight\, and a farmer is chosen to slay him.
+2301,Smith of Wootton Major & Farmer Giles of Ham,John Ronald Reuel Tolkien,1969,Del Rey,5.00,Two bewitching fantasies by J.R.R. Tolkien\, beloved author of THE HOBBIT. In SMITH OF WOOTTON MAJOR\, Tolkien explores the gift of fantasy\, and what it means to the life and character of the man who receives it. And FARMER GILES OF HAM tells a delightfully ribald mock-heroic tale\, where a dragon who invades a town refuses to fight\, and a farmer is chosen to slay him.
+2236,Steering the Craft,Ursula K. Le Guin,2015,Houghton Mifflin Harcourt,4.73,A revised and updated guide to the essentials of a writer's craft\, presented by a brilliant practitioner of the art Completely revised and rewritten to address the challenges and opportunities of the modern era\, this handbook is a short\, deceptively simple guide to the craft of writing. Le Guin lays out ten chapters that address the most fundamental components of narrative\, from the sound of language to sentence construction to point of view. Each chapter combines illustrative examples from the global canon with Le Guin's own witty commentary and an exercise that the writer can do solo or in a group. She also offers a comprehensive guide to working in writing groups\, both actual and online. Masterly and concise\, Steering the Craft deserves a place on every writer's shelf.
+4724,THE UNVANQUISHED,William Faulkner,2011,Vintage,3.50,Set in Mississippi during the Civil War and Reconstruction\, THE UNVANQUISHED focuses on the Sartoris family\, who\, with their code of personal responsibility and courage\, stand for the best of the Old South's traditions.
+5948,That We Are Gentle Creatures,Fyodor Dostoevsky,2009,OUP Oxford,4.33,In the stories in this volume Dostoevsky explores both the figure of the dreamer divorced from reality and also his own ambiguous attitude to utopianism\, themes central to many of his great novels. In White Nights the apparent idyll of the dreamer's romantic fantasies disguises profound loneliness and estrangement from 'living life'. Despite his sentimental friendship with Nastenka\, his final withdrawal into the world of the imagination anticipates the retreat into the 'underground' of many of Dostoevsky's later intellectual heroes. A Gentle Creature and The Dream of a Ridiculous Man show how such withdrawal from reality can end in spiritual desolation and moral indifference and how\, in Dostoevsky's view\, the tragedy of the alienated individual can be resolved only by the rediscovery of a sense of compassion and responsibility towards fellow human beings. This new translation captures the power and lyricism of Dostoevsky's writing\, while the introduction examines the stories in relation to one another and to his novels. ABOUT THE SERIES: For over 100 years Oxford World's Classics has made available the widest range of literature from around the globe. Each affordable volume reflects Oxford's commitment to scholarship\, providing the most accurate text plus a wealth of other valuable features\, including expert introductions by leading authorities\, helpful notes to clarify the text\, up-to-date bibliographies for further study\, and much more.
+1937,The Best Short Stories of Dostoevsky (Modern Library),Fyodor Dostoevsky,2012,Modern Library,4.33,This collection\, unique to the Modern Library\, gathers seven of Dostoevsky's key works and shows him to be equally adept at the short story as with the novel. Exploring many of the same themes as in his longer works\, these small masterpieces move from the tender and romantic White Nights\, an archetypal nineteenth-century morality tale of pathos and loss\, to the famous Notes from the Underground\, a story of guilt\, ineffectiveness\, and uncompromising cynicism\, and the first major work of existential literature. Among Dostoevsky's prototypical characters is Yemelyan in The Honest Thief\, whose tragedy turns on an inability to resist crime. Presented in chronological order\, in David Magarshack's celebrated translation\, this is the definitive edition of Dostoevsky's best stories.
+2776,The Devil and Other Stories (Oxford World's Classics),Leo Tolstoy,2003,OUP Oxford,5.00,'It is impossible to explain why Yevgeny chose Liza Annenskaya\, as it is always impossible to explain why a man chooses this and not that woman.' This collection of eleven stories spans virtually the whole of Tolstoy's creative life. While each is unique in form\, as a group they are representative of his style\, and touch on the central themes that surface in War and Peace and Anna Karenina. Stories as different as 'The Snowstorm'\, 'Lucerne'\, 'The Diary of a Madman'\, and 'The Devil' are grounded in autobiographical experience. They deal with journeys of self-discovery and the moral and religious questioning that characterizes Tolstoy's works of criticism and philosophy. 'Strider' and 'Father Sergy'\, as well as reflecting Tolstoy's own experiences\, also reveal profound psychological insights. These stories range over much of the Russian world of the nineteenth century\, from the nobility to the peasantry\, the military to the clergy\, from merchants and cobblers to a horse and a tree. Together they present a fascinating picture of Tolstoy's skill and artistry. ABOUT THE SERIES: For over 100 years Oxford World's Classics has made available the widest range of literature from around the globe. Each affordable volume reflects Oxford's commitment to scholarship\, providing the most accurate text plus a wealth of other valuable features\, including expert introductions by leading authorities\, helpful notes to clarify the text\, up-to-date bibliographies for further study\, and much more.
+4231,The Dispossessed,Ursula K. Le Guin,1974,Harpercollins,4.26,Frequently reissued with the same ISBN\, but with slightly differing bibliographical details.
+7480,The Hobbit,J. R. R. Tolkien,2012,Mariner Books,4.64,Celebrating 75 years of one of the world's most treasured classics with an all new trade paperback edition. Repackaged with new cover art. 500\,000 first printing.
+6405,The Hobbit or There and Back Again,J. R. R. Tolkien,2012,Mariner Books,4.63,Celebrating 75 years of one of the world's most treasured classics with an all new trade paperback edition. Repackaged with new cover art. 500\,000 first printing.
+2540,The Inspector General (Language - Russian) (Russian Edition),[Nicolai Gogol, Thomas Seltzer],2014,CreateSpace,3.50,The Inspector-General is a national institution. To place a purely literary valuation upon it and call it the greatest of Russian comedies would not convey the significance of its position either in Russian literature or in Russian life itself. There is no other single work in the modern literature of any language that carries with it the wealth of associations which the Inspector-General does to the educated Russian.
+2951,The Insulted and Injured,Fyodor Dostoevsky,2011,Wm. B. Eerdmans Publishing,4.00,The Insulted and Injured\, which came out in 1861\, was Fyodor Dostoevsky's first major work of fiction after his Siberian exile and the first of the long novels that made him famous. Set in nineteenth-century Petersburg\, this gripping novel features a vividly drawn set of characters - including Vanya (Dostoevsky's semi-autobiographical hero)\, Natasha (the woman he loves)\, and Alyosha (Natasha's aristocratic lover) - all suffering from the cruelly selfish machinations of Alyosha's father\, the dark and powerful Prince Valkovsky. Boris Jakim's fresh English-language rendering of this gem in the Doestoevsky canon is both more colorful and more accurate than any earlier translation. --from back cover.
+2130,The J. R. R. Tolkien Audio Collection,[John Ronald Reuel Tolkien, Christopher Tolkien],2002,HarperCollins Publishers,4.89,For generations\, J R R Tolkien's words have brought to thrilling life a world of hobbits\, magic\, and historic myth\, woken from its foggy slumber within our minds. Here\, he tells the tales in his own voice.
+9801,The Karamazov Brothers (Oxford World's Classics),Fyodor Dostoevsky,2008,Oxford University Press,4.40,A remarkable work showing the author's power to depict Russian character and his understanding of human nature. Driven by intense\, uncontrollable emotions of rage and revenge\, the four Karamazov brothers all become involved in the brutal murder of their despicable father.
+5469,The Lays of Beleriand,[John Ronald Reuel Tolkien, Christopher Tolkien],2002,Harpercollins Pub Limited,4.42,The third volume that contains the early myths and legends which led to the writing of Tolkien's epic tale of war\, The Silmarillion. This\, the third volume of The History of Middle-earth\, gives us a priviledged insight into the creation of the mythology of Middle-earth\, through the alliterative verse tales of two of the most crucial stories in Tolkien's world -- those of Turien and Luthien. The first of the poems is the unpublished Lay of The Children of Hurin\, narrating on a grand scale the tragedy of Turin Turambar. The second is the moving Lay of Leithian\, the chief source of the tale of Beren and Luthien in The Silmarillion\, telling of the Quest of the Silmaril and the encounter with Morgoth in his subterranean fortress. Accompanying the poems are commentaries on the evolution of the history of the Elder Days. Also included is the notable criticism of The Lay of The Leithian by CS Lewis\, who read the poem in 1929.
+2675,The Lord of the Rings - Boxed Set,J.R.R. Tolkien,2012,HarperCollins,4.56,This beautiful gift edition of The Hobbit\, J.R.R. Tolkien's classic prelude to his Lord of the Rings trilogy\, features cover art\, illustrations\, and watercolor paintings by the artist Alan Lee. Bilbo Baggins is a hobbit who enjoys a comfortable\, unambitious life\, rarely traveling any farther than his pantry or cellar. But his contentment is disturbed when the wizard Gandalf and a company of dwarves arrive on his doorstep one day to whisk him away on an adventure. They have launched a plot to raid the treasure hoard guarded by Smaug the Magnificent\, a large and very dangerous dragon. Bilbo reluctantly joins their quest\, unaware that on his journey to the Lonely Mountain he will encounter both a magic ring and a frightening creature known as Gollum. Written for J.R.R. Tolkien's own children\, The Hobbit has sold many millions of copies worldwide and established itself as a modern classic.
+7140,The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1),[J. R. R. Tolkien, Alan Lee],2002,HarperSport,4.75,A selection of stunning poster paintings from the celebrated Tolkien artist Alan Lee - the man behind many of the striking images from The Lord of The Rings movie. The 50 paintings contained within the centenary edition of The Lord of the Rings in 1992 have themselves become classics and Alan Lee's interpretations are hailed as the most faithful to Tolkien's own vision. This new poster collection\, a perfect complement to volume one\, reproduces six more of the most popular paintings from the book in a format suitable either for hanging as posters or mounting and framing.
+5127,The Overcoat, Nikolai Gogol,1992,Courier Corporation,3.75,Four short stories include a satirical tale of Russian bureaucrats and a portrayal of an elderly couple living in the secluded countryside.
+8875,The Two Towers,John Ronald Reuel Tolkien,2007,HarperCollins UK,4.64,The second volume in The Lord of the Rings\, This title is also available as a film.
+4977,The Unvanquished,William Faulkner,2011,Vintage,3.50,Set in Mississippi during the Civil War and Reconstruction\, THE UNVANQUISHED focuses on the Sartoris family\, who\, with their code of personal responsibility and courage\, stand for the best of the Old South's traditions.
+4382,The Wolves of Witchmaker,Carole Guinane,2001,iUniverse,5.00,Polly Lavender is mysteriously lured onto Witchmaker's grounds along with her best friends Tony Rico\, Gracie Reene\, and Zeus\, the wolf they rescued as a pup. The three must quickly learn to master the art of magic because they have been chosen to lead Witchmaker Prep against a threat that has grim consequences.
+7912,The Word For World is Forest,Ursula K. Le Guin,2015,Gollancz,4.22,When the inhabitants of a peaceful world are conquered by the bloodthirsty yumens\, their existence is irrevocably altered. Forced into servitude\, the Athsheans find themselves at the mercy of their brutal masters. Desperation causes the Athsheans\, led by Selver\, to retaliate against their captors\, abandoning their strictures against violence. But in defending their lives\, they have endangered the very foundations of their society. For every blow against the invaders is a blow to the humanity of the Athsheans. And once the killing starts\, there is no turning back.
+1211,The brothers Karamazov,Fyodor Dostoevsky,2003,Bantam Classics,1.00,In 1880 Dostoevsky completed The Brothers Karamazov\, the literary effort for which he had been preparing all his life. Compelling\, profound\, complex\, it is the story of a patricide and of the four sons who each had a motive for murder: Dmitry\, the sensualist\, Ivan\, the intellectual\, Alyosha\, the mystic\, and twisted\, cunning Smerdyakov\, the bastard child. Frequently lurid\, nightmarish\, always brilliant\, the novel plunges the reader into a sordid love triangle\, a pathological obsession\, and a gripping courtroom drama. But throughout the whole\, Dostoevsky searhes for the truth--about man\, about life\, about the existence of God. A terrifying answer to man's eternal questions\, this monumental work remains the crowning achievement of perhaps the finest novelist of all time. From the Paperback edition.
+8086,The grand inquisitor (Milestones of thought),Fyodor Dostoevsky,1981,A&C Black,4.09,Dostoevsky's portrayal of the Catholic Church during the Inquisition is a plea for the power of pure faith\, and a critique of the tyrannies of institutionalized religion. This is an except from the Brothers Karamazov which stands alone as a statement of philiosophy and a warning about the surrender of freedom for the sake of comfort.
+8077,The unvanquished,William Faulkner,2011,Vintage,4.00,Set in Mississippi during the Civil War and Reconstruction\, THE UNVANQUISHED focuses on the Sartoris family\, who\, with their code of personal responsibility and courage\, stand for the best of the Old South's traditions.
+8480,The wind's twelve quarters: Short stories,Ursula K. Le Guin,2017,HarperCollins,5.00,The recipient of numerous literary prizes\, including the National Book Award\, the Kafka Award\, and the Pushcart Prize\, Ursula K. Le Guin is renowned for her lyrical writing\, rich characters\, and diverse worlds. The Wind's Twelve Quarters collects seventeen powerful stories\, each with an introduction by the author\, ranging from fantasy to intriguing scientific concepts\, from medieval settings to the future. Including an insightful foreword by Le Guin\, describing her experience\, her inspirations\, and her approach to writing\, this stunning collection explores human values\, relationships\, and survival\, and showcases the myriad talents of one of the most provocative writers of our time.
+2847,To Love A Dark Stranger (Lovegram Historical Romance),Colleen Faulkner,1997,Zebra Books,5.00,Bestselling author Colleen Faulkner's tumultuous saga of royal intrigue and forbidden desire sweeps from the magnificent estates of the aristocracy to the shadowy streets of London to King Charles II's glittering Restoration court.
+3293,Universe by Design,Danny Faulkner,2004,New Leaf Publishing Group,4.25,Views the stars and planets from a creationist standpoint\, addresses common misconceptions and difficulties about relativity and cosmology\, and discusses problems with the big bang theory with many analogies\, examples\, diagrams\, and illustrations. Original.
+5327,War and Peace,Leo Tolstoy,2016,Lulu.com,3.84,Covering the period from the French invasion under Napoleon into Russia. Although not covering solely the war itself\, the serialized novel does cover the effects the war had on Russian society from the common person right up to the Tsar himself. The book starts to move more to a philosophical consideration on war and peace near the end making the book as a whole an important piece of literature.
+4536,War and Peace (Signet Classics),[Leo Tolstoy, Pat Conroy, John Hockenberry],2012,Signet Classics,4.75,Presents the classical epic of the Napoleonic Wars and their effects on four Russian families.
+9032,War and Peace: A Novel (6 Volumes),Tolstoy Leo,2013,Hardpress Publishing,3.81,Unlike some other reproductions of classic texts (1) We have not used OCR(Optical Character Recognition)\, as this leads to bad quality books with introduced typos. (2) In books where there are images such as portraits\, maps\, sketches etc We have endeavoured to keep the quality of these images\, so they represent accurately the original artefact. Although occasionally there may be certain imperfections with these old texts\, we feel they deserve to be made available for future generations to enjoy.
+5119,William Faulkner,William Faulkner,2011,Vintage,4.00,This invaluable volume\, which has been republished to commemorate the one-hundredth anniversary of Faulkner's birth\, contains some of the greatest short fiction by a writer who defined the course of American literature. Its forty-five stories fall into three categories: those not included in Faulkner's earlier collections\, previously unpublished short fiction\, and stories that were later expanded into such novels as The Unvanquished\, The Hamlet\, and Go Down\, Moses. With its Introduction and extensive notes by the biographer Joseph Blotner\, Uncollected Stories of William Faulkner is an essential addition to its author's canon--as well as a book of some of the most haunting\, harrowing\, and atmospheric short fiction written in the twentieth century.
+8615,Winter notes on summer impressions,Fyodor Dostoevsky,2018,Alma Books,4.75,In June 1862\, Dostoevsky left Petersburg on his first excursion to Western Europe. Ostensibly making the trip to consult Western specialists about his epilepsy\, he also wished to see first-hand the source of the Western ideas he believed were corrupting Russia. Over the course of his journey he visited a number of major cities\, including Berlin\, Paris\, London\, Florence\, Milan and Vienna.His record of the trip\, Winter Notes on Summer Impressions - first published in the February 1863 issue of Vremya\, the periodical he edited - is the chrysalis out of which many elements of his later masterpieces developed.
+6478,Woman-The Full Story: A Dynamic Celebration of Freedoms,Michele Guinness,2003,Zondervan,5.00,What does it mean to be a woman today? What have women inherited from their radical\, risk-taking sisters of the past? And how does God view this half of humanity? Michele Guinness invites us on an adventure of discovery\, exploring the biblical texts\, the annals of history and the experiences of women today in search of the challenges and achievements\, failures and joys\, of women throughout the ages.
+8678,Worlds of Exile and Illusion: Three Complete Novels of the Hainish Series in One Volume--Rocannon's World\, Planet of Exile\, City of Illusions,Ursula K. Le Guin,2016,Orb Books,4.41,Worlds of Exile and Illusion contains three novels in the Hainish Series from Ursula K. Le Guin\, one of the greatest science fiction writers and many times the winner of the Hugo and Nebula Awards. Her career as a novelist was launched by the three novels contained here. These books\, Rocannon's World\, Planet of Exile\, and City of Illusions\, are set in the same universe as Le Guin's groundbreaking classic\, The Left Hand of Darkness. At the Publisher's request\, this title is being sold without Digital Rights Management Software (DRM) applied.
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec
index 7e2afb9267e5b..b8569ead94509 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec
@@ -314,6 +314,21 @@ FROM sample_data
 3              |2025-10-01T00:00:00.000Z
 ;
 
+bucketByYearLowBucketCount#[skip:-8.13.99, reason:BUCKET extended in 8.14]
+FROM employees
+| WHERE hire_date >= "1985-02-18T00:00:00.000Z" AND hire_date <= "1988-10-18T00:00:00.000Z"
+| STATS c = COUNT(*) BY b = BUCKET(hire_date, 3, "1985-02-18T00:00:00.000Z", "1988-10-18T00:00:00.000Z")
+| SORT b
+;
+
+// Note: we don't bucket to anything longer than 1 year (like 2 years), so even if requesting 3 buckets, we still get 4
+       c:long  |           b:date
+11             |1985-01-01T00:00:00.000Z
+11             |1986-01-01T00:00:00.000Z
+15             |1987-01-01T00:00:00.000Z
+9              |1988-01-01T00:00:00.000Z
+;
+
 //
 // Numeric bucketing
 //
@@ -393,6 +408,17 @@ ROW long = TO_LONG(100), double = 99., int = 100
 99.0           |0.0            |99.0
 ;
 
+// identical results as above
+bucketNumericMixedTypesIntegerSpans
+required_capability: bucket_whole_number_as_span
+ROW long = TO_LONG(100), double = 99., int = 100
+| STATS BY b1 = BUCKET(long, double::int), b2 = BUCKET(double, long), b3 = BUCKET(int, 49.5)
+;
+
+      b1:double|      b2:double|      b3:double
+99.0           |0.0            |99.0
+;
+
 bucketWithFloats#[skip:-8.13.99, reason:BUCKET renamed in 8.14]
 FROM employees
 | WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z"
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/comparison.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/comparison.csv-spec
index f5bc5229283db..e0b921947e16d 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/comparison.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/comparison.csv-spec
@@ -116,3 +116,100 @@ from employees
 emp_no:integer
 10001
 ;
+
+rangeInteger
+from employees
+| where emp_no > 10001 and emp_no < 10003
+| keep emp_no, first_name
+;
+
+emp_no:integer |first_name:keyword
+10002          |Bezalel
+;
+
+rangeLong
+from employees
+| where languages.long > 1 and languages.long < 3
+| keep emp_no, first_name
+| sort emp_no
+| limit 2
+;
+
+emp_no:integer |first_name:keyword
+10001          |Georgi
+10008          |Saniya
+;
+
+rangeDouble
+from employees
+| where height > 1.4 and height < 2.0
+| keep emp_no, first_name
+| sort emp_no
+| limit 2
+;
+
+emp_no:integer |first_name:keyword
+10003          |Parto
+10004          |Chirstian
+;
+
+rangeUnsignedLong
+from ul_logs
+| where bytes_out >= to_ul(4747671420480199905) and bytes_out <= to_ul(12749081495402663265)
+| keep id
+| sort id
+| limit 2
+;
+
+id:integer
+1
+3
+;
+
+
+rangeKeyword
+from employees
+| where first_name >= "A" and first_name <= "D"
+| keep emp_no, first_name
+| sort emp_no
+| limit 2
+;
+
+emp_no:integer |first_name:keyword
+10002          |Bezalel
+10004          |Chirstian
+;
+
+rangeVersion
+required_capability: string_literal_auto_casting_extended
+from apps
+| where version > "2" and version < "4"
+| keep id, version
+| sort id
+;
+
+id:integer |version:version
+2          |2.1
+3          |2.3.4
+4          |2.12.0
+;
+
+rangeDateTime
+from employees
+| where birth_date >= "1952-01-01" and birth_date <= "1952-12-31"
+| stats cnt = count(*)
+;
+
+cnt:long
+8
+;
+
+rangeMixed
+from employees
+| where birth_date >= "1952-01-01" and birth_date <= "1952-12-31" and hire_date >= "1980-01-01" and hire_date <= "1989-12-31"
+| stats cnt = count(*)
+;
+
+cnt:long
+5
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/convert.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/convert.csv-spec
index 3ef1a322eb94a..42b5c0344b559 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/convert.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/convert.csv-spec
@@ -180,3 +180,37 @@ ver:version
 1.2.3
 //end::docsCastOperator-result[]
 ;
+
+mixedNumericTypesInLiterals
+required_capability: mixed_numeric_types_in_coalesce
+from employees
+| where languages.long in (1, 2.0, null)
+| keep emp_no, languages
+| sort emp_no
+| limit 10
+;
+
+emp_no:integer | languages:integer
+10001          | 2
+10005          | 1
+10008          | 2
+10009          | 1
+10013          | 1
+10016          | 2
+10017          | 2
+10018          | 2
+10019          | 1
+10033          | 1
+;
+
+mixedNumericTypesInFields
+required_capability: mixed_numeric_types_in_coalesce
+from employees
+| where languages in (7.0, height)
+| keep emp_no, languages, height
+| sort emp_no
+;
+
+emp_no:integer | languages:integer | height:double
+10037          | 2                 | 2.0
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv
new file mode 100644
index 0000000000000..7e857f5243f58
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv
@@ -0,0 +1,9 @@
+millis:date,nanos:date_nanos
+2023-10-23T13:55:01.543Z,2023-10-23T13:55:01.543123456Z
+2023-10-23T13:53:55.832Z,2023-10-23T13:53:55.832987654Z
+2023-10-23T13:52:55.015Z,2023-10-23T13:52:55.015787878Z
+2023-10-23T13:51:54.732Z,2023-10-23T13:51:54.732102837Z
+2023-10-23T13:33:34.937Z,2023-10-23T13:33:34.937193000Z
+2023-10-23T12:27:28.948Z,2023-10-23T12:27:28.948000000Z
+2023-10-23T12:15:03.360Z,2023-10-23T12:15:03.360103847Z
+1999-10-23T12:15:03.360Z,[2023-03-23T12:15:03.360103847Z, 2023-02-23T13:33:34.937193000Z, 2023-01-23T13:55:01.543123456Z]
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec
new file mode 100644
index 0000000000000..b77689e1b5768
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec
@@ -0,0 +1,72 @@
+simple
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis DESC | KEEP millis, nanos | LIMIT 1;
+
+millis:date              | nanos:date_nanos
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
+;
+
+sort by nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT nanos DESC | KEEP millis, nanos | LIMIT 1;
+
+millis:date              | nanos:date_nanos
+2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z
+;
+
+sort by nanos asc
+required_capability: date_nanos_type
+
+FROM date_nanos | WHERE millis > "2020-02-02" | SORT nanos ASC | KEEP millis, nanos | LIMIT 1;
+
+millis:date              | nanos:date_nanos
+2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z
+;
+
+mv_max on date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis ASC | EVAL nanos = MV_MAX(nanos) | KEEP nanos | LIMIT 1;
+
+
+nanos:date_nanos
+2023-03-23T12:15:03.360103847Z
+;
+
+mv_min on date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis ASC | EVAL nanos = MV_MIN(nanos) | KEEP nanos | LIMIT 1;
+
+nanos:date_nanos
+2023-01-23T13:55:01.543123456Z
+;
+
+mv_count on date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis ASC | EVAL ct = MV_COUNT(nanos) | KEEP ct | LIMIT 1;
+
+ct:integer
+3
+;
+
+mv_first on date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis ASC | EVAL nanos = MV_FIRST(nanos) | KEEP nanos | LIMIT 1;
+
+nanos:date_nanos
+2023-01-23T13:55:01.543123456Z
+;
+
+mv_last on date nanos
+required_capability: date_nanos_type
+
+FROM date_nanos | SORT millis asc | EVAL nanos = MV_LAST(nanos) | KEEP nanos | LIMIT 1;
+
+nanos:date_nanos
+2023-03-23T12:15:03.360103847Z
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec
index 8c4e797b7982d..38f09d2e3c56e 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec
@@ -75,6 +75,39 @@ first_name:keyword | last_name:keyword | name:keyword    | foo:keyword
 Georgi             | Facello           | Georgi1 Facello | Facello
 ;
 
+shadowingWhenPushedDownPastRename
+required_capability: fixed_pushdown_past_project
+ROW city = "Zürich", long_city_name = "Zurich, the largest city in Switzerland"
+| RENAME city AS c
+| DISSECT long_city_name "Zurich, the %{city} city in Switzerland"
+;
+
+c:keyword | long_city_name:keyword                  | city:keyword
+Zürich    | Zurich, the largest city in Switzerland | largest
+;
+
+shadowingWhenPushedDownPastRename2
+required_capability: fixed_pushdown_past_project
+ROW city = "Zürich", long_city_name = "Zurich, the largest city in Switzerland"
+| RENAME city AS c
+| DISSECT long_city_name "Zurich, the %{city} city in %{foo}"
+;
+
+c:keyword | long_city_name:keyword                  | city:keyword | foo:keyword
+Zürich    | Zurich, the largest city in Switzerland | largest      | Switzerland
+;
+
+shadowingWhenPushedDownPastRename3
+required_capability: fixed_pushdown_past_project
+ROW city = "Zürich", long_city_name = "Zurich, the largest city in Switzerland"
+| RENAME long_city_name AS c
+| DISSECT c "Zurich, the %{long_city_name} city in Switzerland"
+;
+
+city:keyword | c:keyword                               | long_city_name:keyword
+Zürich       | Zurich, the largest city in Switzerland | largest
+;
+
 
 complexPattern
 ROW a = "1953-01-23T12:15:00Z - some text - 127.0.0.1;" 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec
index ab2ddb84ed969..925c08f317125 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec
@@ -174,6 +174,42 @@ city:keyword | airport:text
 Zürich       | Zurich Int'l
 ;
 
+shadowingWhenPushedDownPastRename
+required_capability: enrich_load
+required_capability: fixed_pushdown_past_project
+ROW city = "Zürich", airport = "ZRH"
+| RENAME airport AS a
+| ENRICH city_names ON city WITH airport
+;
+
+city:keyword | a:keyword | airport:text
+Zürich       | ZRH       | Zurich Int'l
+;
+
+shadowingWhenPushedDownPastRename2
+required_capability: enrich_load
+required_capability: fixed_pushdown_past_project
+ROW city = "Zürich", airport = "ZRH"
+| RENAME airport AS a
+| ENRICH city_names ON city WITH airport, region
+;
+
+city:keyword | a:keyword | airport:text | region:text
+Zürich       | ZRH       | Zurich Int'l | Bezirk Zürich
+;
+
+shadowingWhenPushedDownPastRename3
+required_capability: enrich_load
+required_capability: fixed_pushdown_past_project
+ROW city = "Zürich", airport = "ZRH"
+| RENAME city as c
+| ENRICH city_names ON c WITH city = airport
+;
+
+c:keyword | airport:keyword | city:text
+Zürich    | ZRH             | Zurich Int'l
+;
+
 simple
 required_capability: enrich_load
 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec
index 770358e5120da..61a0ccd4af0c5 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec
@@ -55,6 +55,45 @@ x:integer
 9999
 ;
 
+shadowingWhenPushedDownPastRename
+required_capability: fixed_pushdown_past_project
+FROM employees
+| WHERE emp_no < 10002
+| KEEP emp_no, languages
+| RENAME emp_no AS z
+| EVAL emp_no = 3
+;
+
+z:integer | languages:integer | emp_no:integer
+    10001 |                 2 |              3
+;
+
+shadowingWhenPushedDownPastRename2
+required_capability: fixed_pushdown_past_project
+FROM employees
+| WHERE emp_no < 10002
+| KEEP emp_no, languages
+| RENAME emp_no AS z
+| EVAL emp_no = z + 1, emp_no = emp_no + languages, a = 0, languages = -1
+;
+
+z:integer | emp_no:integer | a:integer | languages:integer
+    10001 |          10004 |         0 |                -1
+;
+
+shadowingWhenPushedDownPastRename3
+required_capability: fixed_pushdown_past_project
+FROM employees
+| WHERE emp_no < 10002
+| KEEP emp_no, languages
+| RENAME emp_no AS z
+| EVAL emp_no = z + 1
+;
+
+z:integer | languages:integer | emp_no:integer
+    10001 |                 2 |          10002
+;
+
 
 withMath
 row a = 1 | eval b = 2 + 3;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec
index d9857e8c122ef..98c88d06caa75 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec
@@ -76,6 +76,39 @@ San Francisco     | CA 94108         | ["CA", "94108"]
 Tokyo             | 100-7014         | null
 ;
 
+shadowingWhenPushedDownPastRename
+required_capability: fixed_pushdown_past_project
+ROW city = "Zürich", long_city_name = "Zürich, the largest city in Switzerland"
+| RENAME city AS c
+| GROK long_city_name "Zürich, the %{WORD:city} %{WORD:city} %{WORD:city} %{WORD:city}"
+;
+
+c:keyword | long_city_name:keyword                        | city:keyword
+Zürich    | Zürich, the largest city in Switzerland       | ["largest", "city", "in", "Switzerland"]
+;
+
+shadowingWhenPushedDownPastRename2
+required_capability: fixed_pushdown_past_project
+ROW city = "Zürich", long_city_name = "Zürich, the largest city in Switzerland"
+| RENAME city AS c
+| GROK long_city_name "Zürich, the %{WORD:city} %{WORD:foo} %{WORD:city} %{WORD:foo}"
+;
+
+c:keyword | long_city_name:keyword                        | city:keyword       | foo:keyword
+Zürich    | Zürich, the largest city in Switzerland       | ["largest", "in"]  | ["city", "Switzerland"]
+;
+
+shadowingWhenPushedDownPastRename3
+required_capability: fixed_pushdown_past_project
+ROW city = "Zürich", long_city_name = "Zürich, the largest city in Switzerland"
+| RENAME long_city_name AS c
+| GROK c "Zürich, the %{WORD:long_city_name} %{WORD:long_city_name} %{WORD:long_city_name} %{WORD:long_city_name}"
+;
+
+city:keyword | c:keyword                               | long_city_name:keyword
+Zürich       | Zürich, the largest city in Switzerland | ["largest", "city", "in", "Switzerland"]
+;
+
 complexPattern
 ROW a = "1953-01-23T12:15:00Z 127.0.0.1 some.email@foo.com 42" 
 | GROK a "%{TIMESTAMP_ISO8601:date} %{IP:ip} %{EMAILADDRESS:email} %{NUMBER:num:int}" 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec
new file mode 100644
index 0000000000000..3f2e14f74174b
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec
@@ -0,0 +1,655 @@
+maxOfInt
+required_capability: inlinestats
+
+// tag::max-languages[]
+FROM employees
+| KEEP emp_no, languages
+| INLINESTATS max_lang = MAX(languages) 
+| WHERE max_lang == languages
+| SORT emp_no ASC
+| LIMIT 5
+// end::max-languages[]
+;
+
+// tag::max-languages-result[]
+emp_no:integer | languages:integer | max_lang:integer
+         10002 |                 5 | 5
+         10004 |                 5 | 5
+         10011 |                 5 | 5
+         10012 |                 5 | 5
+         10014 |                 5 | 5
+// end::max-languages-result[]
+;
+
+maxOfIntByKeyword
+required_capability: inlinestats
+
+FROM employees
+| KEEP emp_no, languages, gender
+| INLINESTATS max_lang = MAX(languages) BY gender 
+| WHERE max_lang == languages
+| SORT emp_no ASC
+| LIMIT 5;
+
+emp_no:integer | languages:integer | gender:keyword | max_lang:integer
+         10002 |                 5 | F              | 5
+         10004 |                 5 | M              | 5
+         10011 |                 5 | null           | 5
+         10012 |                 5 | null           | 5
+         10014 |                 5 | null           | 5
+;
+
+maxOfLongByKeyword
+required_capability: inlinestats
+
+FROM employees
+| KEEP emp_no, avg_worked_seconds, gender
+| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY gender 
+| WHERE max_avg_worked_seconds == avg_worked_seconds
+| SORT emp_no ASC;
+
+emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_seconds:long
+         10007 |               393084805 | F              | 393084805
+         10015 |               390266432 | null           | 390266432
+         10030 |               394597613 | M              | 394597613
+;
+
+maxOfLong
+required_capability: inlinestats
+
+FROM employees
+| KEEP emp_no, avg_worked_seconds, gender
+| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) 
+| WHERE max_avg_worked_seconds == avg_worked_seconds
+| SORT emp_no ASC;
+
+emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_seconds:long
+         10030 |               394597613 | M              | 394597613
+;
+
+maxOfLongByCalculatedKeyword
+required_capability: inlinestats_v2
+
+// tag::longest-tenured-by-first[]
+FROM employees
+| KEEP emp_no, avg_worked_seconds, last_name
+| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY SUBSTRING(last_name, 0, 1)
+| WHERE max_avg_worked_seconds == avg_worked_seconds
+| SORT last_name ASC
+| LIMIT 5
+// end::longest-tenured-by-first[]
+;
+
+// tag::longest-tenured-by-first-result[]
+emp_no:integer | avg_worked_seconds:long | last_name:keyword | SUBSTRING(last_name, 0, 1):keyword | max_avg_worked_seconds:long
+         10065 |               372660279 | Awdeh             | A                                  | 372660279
+         10074 |               382397583 | Bernatsky         | B                                  | 382397583
+         10044 |               387408356 | Casley            | C                                  | 387408356
+         10030 |               394597613 | Demeyer           | D                                  | 394597613
+         10087 |               305782871 | Eugenio           | E                                  | 305782871
+// end::longest-tenured-by-first-result[]
+;
+
+maxOfLongByCalculatedNamedKeyword
+required_capability: inlinestats_v2
+
+FROM employees
+| KEEP emp_no, avg_worked_seconds, last_name
+| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY l = SUBSTRING(last_name, 0, 1)
+| WHERE max_avg_worked_seconds == avg_worked_seconds
+| SORT last_name ASC
+| LIMIT 5
+;
+
+emp_no:integer | avg_worked_seconds:long | last_name:keyword | l:keyword | max_avg_worked_seconds:long
+         10065 |               372660279 | Awdeh             | A         | 372660279
+         10074 |               382397583 | Bernatsky         | B         | 382397583
+         10044 |               387408356 | Casley            | C         | 387408356
+         10030 |               394597613 | Demeyer           | D         | 394597613
+         10087 |               305782871 | Eugenio           | E         | 305782871
+;
+
+maxOfLongByCalculatedDroppedKeyword
+required_capability: inlinestats_v2
+
+FROM employees
+| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY l = SUBSTRING(last_name, 0, 1)
+| WHERE max_avg_worked_seconds == avg_worked_seconds
+| KEEP emp_no, avg_worked_seconds, last_name, max_avg_worked_seconds  
+| SORT last_name ASC
+| LIMIT 5
+;
+
+emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_seconds:long
+         10065 |               372660279 | Awdeh             | 372660279
+         10074 |               382397583 | Bernatsky         | 382397583
+         10044 |               387408356 | Casley            | 387408356
+         10030 |               394597613 | Demeyer           | 394597613
+         10087 |               305782871 | Eugenio           | 305782871
+;
+
+maxOfLongByEvaledKeyword
+required_capability: inlinestats
+
+FROM employees
+| EVAL l = SUBSTRING(last_name, 0, 1)
+| KEEP emp_no, avg_worked_seconds, l
+| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY l 
+| WHERE max_avg_worked_seconds == avg_worked_seconds
+| SORT l ASC
+| LIMIT 5
+;
+
+emp_no:integer | avg_worked_seconds:long | l:keyword | max_avg_worked_seconds:long
+         10065 |               372660279 | A         | 372660279
+         10074 |               382397583 | B         | 382397583
+         10044 |               387408356 | C         | 387408356
+         10030 |               394597613 | D         | 394597613
+         10087 |               305782871 | E         | 305782871
+;
+
+maxOfLongByInt
+required_capability: inlinestats
+
+FROM employees
+| KEEP emp_no, avg_worked_seconds, languages
+| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY languages
+| WHERE max_avg_worked_seconds == avg_worked_seconds
+| SORT languages ASC;
+
+emp_no:integer | avg_worked_seconds:long | languages:integer | max_avg_worked_seconds:long
+         10044 |               387408356 |                 1 | 387408356
+         10099 |               377713748 |                 2 | 377713748
+         10030 |               394597613 |                 3 | 394597613
+         10007 |               393084805 |                 4 | 393084805
+         10015 |               390266432 |                 5 | 390266432
+         10027 |               374037782 |              null | 374037782
+;
+
+maxOfLongByIntDouble
+required_capability: inlinestats
+
+FROM employees
+| KEEP emp_no, avg_worked_seconds, languages, height
+| EVAL height=ROUND(height, 1)
+| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY languages, height
+| WHERE max_avg_worked_seconds == avg_worked_seconds
+| SORT languages, height ASC
+| LIMIT 4;
+
+emp_no:integer | avg_worked_seconds:long | languages:integer | height:double | max_avg_worked_seconds:long
+         10083 |               331236443 |                 1 |           1.4 | 331236443
+         10084 |               359067056 |                 1 |           1.5 | 359067056
+         10033 |               208374744 |                 1 |           1.6 | 208374744
+         10086 |               328580163 |                 1 |           1.7 | 328580163
+;
+
+
+two
+required_capability: inlinestats
+
+FROM employees
+| KEEP emp_no, languages, avg_worked_seconds, gender
+| INLINESTATS avg_avg_worked_seconds = AVG(avg_worked_seconds) BY languages
+| WHERE avg_worked_seconds > avg_avg_worked_seconds
+| INLINESTATS max_languages = MAX(languages) BY gender
+| SORT emp_no ASC
+| LIMIT 3;
+
+emp_no:integer | languages:integer | avg_worked_seconds:long | gender:keyword | avg_avg_worked_seconds:double | max_languages:integer
+         10002 |                 5 |               328922887 | F              | 3.133013149047619E8           | 5
+         10006 |                 3 |               372957040 | F              | 2.978159518235294E8           | 5
+         10007 |                 4 |               393084805 | F              | 2.863684210555556E8           | 5
+;
+
+byMultivaluedSimple
+required_capability: inlinestats
+
+// tag::mv-group[]
+FROM airports
+| INLINESTATS min_scalerank=MIN(scalerank) BY type
+| EVAL type=MV_SORT(type), min_scalerank=MV_SORT(min_scalerank)
+| KEEP abbrev, type, scalerank, min_scalerank
+| WHERE abbrev == "GWL"
+// end::mv-group[]
+;
+
+// tag::mv-group-result[]
+abbrev:keyword |  type:keyword   | scalerank:integer | min_scalerank:integer
+           GWL | [mid, military] | 9                 | [2, 4]
+// end::mv-group-result[]
+;
+
+byMultivaluedMvExpand
+required_capability: inlinestats
+
+// tag::mv-expand[]
+FROM airports
+| KEEP abbrev, type, scalerank
+| MV_EXPAND type
+| INLINESTATS min_scalerank=MIN(scalerank) BY type
+| SORT min_scalerank ASC
+| WHERE abbrev == "GWL"
+// end::mv-expand[]
+;
+
+// tag::mv-expand-result[]
+abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer
+           GWL |          mid | 9                 | 2
+           GWL |     military | 9                 | 4
+// end::mv-expand-result[]
+;
+
+byMvExpand
+required_capability: inlinestats
+
+// tag::extreme-airports[]
+FROM airports
+| MV_EXPAND type
+| EVAL lat = ST_Y(location)
+| INLINESTATS most_northern=MAX(lat), most_southern=MIN(lat) BY type
+| WHERE lat == most_northern OR lat == most_southern
+| SORT lat DESC
+| KEEP type, name, location
+// end::extreme-airports[]
+;
+
+// tag::extreme-airports-result[]
+ type:keyword |           name:text           | location:geo_point
+          mid |             Svalbard Longyear | POINT (15.495229 78.246717)
+        major |                Tromsø Langnes | POINT (18.9072624292132 69.6796790473478)
+     military | Severomorsk-3 (Murmansk N.E.) | POINT (33.2903527616285 69.0168711826804)
+    spaceport |           Baikonur Cosmodrome | POINT (63.307354423875 45.9635739403124)
+        small |                       Dhamial | POINT (73.0320498392002 33.5614146278861)
+        small |                      Sahnewal | POINT (75.9570722403652 30.8503598561702)
+    spaceport |       Centre Spatial Guyanais | POINT (-52.7684296893452 5.23941001258035)
+     military |         Santos Air Force Base | POINT (-46.3052704931003 -23.9237590410637)
+        major |            Christchurch Int'l | POINT (172.538675565223 -43.4885486784104)
+          mid |          Hermes Quijada Int'l | POINT (-67.7530268462675 -53.7814746058316)
+// end::extreme-airports-result[]
+;
+
+brokenwhy-Ignore
+required_capability: inlinestats
+
+FROM airports
+| INLINESTATS min_scalerank=MIN(scalerank) BY type
+| MV_EXPAND type
+| WHERE scalerank == MV_MIN(scalerank);
+
+abbrev:keyword |  type:keyword   | scalerank:integer | min_scalerank:integer
+           GWL | [mid, military] | 9                 | [2, 4]
+;
+
+afterStats
+required_capability: inlinestats
+
+FROM airports
+| STATS count=COUNT(*) BY country
+| INLINESTATS avg=AVG(count)
+| WHERE count > avg * 3
+| SORT count DESC, country ASC
+;
+
+count:long | country:keyword | avg:double
+       129 |   United States | 4.455
+        50 |           India | 4.455
+        45 |          Mexico | 4.455
+        41 |           China | 4.455
+        37 |          Canada | 4.455
+        31 |          Brazil | 4.455
+        26 |          Russia | 4.455
+        19 |            null | 4.455
+        17 |       Australia | 4.455
+        17 |  United Kingdom | 4.455
+;
+
+afterWhere
+required_capability: inlinestats
+
+FROM airports
+| WHERE country != "United States"
+| INLINESTATS count=COUNT(*) BY country
+| SORT count DESC, abbrev ASC
+| KEEP abbrev, country, count
+| LIMIT 4
+;
+
+abbrev:keyword | country:keyword | count:long
+           AGR |           India | 50
+           AMD |           India | 50
+           BBI |           India | 50
+           BDQ |           India | 50
+;
+
+afterLookup
+required_capability: inlinestats
+
+FROM airports
+| RENAME scalerank AS int
+| LOOKUP int_number_names ON int
+| RENAME name as scalerank
+| DROP int
+| INLINESTATS count=COUNT(*) BY scalerank
+| SORT abbrev ASC
+| KEEP abbrev, scalerank
+| LIMIT 4
+;
+
+abbrev:keyword | scalerank:keyword
+           ABJ |              four
+           ABQ |               six
+           ABV |              five
+           ACA |              four
+;
+
+afterEnrich
+required_capability: inlinestats
+required_capability: enrich_load
+
+FROM airports
+| KEEP abbrev, city
+| WHERE abbrev NOT IN ("ADJ", "ATQ") // Skip airports in regions with right-to-left text which the test file isn't good with
+| ENRICH city_names ON city WITH region
+| WHERE MV_COUNT(region) == 1
+| INLINESTATS COUNT(*) BY region
+| SORT abbrev ASC
+| WHERE `COUNT(*)` > 1
+| LIMIT 3
+;
+
+abbrev:keyword | city:keyword |       region:text | "COUNT(*)":long 
+           ALA |       Almaty |     Жетісу ауданы | 2
+           BXJ |       Almaty |     Жетісу ауданы | 2
+           FUK |      Fukuoka |             中央区 | 2
+;
+
+beforeStats
+required_capability: inlinestats
+
+FROM airports
+| EVAL lat = ST_Y(location)
+| INLINESTATS avg_lat=AVG(lat)
+| STATS northern=COUNT(lat > avg_lat OR NULL), southern=COUNT(lat < avg_lat OR NULL)
+;
+
+northern:long | southern:long
+          520 | 371
+;
+
+beforeKeepSort
+required_capability: inlinestats
+
+FROM employees
+| INLINESTATS max_salary = MAX(salary) by languages
+| KEEP emp_no, languages, max_salary
+| SORT emp_no ASC
+| LIMIT 3;
+
+emp_no:integer | languages:integer | max_salary:integer
+         10001 |                 2 | 73578
+         10002 |                 5 | 66817
+         10003 |                 4 | 74572
+;
+
+beforeKeepWhere
+required_capability: inlinestats
+
+FROM employees
+| INLINESTATS max_salary = MAX(salary) by languages
+| KEEP emp_no, languages, max_salary
+| WHERE emp_no == 10003;
+
+ignoreOrder:true
+emp_no:integer | languages:integer | max_salary:integer
+         10003 |                 4 | 74572
+;
+
+beforeEnrich
+required_capability: inlinestats
+required_capability: enrich_load
+
+FROM airports
+| KEEP abbrev, type, city
+| INLINESTATS COUNT(*) BY type
+| ENRICH city_names ON city WITH region
+| WHERE MV_COUNT(region) == 1
+| SORT abbrev ASC
+| LIMIT 3
+;
+
+abbrev:keyword | type:keyword |    city:keyword    | "COUNT(*)":long | region:text
+           ABJ |          mid |            Abidjan |             499 | Abidjan
+           ABV |        major |              Abuja |             385 | Municipal Area Council
+           ACA |        major | Acapulco de Juárez |             385 | Acapulco de Juárez
+;
+
+beforeAndAfterEnrich
+required_capability: inlinestats
+required_capability: enrich_load
+
+FROM airports
+| KEEP abbrev, type, city
+| INLINESTATS COUNT(*) BY type
+| ENRICH city_names ON city WITH region
+| WHERE MV_COUNT(region) == 1
+| INLINESTATS count_region=COUNT(*) BY region 
+| SORT abbrev ASC
+| LIMIT 3
+;
+
+abbrev:keyword | type:keyword |    city:keyword    | "COUNT(*)":long | region:text            | count_region:long
+           ABJ |          mid |            Abidjan |             499 | Abidjan                | 1
+           ABV |        major |              Abuja |             385 | Municipal Area Council | 1
+           ACA |        major | Acapulco de Juárez |             385 | Acapulco de Juárez     | 1
+;
+
+
+shadowing
+required_capability: inlinestats
+
+ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right"
+| INLINESTATS env=VALUES(right) BY client_ip
+;
+
+left:keyword | client_ip:keyword | right:keyword | env:keyword
+left         | 172.21.0.5        | right         | right
+;
+
+shadowingMulti
+required_capability: inlinestats
+
+ROW left = "left", airport = "Zurich Airport ZRH", city = "Zürich", middle = "middle", region = "North-East Switzerland", right = "right"
+| INLINESTATS airport=VALUES(left), region=VALUES(left), city_boundary=VALUES(left) BY city
+;
+
+left:keyword | city:keyword | middle:keyword | right:keyword | airport:keyword | region:keyword | city_boundary:keyword
+left         | Zürich       | middle         | right         |            left |           left | left
+;
+
+shadowingSelf
+required_capability: inlinestats
+
+ROW city="Raleigh"
+| INLINESTATS city=COUNT(city)
+;
+
+city:long
+1
+;
+
+shadowingSelfBySelf-Ignore
+required_capability: inlinestats
+
+ROW city="Raleigh"
+| INLINESTATS city=COUNT(city) BY city
+;
+
+city:long
+1
+;
+
+shadowingInternal-Ignore
+required_capability: inlinestats
+
+ROW city = "Zürich"
+| INLINESTATS x=VALUES(city), x=VALUES(city)
+;
+
+city:keyword | x:keyword
+Zürich       | Zürich
+;
+
+byConstant-Ignore
+required_capability: inlinestats
+
+FROM employees
+| KEEP emp_no, languages
+| INLINESTATS max_lang = MAX(languages) BY y=1 
+| WHERE max_lang == languages
+| SORT emp_no ASC
+| LIMIT 5
+;
+
+emp_no:integer | languages:integer | max_lang:integer | y:integer
+         10002 |                 5 |                5 | 1
+         10004 |                 5 |                5 | 1
+         10011 |                 5 |                5 | 1
+         10012 |                 5 |                5 | 1
+         10014 |                 5 |                5 | 1
+;
+
+aggConstant
+required_capability: inlinestats
+
+FROM employees
+| KEEP emp_no
+| INLINESTATS one = MAX(1) BY emp_no
+| SORT emp_no ASC
+| LIMIT 5
+;
+
+emp_no:integer | one:integer
+         10001 | 1
+         10002 | 1
+         10003 | 1
+         10004 | 1
+         10005 | 1
+;
+
+percentile
+required_capability: inlinestats
+
+FROM employees
+| KEEP emp_no, salary
+| INLINESTATS ninety_fifth_salary = PERCENTILE(salary, 95)
+| WHERE salary > ninety_fifth_salary
+| SORT emp_no ASC
+| LIMIT 5
+;
+
+emp_no:integer | salary:integer | ninety_fifth_salary:double
+         10007 |          74572 | 73584.95
+         10019 |          73717 | 73584.95
+         10027 |          73851 | 73584.95
+         10029 |          74999 | 73584.95
+         10045 |          74970 | 73584.95
+;
+
+byTwoCalculated
+required_capability: inlinestats_v2
+
+FROM airports
+| WHERE abbrev IS NOT NULL
+| KEEP abbrev, scalerank, location
+| INLINESTATS min_sl=MIN(scalerank)
+           BY lat_10 = ROUND(ST_Y(location), -1)
+            , lon_10 = ROUND(ST_X(location), -1)
+| SORT abbrev DESC
+| LIMIT 3
+;
+
+abbrev:keyword | scalerank:integer |             location:geo_point             | lat_10:double | lon_10:double | min_sl:integer
+           ZRH |                 3 | POINT(8.56221279534765 47.4523895064915)   |            50 |            10 | 2
+           ZNZ |                 4 | POINT (39.2223319841558 -6.21857034620282) |           -10 |            40 | 4
+           ZLO |                 7 | POINT (-104.560095200097 19.1480860285854) |            20 |          -100 | 2
+;
+
+byTwoCalculatedSecondOverwrites
+required_capability: inlinestats_v2
+
+FROM airports
+| WHERE abbrev IS NOT NULL
+| KEEP abbrev, scalerank, location
+| INLINESTATS min_sl=MIN(scalerank)
+           BY x = ROUND(ST_Y(location), -1)
+            , x = ROUND(ST_X(location), -1)
+| SORT abbrev DESC
+| LIMIT 3
+;
+
+abbrev:keyword | scalerank:integer |             location:geo_point             | x:double | min_sl:integer
+           ZRH |                 3 | POINT(8.56221279534765 47.4523895064915)   |       10 | 2
+           ZNZ |                 4 | POINT (39.2223319841558 -6.21857034620282) |       40 | 2
+           ZLO |                 7 | POINT (-104.560095200097 19.1480860285854) |     -100 | 2
+;
+
+byTwoCalculatedSecondOverwritesReferencingFirst
+required_capability: inlinestats_v2
+
+FROM airports
+| WHERE abbrev IS NOT NULL
+| KEEP abbrev, scalerank, location
+| EVAL x = ST_X(location)
+| INLINESTATS min_sl=MIN(scalerank)
+           BY x = ROUND(x, -1)
+            , x = ROUND(x, -1)
+| SORT abbrev DESC
+| LIMIT 3
+;
+
+abbrev:keyword | scalerank:integer |             location:geo_point             | x:double | min_sl:integer
+           ZRH |                 3 | POINT(8.56221279534765 47.4523895064915)   |       10 | 2
+           ZNZ |                 4 | POINT (39.2223319841558 -6.21857034620282) |       40 | 2
+           ZLO |                 7 | POINT (-104.560095200097 19.1480860285854) |     -100 | 2
+;
+
+
+groupShadowsAgg
+required_capability: inlinestats_v2
+
+FROM airports
+| WHERE abbrev IS NOT NULL
+| KEEP abbrev, scalerank, location
+| INLINESTATS min_sl=MIN(scalerank)
+            , lat_10 = ROUND(ST_Y(location), -1)
+           BY lat_10 = ROUND(ST_Y(location), -1)
+            , lon_10 = ROUND(ST_X(location), -1)
+| SORT abbrev DESC
+| LIMIT 3
+;
+
+abbrev:keyword | scalerank:integer |             location:geo_point             | lat_10:double | lon_10:double | min_sl:integer
+           ZRH |                 3 | POINT(8.56221279534765 47.4523895064915)   |            50 |            10 | 2
+           ZNZ |                 4 | POINT (39.2223319841558 -6.21857034620282) |           -10 |            40 | 4
+           ZLO |                 7 | POINT (-104.560095200097 19.1480860285854) |            20 |          -100 | 2
+;
+
+groupShadowsField
+required_capability: inlinestats_v2
+
+  FROM employees
+| KEEP emp_no, salary, hire_date
+| INLINESTATS avg_salary = AVG(salary)
+           BY hire_date = DATE_TRUNC(1 year, hire_date)
+| WHERE salary > avg_salary
+| SORT emp_no ASC
+| LIMIT 4
+;
+
+emp_no:integer | salary:integer |  hire_date:datetime  | avg_salary:double
+         10001 |          57305 | 1986-01-01T00:00:00Z | 43869.63636363636
+         10002 |          56371 | 1985-01-01T00:00:00Z | 51831.818181818184
+         10003 |          61805 | 1986-01-01T00:00:00Z | 43869.63636363636
+         10005 |          63528 | 1989-01-01T00:00:00Z | 53487.07692307692
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ints.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ints.csv-spec
index c8cb6cf88a4f0..b399734151412 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ints.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ints.csv-spec
@@ -986,3 +986,36 @@ s:double | emp_no:integer | salary:integer
 1.0     | 10002          | 56371
 1.0     | 10041          | 56415
 ;
+
+CombineBinaryComparisonsMv
+required_capability: combine_binary_comparisons
+
+row x = [1,2,3]
+| where 12 * (-x - 5) >= -120 OR x < 5
+;
+warning:Line 2:15: evaluation of [-x] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:15: java.lang.IllegalArgumentException: single-value function encountered multi-value
+warning:Line 2:34: evaluation of [x < 5] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:34: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+x:integer
+;
+
+CombineBinaryComparisonsEmp
+required_capability: combine_binary_comparisons
+
+from employees
+| where salary_change.int == 2 OR salary_change.int > 1
+| keep emp_no, salary_change.int
+| sort emp_no
+;
+warning:Line 2:35: evaluation of [salary_change.int > 1] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:35: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+emp_no:integer |salary_change.int:integer
+10044 | 8
+10046 | 2
+10066 | 5
+10079 | 7
+10086 | 13
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
index 697b1c899d65e..0fb6994ef759f 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
@@ -282,6 +282,78 @@ str1:keyword |str2:keyword |ip1:ip  |ip2:ip
 // end::to_ip-result[]
 ;
 
+cdirMatchOrsIPs
+required_capability: combine_disjunctive_cidrmatches
+
+FROM hosts
+| WHERE CIDR_MATCH(ip1, "127.0.0.2/32") or CIDR_MATCH(ip0, "127.0.0.1") or CIDR_MATCH(ip1, "127.0.0.3/32") or CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9")
+| KEEP card, host, ip0, ip1
+| sort host, card, ip0, ip1
+;
+warning:Line 2:20: evaluation of [ip1] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:20: java.lang.IllegalArgumentException: single-value function encountered multi-value
+warning:Line 2:55: evaluation of [ip0] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:55: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+card:keyword   |host:keyword   |ip0:ip                   |ip1:ip
+eth0           |alpha          |127.0.0.1                |127.0.0.1
+eth0           |beta           |127.0.0.1                |::1
+eth1           |beta           |127.0.0.1                |127.0.0.2
+eth1           |beta           |127.0.0.1                |128.0.0.1
+eth0           |gamma          |fe80::cae2:65ff:fece:feb9|127.0.0.3
+lo0            |gamma          |fe80::cae2:65ff:fece:feb9|fe81::cae2:65ff:fece:feb9
+;
+
+cdirMatchEqualsInsOrs
+required_capability: combine_disjunctive_cidrmatches
+
+FROM hosts
+| WHERE host == "alpha" OR host == "gamma" OR CIDR_MATCH(ip1, "127.0.0.2/32") OR CIDR_MATCH(ip0, "127.0.0.1") OR card IN ("eth0", "eth1") OR CIDR_MATCH(ip1, "127.0.0.3/32") OR card == "lo0" OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta"
+| KEEP card, host, ip0, ip1
+| sort host, card, ip0, ip1
+;
+warning:Line 2:58: evaluation of [ip1] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:58: java.lang.IllegalArgumentException: single-value function encountered multi-value
+warning:Line 2:93: evaluation of [ip0] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:93: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+card:keyword   |host:keyword   |ip0:ip                                                                            |ip1:ip
+eth0           |alpha          |127.0.0.1                                                                         |127.0.0.1
+eth1           |alpha          |::1                                                                               |::1
+eth0           |beta           |127.0.0.1                                                                         |::1
+eth1           |beta           |127.0.0.1                                                                         |127.0.0.2
+eth1           |beta           |127.0.0.1                                                                         |128.0.0.1
+eth0           |epsilon        |[fe80::cae2:65ff:fece:feb9, fe80::cae2:65ff:fece:fec0, fe80::cae2:65ff:fece:fec1] |fe80::cae2:65ff:fece:fec1
+eth1           |epsilon        |null                                                                              |[127.0.0.1, 127.0.0.2, 127.0.0.3]
+eth0           |gamma          |fe80::cae2:65ff:fece:feb9                                                         |127.0.0.3
+lo0            |gamma          |fe80::cae2:65ff:fece:feb9                                                         |fe81::cae2:65ff:fece:feb9
+;
+
+cdirMatchEqualsInsOrsIPs
+required_capability: combine_disjunctive_cidrmatches
+
+FROM hosts
+| WHERE host == "alpha" OR host == "gamma" OR CIDR_MATCH(ip1, "127.0.0.2/32") OR ip0 == "127.0.0.1" OR card IN ("eth0", "eth1") OR ip1 IN ("127.0.0.3", "127.0.0.1") OR card == "lo0" OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta"
+| KEEP card, host, ip0, ip1
+| sort host, card, ip0, ip1
+;
+warning:Line 2:58: evaluation of [ip1] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:58: java.lang.IllegalArgumentException: single-value function encountered multi-value
+warning:Line 2:82: evaluation of [ip0] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:82: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+card:keyword   |host:keyword   |ip0:ip                                                                            |ip1:ip
+eth0           |alpha          |127.0.0.1                                                                         |127.0.0.1
+eth1           |alpha          |::1                                                                               |::1
+eth0           |beta           |127.0.0.1                                                                         |::1
+eth1           |beta           |127.0.0.1                                                                         |127.0.0.2
+eth1           |beta           |127.0.0.1                                                                         |128.0.0.1
+eth0           |epsilon        |[fe80::cae2:65ff:fece:feb9, fe80::cae2:65ff:fece:fec0, fe80::cae2:65ff:fece:fec1] |fe80::cae2:65ff:fece:fec1
+eth1           |epsilon        |null                                                                              |[127.0.0.1, 127.0.0.2, 127.0.0.3]
+eth0           |gamma          |fe80::cae2:65ff:fece:feb9                                                         |127.0.0.3
+lo0            |gamma          |fe80::cae2:65ff:fece:feb9                                                         |fe81::cae2:65ff:fece:feb9
+;
+
 pushDownIP
 from hosts | where ip1 == to_ip("::1") | keep card, host, ip0, ip1;
 ignoreOrder:true
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-alerts.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-alerts.json
new file mode 100644
index 0000000000000..a5ceac91abe50
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-alerts.json
@@ -0,0 +1,10 @@
+{
+  "properties": {
+    "host.name": {
+      "type": "keyword"
+    },
+    "kibana.alert.risk_score": {
+      "type": "double"
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-books.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-books.json
new file mode 100644
index 0000000000000..29e3c94c579b1
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-books.json
@@ -0,0 +1,30 @@
+{
+  "properties": {
+    "book_no": {
+      "type": "keyword"
+    },
+    "title": {
+      "type": "text"
+    },
+    "author": {
+      "type": "text",
+      "fields": {
+        "keyword": {
+          "type": "keyword"
+        }
+      }
+    },
+    "description": {
+      "type": "text"
+    },
+    "publisher": {
+      "type": "text"
+    },
+    "ratings": {
+      "type": "float"
+    },
+    "year": {
+      "type": "integer"
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json
new file mode 100644
index 0000000000000..506290d90b4b0
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json
@@ -0,0 +1,10 @@
+{
+    "properties": {
+        "millis": {
+            "type": "date"
+        },
+        "nanos": {
+            "type": "date_nanos"
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-missing_ip_sample_data.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-missing_ip_sample_data.json
new file mode 100644
index 0000000000000..6f3796dd7715d
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-missing_ip_sample_data.json
@@ -0,0 +1,13 @@
+{
+    "properties": {
+        "@timestamp": {
+            "type": "date"
+        },
+        "event_duration": {
+            "type": "long"
+        },
+        "message": {
+            "type": "keyword"
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec
new file mode 100644
index 0000000000000..56eded5ce4603
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec
@@ -0,0 +1,92 @@
+###############################################
+# Tests for MATCH operator
+#
+
+singleMatchWithTextField
+required_capability: match_operator
+from books | where author match "William Faulkner" | keep book_no, author | sort book_no | LIMIT 5;
+
+book_no:keyword | author:text
+2378            | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott]
+2713            | William Faulkner
+2847            | Colleen Faulkner
+2883            | William Faulkner
+3293            | Danny Faulkner
+;
+
+singleMatchWithKeywordField
+required_capability: match_operator
+from books | where author.keyword match "William Faulkner" | keep book_no, author | sort book_no;
+
+book_no:keyword | author:text
+2713            | William Faulkner
+2883            | William Faulkner
+4724            | William Faulkner
+4977            | William Faulkner
+5119            | William Faulkner
+5404            | William Faulkner
+5578            | William Faulkner
+8077            | William Faulkner
+9896            | William Faulkner
+;
+
+multipleMatch
+required_capability: match_operator
+from books
+| where (description match "Sauron" OR description match "Dark Lord") AND
+  (author match "J. R. R. Tolkien" OR author match "John Ronald Reuel Tolkien")
+| keep book_no, title, author
+| sort book_no
+| limit 4
+;
+
+book_no:keyword | title:text                                                       | author:text
+1463            | Realms of Tolkien: Images of Middle-earth                        | J. R. R. Tolkien
+2675            | The Lord of the Rings - Boxed Set                                | J.R.R. Tolkien
+2714            | Return of the King Being the Third Part of The Lord of the Rings | J. R. R. Tolkien
+2936            | Fellowship of the Ring 2ND Edition                               | John Ronald Reuel Tolkien
+;
+
+multipleWhereWithMatch
+required_capability: match_operator
+from books
+| where title match "short stories"
+| where author match "Ursula K. Le Guin"
+| keep book_no, title, author
+| sort book_no
+;
+
+book_no:keyword | title:text                                | author:text
+8480            | The wind's twelve quarters: Short stories | Ursula K. Le Guin
+;
+
+combinedMatchWithFunctions
+required_capability: match_operator
+from books
+| where title match "Tolkien" AND author match "Tolkien" AND year > 2000
+| where mv_count(author) == 1
+| keep book_no, title, author, year
+| sort book_no
+;
+
+book_no:keyword | title:text               | author:text    | year:integer
+5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014
+;
+
+matchWithStats
+required_capability: match_operator
+from books
+| where author match "faulkner" AND year > 1990
+| where mv_count(author) == 1
+| stats count(*) BY author.keyword
+| sort author.keyword
+;
+
+count(*): long  | author.keyword:keyword
+1               | Bettilu Stein Faulkner
+2               | Colleen Faulkner
+1               | Danny Faulkner
+1               | Keith Faulkner
+1               | Paul Faulkner
+8               | William Faulkner
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match.csv-spec
new file mode 100644
index 0000000000000..2bc2a865c0052
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match.csv-spec
@@ -0,0 +1,47 @@
+matchKeywordField
+required_capability: match_command
+
+from books | match "author.keyword: *Stein*"  | keep book_no, author | sort book_no;
+
+book_no:keyword | author:text
+7381            | Bettilu Stein Faulkner
+;
+
+matchMultipleTextFields
+required_capability: match_command
+
+from books | match "title:Return* AND author:*Tolkien"  | keep book_no, title | sort book_no;
+
+book_no:keyword | title:text
+2714           | Return of the King Being the Third Part of The Lord of the Rings
+7350           | Return of the Shadow
+;
+
+matchAllFields
+required_capability: match_command
+
+from books | match "dark AND lord AND Sauron" | keep book_no, title | sort book_no;
+
+book_no:keyword | title:text
+2714            | Return of the King Being the Third Part of The Lord of the Rings
+2936            | Fellowship of the Ring 2ND Edition
+;
+
+matchWithWhereFunctionsAndStats
+required_capability: match_command
+
+from books
+| match "Faulkner AND ratings:>4.0"
+| where year > 1950 and mv_count(author) == 1
+| stats count(*) BY author.keyword
+| sort author.keyword
+;
+
+count(*): long  | author.keyword:keyword
+1               | Bettilu Stein Faulkner
+2               | Colleen Faulkner
+1               | Danny Faulkner
+1               | Keith Faulkner
+1               | Paul Faulkner
+1               | William Faulkner
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
index dc2860a5ad8dd..f1f66a9cb990c 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
@@ -1,4 +1,5 @@
 metaFunctionsSynopsis#[skip:-8.15.99]
+required_capability: date_nanos_type
 meta functions | keep synopsis;
 
 synopsis:keyword
@@ -8,8 +9,8 @@ synopsis:keyword
 "double atan(number:double|integer|long|unsigned_long)"
 "double atan2(y_coordinate:double|integer|long|unsigned_long, x_coordinate:double|integer|long|unsigned_long)"
 "double avg(number:double|integer|long)"
-"double|date bin(field:integer|long|double|date, buckets:integer|double|date_period|time_duration, ?from:integer|long|double|date, ?to:integer|long|double|date)"
-"double|date bucket(field:integer|long|double|date, buckets:integer|double|date_period|time_duration, ?from:integer|long|double|date, ?to:integer|long|double|date)"
+"double|date bin(field:integer|long|double|date, buckets:integer|long|double|date_period|time_duration, ?from:integer|long|double|date|keyword|text, ?to:integer|long|double|date|keyword|text)"
+"double|date bucket(field:integer|long|double|date, buckets:integer|long|double|date_period|time_duration, ?from:integer|long|double|date|keyword|text, ?to:integer|long|double|date|keyword|text)"
 "boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version case(condition:boolean, trueValue...:boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version)"
 "double cbrt(number:double|integer|long|unsigned_long)"
 "double|integer|long|unsigned_long ceil(number:double|integer|long|unsigned_long)"
@@ -19,7 +20,7 @@ synopsis:keyword
 "double cos(angle:double|integer|long|unsigned_long)"
 "double cosh(angle:double|integer|long|unsigned_long)"
 "long count(?field:boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version)"
-"long count_distinct(field:boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|version, ?precision:integer)"
+"long count_distinct(field:boolean|date|double|integer|ip|keyword|long|text|version, ?precision:integer|long|unsigned_long)"
 "integer date_diff(unit:keyword|text, startTimestamp:date, endTimestamp:date)"
 "long date_extract(datePart:keyword|text, date:date)"
 "keyword date_format(?dateFormat:keyword|text, date:date)"
@@ -39,20 +40,22 @@ double e()
 "double log(?base:integer|unsigned_long|long|double, number:integer|unsigned_long|long|double)"
 "double log10(number:double|integer|long|unsigned_long)"
 "keyword|text ltrim(string:keyword|text)"
-"boolean|double|integer|long|date|ip max(field:boolean|double|integer|long|date|ip)"
-"double|integer|long median(number:double|integer|long)"
-"double|integer|long median_absolute_deviation(number:double|integer|long)"
-"boolean|double|integer|long|date|ip min(field:boolean|double|integer|long|date|ip)"
+"boolean|double|integer|long|date|ip|keyword|text|long|version max(field:boolean|double|integer|long|date|ip|keyword|text|long|version)"
+"double median(number:double|integer|long)"
+"double median_absolute_deviation(number:double|integer|long)"
+"boolean|double|integer|long|date|ip|keyword|text|long|version min(field:boolean|double|integer|long|date|ip|keyword|text|long|version)"
 "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_append(field1:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version, field2:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version)"
 "double mv_avg(number:double|integer|long|unsigned_long)"
 "keyword mv_concat(string:text|keyword, delim:text|keyword)"
-"integer mv_count(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
+"integer mv_count(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
 "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_dedupe(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version)"
-"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version mv_first(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
-"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version mv_last(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
-"boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version mv_max(field:boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version)"
+"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version mv_first(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
+"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version mv_last(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
+"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version mv_max(field:boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version)"
 "double|integer|long|unsigned_long mv_median(number:double|integer|long|unsigned_long)"
-"boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version mv_min(field:boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version)"
+"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version mv_min(field:boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version)"
+"double|integer|long mv_percentile(number:double|integer|long, percentile:double|integer|long)"
+"double mv_pseries_weighted_sum(number:double, p:double)"
 "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_slice(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version, start:integer, ?end:integer)"
 "boolean|date|double|integer|ip|keyword|long|text|version mv_sort(field:boolean|date|double|integer|ip|keyword|long|text|version, ?order:keyword)"
 "double|integer|long|unsigned_long mv_sum(number:double|integer|long|unsigned_long)"
@@ -81,7 +84,7 @@ double pi()
 "double st_y(point:geo_point|cartesian_point)"
 "boolean starts_with(str:keyword|text, prefix:keyword|text)"
 "keyword substring(string:keyword|text, start:integer, ?length:integer)"
-"long sum(number:double|integer|long)"
+"long|double sum(number:double|integer|long)"
 "double tan(angle:double|integer|long|unsigned_long)"
 "double tanh(angle:double|integer|long|unsigned_long)"
 double tau()
@@ -90,34 +93,35 @@ double tau()
 "boolean to_boolean(field:boolean|keyword|text|double|long|unsigned_long|integer)"
 "cartesian_point to_cartesianpoint(field:cartesian_point|keyword|text)"
 "cartesian_shape to_cartesianshape(field:cartesian_point|cartesian_shape|keyword|text)"
-"date to_datetime(field:date|keyword|text|double|long|unsigned_long|integer)"
+"date to_datetime(field:date|date_nanos|keyword|text|double|long|unsigned_long|integer)"
 "double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long)"
 "double to_degrees(number:double|integer|long|unsigned_long)"
 "double to_double(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long)"
-"date to_dt(field:date|keyword|text|double|long|unsigned_long|integer)"
+"date to_dt(field:date|date_nanos|keyword|text|double|long|unsigned_long|integer)"
 "geo_point to_geopoint(field:geo_point|keyword|text)"
 "geo_shape to_geoshape(field:geo_point|geo_shape|keyword|text)"
 "integer to_int(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)"
 "integer to_integer(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)"
 "ip to_ip(field:ip|keyword|text)"
-"long to_long(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long)"
+"long to_long(field:boolean|date|date_nanos|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long)"
 "keyword|text to_lower(str:keyword|text)"
 "double to_radians(number:double|integer|long|unsigned_long)"
-"keyword to_str(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
-"keyword to_string(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
+"keyword to_str(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
+"keyword to_string(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
 "unsigned_long to_ul(field:boolean|date|keyword|text|double|long|unsigned_long|integer)"
 "unsigned_long to_ulong(field:boolean|date|keyword|text|double|long|unsigned_long|integer)"
 "unsigned_long to_unsigned_long(field:boolean|date|keyword|text|double|long|unsigned_long|integer)"
 "keyword|text to_upper(str:keyword|text)"
 "version to_ver(field:keyword|text|version)"
 "version to_version(field:keyword|text|version)"
-"boolean|double|integer|long|date top(field:boolean|double|integer|long|date, limit:integer, order:keyword)"
+"boolean|double|integer|long|date|ip top(field:boolean|double|integer|long|date|ip, limit:integer, order:keyword)"
 "keyword|text trim(string:keyword|text)"
 "boolean|date|double|integer|ip|keyword|long|text|version values(field:boolean|date|double|integer|ip|keyword|long|text|version)"
 "double weighted_avg(number:double|integer|long, weight:double|integer|long)"
 ;
 
 metaFunctionsArgs#[skip:-8.15.99]
+required_capability: date_nanos_type
   META functions
 | EVAL name = SUBSTRING(name, 0, 14)
 | KEEP name, argNames, argTypes, argDescriptions;
@@ -129,8 +133,8 @@ asin          |number                              |"double|integer|long|unsigne
 atan          |number                              |"double|integer|long|unsigned_long"                                                                                               |Numeric expression. If `null`, the function returns `null`.
 atan2         |[y_coordinate, x_coordinate]        |["double|integer|long|unsigned_long", "double|integer|long|unsigned_long"]                                                        |[y coordinate. If `null`\, the function returns `null`., x coordinate. If `null`\, the function returns `null`.]
 avg           |number                              |"double|integer|long"                                                                                                             |[""]
-bin           |[field, buckets, from, to]          |["integer|long|double|date", "integer|double|date_period|time_duration", "integer|long|double|date", "integer|long|double|date"]  |[Numeric or date expression from which to derive buckets., Target number of buckets., Start of the range. Can be a number or a date expressed as a string., End of the range. Can be a number or a date expressed as a string.]
-bucket        |[field, buckets, from, to]          |["integer|long|double|date", "integer|double|date_period|time_duration", "integer|long|double|date", "integer|long|double|date"]  |[Numeric or date expression from which to derive buckets., Target number of buckets., Start of the range. Can be a number or a date expressed as a string., End of the range. Can be a number or a date expressed as a string.]
+bin           |[field, buckets, from, to]          |["integer|long|double|date", "integer|long|double|date_period|time_duration", "integer|long|double|date|keyword|text", "integer|long|double|date|keyword|text"]  |[Numeric or date expression from which to derive buckets., Target number of buckets\, or desired bucket size if `from` and `to` parameters are omitted., Start of the range. Can be a number\, a date or a date expressed as a string., End of the range. Can be a number\, a date or a date expressed as a string.]
+bucket        |[field, buckets, from, to]          |["integer|long|double|date", "integer|long|double|date_period|time_duration", "integer|long|double|date|keyword|text", "integer|long|double|date|keyword|text"]  |[Numeric or date expression from which to derive buckets., Target number of buckets\, or desired bucket size if `from` and `to` parameters are omitted., Start of the range. Can be a number\, a date or a date expressed as a string., End of the range. Can be a number\, a date or a date expressed as a string.]
 case          |[condition, trueValue]              |[boolean, "boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version"]                     |[A condition., The value that's returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches.]
 cbrt          |number                              |"double|integer|long|unsigned_long"                                                                                               |"Numeric expression. If `null`, the function returns `null`."
 ceil          |number                              |"double|integer|long|unsigned_long"                                                                                               |Numeric expression. If `null`, the function returns `null`.
@@ -139,8 +143,8 @@ coalesce      |first                               |"boolean|cartesian_point|car
 concat        |[string1, string2]                  |["keyword|text", "keyword|text"]                                                                                                  |[Strings to concatenate., Strings to concatenate.]
 cos           |angle                               |"double|integer|long|unsigned_long"                                                                                               |An angle, in radians. If `null`, the function returns `null`.
 cosh          |angle                               |"double|integer|long|unsigned_long"                                                                                               |An angle, in radians. If `null`, the function returns `null`.
-count         |field                               |"boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version"                                |Column or literal for which to count the number of values.
-count_distinct|[field, precision]                  |["boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|version", integer]                                   |[Column or literal for which to count the number of distinct values., ]
+count         |field                               |"boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version"                                |Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows).
+count_distinct|[field, precision]                  |["boolean|date|double|integer|ip|keyword|long|text|version", "integer|long|unsigned_long"]                                        |[Column or literal for which to count the number of distinct values., Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000.]
 date_diff     |[unit, startTimestamp, endTimestamp]|["keyword|text", date, date]                                                                                                      |[Time difference unit, A string representing a start timestamp, A string representing an end timestamp]
 date_extract  |[datePart, date]                    |["keyword|text", date]                                                                                                            |[Part of the date to extract.  Can be: `aligned_day_of_week_in_month`\, `aligned_day_of_week_in_year`\, `aligned_week_of_month`\, `aligned_week_of_year`\, `ampm_of_day`\, `clock_hour_of_ampm`\, `clock_hour_of_day`\, `day_of_month`\, `day_of_week`\, `day_of_year`\, `epoch_day`\, `era`\, `hour_of_ampm`\, `hour_of_day`\, `instant_seconds`\, `micro_of_day`\, `micro_of_second`\, `milli_of_day`\, `milli_of_second`\, `minute_of_day`\, `minute_of_hour`\, `month_of_year`\, `nano_of_day`\, `nano_of_second`\, `offset_seconds`\, `proleptic_month`\, `second_of_day`\, `second_of_minute`\, `year`\, or `year_of_era`. Refer to https://docs.oracle.com/javase/8/docs/api/java/time/temporal/ChronoField.html[java.time.temporal.ChronoField] for a description of these values.  If `null`\, the function returns `null`., Date expression. If `null`\, the function returns `null`.]
 date_format   |[dateFormat, date]                  |["keyword|text", date]                                                                                                            |[Date format (optional).  If no format is specified\, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If `null`\, the function returns `null`., Date expression. If `null`\, the function returns `null`.]
@@ -160,20 +164,22 @@ locate        |[string, substring, start]          |["keyword|text", "keyword|te
 log           |[base, number]                      |["integer|unsigned_long|long|double", "integer|unsigned_long|long|double"]                                                        |["Base of logarithm. If `null`\, the function returns `null`. If not provided\, this function returns the natural logarithm (base e) of a value.", "Numeric expression. If `null`\, the function returns `null`."]
 log10         |number                              |"double|integer|long|unsigned_long"                                                                                               |Numeric expression. If `null`, the function returns `null`.
 ltrim         |string                              |"keyword|text"                                                                                                                    |String expression. If `null`, the function returns `null`.
-max           |field                               |"boolean|double|integer|long|date|ip"                                                                                             |[""]
+max           |field                               |"boolean|double|integer|long|date|ip|keyword|text|long|version"                                                                   |[""]
 median        |number                              |"double|integer|long"                                                                                                             |[""]
 median_absolut|number                              |"double|integer|long"                                                                                                             |[""]
-min           |field                               |"boolean|double|integer|long|date|ip"                                                                                             |[""]
+min           |field                               |"boolean|double|integer|long|date|ip|keyword|text|long|version"                                                                   |[""]
 mv_append     |[field1, field2]                    |["boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version", "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"]                 | ["", ""]
 mv_avg        |number                              |"double|integer|long|unsigned_long"                                                                                               |Multivalue expression.
 mv_concat     |[string, delim]                     |["text|keyword", "text|keyword"]                                                                                                  |[Multivalue expression., Delimiter.]
-mv_count      |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Multivalue expression.
+mv_count      |field                               |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Multivalue expression.
 mv_dedupe     |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"                    |Multivalue expression.
-mv_first      |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Multivalue expression.
-mv_last       |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Multivalue expression.
-mv_max        |field                               |"boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version"                                                          |Multivalue expression.
+mv_first      |field                               |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Multivalue expression.
+mv_last       |field                               |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Multivalue expression.
+mv_max        |field                               |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version"                                               |Multivalue expression.
 mv_median     |number                              |"double|integer|long|unsigned_long"                                                                                               |Multivalue expression.
-mv_min        |field                               |"boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version"                                                          |Multivalue expression.
+mv_min        |field                               |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version"                                               |Multivalue expression.
+mv_percentile |[number, percentile]                |["double|integer|long", "double|integer|long"]                                                                                    |[Multivalue expression., The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead.]
+mv_pseries_wei|[number, p]                         |[double, double]                                                                                                                  |[Multivalue expression., It is a constant number that represents the 'p' parameter in the P-Series. It impacts every element's contribution to the weighted sum.]
 mv_slice      |[field, start, end]                 |["boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version", integer, integer]|[Multivalue expression. If `null`\, the function returns `null`., Start position. If `null`\, the function returns `null`. The start argument can be negative. An index of -1 is used to specify the last value in the list., End position(included). Optional; if omitted\, the position at `start` is returned. The end argument can be negative. An index of -1 is used to specify the last value in the list.]
 mv_sort       |[field, order]                      |["boolean|date|double|integer|ip|keyword|long|text|version", keyword]                                                             |[Multivalue expression. If `null`\, the function returns `null`., Sort order. The valid options are ASC and DESC\, the default is ASC.]
 mv_sum        |number                              |"double|integer|long|unsigned_long"                                                                                               |Multivalue expression.
@@ -211,28 +217,28 @@ to_bool       |field                               |"boolean|keyword|text|double
 to_boolean    |field                               |"boolean|keyword|text|double|long|unsigned_long|integer"                                                                          |Input value. The input can be a single- or multi-valued column or an expression.
 to_cartesianpo|field                               |"cartesian_point|keyword|text"                                                                                                    |Input value. The input can be a single- or multi-valued column or an expression.
 to_cartesiansh|field                               |"cartesian_point|cartesian_shape|keyword|text"                                                                                    |Input value. The input can be a single- or multi-valued column or an expression.
-to_datetime   |field                               |"date|keyword|text|double|long|unsigned_long|integer"                                                                             |Input value. The input can be a single- or multi-valued column or an expression.
+to_datetime   |field                               |"date|date_nanos|keyword|text|double|long|unsigned_long|integer"                                                                             |Input value. The input can be a single- or multi-valued column or an expression.
 to_dbl        |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long"                         |Input value. The input can be a single- or multi-valued column or an expression.
 to_degrees    |number                              |"double|integer|long|unsigned_long"                                                                                               |Input value. The input can be a single- or multi-valued column or an expression.
 to_double     |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long"                         |Input value. The input can be a single- or multi-valued column or an expression.
-to_dt         |field                               |"date|keyword|text|double|long|unsigned_long|integer"                                                                             |Input value. The input can be a single- or multi-valued column or an expression.
+to_dt         |field                               |"date|date_nanos|keyword|text|double|long|unsigned_long|integer"                                                                             |Input value. The input can be a single- or multi-valued column or an expression.
 to_geopoint   |field                               |"geo_point|keyword|text"                                                                                                          |Input value. The input can be a single- or multi-valued column or an expression.
 to_geoshape   |field                               |"geo_point|geo_shape|keyword|text"                                                                                                |Input value. The input can be a single- or multi-valued column or an expression.
 to_int        |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer"                                                     |Input value. The input can be a single- or multi-valued column or an expression.
 to_integer    |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer"                                                     |Input value. The input can be a single- or multi-valued column or an expression.
 to_ip         |field                               |"ip|keyword|text"                                                                                                                 |Input value. The input can be a single- or multi-valued column or an expression.
-to_long       |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long"                                        |Input value. The input can be a single- or multi-valued column or an expression.
+to_long       |field                               |"boolean|date|date_nanos|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long"                                        |Input value. The input can be a single- or multi-valued column or an expression.
 to_lower      |str                                 |"keyword|text"                                                                                                                    |String expression. If `null`, the function returns `null`.
 to_radians    |number                              |"double|integer|long|unsigned_long"                                                                                               |Input value. The input can be a single- or multi-valued column or an expression.
-to_str        |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Input value. The input can be a single- or multi-valued column or an expression.
-to_string     |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Input value. The input can be a single- or multi-valued column or an expression.
+to_str        |field                               |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Input value. The input can be a single- or multi-valued column or an expression.
+to_string     |field                               |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Input value. The input can be a single- or multi-valued column or an expression.
 to_ul         |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer"                                                                     |Input value. The input can be a single- or multi-valued column or an expression.
 to_ulong      |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer"                                                                     |Input value. The input can be a single- or multi-valued column or an expression.
 to_unsigned_lo|field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer"                                                                     |Input value. The input can be a single- or multi-valued column or an expression.
 to_upper      |str                                 |"keyword|text"                                                                                                                    |String expression. If `null`, the function returns `null`.
 to_ver        |field                               |"keyword|text|version"                                                                                                            |Input value. The input can be a single- or multi-valued column or an expression.
 to_version    |field                               |"keyword|text|version"                                                                                                            |Input value. The input can be a single- or multi-valued column or an expression.
-top           |[field, limit, order]               |["boolean|double|integer|long|date", integer, keyword]                                                                            |[The field to collect the top values for.,The maximum number of values to collect.,The order to calculate the top values. Either `asc` or `desc`.]
+top           |[field, limit, order]               |["boolean|double|integer|long|date|ip", integer, keyword]                                                                         |[The field to collect the top values for.,The maximum number of values to collect.,The order to calculate the top values. Either `asc` or `desc`.]
 trim          |string                              |"keyword|text"                                                                                                                    |String expression. If `null`, the function returns `null`.
 values        |field                               |"boolean|date|double|integer|ip|keyword|long|text|version"                                                                        |[""]
 weighted_avg  |[number, weight]                    |["double|integer|long", "double|integer|long"]                                                                                    |[A numeric value., A numeric weight.]
@@ -283,8 +289,8 @@ log           |Returns the logarithm of a value to a base. The input can be any
 log10         |Returns the logarithm of a value to base 10. The input can be any numeric value, the return value is always a double.  Logs of 0 and negative numbers return `null` as well as a warning.
 ltrim         |Removes leading whitespaces from a string.
 max           |The maximum value of a field.
-median        |The value that is greater than half of all values and less than half of all values.
-median_absolut|The median absolute deviation, a measure of variability.
+median        |The value that is greater than half of all values and less than half of all values, also known as the 50% <<esql-percentile>>.
+median_absolut|"Returns the median absolute deviation, a measure of variability. It is a robust statistic, meaning that it is useful for describing data that may have outliers, or may not be normally distributed. For such data it can be more descriptive than standard deviation.  It is calculated as the median of each data point's deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`."
 min           |The minimum value of a field.
 mv_append     |Concatenates values of two multi-value fields.
 mv_avg        |Converts a multivalued field into a single valued field containing the average of all of the values.
@@ -296,6 +302,8 @@ mv_last       |Converts a multivalue expression into a single valued column cont
 mv_max        |Converts a multivalued expression into a single valued column containing the maximum value.
 mv_median     |Converts a multivalued field into a single valued field containing the median value.
 mv_min        |Converts a multivalued expression into a single valued column containing the minimum value.
+mv_percentile |Converts a multivalued field into a single valued field containing the value at which a certain percentage of observed values occur.
+mv_pseries_wei|Converts a multivalued expression into a single-valued column by multiplying every element on the input list by its corresponding term in P-Series and computing the sum.
 mv_slice      |Returns a subset of the multivalued field using the start and end index values.
 mv_sort       |Sorts a multivalued field in lexicographical order.
 mv_sum        |Converts a multivalued field into a single valued field containing the sum of all of the values.
@@ -314,7 +322,7 @@ sin           |Returns ths {wikipedia}/Sine_and_cosine[Sine] trigonometric funct
 sinh          |Returns the {wikipedia}/Hyperbolic_functions[hyperbolic sine] of an angle.
 split         |Split a single valued string into multiple strings.
 sqrt          |Returns the square root of a number. The input can be any numeric value, the return value is always a double. Square roots of negative numbers and infinities are null.
-st_centroid_ag|The centroid of a spatial field.
+st_centroid_ag|Calculate the spatial centroid over a field with spatial point geometry type.
 st_contains   |Returns whether the first geometry contains the second geometry. This is the inverse of the <<esql-st_within,ST_WITHIN>> function.
 st_disjoint   |Returns whether the two geometries or geometry columns are disjoint. This is the inverse of the <<esql-st_intersects,ST_INTERSECTS>> function. In mathematical terms: ST_Disjoint(A, B) ⇔ A ⋂ B = ∅
 st_distance   |Computes the distance between two points. For cartesian geometries, this is the pythagorean distance in the same units as the original coordinates. For geographic geometries, this is the circular distance along the great circle in meters.
@@ -324,7 +332,7 @@ st_x          |Extracts the `x` coordinate from the supplied point. If the point
 st_y          |Extracts the `y` coordinate from the supplied point. If the points is of type `geo_point` this is equivalent to extracting the `latitude` value.
 starts_with   |Returns a boolean that indicates whether a keyword string starts with another string.
 substring     |Returns a substring of a string, specified by a start position and an optional length.
-sum           |The sum of a numeric field.
+sum           |The sum of a numeric expression.
 tan           |Returns the {wikipedia}/Sine_and_cosine[Tangent] trigonometric function of an angle.
 tanh          |Returns the {wikipedia}/Hyperbolic_functions[Tangent] hyperbolic function of an angle.
 tau           |Returns the https://tauday.com/tau-manifesto[ratio] of a circle's circumference to its radius.
@@ -356,11 +364,12 @@ to_ver        |Converts an input string to a version value.
 to_version    |Converts an input string to a version value.
 top           |Collects the top values for a field. Includes repeated values.
 trim          |Removes leading and trailing whitespaces from a string.
-values        |Collect values for a field.
-weighted_avg  |The weighted average of a numeric field.
+values        |Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed. If you need the values returned in order use <<esql-mv_sort>>.
+weighted_avg  |The weighted average of a numeric expression.
 ;
 
 metaFunctionsRemaining#[skip:-8.15.99]
+required_capability: date_nanos_type
   META functions
 | EVAL name = SUBSTRING(name, 0, 14)
 | KEEP name, *
@@ -405,20 +414,22 @@ locate        |integer
 log           |double                                                                                                                      |[true, false]               |false           |false
 log10         |double                                                                                                                      |false                       |false           |false
 ltrim         |"keyword|text"                                                                                                              |false                       |false           |false
-max           |"boolean|double|integer|long|date|ip"                                                                                       |false                       |false           |true
-median        |"double|integer|long"                                                                                                       |false                       |false           |true
-median_absolut|"double|integer|long"                                                                                                       |false                       |false           |true
-min           |"boolean|double|integer|long|date|ip"                                                                                       |false                       |false           |true
+max           |"boolean|double|integer|long|date|ip|keyword|text|long|version"                                                             |false                       |false           |true
+median        |double                                                                                                                      |false                       |false           |true
+median_absolut|double                                                                                                                      |false                       |false           |true
+min           |"boolean|double|integer|long|date|ip|keyword|text|long|version"                                                             |false                       |false           |true
 mv_append     |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"              |[false, false]              |false 		     |false
 mv_avg        |double                                                                                                                      |false                       |false           |false
 mv_concat     |keyword                                                                                                                     |[false, false]              |false           |false
 mv_count      |integer                                                                                                                     |false                       |false           |false
 mv_dedupe     |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"              |false                       |false           |false
-mv_first      |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"|false                       |false           |false
-mv_last       |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"|false                       |false           |false
-mv_max        |"boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version"                                                    |false                       |false           |false
+mv_first      |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"|false                       |false           |false
+mv_last       |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"|false                       |false           |false
+mv_max        |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version"                                         |false                       |false           |false
 mv_median     |"double|integer|long|unsigned_long"                                                                                         |false                       |false           |false
-mv_min        |"boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version"                                                    |false                       |false           |false
+mv_min        |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version"                                         |false                       |false           |false
+mv_percentile |"double|integer|long"                                                                                                       |[false, false]              |false           |false
+mv_pseries_wei|"double"                                                                                                                    |[false, false]              |false           |false 
 mv_slice      |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"              |[false, false, true]        |false           |false
 mv_sort       |"boolean|date|double|integer|ip|keyword|long|text|version"                                                                  |[false, true]               |false           |false
 mv_sum        |"double|integer|long|unsigned_long"                                                                                         |false                       |false           |false
@@ -447,7 +458,7 @@ st_x          |double
 st_y          |double                                                                                                                      |false                       |false           |false
 starts_with   |boolean                                                                                                                     |[false, false]              |false           |false
 substring     |keyword                                                                                                                     |[false, false, true]        |false           |false
-sum           |long                                                                                                                        |false                       |false           |true
+sum           |"long|double"                                                                                                               |false                       |false           |true
 tan           |double                                                                                                                      |false                       |false           |false
 tanh          |double                                                                                                                      |false                       |false           |false
 tau           |double                                                                                                                      |null                        |false           |false
@@ -477,7 +488,7 @@ to_unsigned_lo|unsigned_long
 to_upper      |"keyword|text"                                                                                                              |false                       |false           |false
 to_ver        |version                                                                                                                     |false                       |false           |false
 to_version    |version                                                                                                                     |false                       |false           |false
-top           |"boolean|double|integer|long|date"                                                                                          |[false, false, false]       |false           |true
+top           |"boolean|double|integer|long|date|ip"                                                                                       |[false, false, false]       |false           |true
 trim          |"keyword|text"                                                                                                              |false                       |false           |false
 values        |"boolean|date|double|integer|ip|keyword|long|text|version"                                                                  |false                       |false           |true
 weighted_avg  |"double"                                                                                                                    |[false, false]              |false           |true
@@ -497,5 +508,5 @@ countFunctions#[skip:-8.15.99]
 meta functions |  stats  a = count(*), b = count(*), c = count(*) |  mv_expand c;
 
 a:long | b:long | c:long
-113    | 113    | 113
+115    | 115    | 115
 ;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/missing_ip_sample_data.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/missing_ip_sample_data.csv
new file mode 100644
index 0000000000000..e8e9ddcaee83b
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/missing_ip_sample_data.csv
@@ -0,0 +1,8 @@
+@timestamp:date,event_duration:long,message:keyword
+2023-10-23T13:55:01.543Z,1756467,Connected to 10.1.0.1
+2023-10-23T13:53:55.832Z,5033755,Connection error
+2023-10-23T13:52:55.015Z,8268153,Connection error
+2023-10-23T13:51:54.732Z,725448,Connection error
+2023-10-23T13:33:34.937Z,1232382,Disconnected
+2023-10-23T12:27:28.948Z,2764889,Connected to 10.1.0.2
+2023-10-23T12:15:03.360Z,3450233,Connected to 10.1.0.3
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_percentile.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_percentile.csv-spec
new file mode 100644
index 0000000000000..c51e62e865ea2
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_percentile.csv-spec
@@ -0,0 +1,163 @@
+default
+required_capability: fn_mv_percentile
+
+// tag::example[]
+ROW values = [5, 5, 10, 12, 5000]
+| EVAL p50 = MV_PERCENTILE(values, 50), median = MV_MEDIAN(values)
+// end::example[]
+;
+
+// tag::example-result[]
+values:integer       | p50:integer | median:integer
+[5, 5, 10, 12, 5000] | 10          | 10
+// end::example-result[]
+;
+
+p0
+required_capability: fn_mv_percentile
+
+ROW a = [5, 5, 10, 12, 5000]
+| EVAL pInt = MV_PERCENTILE(a, 0), pLong = MV_PERCENTILE(a, 0::long), pDouble = MV_PERCENTILE(a, 0.0)
+| KEEP pInt, pLong, pDouble
+;
+
+pInt:integer | pLong:integer | pDouble:integer
+5            | 5             | 5
+;
+
+p100
+required_capability: fn_mv_percentile
+
+ROW a = [5, 5, 10, 12, 5000]
+| EVAL pInt = MV_PERCENTILE(a, 100), pLong = MV_PERCENTILE(a, 100::long), pDouble = MV_PERCENTILE(a, 100.0)
+| KEEP pInt, pLong, pDouble
+;
+
+pInt:integer | pLong:integer | pDouble:integer
+5000         | 5000          | 5000
+;
+
+fractionInt
+required_capability: fn_mv_percentile
+
+ROW a = [0, 10]
+| EVAL pInt = MV_PERCENTILE(a, 75), pLong = MV_PERCENTILE(a, 75::long), pDouble = MV_PERCENTILE(a, 75.0)
+| KEEP pInt, pLong, pDouble
+;
+
+pInt:integer | pLong:integer | pDouble:integer
+7            | 7             | 7
+;
+
+fractionLong
+required_capability: fn_mv_percentile
+
+ROW a = to_long([0, 10])
+| EVAL pInt = MV_PERCENTILE(a, 75), pLong = MV_PERCENTILE(a, 75::long), pDouble = MV_PERCENTILE(a, 75.0)
+| KEEP pInt, pLong, pDouble
+;
+
+pInt:long | pLong:long | pDouble:long
+7         | 7          | 7
+;
+
+fractionDouble
+required_capability: fn_mv_percentile
+
+ROW a = [0., 10.]
+| EVAL pInt = MV_PERCENTILE(a, 75), pLong = MV_PERCENTILE(a, 75::long), pDouble = MV_PERCENTILE(a, 75.0)
+| KEEP pInt, pLong, pDouble
+;
+
+pInt:double | pLong:double | pDouble:double
+7.5         | 7.5          | 7.5
+;
+
+singleValue
+required_capability: fn_mv_percentile
+
+ROW integer = 5, long = 5::long, double = 5.0
+| EVAL
+    integer = MV_PERCENTILE(integer, 75),
+    long = MV_PERCENTILE(long, 75),
+    double = MV_PERCENTILE(double, 75)
+;
+
+integer:integer | long:long | double:double
+5               | 5         | 5
+;
+
+fromIndex
+required_capability: fn_mv_percentile
+
+FROM employees
+| EVAL
+    integer = MV_PERCENTILE(salary_change.int, 75),
+    long = MV_PERCENTILE(salary_change.long, 75),
+    double = MV_PERCENTILE(salary_change, 75)
+| KEEP emp_no, integer, long, double
+| SORT double
+| LIMIT 3
+;
+
+emp_no:integer | integer:integer | long:long | double:double
+10034          | -8              | -8        | -8.46
+10037          | -7              | -7        | -7.08
+10039          | -6              | -6        | -6.9
+;
+
+fromIndexPercentile
+required_capability: fn_mv_percentile
+
+FROM employees
+| SORT emp_no
+| LIMIT 1
+| EVAL
+    integer = MV_PERCENTILE(salary_change.int, languages),
+    long = MV_PERCENTILE(salary_change.long, languages.long),
+    double = MV_PERCENTILE(salary_change, height),
+    null_value = MV_PERCENTILE(salary_change, emp_no)
+| KEEP integer, long, double, null_value
+;
+warning:Line 8:14: evaluation of [MV_PERCENTILE(salary_change, emp_no)] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 8:14: java.lang.IllegalArgumentException: Percentile parameter must be a number between 0 and 100, found [10001.0]
+
+integer:integer | long:long | double:double | null_value:double
+1               | 1         | 1.19          | null
+;
+
+multipleExpressions
+required_capability: fn_mv_percentile
+
+ROW x = [0, 5, 10]
+| EVAL
+    MV_PERCENTILE(x, 75),
+    a = MV_PERCENTILE(x, 75),
+    b = MV_PERCENTILE(TO_DOUBLE([0, 5, 10]), 75),
+    c = MV_PERCENTILE(CASE(true, x, [0, 1]), 75)
+;
+
+x:integer  | MV_PERCENTILE(x, 75):integer | a:integer | b:double | c:integer
+[0, 5, 10] | 7                            | 7         | 7.5      | 7
+;
+
+nullsAndFolds
+required_capability: fn_mv_percentile
+
+ROW x = [5, 5, 10, 12, 5000], n = null, y = 50
+| EVAL evalNull = null / 2, evalValue = 31 + 1
+| LIMIT 1
+| EVAL
+    a = mv_percentile(y, 90),
+    b = mv_percentile(x, y),
+    c = mv_percentile(null, null),
+    d = mv_percentile(null, y),
+    e = mv_percentile(evalNull, y),
+    f = mv_percentile(evalValue, y),
+    g = mv_percentile(n, y)
+| KEEP a, b, c, d, e, f, g
+;
+
+a:integer | b:integer | c:null | d:null | e:integer | f:integer | g:null
+50        | 10        | null   | null   | null      | 32        | null
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_pseries_weighted_sum.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_pseries_weighted_sum.csv-spec
new file mode 100644
index 0000000000000..df8771ad7a832
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_pseries_weighted_sum.csv-spec
@@ -0,0 +1,90 @@
+default
+required_capability: mv_pseries_weighted_sum
+
+// tag::example[]
+ROW a = [70.0, 45.0, 21.0, 21.0, 21.0]
+| EVAL sum = MV_PSERIES_WEIGHTED_SUM(a, 1.5)
+| KEEP sum
+// end::example[]
+;
+
+// tag::example-result[]
+sum:double
+94.45465156212452
+// end::example-result[]
+;
+
+oneElement
+required_capability: mv_pseries_weighted_sum
+
+ROW data = [3.0]
+| EVAL score = MV_PSERIES_WEIGHTED_SUM(data, 9999.9)
+| KEEP score;
+
+score:double
+3.0
+;
+
+zeroP
+required_capability: mv_pseries_weighted_sum
+
+ROW data = [3.0, 10.0, 15.0]
+| EVAL score = MV_PSERIES_WEIGHTED_SUM(data, 0.0)
+| KEEP score;
+
+score:double
+28.0
+;
+
+negativeP
+required_capability: mv_pseries_weighted_sum
+
+ROW data = [10.0, 5.0, 3.0]
+| EVAL score = MV_PSERIES_WEIGHTED_SUM(data, -2.0)
+| KEEP score;
+
+score:double
+57.0
+;
+
+composed
+required_capability: mv_pseries_weighted_sum
+
+ROW data = [21.0, 45.0, 21.0, 70.0, 21.0]
+| EVAL sorted = MV_SORT(data, "desc")
+| EVAL score = MV_PSERIES_WEIGHTED_SUM(sorted, 1.5)
+| EVAL normalized_score = ROUND(100 * score / 261.2, 2)
+| KEEP normalized_score, score;
+
+normalized_score:double|score:double
+36.16                  |94.45465156212452
+;
+
+multivalueAggregation
+required_capability: mv_pseries_weighted_sum
+
+FROM alerts
+| WHERE host.name is not null
+| SORT host.name, kibana.alert.risk_score
+| STATS score=MV_PSERIES_WEIGHTED_SUM(
+    TOP(kibana.alert.risk_score, 10000, "desc"), 1.5
+) BY host.name
+| EVAL normalized_score = ROUND(100 * score / 261.2, 2)
+| KEEP host.name, normalized_score, score
+| SORT normalized_score DESC;
+
+host.name:keyword|normalized_score:double|score:double
+test-host-1      |36.16                  |94.45465156212452
+test-host-2      |13.03                  |34.036822671263614
+;
+
+asArgument
+required_capability: mv_pseries_weighted_sum
+
+ROW data = [70.0, 45.0, 21.0, 21.0, 21.0]
+| EVAL score = ROUND(MV_PSERIES_WEIGHTED_SUM(data, 1.5), 1)
+| KEEP score;
+
+score:double
+94.5
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec
index 92537ed1221e8..9914d073a589d 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec
@@ -142,3 +142,52 @@ emp_no:integer | first_name:keyword | foo:boolean
          10002 | Bezalel            | true
          10003 | Parto              | true
 ;
+
+coalesceMixedNumeric
+required_capability: mixed_numeric_types_in_coalesce
+FROM employees
+| WHERE emp_no <= 10021 and emp_no >= 10018
+| EVAL x = coalesce(languages.long, 0), y = coalesce(height, 0), z = coalesce(languages::double, 0)
+| SORT emp_no ASC
+| KEEP emp_no, languages, x, height, y, z
+;
+
+emp_no:integer | languages:integer | x:long | height:double | y:double | z:double
+         10018 | 2                 | 2      | 1.97          | 1.97     | 2.0
+         10019 | 1                 | 1      | 2.06          | 2.06     | 1.0
+         10020 | null              | 0      | 1.41          | 1.41     | 0.0
+         10021 | null              | 0      | 1.47          | 1.47     | 0.0
+;
+
+coalesceMixedNumericWithNull
+required_capability: mixed_numeric_types_in_coalesce
+FROM employees
+| WHERE emp_no <= 10021 and emp_no >= 10018
+| EVAL x = coalesce(languages.long, null, 0), y = coalesce(null, height, 0),  z = coalesce(languages::double, null, 0)
+| SORT emp_no ASC
+| KEEP emp_no, languages.long, x, height, y, z
+;
+
+emp_no:integer | languages.long:long | x:long | height:double | y:double | z:double
+         10018 | 2                   | 2      | 1.97          | 1.97     | 2.0
+         10019 | 1                   | 1      | 2.06          | 2.06     | 1.0
+         10020 | null                | 0      | 1.41          | 1.41     | 0.0
+         10021 | null                | 0      | 1.47          | 1.47     | 0.0
+;
+
+coalesceMixedNumericFields
+required_capability: mixed_numeric_types_in_coalesce
+FROM employees
+| WHERE emp_no <= 10021 and emp_no >= 10018
+| EVAL x = coalesce(height, languages.long, 0), y = coalesce(height, null, languages, 0),
+z = coalesce(languages::double, null, salary, height, 0)
+| SORT emp_no ASC
+| KEEP emp_no, languages, height, x, y, z, salary
+;
+
+emp_no:integer | languages:integer | height:double | x:double | y:double | z:double | salary:integer
+         10018 | 2                 | 1.97          | 1.97     | 1.97     | 2.0      | 56760
+         10019 | 1                 | 2.06          | 2.06     | 2.06     | 1.0      | 73717
+         10020 | null              | 1.41          | 1.41     | 1.41     | 40031.0  | 40031
+         10021 | null              | 1.47          | 1.47     | 1.47     | 60408.0  | 60408
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec
index 02067e9dbe490..35416c7945128 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec
@@ -487,6 +487,17 @@ POINT (42.97109629958868 14.7552534006536)    | 1
 ###############################################
 # Tests for ST_INTERSECTS on GEO_POINT type
 
+literalGeoPointIntersectsLiteralPolygon
+required_capability: st_intersects
+
+ROW pt = TO_GEOPOINT("POINT(0 85)"), polygon = TO_GEOSHAPE("POLYGON((-10 70, 10 70, 10 85, -10 85, -10 70))")
+| EVAL intersects = ST_INTERSECTS(pt, polygon)
+;
+
+pt:geo_point  | polygon:geo_shape                               | intersects:boolean
+POINT(0 85)   | POLYGON((-10 70, 10 70, 10 85, -10 85, -10 70)) | true
+;
+
 pointIntersectsLiteralPolygon
 required_capability: st_intersects
 
@@ -889,6 +900,34 @@ wkt:keyword    | pt:geo_point  | distance:double
 "POINT(1 -1)"  | POINT(1 -1)   | 157249.59498573805
 ;
 
+literalGeoPointDistanceOneDegree
+required_capability: st_distance
+
+ROW wkt = ["POINT(1 0)", "POINT(-1 0)", "POINT(0 1)", "POINT(0 -1)"]
+| MV_EXPAND wkt
+| EVAL pt = TO_GEOPOINT(wkt)
+| EVAL distance = ST_DISTANCE(pt, TO_GEOPOINT("POINT(0 0)"))
+;
+
+wkt:keyword    | pt:geo_point  | distance:double
+"POINT(1 0)"   | POINT(1 0)    | 111195.07310665186
+"POINT(-1 0)"  | POINT(-1 0)   | 111195.08242688453
+"POINT(0 1)"   | POINT(0 1)    | 111195.07776676829
+"POINT(0 -1)"  | POINT(0 -1)   | 111195.08242688453
+;
+
+twoCitiesPointDistanceGeo
+required_capability: st_distance
+required_capability: spatial_functions_fix_crstype_folding
+
+ROW p1 = TO_GEOPOINT("POINT(-90.82814 29.79511)"), p2 = TO_GEOPOINT("POINT(-90.79731509999999 29.8835389)")
+| EVAL d = ST_DISTANCE(p1, p2)
+;
+
+p1:geo_point                | p2:geo_point                           | d:double
+POINT (-90.82814 29.79511)  | POINT (-90.79731509999999 29.8835389)  | 10272.529272836206
+;
+
 airportCityLocationPointDistance
 required_capability: st_distance
 
@@ -1433,6 +1472,17 @@ POINT (726480.0130685265 3359566.331716279) | 849
 ###############################################
 # Tests for ST_INTERSECTS on CARTESIAN_POINT type
 
+literalCartesianPointIntersectsLiteralPolygon
+required_capability: st_intersects
+
+ROW pt = TO_CARTESIANPOINT("POINT(0 85)"), polygon = TO_CARTESIANSHAPE("POLYGON((-10 70, 10 70, 10 85, -10 85, -10 70))")
+| EVAL intersects = ST_INTERSECTS(pt, polygon)
+;
+
+pt:cartesian_point  | polygon:cartesian_shape                         | intersects:boolean
+POINT(0 85)         | POLYGON((-10 70, 10 70, 10 85, -10 85, -10 70)) | true
+;
+
 cartesianCentroidFromAirportsAfterIntersectsPredicate
 required_capability: st_intersects
 
@@ -1996,6 +2046,33 @@ wkt:keyword    | pt:cartesian_point  | distance:double
 "POINT(1 -1)"  | POINT(1 -1)         | 1.4142135623730951
 ;
 
+literalCartesianPointDistanceOneUnit
+required_capability: st_distance
+
+ROW wkt = ["POINT(1 0)", "POINT(-1 0)", "POINT(0 1)", "POINT(0 -1)"]
+| MV_EXPAND wkt
+| EVAL pt = TO_CARTESIANPOINT(wkt)
+| EVAL distance = ST_DISTANCE(pt, TO_CARTESIANPOINT("POINT(0 0)"))
+;
+
+wkt:keyword    | pt:cartesian_point  | distance:double
+"POINT(1 0)"   | POINT(1 0)          | 1.0
+"POINT(-1 0)"  | POINT(-1 0)         | 1.0
+"POINT(0 1)"   | POINT(0 1)          | 1.0
+"POINT(0 -1)"  | POINT(0 -1)         | 1.0
+;
+
+twoCitiesPointDistanceCartesian
+required_capability: st_distance
+
+ROW p1 = TO_CARTESIANPOINT("POINT(-90.82814 29.79511)"), p2 = TO_CARTESIANPOINT("POINT(-90.79731509999999 29.8835389)")
+| EVAL d = ST_DISTANCE(p1, p2)
+;
+
+p1:cartesian_point          | p2:cartesian_point                     | d:double
+POINT (-90.82814 29.79511)  | POINT (-90.79731509999999 29.8835389)  | 0.09364744959271905
+;
+
 airportCartesianCityLocationPointDistance
 required_capability: st_distance
 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec
index 3740d05ad97a2..3be846630d5b8 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec
@@ -67,12 +67,173 @@ required_capability: agg_max_min_ip_support
 from hosts
 | eval x = ip0
 | where host > "alpha"
-| stats max(ip0), a = max(ip0), b = max(x), c = max(case(host == "beta", ip0, ip1)) by host;
+| stats max(ip0), a = max(ip0), b = max(x), c = max(case(host == "beta", ip0, ip1)) by host
+| sort host asc;
 
 max(ip0):ip | a:ip | b:ip | c:ip | host:keyword
 127.0.0.1 | 127.0.0.1 | 127.0.0.1 | 127.0.0.1 | beta
-fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | gamma
 fe82::cae2:65ff:fece:fec0 | fe82::cae2:65ff:fece:fec0 | fe82::cae2:65ff:fece:fec0 | fe82::cae2:65ff:fece:fec0 | epsilon
+fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | gamma
+;
+
+maxOfVersion
+required_capability: agg_max_min_string_support
+from apps
+| eval x = version
+| where id > 2
+| stats max(version), a = max(version), b = max(x), c = max(case(name == "iiiii", "100.0.0"::version, version));
+
+max(version):version | a:version | b:version | c:version
+bad                  | bad       | bad       | 100.0.0
+;
+
+maxOfVersionGrouping
+required_capability: agg_max_min_string_support
+from apps
+| eval x = version
+| where id > 2
+| stats max(version), a = max(version), b = max(x), c = max(case(name == "ccccc", "100.0.0"::version, version)) by name
+| sort name asc
+| limit 3;
+
+max(version):version | a:version | b:version | c:version | name:keyword
+1.2.3.4              | 1.2.3.4   | 1.2.3.4   | 1.2.3.4   | aaaaa
+2.3.4                | 2.3.4     | 2.3.4     | 100.0.0   | ccccc
+2.12.0               | 2.12.0    | 2.12.0    | 2.12.0    | ddddd
+;
+
+maxOfKeyword
+required_capability: agg_max_min_string_support
+from airports
+| eval x = abbrev
+| where scalerank >= 9
+| stats max(abbrev), a = max(abbrev), b = max(x), c = max(case(mv_first(type) == "small", "___"::keyword, abbrev));
+
+max(abbrev):keyword | a:keyword | b:keyword | c:keyword
+ZAH                 | ZAH       | ZAH       | ___
+;
+
+maxOfKeywordGrouping
+required_capability: agg_max_min_string_support
+from airports
+| eval x = abbrev
+| where scalerank >= 9
+| stats max(abbrev), a = max(abbrev), b = max(x), c = max(case(mv_first(type) == "small", "___"::keyword, abbrev)) by type
+| sort type asc
+| limit 4;
+
+max(abbrev):keyword | a:keyword | b:keyword | c:keyword | type:keyword
+IXC                 | IXC       | IXC       | IXC       | major
+ZAH                 | ZAH       | ZAH       | ZAH       | mid
+VIBY                | VIBY      | VIBY      | VIBY      | military
+OPQS                | OPQS      | OPQS      | ___       | small
+;
+
+maxOfText
+required_capability: agg_max_min_string_support
+from airports
+| eval x = name
+| where scalerank >= 9
+| stats max(name), a = max(name), b = max(x);
+
+max(name):text   | a:text           | b:text
+Zaporozhye Int'l | Zaporozhye Int'l | Zaporozhye Int'l
+;
+
+maxOfTextGrouping
+required_capability: agg_max_min_string_support
+from airports
+| eval x = name
+| where scalerank >= 9
+| stats max(name), a = max(name), b = max(x) by type
+| sort type asc
+| limit 4;
+
+max(name):text   | a:text           | b:text           | type:keyword
+Cheongju Int'l   | Cheongju Int'l   | Cheongju Int'l   | major
+Zaporozhye Int'l | Zaporozhye Int'l | Zaporozhye Int'l | mid
+Zaporozhye Int'l | Zaporozhye Int'l | Zaporozhye Int'l | military
+Sahnewal         | Sahnewal         | Sahnewal         | small
+;
+
+minOfVersion
+required_capability: agg_max_min_string_support
+from apps
+| eval x = version
+| where id > 2
+| stats min(version), a = min(version), b = min(x), c = min(case(name == "iiiii", "1.0"::version, version));
+
+min(version):version | a:version | b:version | c:version
+1.2.3.4              | 1.2.3.4   | 1.2.3.4   | 1.0
+;
+
+minOfVersionGrouping
+required_capability: agg_max_min_string_support
+from apps
+| eval x = version
+| where id > 2
+| stats min(version), a = min(version), b = min(x), c = min(case(name == "ccccc", "100.0.0"::version, version)) by name
+| sort name asc
+| limit 3;
+
+min(version):version | a:version | b:version | c:version | name:keyword
+1.2.3.4              | 1.2.3.4   | 1.2.3.4   | 1.2.3.4   | aaaaa
+2.3.4                | 2.3.4     | 2.3.4     | 100.0.0   | ccccc
+2.12.0               | 2.12.0    | 2.12.0    | 2.12.0    | ddddd
+;
+
+minOfKeyword
+required_capability: agg_max_min_string_support
+from airports
+| eval x = abbrev
+| where scalerank >= 9
+| stats min(abbrev), a = min(abbrev), b = min(x), c = max(case(mv_first(type) == "small", "___"::keyword, abbrev));
+
+min(abbrev):keyword | a:keyword | b:keyword | c:keyword
+AWZ                 | AWZ       | AWZ       | ___
+;
+
+minOfKeywordGrouping
+required_capability: agg_max_min_string_support
+from airports
+| eval x = abbrev
+| where scalerank >= 9
+| stats min(abbrev), a = min(abbrev), b = min(x), c = min(case(mv_first(type) == "small", "___"::keyword, abbrev)) by type
+| sort type asc
+| limit 4;
+
+min(abbrev):keyword | a:keyword | b:keyword | c:keyword | type:keyword
+CJJ                 | CJJ       | CJJ       | CJJ       | major
+AWZ                 | AWZ       | AWZ       | AWZ       | mid
+GWL                 | GWL       | GWL       | GWL       | military
+LUH                 | LUH       | LUH       | ___       | small
+;
+
+minOfText
+required_capability: agg_max_min_string_support
+from airports
+| eval x = name
+| where scalerank >= 9
+| stats min(name), a = min(name), b = min(x);
+
+min(name):text      | a:text              | b:text
+Abdul Rachman Saleh | Abdul Rachman Saleh | Abdul Rachman Saleh
+;
+
+minOfTextGrouping
+required_capability: agg_max_min_string_support
+from airports
+| eval x = name
+| where scalerank >= 9
+| stats min(name), a = min(name), b = min(x) by type
+| sort type asc
+| limit 4;
+
+min(name):text      | a:text              | b:text              | type:keyword
+Chandigarh Int'l    | Chandigarh Int'l    | Chandigarh Int'l    | major
+Abdul Rachman Saleh | Abdul Rachman Saleh | Abdul Rachman Saleh | mid
+Abdul Rachman Saleh | Abdul Rachman Saleh | Abdul Rachman Saleh | military
+Dhamial             | Dhamial             | Dhamial             | small
 ;
 
 minOfBooleanExpression
@@ -111,12 +272,13 @@ required_capability: agg_max_min_ip_support
 from hosts
 | eval x = ip0
 | where host > "alpha"
-| stats min(ip0), a = min(ip0), b = min(x), c = min(case(host == "beta", ip0, ip1)) by host;
+| stats min(ip0), a = min(ip0), b = min(x), c = min(case(host == "beta", ip0, ip1)) by host
+| sort host asc;
 
 min(ip0):ip | a:ip | b:ip | c:ip | host:keyword
 127.0.0.1 | 127.0.0.1 | 127.0.0.1 | 127.0.0.1 | beta
-fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | gamma
 fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | 127.0.0.1 | epsilon
+fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | 127.0.0.3 | gamma
 ;
 
 maxOfShort
@@ -663,6 +825,65 @@ ca:l | cx:l | l:i
 1    | 1    | null
 ;
 
+///////////////////////////////////////////////////////////////
+// Test edge case interaction with push down past a rename
+// https://github.com/elastic/elasticsearch/issues/108008
+///////////////////////////////////////////////////////////////
+
+countSameFieldWithEval
+required_capability: fixed_pushdown_past_project
+from employees | stats  b = count(gender), c = count(gender) by gender | eval b = gender | sort c asc
+;
+
+c:l | gender:s | b:s
+0   | null     | null
+33  | F        | F
+57  | M        | M
+;
+
+countSameFieldWithDissect
+required_capability: fixed_pushdown_past_project
+from employees | stats b = count(gender), c = count(gender) by gender | dissect gender "%{b}" | sort c asc
+;
+
+c:l | gender:s | b:s
+0   | null     | null
+33  | F        | F
+57  | M        | M
+;
+
+countSameFieldWithGrok
+required_capability: fixed_pushdown_past_project
+from employees | stats  b = count(gender), c = count(gender) by gender | grok gender "%{USERNAME:b}" | sort c asc
+;
+
+c:l | gender:s | b:s
+0   | null     | null
+33  | F        | F
+57  | M        | M
+;
+
+countSameFieldWithEnrich
+required_capability: fixed_pushdown_past_project
+required_capability: enrich_load
+from employees | stats  b = count(gender), c = count(gender) by gender | enrich languages_policy on gender with b = language_name | sort c asc
+;
+
+c:l | gender:s | b:s
+0   | null     | null
+33  | F        | null
+57  | M        | null
+;
+
+countSameFieldWithEnrichLimit0
+required_capability: fixed_pushdown_past_project
+from employees | stats  b = count(gender), c = count(gender) by gender | enrich languages_policy on gender with b = language_name | sort c asc | limit 0
+;
+
+c:l | gender:s | b:s
+;
+///////////////////////////////////////////////////////////////
+
 aggsWithoutStats
 from employees | stats by gender | sort gender;
 
@@ -1397,6 +1618,37 @@ m:i | o:i   | l:i | s:i
 1   | 39729 | 1   | 39729
 ;
 
+byTwoCalculatedSecondOverwrites
+FROM employees
+| STATS m = MAX(salary) by l = salary + 1, l = languages + 1
+| SORT m
+| LIMIT 5
+;
+
+ m:i  | l:i
+66817 | 6
+73578 | 3
+73717 | 2
+74572 | 5
+74970 | 4
+;
+
+byTwoCalculatedSecondOverwritesReferencingFirst
+FROM employees
+| EVAL l = languages
+| STATS m = MAX(salary) by l = l + 1, l = l + 1
+| SORT m
+| LIMIT 5
+;
+
+ m:i  | l:i
+66817 | 6
+73578 | 3
+73717 | 2
+74572 | 5
+74970 | 4
+;
+
 nestedAggsOverGroupingWithAliasAndProjection#[skip:-8.13.99,reason:supported in 8.14]
 FROM employees
 | STATS e = length(f) + 1, c = count(*) by f = first_name
@@ -1957,6 +2209,19 @@ M
 null
 ;
 
+shadowingInternalWithGroup2#[skip:-8.14.1,reason:implemented in 8.14]
+FROM employees
+| STATS x = MAX(emp_no), y = count(x) BY x = emp_no, x = gender
+| SORT x ASC
+;
+
+y:long | x:keyword
+    33 | F
+    57 | M
+     0 | null
+;
+
+
 shadowingTheGroup
 FROM employees
 | STATS gender = MAX(emp_no), gender = MIN(emp_no) BY gender
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_percentile.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_percentile.csv-spec
index db386e877b9c3..2ac7a0cf6217a 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_percentile.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_percentile.csv-spec
@@ -195,3 +195,39 @@ p80_max_salary_change:double
 12.132
 // end::docsStatsPercentileNestedExpression-result[]
 ;
+
+constantsFrom
+required_capability: fn_mv_percentile
+from employees
+| eval single = 7, mv = [1, 7, 10]
+| stats
+    eval_single = percentile(single, 50),
+    eval_mv = percentile(mv, 50),
+    constant_single = percentile(5, 50),
+    constant_mv = percentile([1, 5, 10], 50);
+
+eval_single:double | eval_mv:double | constant_single:double | constant_mv:double
+7                  | 7              | 5                      | 5
+;
+
+constantsRow
+required_capability: fn_mv_percentile
+row single=7, mv=[1, 7, 10]
+| stats
+    eval_single = percentile(single, 50),
+    eval_mv = percentile(mv, 50),
+    constant_single = percentile(5, 50),
+    constant_mv = percentile([1, 5, 10], 50);
+
+eval_single:double | eval_mv:double | constant_single:double | constant_mv:double
+7                  | 7              | 5                      | 5
+;
+
+singleConstant
+required_capability: fn_mv_percentile
+row a=0
+| stats constant_single = percentile(5, 50);
+
+constant_single:double
+5
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_top.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_top.csv-spec
index 944c45992fe5a..86f91adf506d1 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_top.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_top.csv-spec
@@ -190,3 +190,37 @@ ROW constant = true, mv = [true, false]
 constant_asc:boolean | constant_desc:boolean | mv_asc:boolean | mv_desc:boolean
 true                 | true                  | [false, true]  | [true, false]
 ;
+
+topIps
+required_capability: agg_top
+required_capability: agg_top_ip_support
+from hosts
+| eval x = ip0
+| where host > "alpha"
+| stats
+    a = TOP(ip0, 2, "desc"),
+    b = TOP(x, 2, "desc"),
+    c = TOP(case(host == "beta", ip0, ip1), 2, "desc");
+
+a:ip | b:ip | c:ip
+[fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9] | [fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9] | [fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9]
+;
+
+topIpsGrouping
+required_capability: agg_top
+required_capability: agg_top_ip_support
+from hosts
+| eval x = ip0
+| where host > "alpha"
+| stats
+    a = TOP(ip0, 2, "desc"),
+    b = TOP(x, 2, "desc"),
+    c = TOP(case(host == "beta", ip0, ip1), 2, "desc")
+    by host
+| sort host asc;
+
+a:ip | b:ip | c:ip | host:keyword
+[127.0.0.1, 127.0.0.1] | [127.0.0.1, 127.0.0.1] | [127.0.0.1, 127.0.0.1] | beta
+[fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9] | [fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9] | [fe82::cae2:65ff:fece:fec0, fe81::cae2:65ff:fece:feb9] | epsilon
+[fe80::cae2:65ff:fece:feb9, fe80::cae2:65ff:fece:feb9] | [fe80::cae2:65ff:fece:feb9, fe80::cae2:65ff:fece:feb9] | [fe81::cae2:65ff:fece:feb9, 127.0.0.3] | gamma
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec
index eaf27dca83b3e..c6a2d47a78dc9 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec
@@ -335,6 +335,27 @@ count:long  |  client_ip:ip
 2           |  172.21.2.162
 ;
 
+statsUnionAggInline
+required_capability: union_types
+required_capability: inlinestats
+
+FROM sample_data, sample_data_str
+| STATS
+    count = COUNT(CIDR_MATCH(TO_IP(client_ip), "172.21.0.0/24") OR NULL)
+  BY
+    @timestamp = DATE_TRUNC(10 minutes, @timestamp)
+| SORT count DESC, @timestamp ASC
+| LIMIT 4
+;
+
+count:long | @timestamp:date
+         2 | 2023-10-23T13:30:00.000Z
+         0 | 2023-10-23T12:10:00.000Z
+         0 | 2023-10-23T12:20:00.000Z
+         0 | 2023-10-23T13:50:00.000Z
+;
+
+
 multiIndexIpStringStatsInline2
 required_capability: union_types
 required_capability: union_types_agg_cast
@@ -384,6 +405,74 @@ count:long  |  message:keyword
 2           |  Connected to 10.1.0.3
 ;
 
+multiIndexMissingIpToString
+required_capability: union_types
+required_capability: union_types_missing_field
+
+FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
+| EVAL client_ip = TO_STRING(client_ip)
+| KEEP _index, @timestamp, client_ip, event_duration, message
+| SORT _index ASC, @timestamp DESC
+;
+
+_index:keyword         | @timestamp:date           |  client_ip:keyword  |  event_duration:long  |  message:keyword
+missing_ip_sample_data | 2023-10-23T13:55:01.543Z  |  null               |  1756467              |  Connected to 10.1.0.1
+missing_ip_sample_data | 2023-10-23T13:53:55.832Z  |  null               |  5033755              |  Connection error
+missing_ip_sample_data | 2023-10-23T13:52:55.015Z  |  null               |  8268153              |  Connection error
+missing_ip_sample_data | 2023-10-23T13:51:54.732Z  |  null               |  725448               |  Connection error
+missing_ip_sample_data | 2023-10-23T13:33:34.937Z  |  null               |  1232382              |  Disconnected
+missing_ip_sample_data | 2023-10-23T12:27:28.948Z  |  null               |  2764889              |  Connected to 10.1.0.2
+missing_ip_sample_data | 2023-10-23T12:15:03.360Z  |  null               |  3450233              |  Connected to 10.1.0.3
+sample_data            | 2023-10-23T13:55:01.543Z  |  172.21.3.15        |  1756467              |  Connected to 10.1.0.1
+sample_data            | 2023-10-23T13:53:55.832Z  |  172.21.3.15        |  5033755              |  Connection error
+sample_data            | 2023-10-23T13:52:55.015Z  |  172.21.3.15        |  8268153              |  Connection error
+sample_data            | 2023-10-23T13:51:54.732Z  |  172.21.3.15        |  725448               |  Connection error
+sample_data            | 2023-10-23T13:33:34.937Z  |  172.21.0.5         |  1232382              |  Disconnected
+sample_data            | 2023-10-23T12:27:28.948Z  |  172.21.2.113       |  2764889              |  Connected to 10.1.0.2
+sample_data            | 2023-10-23T12:15:03.360Z  |  172.21.2.162       |  3450233              |  Connected to 10.1.0.3
+sample_data_str        | 2023-10-23T13:55:01.543Z  |  172.21.3.15        |  1756467              |  Connected to 10.1.0.1
+sample_data_str        | 2023-10-23T13:53:55.832Z  |  172.21.3.15        |  5033755              |  Connection error
+sample_data_str        | 2023-10-23T13:52:55.015Z  |  172.21.3.15        |  8268153              |  Connection error
+sample_data_str        | 2023-10-23T13:51:54.732Z  |  172.21.3.15        |  725448               |  Connection error
+sample_data_str        | 2023-10-23T13:33:34.937Z  |  172.21.0.5         |  1232382              |  Disconnected
+sample_data_str        | 2023-10-23T12:27:28.948Z  |  172.21.2.113       |  2764889              |  Connected to 10.1.0.2
+sample_data_str        | 2023-10-23T12:15:03.360Z  |  172.21.2.162       |  3450233              |  Connected to 10.1.0.3
+;
+
+multiIndexMissingIpToIp
+required_capability: union_types
+required_capability: union_types_missing_field
+
+FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
+| EVAL client_ip = TO_IP(client_ip)
+| KEEP _index, @timestamp, client_ip, event_duration, message
+| SORT _index ASC, @timestamp DESC
+;
+
+_index:keyword         | @timestamp:date           |  client_ip:ip  |  event_duration:long  |  message:keyword
+missing_ip_sample_data | 2023-10-23T13:55:01.543Z  |  null          |  1756467              |  Connected to 10.1.0.1
+missing_ip_sample_data | 2023-10-23T13:53:55.832Z  |  null          |  5033755              |  Connection error
+missing_ip_sample_data | 2023-10-23T13:52:55.015Z  |  null          |  8268153              |  Connection error
+missing_ip_sample_data | 2023-10-23T13:51:54.732Z  |  null          |  725448               |  Connection error
+missing_ip_sample_data | 2023-10-23T13:33:34.937Z  |  null          |  1232382              |  Disconnected
+missing_ip_sample_data | 2023-10-23T12:27:28.948Z  |  null          |  2764889              |  Connected to 10.1.0.2
+missing_ip_sample_data | 2023-10-23T12:15:03.360Z  |  null          |  3450233              |  Connected to 10.1.0.3
+sample_data            | 2023-10-23T13:55:01.543Z  |  172.21.3.15   |  1756467              |  Connected to 10.1.0.1
+sample_data            | 2023-10-23T13:53:55.832Z  |  172.21.3.15   |  5033755              |  Connection error
+sample_data            | 2023-10-23T13:52:55.015Z  |  172.21.3.15   |  8268153              |  Connection error
+sample_data            | 2023-10-23T13:51:54.732Z  |  172.21.3.15   |  725448               |  Connection error
+sample_data            | 2023-10-23T13:33:34.937Z  |  172.21.0.5    |  1232382              |  Disconnected
+sample_data            | 2023-10-23T12:27:28.948Z  |  172.21.2.113  |  2764889              |  Connected to 10.1.0.2
+sample_data            | 2023-10-23T12:15:03.360Z  |  172.21.2.162  |  3450233              |  Connected to 10.1.0.3
+sample_data_str        | 2023-10-23T13:55:01.543Z  |  172.21.3.15   |  1756467              |  Connected to 10.1.0.1
+sample_data_str        | 2023-10-23T13:53:55.832Z  |  172.21.3.15   |  5033755              |  Connection error
+sample_data_str        | 2023-10-23T13:52:55.015Z  |  172.21.3.15   |  8268153              |  Connection error
+sample_data_str        | 2023-10-23T13:51:54.732Z  |  172.21.3.15   |  725448               |  Connection error
+sample_data_str        | 2023-10-23T13:33:34.937Z  |  172.21.0.5    |  1232382              |  Disconnected
+sample_data_str        | 2023-10-23T12:27:28.948Z  |  172.21.2.113  |  2764889              |  Connected to 10.1.0.2
+sample_data_str        | 2023-10-23T12:15:03.360Z  |  172.21.2.162  |  3450233              |  Connected to 10.1.0.3
+;
+
 multiIndexTsLong
 required_capability: union_types
 required_capability: metadata_fields
@@ -954,3 +1043,311 @@ event_duration:long | _index:keyword      | ts:date                   | ts_str:k
 8268153             | sample_data_str     | 2023-10-23T13:52:55.015Z  | 2023-10-23T13:52:55.015Z  | 1698069175015  |  172.21.3.15   |  172.21.3.15
 8268153             | sample_data_ts_long | 2023-10-23T13:52:55.015Z  | 1698069175015             | 1698069175015  |  172.21.3.15   |  172.21.3.15
 ;
+
+
+inlineStatsUnionGroup-Ignore
+required_capability: union_types
+required_capability: inlinestats
+
+FROM sample_data, sample_data_ts_long
+| INLINESTATS count = COUNT(*)
+           BY @timestamp = SUBSTRING(TO_STRING(@timestamp), 0, 7)
+| SORT client_ip ASC, @timestamp ASC
+| LIMIT 4
+;
+
+client_ip:ip | event_duration:long |    message:keyword    | @timestamp:keyword | count:long
+  172.21.0.5 |             1232382 | Disconnected          |            1698068 | 1
+  172.21.0.5 |             1232382 | Disconnected          |            2023-10 | 7
+172.21.2.113 |             2764889 | Connected to 10.1.0.2 |            1698064 | 1
+172.21.2.113 |             2764889 | Connected to 10.1.0.2 |            2023-10 | 7
+;
+
+inlineStatsUnionGroupWithEval-Ignore
+required_capability: union_types
+required_capability: inlinestats
+
+FROM sample_data, sample_data_ts_long
+| EVAL @timestamp = SUBSTRING(TO_STRING(@timestamp), 0, 7)
+| INLINESTATS count = COUNT(*) BY @timestamp
+| SORT client_ip ASC, @timestamp ASC
+| LIMIT 4
+;
+
+client_ip:ip | event_duration:long |    message:keyword    | @timestamp:keyword | count:long
+  172.21.0.5 |             1232382 | Disconnected          |            1698068 | 1
+  172.21.0.5 |             1232382 | Disconnected          |            2023-10 | 7
+172.21.2.113 |             2764889 | Connected to 10.1.0.2 |            1698064 | 1
+172.21.2.113 |             2764889 | Connected to 10.1.0.2 |            2023-10 | 7
+;
+
+inlineStatsUnionGroupTogether-Ignore
+required_capability: union_types
+required_capability: inlinestats
+
+FROM sample_data, sample_data_ts_long
+| INLINESTATS count = COUNT(*)
+           BY @timestamp = TO_STRING(TO_DATETIME(@timestamp))
+| SORT client_ip ASC, @timestamp ASC
+| LIMIT 4
+;
+
+client_ip:ip | event_duration:long |    message:keyword    |    @timestamp:keyword    | count:long
+  172.21.0.5 |             1232382 | Disconnected          | 2023-10-23T13:33:34.937Z | 2
+  172.21.0.5 |             1232382 | Disconnected          | 2023-10-23T13:33:34.937Z | 2
+172.21.2.113 |             2764889 | Connected to 10.1.0.2 | 2023-10-23T12:27:28.948Z | 2
+172.21.2.113 |             2764889 | Connected to 10.1.0.2 | 2023-10-23T12:27:28.948Z | 2
+;
+
+# Once INLINESTATS supports expressions in agg functions and groups, convert the group in the inlinestats
+
+multiIndexIndirectUseOfUnionTypesInSort
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+FROM sample_data, sample_data_ts_long
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+@timestamp:null | client_ip:ip | event_duration:long | message:keyword
+           null | 172.21.0.5   |             1232382 | Disconnected
+;
+
+multiIndexIndirectUseOfUnionTypesInEval
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+FROM sample_data, sample_data_ts_long
+| EVAL foo = event_duration > 1232381
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:boolean
+           null | 172.21.0.5   |             1232382 | Disconnected    | true
+;
+
+multiIndexIndirectUseOfUnionTypesInRename
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+required_capability: union_types_fix_rename_resolution
+FROM sample_data, sample_data_ts_long
+| RENAME message AS event_message
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+@timestamp:null | client_ip:ip | event_duration:long | event_message:keyword
+           null | 172.21.0.5   |             1232382 | Disconnected
+;
+
+multiIndexIndirectUseOfUnionTypesInKeep
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+FROM sample_data, sample_data_ts_long
+| KEEP client_ip, event_duration, message
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+client_ip:ip | event_duration:long | message:keyword
+172.21.0.5   |             1232382 | Disconnected
+;
+
+multiIndexIndirectUseOfUnionTypesInWildcardKeep
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+required_capability: union_types_fix_rename_resolution
+FROM sample_data, sample_data_ts_long
+| KEEP *
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+@timestamp:null | client_ip:ip | event_duration:long | message:keyword
+           null | 172.21.0.5   |             1232382 | Disconnected
+;
+
+multiIndexIndirectUseOfUnionTypesInWildcardKeep2
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+required_capability: union_types_fix_rename_resolution
+FROM sample_data, sample_data_ts_long
+| KEEP *e*
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+@timestamp:null | client_ip:ip | event_duration:long | message:keyword
+           null | 172.21.0.5   |             1232382 | Disconnected
+;
+
+
+multiIndexUseOfUnionTypesInKeep
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+required_capability: union_types_fix_rename_resolution
+FROM sample_data, sample_data_ts_long
+| KEEP @timestamp
+| LIMIT 1
+;
+
+@timestamp:null
+null
+;
+
+multiIndexUseOfUnionTypesInDrop
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+required_capability: union_types_fix_rename_resolution
+FROM sample_data, sample_data_ts_long
+| DROP @timestamp
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+client_ip:ip | event_duration:long | message:keyword
+172.21.0.5   |             1232382 | Disconnected
+;
+
+multiIndexIndirectUseOfUnionTypesInWildcardDrop
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+required_capability: union_types_fix_rename_resolution
+FROM sample_data, sample_data_ts_long
+| DROP *time*
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+client_ip:ip | event_duration:long | message:keyword
+172.21.0.5   |             1232382 | Disconnected
+;
+
+multiIndexIndirectUseOfUnionTypesInWhere
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+FROM sample_data, sample_data_ts_long
+| WHERE message == "Disconnected"
+;
+
+@timestamp:null | client_ip:ip | event_duration:long | message:keyword
+           null | 172.21.0.5   |             1232382 | Disconnected
+           null | 172.21.0.5   |             1232382 | Disconnected
+;
+
+multiIndexIndirectUseOfUnionTypesInDissect
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+FROM sample_data, sample_data_ts_long
+| DISSECT message "%{foo}"
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:keyword
+           null | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
+;
+
+multiIndexIndirectUseOfUnionTypesInGrok
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+FROM sample_data, sample_data_ts_long
+| GROK message "%{WORD:foo}"
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:keyword
+           null | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
+;
+
+multiIndexIndirectUseOfUnionTypesInEnrich
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+required_capability: enrich_load
+FROM sample_data, sample_data_ts_long
+| EVAL client_ip = client_ip::keyword
+| ENRICH clientip_policy ON client_ip WITH env
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+@timestamp:null | event_duration:long | message:keyword | client_ip:keyword | env:keyword
+           null |             1232382 | Disconnected    | 172.21.0.5        | Development
+;
+
+multiIndexIndirectUseOfUnionTypesInStats
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+FROM sample_data, sample_data_ts_long
+| STATS foo = max(event_duration) BY client_ip
+| SORT client_ip ASC
+;
+
+foo:long | client_ip:ip
+ 1232382 | 172.21.0.5
+ 2764889 | 172.21.2.113
+ 3450233 | 172.21.2.162
+ 8268153 | 172.21.3.15
+;
+
+multiIndexIndirectUseOfUnionTypesInInlineStats
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+required_capability: inlinestats
+FROM sample_data, sample_data_ts_long
+| INLINESTATS foo = max(event_duration)
+| SORT client_ip ASC
+| LIMIT 1
+;
+
+@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:long
+           null | 172.21.0.5   |             1232382 | Disconnected    |  8268153
+;
+
+multiIndexIndirectUseOfUnionTypesInLookup
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+required_capability: lookup_v4
+FROM sample_data, sample_data_ts_long
+| SORT client_ip ASC
+| LIMIT 1
+| EVAL int = (event_duration - 1232380)::integer
+| LOOKUP int_number_names ON int
+;
+
+@timestamp:null | client_ip:ip | event_duration:long | message:keyword | int:integer | name:keyword
+           null | 172.21.0.5   |             1232382 | Disconnected    |           2 | two
+;
+
+multiIndexIndirectUseOfUnionTypesInMvExpand
+// TODO: `union_types` is required only because this makes the test skip in the csv tests; better solution:
+// make the csv tests work with multiple indices.
+required_capability: union_types
+FROM sample_data, sample_data_ts_long
+| EVAL foo = MV_APPEND(message, message)
+| SORT client_ip ASC
+| LIMIT 1
+| MV_EXPAND foo
+;
+
+@timestamp:null | client_ip:ip | event_duration:long | message:keyword | foo:keyword
+           null | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
+           null | 172.21.0.5   |             1232382 | Disconnected    | Disconnected
+;
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java
index 7a5072120e5af..8de65847c3f85 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractPausableIntegTestCase.java
@@ -82,12 +82,7 @@ public void setupIndex() throws IOException {
             mapping.endObject();
         }
         mapping.endObject();
-        client().admin()
-            .indices()
-            .prepareCreate("test")
-            .setSettings(Map.of("number_of_shards", 1, "number_of_replicas", 0))
-            .setMapping(mapping.endObject())
-            .get();
+        client().admin().indices().prepareCreate("test").setSettings(indexSettings(1, 0)).setMapping(mapping.endObject()).get();
 
         BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
         for (int i = 0; i < numberOfDocs(); i++) {
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
index 147e62f7ee3bc..3b6cbfb7526a8 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
@@ -12,10 +12,10 @@
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.ingest.common.IngestCommonPlugin;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.Plugin;
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java
index cdfa6eb2d03f3..dab99a0f719dd 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java
@@ -7,16 +7,17 @@
 
 package org.elasticsearch.xpack.esql.action;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.operator.DriverProfile;
 import org.elasticsearch.compute.operator.DriverStatus;
@@ -24,12 +25,16 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.ingest.common.IngestCommonPlugin;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse;
 import org.elasticsearch.reindex.ReindexPlugin;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.RemoteTransportException;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
@@ -43,6 +48,7 @@
 import org.elasticsearch.xpack.enrich.EnrichPlugin;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.enrich.EnrichLookupService;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.junit.After;
@@ -82,6 +88,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         plugins.add(IngestCommonPlugin.class);
         plugins.add(ReindexPlugin.class);
         plugins.add(InternalTransportSettingPlugin.class);
+        plugins.add(MockTransportService.TestPlugin.class);
         return plugins;
     }
 
@@ -420,6 +427,24 @@ public void testManyDocuments() {
         }
     }
 
+    public void testRejection() {
+        for (var ts : internalCluster().getInstances(TransportService.class)) {
+            ((MockTransportService) ts).addRequestHandlingBehavior(EnrichLookupService.LOOKUP_ACTION_NAME, (h, r, channel, t) -> {
+                EsRejectedExecutionException ex = new EsRejectedExecutionException("test", false);
+                channel.sendResponse(new RemoteTransportException("test", ex));
+            });
+        }
+        try {
+            String query = "FROM listen* | " + enrichSongCommand();
+            Exception error = expectThrows(Exception.class, () -> run(query).close());
+            assertThat(ExceptionsHelper.status(error), equalTo(RestStatus.TOO_MANY_REQUESTS));
+        } finally {
+            for (var ts : internalCluster().getInstances(TransportService.class)) {
+                ((MockTransportService) ts).clearAllRules();
+            }
+        }
+    }
+
     public static class LocalStateEnrich extends LocalStateCompositeXPackPlugin {
 
         public LocalStateEnrich(final Settings settings, final Path configPath) throws Exception {
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java
index fb598cb855013..1ce92ded8acc6 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java
@@ -8,14 +8,24 @@
 package org.elasticsearch.xpack.esql.action;
 
 import org.apache.lucene.tests.util.LuceneTestCase;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.compute.operator.exchange.ExchangeService;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.search.SearchService;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.RemoteTransportException;
+import org.elasticsearch.transport.TransportChannel;
+import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
 import org.hamcrest.Matchers;
 import org.junit.Before;
@@ -27,6 +37,10 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 
 /**
  * Make sures that we can run many concurrent requests with large number of shards with any data_partitioning.
@@ -38,6 +52,7 @@ public class ManyShardsIT extends AbstractEsqlIntegTestCase {
     protected Collection<Class<? extends Plugin>> getMockPlugins() {
         var plugins = new ArrayList<>(super.getMockPlugins());
         plugins.add(MockSearchService.TestPlugin.class);
+        plugins.add(MockTransportService.TestPlugin.class);
         return plugins;
     }
 
@@ -97,6 +112,51 @@ public void testConcurrentQueries() throws Exception {
         }
     }
 
+    public void testRejection() throws Exception {
+        String[] nodes = internalCluster().getNodeNames();
+        for (String node : nodes) {
+            MockTransportService ts = (MockTransportService) internalCluster().getInstance(TransportService.class, node);
+            ts.addRequestHandlingBehavior(ExchangeService.EXCHANGE_ACTION_NAME, (handler, request, channel, task) -> {
+                handler.messageReceived(request, new TransportChannel() {
+                    @Override
+                    public String getProfileName() {
+                        return channel.getProfileName();
+                    }
+
+                    @Override
+                    public void sendResponse(TransportResponse response) {
+                        channel.sendResponse(new RemoteTransportException("simulated", new EsRejectedExecutionException("test queue")));
+                    }
+
+                    @Override
+                    public void sendResponse(Exception exception) {
+                        channel.sendResponse(exception);
+                    }
+                }, task);
+            });
+        }
+        try {
+            AtomicReference<Exception> failure = new AtomicReference<>();
+            EsqlQueryRequest request = new EsqlQueryRequest();
+            request.query("from test-* | stats count(user) by tags");
+            request.acceptedPragmaRisks(true);
+            request.pragmas(randomPragmas());
+            CountDownLatch queryLatch = new CountDownLatch(1);
+            client().execute(EsqlQueryAction.INSTANCE, request, ActionListener.runAfter(ActionListener.wrap(r -> {
+                r.close();
+                throw new AssertionError("expected failure");
+            }, failure::set), queryLatch::countDown));
+            assertTrue(queryLatch.await(10, TimeUnit.SECONDS));
+            assertThat(failure.get(), instanceOf(EsRejectedExecutionException.class));
+            assertThat(ExceptionsHelper.status(failure.get()), equalTo(RestStatus.TOO_MANY_REQUESTS));
+            assertThat(failure.get().getMessage(), equalTo("test queue"));
+        } finally {
+            for (String node : nodes) {
+                ((MockTransportService) internalCluster().getInstance(TransportService.class, node)).clearAllRules();
+            }
+        }
+    }
+
     static class SearchContextCounter {
         private final int maxAllowed;
         private final AtomicInteger current = new AtomicInteger();
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java
new file mode 100644
index 0000000000000..ab21a2bb8b995
--- /dev/null
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java
@@ -0,0 +1,233 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plugin;
+
+import org.elasticsearch.Build;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.esql.VerificationException;
+import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
+import org.elasticsearch.xpack.esql.action.ColumnInfoImpl;
+import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
+import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.junit.Before;
+
+import java.util.List;
+
+import static org.elasticsearch.test.ListMatcher.matchesList;
+import static org.elasticsearch.test.MapMatcher.assertMap;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+//@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug")
+public class MatchOperatorIT extends AbstractEsqlIntegTestCase {
+
+    @Before
+    public void setupIndex() {
+        createAndPopulateIndex();
+    }
+
+    @Override
+    protected EsqlQueryResponse run(EsqlQueryRequest request) {
+        assumeTrue("match operator available in snapshot builds only", Build.current().isSnapshot());
+        return super.run(request);
+    }
+
+    public void testSimpleWhereMatch() {
+        var query = """
+            FROM test
+            | WHERE content MATCH "fox"
+            | KEEP id
+            | SORT id
+            """;
+
+        try (var resp = run(query)) {
+            assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of("id")));
+            assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of("INTEGER")));
+            // values
+            List<List<Object>> values = getValuesList(resp);
+            assertMap(values, matchesList().item(List.of(1)).item(List.of(6)));
+        }
+    }
+
+    public void testCombinedWhereMatch() {
+        var query = """
+            FROM test
+            | WHERE content MATCH "fox" AND id > 5
+            | KEEP id
+            | SORT id
+            """;
+
+        try (var resp = run(query)) {
+            assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of(("id"))));
+            assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of(("INTEGER"))));
+            // values
+            List<List<Object>> values = getValuesList(resp);
+            assertMap(values, matchesList().item(List.of(6)));
+        }
+    }
+
+    public void testMultipleMatch() {
+        var query = """
+            FROM test
+            | WHERE content MATCH "fox" OR content MATCH "brown"
+            | KEEP id
+            | SORT id
+            """;
+
+        try (var resp = run(query)) {
+            assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of(("id"))));
+            assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of(("INTEGER"))));
+            // values
+            List<List<Object>> values = getValuesList(resp);
+            assertThat(values.size(), equalTo(5));
+            assertMap(values, matchesList().item(List.of(1)).item(List.of(2)).item(List.of(3)).item(List.of(4)).item(List.of(6)));
+        }
+    }
+
+    public void testMultipleWhereMatch() {
+        var query = """
+            FROM test
+            | WHERE content MATCH "fox" OR content MATCH "brown"
+            | EVAL summary = CONCAT("document with id: ", to_str(id), "and content: ", content)
+            | SORT summary
+            | LIMIT 4
+            | WHERE content MATCH "brown fox"
+            | KEEP id
+            """;
+
+        // TODO: this should not raise an error;
+        var error = expectThrows(ElasticsearchException.class, () -> run(query));
+        assertThat(error.getMessage(), containsString("Unsupported expression [content MATCH \"brown fox\"]"));
+    }
+
+    public void testNotWhereMatch() {
+        var query = """
+            FROM test
+            | WHERE NOT content MATCH "brown fox"
+            | KEEP id
+            | SORT id
+            """;
+
+        try (var resp = run(query)) {
+            assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of(("id"))));
+            assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of(("INTEGER"))));
+            // values
+            List<List<Object>> values = getValuesList(resp);
+            assertMap(values, matchesList().item(List.of(5)));
+        }
+    }
+
+    public void testNonExistingColumn() {
+        var query = """
+            FROM test
+            | WHERE something MATCH "fox"
+            """;
+
+        var error = expectThrows(VerificationException.class, () -> run(query));
+        assertThat(error.getMessage(), containsString("Unknown column [something]"));
+    }
+
+    public void testWhereMatchEvalColumn() {
+        var query = """
+            FROM test
+            | EVAL upper_content = to_upper(content)
+            | WHERE upper_content MATCH "FOX"
+            | KEEP id
+            """;
+
+        var error = expectThrows(VerificationException.class, () -> run(query));
+        assertThat(error.getMessage(), containsString("MATCH requires a mapped index field, found [upper_content]"));
+    }
+
+    public void testWhereMatchOverWrittenColumn() {
+        var query = """
+            FROM test
+            | DROP content
+            | EVAL content = CONCAT("document with ID ", to_str(id))
+            | WHERE content MATCH "document"
+            """;
+
+        var error = expectThrows(VerificationException.class, () -> run(query));
+        assertThat(error.getMessage(), containsString("MATCH requires a mapped index field, found [content]"));
+    }
+
+    public void testWhereMatchAfterStats() {
+        var query = """
+            FROM test
+            | STATS count(*)
+            | WHERE content match "fox"
+            """;
+
+        var error = expectThrows(VerificationException.class, () -> run(query));
+        assertThat(error.getMessage(), containsString("Unknown column [content]"));
+    }
+
+    public void testWhereMatchWithFunctions() {
+        var query = """
+            FROM test
+            | WHERE content MATCH "fox" OR to_upper(content) == "FOX"
+            """;
+        var error = expectThrows(ElasticsearchException.class, () -> run(query));
+        assertThat(error.getMessage(), containsString(" Invalid condition using MATCH"));
+    }
+
+    public void testWhereMatchWithRow() {
+        var query = """
+            ROW content = "a brown fox"
+            | WHERE content MATCH "fox"
+            """;
+
+        var error = expectThrows(ElasticsearchException.class, () -> run(query));
+        assertThat(error.getMessage(), containsString("MATCH requires a mapped index field, found [content]"));
+    }
+
+    public void testMatchWithinEval() {
+        var query = """
+            FROM test
+            | EVAL matches_query = content MATCH "fox"
+            """;
+
+        var error = expectThrows(VerificationException.class, () -> run(query));
+        assertThat(error.getMessage(), containsString("EVAL does not support MATCH expressions"));
+    }
+
+    public void testMatchWithNonTextField() {
+        var query = """
+            FROM test
+            | WHERE id MATCH "fox"
+            """;
+
+        var error = expectThrows(VerificationException.class, () -> run(query));
+        assertThat(error.getMessage(), containsString(" MATCH requires a text or keyword field, but [id] has type [integer]"));
+    }
+
+    private void createAndPopulateIndex() {
+        var indexName = "test";
+        var client = client().admin().indices();
+        var CreateRequest = client.prepareCreate(indexName)
+            .setSettings(Settings.builder().put("index.number_of_shards", 1))
+            .setMapping("id", "type=integer", "content", "type=text");
+        assertAcked(CreateRequest);
+        client().prepareBulk()
+            .add(new IndexRequest(indexName).id("1").source("id", 1, "content", "This is a brown fox"))
+            .add(new IndexRequest(indexName).id("2").source("id", 2, "content", "This is a brown dog"))
+            .add(new IndexRequest(indexName).id("3").source("id", 3, "content", "This dog is really brown"))
+            .add(new IndexRequest(indexName).id("4").source("id", 4, "content", "The dog is brown but this document is very very long"))
+            .add(new IndexRequest(indexName).id("5").source("id", 5, "content", "There is also a white cat"))
+            .add(new IndexRequest(indexName).id("6").source("id", 6, "content", "The quick brown fox jumps over the lazy dog"))
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+        ensureYellow(indexName);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4
index 25b5cae1acdd0..6570a25469971 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4
@@ -1,5 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
 lexer grammar EsqlBaseLexer;
 
+@header {
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+}
+
+options {
+  superClass=LexerConfig;
+  caseInsensitive=true;
+}
+
+/*
+ * Before modifying this file, please read the section above as changes here
+ * have significant impact in the ANTLR generated code and its consumption upstream
+ * (including Kibana).
+ *
+ * A. To add a development token (only available behind in snapshot/dev builds)
+ *
+ * Since the tokens/modes are in development, simply define them under the
+ * "// in development section" and follow the section comments in that section.
+ * That is use the DEV_ prefix and use the {this.isDevVersion()}? conditional.
+ * They are defined at the end of the file, to minimize the impact on the existing
+ * token types.
+ *
+ * B. To add a new (production-ready) token
+ *
+ * Be sure to go through step A (add a development token).
+ * Make sure to remove the prefix and conditional before promoting the tokens in
+ * production.
+ * Since tokens types (numbers) are generated by ANTLR in a continuous fashion,
+ * it is desirable to avoid changing these values hence where possible, add
+ * add them at the end of their respective section.
+ * Note that the use of lexing modes prevents this since any addition to a mode
+ * (regardless where it occurs) shifts all the declarations that follow in other modes.
+ *
+ * C. Renaming a token
+ *
+ * Avoid renaming the token. But if you really have to, please check with the
+ * Kibana team as they might be using the generated ANTLR "dictionary".
+ *
+ * D. To remove a token
+ *
+ * If the tokens haven't made it to production (and make sure to double check),
+ * simply remove them from the grammar.
+ * If the tokens get promoted to release, check with the Kibana team the impact
+ * they have on the UI (auto-completion, etc...)
+ */
+
 DISSECT : 'dissect'           -> pushMode(EXPRESSION_MODE);
 DROP : 'drop'                 -> pushMode(PROJECT_MODE);
 ENRICH : 'enrich'             -> pushMode(ENRICH_MODE);
@@ -7,12 +64,9 @@ EVAL : 'eval'                 -> pushMode(EXPRESSION_MODE);
 EXPLAIN : 'explain'           -> pushMode(EXPLAIN_MODE);
 FROM : 'from'                 -> pushMode(FROM_MODE);
 GROK : 'grok'                 -> pushMode(EXPRESSION_MODE);
-INLINESTATS : 'inlinestats'   -> pushMode(EXPRESSION_MODE);
 KEEP : 'keep'                 -> pushMode(PROJECT_MODE);
 LIMIT : 'limit'               -> pushMode(EXPRESSION_MODE);
-LOOKUP : 'lookup'             -> pushMode(LOOKUP_MODE);
 META : 'meta'                 -> pushMode(META_MODE);
-METRICS : 'metrics'           -> pushMode(METRICS_MODE);
 MV_EXPAND : 'mv_expand'       -> pushMode(MVEXPAND_MODE);
 RENAME : 'rename'             -> pushMode(RENAME_MODE);
 ROW : 'row'                   -> pushMode(EXPRESSION_MODE);
@@ -20,7 +74,26 @@ SHOW : 'show'                 -> pushMode(SHOW_MODE);
 SORT : 'sort'                 -> pushMode(EXPRESSION_MODE);
 STATS : 'stats'               -> pushMode(EXPRESSION_MODE);
 WHERE : 'where'               -> pushMode(EXPRESSION_MODE);
-UNKNOWN_CMD : ~[ \r\n\t[\]/]+ -> pushMode(EXPRESSION_MODE);
+//
+// in development
+//
+// Before adding a new in-development command, to sandbox the behavior when running in production environments
+//
+// For example: to add myCommand use the following declaration:
+// DEV_MYCOMMAND : {this.isDevVersion()}? 'mycommand' -> ...
+//
+// Once the command has been stabilized, remove the DEV_ prefix and the {}? conditional and move the command to the
+// main section while preserving alphabetical order:
+// MYCOMMAND : 'mycommand' -> ...
+DEV_INLINESTATS : {this.isDevVersion()}? 'inlinestats'   -> pushMode(EXPRESSION_MODE);
+DEV_LOOKUP :      {this.isDevVersion()}? 'lookup'        -> pushMode(LOOKUP_MODE);
+DEV_MATCH :       {this.isDevVersion()}? 'match'         -> pushMode(EXPRESSION_MODE);
+DEV_METRICS :     {this.isDevVersion()}? 'metrics'       -> pushMode(METRICS_MODE);
+
+//
+// Catch-all for unrecognized commands - don't define any beyond this line
+//
+UNKNOWN_CMD : ~[ \r\n\t[\]/]+ -> pushMode(EXPRESSION_MODE) ;
 
 LINE_COMMENT
     : '//' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN)
@@ -34,27 +107,6 @@ WS
     : [ \r\n\t]+ -> channel(HIDDEN)
     ;
 
-// in 8.14 ` were not allowed
-// this has been relaxed in 8.15 since " is used for quoting
-fragment UNQUOTED_SOURCE_PART
-    : ~[:"=|,[\]/ \t\r\n]
-    | '/' ~[*/] // allow single / but not followed by another / or * which would start a comment -- used in index pattern date spec
-    ;
-
-UNQUOTED_SOURCE
-    : UNQUOTED_SOURCE_PART+
-    ;
-
-//
-// Explain
-//
-mode EXPLAIN_MODE;
-EXPLAIN_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE);
-EXPLAIN_PIPE : PIPE -> type(PIPE), popMode;
-EXPLAIN_WS : WS -> channel(HIDDEN);
-EXPLAIN_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN);
-EXPLAIN_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN);
-
 //
 // Expression - used by most command
 //
@@ -67,7 +119,7 @@ fragment DIGIT
     ;
 
 fragment LETTER
-    : [A-Za-z]
+    : [a-z]
     ;
 
 fragment ESCAPE_SEQUENCE
@@ -79,7 +131,7 @@ fragment UNESCAPED_CHARS
     ;
 
 fragment EXPONENT
-    : [Ee] [+-]? DIGIT+
+    : [e] [+-]? DIGIT+
     ;
 
 fragment ASPERAND
@@ -130,11 +182,11 @@ DESC : 'desc';
 DOT : '.';
 FALSE : 'false';
 FIRST : 'first';
-LAST : 'last';
-LP : '(';
 IN: 'in';
 IS: 'is';
+LAST : 'last';
 LIKE: 'like';
+LP : '(';
 NOT : 'not';
 NULL : 'null';
 NULLS : 'nulls';
@@ -158,8 +210,11 @@ ASTERISK : '*';
 SLASH : '/';
 PERCENT : '%';
 
+// move it in the main section if the feature gets promoted
+DEV_MATCH_OP : {this.isDevVersion()}? DEV_MATCH -> type(DEV_MATCH);
+
 NAMED_OR_POSITIONAL_PARAM
-    : PARAM LETTER UNQUOTED_ID_BODY*
+    : PARAM (LETTER | UNDERSCORE) UNQUOTED_ID_BODY*
     | PARAM DIGIT+
     ;
 
@@ -197,6 +252,17 @@ EXPR_MULTILINE_COMMENT
 EXPR_WS
     : WS -> channel(HIDDEN)
     ;
+
+//
+// Explain
+//
+mode EXPLAIN_MODE;
+EXPLAIN_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE);
+EXPLAIN_PIPE : PIPE -> type(PIPE), popMode;
+EXPLAIN_WS : WS -> channel(HIDDEN);
+EXPLAIN_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN);
+EXPLAIN_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN);
+
 //
 // FROM command
 //
@@ -209,6 +275,17 @@ FROM_COMMA : COMMA -> type(COMMA);
 FROM_ASSIGN : ASSIGN -> type(ASSIGN);
 METADATA : 'metadata';
 
+// in 8.14 ` were not allowed
+// this has been relaxed in 8.15 since " is used for quoting
+fragment UNQUOTED_SOURCE_PART
+    : ~[:"=|,[\]/ \t\r\n]
+    | '/' ~[*/] // allow single / but not followed by another / or * which would start a comment -- used in index pattern date spec
+    ;
+
+UNQUOTED_SOURCE
+    : UNQUOTED_SOURCE_PART+
+    ;
+
 FROM_UNQUOTED_SOURCE : UNQUOTED_SOURCE -> type(UNQUOTED_SOURCE);
 FROM_QUOTED_SOURCE : QUOTED_STRING -> type(QUOTED_STRING);
 
@@ -346,50 +423,6 @@ ENRICH_FIELD_WS
     : WS -> channel(HIDDEN)
     ;
 
-// LOOKUP ON key
-mode LOOKUP_MODE;
-LOOKUP_PIPE : PIPE -> type(PIPE), popMode;
-LOOKUP_COLON : COLON -> type(COLON);
-LOOKUP_COMMA : COMMA -> type(COMMA);
-LOOKUP_DOT: DOT -> type(DOT);
-LOOKUP_ON : ON -> type(ON), pushMode(LOOKUP_FIELD_MODE);
-
-LOOKUP_UNQUOTED_SOURCE: UNQUOTED_SOURCE -> type(UNQUOTED_SOURCE);
-LOOKUP_QUOTED_SOURCE : QUOTED_STRING -> type(QUOTED_STRING);
-
-LOOKUP_LINE_COMMENT
-    : LINE_COMMENT -> channel(HIDDEN)
-    ;
-
-LOOKUP_MULTILINE_COMMENT
-    : MULTILINE_COMMENT -> channel(HIDDEN)
-    ;
-
-LOOKUP_WS
-    : WS -> channel(HIDDEN)
-    ;
-
-mode LOOKUP_FIELD_MODE;
-LOOKUP_FIELD_PIPE : PIPE -> type(PIPE), popMode, popMode;
-LOOKUP_FIELD_COMMA : COMMA -> type(COMMA);
-LOOKUP_FIELD_DOT: DOT -> type(DOT);
-
-LOOKUP_FIELD_ID_PATTERN
-    : ID_PATTERN -> type(ID_PATTERN)
-    ;
-
-LOOKUP_FIELD_LINE_COMMENT
-    : LINE_COMMENT -> channel(HIDDEN)
-    ;
-
-LOOKUP_FIELD_MULTILINE_COMMENT
-    : MULTILINE_COMMENT -> channel(HIDDEN)
-    ;
-
-LOOKUP_FIELD_WS
-    : WS -> channel(HIDDEN)
-    ;
-
 mode MVEXPAND_MODE;
 MVEXPAND_PIPE : PIPE -> type(PIPE), popMode;
 MVEXPAND_DOT: DOT -> type(DOT);
@@ -475,6 +508,51 @@ SETTING_WS
     : WS -> channel(HIDDEN)
     ;
 
+//
+// LOOKUP ON key
+//
+mode LOOKUP_MODE;
+LOOKUP_PIPE : PIPE -> type(PIPE), popMode;
+LOOKUP_COLON : COLON -> type(COLON);
+LOOKUP_COMMA : COMMA -> type(COMMA);
+LOOKUP_DOT: DOT -> type(DOT);
+LOOKUP_ON : ON -> type(ON), pushMode(LOOKUP_FIELD_MODE);
+
+LOOKUP_UNQUOTED_SOURCE: UNQUOTED_SOURCE -> type(UNQUOTED_SOURCE);
+LOOKUP_QUOTED_SOURCE : QUOTED_STRING -> type(QUOTED_STRING);
+
+LOOKUP_LINE_COMMENT
+    : LINE_COMMENT -> channel(HIDDEN)
+    ;
+
+LOOKUP_MULTILINE_COMMENT
+    : MULTILINE_COMMENT -> channel(HIDDEN)
+    ;
+
+LOOKUP_WS
+    : WS -> channel(HIDDEN)
+    ;
+
+mode LOOKUP_FIELD_MODE;
+LOOKUP_FIELD_PIPE : PIPE -> type(PIPE), popMode, popMode;
+LOOKUP_FIELD_COMMA : COMMA -> type(COMMA);
+LOOKUP_FIELD_DOT: DOT -> type(DOT);
+
+LOOKUP_FIELD_ID_PATTERN
+    : ID_PATTERN -> type(ID_PATTERN)
+    ;
+
+LOOKUP_FIELD_LINE_COMMENT
+    : LINE_COMMENT -> channel(HIDDEN)
+    ;
+
+LOOKUP_FIELD_MULTILINE_COMMENT
+    : MULTILINE_COMMENT -> channel(HIDDEN)
+    ;
+
+LOOKUP_FIELD_WS
+    : WS -> channel(HIDDEN)
+    ;
 
 //
 // METRICS command
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
index 63eb3a86419a3..747fbbc64cf5f 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
@@ -5,123 +5,124 @@ EVAL=4
 EXPLAIN=5
 FROM=6
 GROK=7
-INLINESTATS=8
-KEEP=9
-LIMIT=10
-LOOKUP=11
-META=12
-METRICS=13
-MV_EXPAND=14
-RENAME=15
-ROW=16
-SHOW=17
-SORT=18
-STATS=19
-WHERE=20
-UNKNOWN_CMD=21
-LINE_COMMENT=22
-MULTILINE_COMMENT=23
-WS=24
-UNQUOTED_SOURCE=25
-EXPLAIN_WS=26
-EXPLAIN_LINE_COMMENT=27
-EXPLAIN_MULTILINE_COMMENT=28
-PIPE=29
-QUOTED_STRING=30
-INTEGER_LITERAL=31
-DECIMAL_LITERAL=32
-BY=33
-AND=34
-ASC=35
-ASSIGN=36
-CAST_OP=37
-COMMA=38
-DESC=39
-DOT=40
-FALSE=41
-FIRST=42
-LAST=43
+KEEP=8
+LIMIT=9
+META=10
+MV_EXPAND=11
+RENAME=12
+ROW=13
+SHOW=14
+SORT=15
+STATS=16
+WHERE=17
+DEV_INLINESTATS=18
+DEV_LOOKUP=19
+DEV_MATCH=20
+DEV_METRICS=21
+UNKNOWN_CMD=22
+LINE_COMMENT=23
+MULTILINE_COMMENT=24
+WS=25
+PIPE=26
+QUOTED_STRING=27
+INTEGER_LITERAL=28
+DECIMAL_LITERAL=29
+BY=30
+AND=31
+ASC=32
+ASSIGN=33
+CAST_OP=34
+COMMA=35
+DESC=36
+DOT=37
+FALSE=38
+FIRST=39
+IN=40
+IS=41
+LAST=42
+LIKE=43
 LP=44
-IN=45
-IS=46
-LIKE=47
-NOT=48
-NULL=49
-NULLS=50
-OR=51
-PARAM=52
-RLIKE=53
-RP=54
-TRUE=55
-EQ=56
-CIEQ=57
-NEQ=58
-LT=59
-LTE=60
-GT=61
-GTE=62
-PLUS=63
-MINUS=64
-ASTERISK=65
-SLASH=66
-PERCENT=67
-NAMED_OR_POSITIONAL_PARAM=68
-OPENING_BRACKET=69
-CLOSING_BRACKET=70
-UNQUOTED_IDENTIFIER=71
-QUOTED_IDENTIFIER=72
-EXPR_LINE_COMMENT=73
-EXPR_MULTILINE_COMMENT=74
-EXPR_WS=75
+NOT=45
+NULL=46
+NULLS=47
+OR=48
+PARAM=49
+RLIKE=50
+RP=51
+TRUE=52
+EQ=53
+CIEQ=54
+NEQ=55
+LT=56
+LTE=57
+GT=58
+GTE=59
+PLUS=60
+MINUS=61
+ASTERISK=62
+SLASH=63
+PERCENT=64
+NAMED_OR_POSITIONAL_PARAM=65
+OPENING_BRACKET=66
+CLOSING_BRACKET=67
+UNQUOTED_IDENTIFIER=68
+QUOTED_IDENTIFIER=69
+EXPR_LINE_COMMENT=70
+EXPR_MULTILINE_COMMENT=71
+EXPR_WS=72
+EXPLAIN_WS=73
+EXPLAIN_LINE_COMMENT=74
+EXPLAIN_MULTILINE_COMMENT=75
 METADATA=76
-FROM_LINE_COMMENT=77
-FROM_MULTILINE_COMMENT=78
-FROM_WS=79
-ID_PATTERN=80
-PROJECT_LINE_COMMENT=81
-PROJECT_MULTILINE_COMMENT=82
-PROJECT_WS=83
-AS=84
-RENAME_LINE_COMMENT=85
-RENAME_MULTILINE_COMMENT=86
-RENAME_WS=87
-ON=88
-WITH=89
-ENRICH_POLICY_NAME=90
-ENRICH_LINE_COMMENT=91
-ENRICH_MULTILINE_COMMENT=92
-ENRICH_WS=93
-ENRICH_FIELD_LINE_COMMENT=94
-ENRICH_FIELD_MULTILINE_COMMENT=95
-ENRICH_FIELD_WS=96
-LOOKUP_LINE_COMMENT=97
-LOOKUP_MULTILINE_COMMENT=98
-LOOKUP_WS=99
-LOOKUP_FIELD_LINE_COMMENT=100
-LOOKUP_FIELD_MULTILINE_COMMENT=101
-LOOKUP_FIELD_WS=102
-MVEXPAND_LINE_COMMENT=103
-MVEXPAND_MULTILINE_COMMENT=104
-MVEXPAND_WS=105
-INFO=106
-SHOW_LINE_COMMENT=107
-SHOW_MULTILINE_COMMENT=108
-SHOW_WS=109
-FUNCTIONS=110
-META_LINE_COMMENT=111
-META_MULTILINE_COMMENT=112
-META_WS=113
-COLON=114
-SETTING=115
-SETTING_LINE_COMMENT=116
-SETTTING_MULTILINE_COMMENT=117
-SETTING_WS=118
-METRICS_LINE_COMMENT=119
-METRICS_MULTILINE_COMMENT=120
-METRICS_WS=121
-CLOSING_METRICS_LINE_COMMENT=122
-CLOSING_METRICS_MULTILINE_COMMENT=123
-CLOSING_METRICS_WS=124
+UNQUOTED_SOURCE=77
+FROM_LINE_COMMENT=78
+FROM_MULTILINE_COMMENT=79
+FROM_WS=80
+ID_PATTERN=81
+PROJECT_LINE_COMMENT=82
+PROJECT_MULTILINE_COMMENT=83
+PROJECT_WS=84
+AS=85
+RENAME_LINE_COMMENT=86
+RENAME_MULTILINE_COMMENT=87
+RENAME_WS=88
+ON=89
+WITH=90
+ENRICH_POLICY_NAME=91
+ENRICH_LINE_COMMENT=92
+ENRICH_MULTILINE_COMMENT=93
+ENRICH_WS=94
+ENRICH_FIELD_LINE_COMMENT=95
+ENRICH_FIELD_MULTILINE_COMMENT=96
+ENRICH_FIELD_WS=97
+MVEXPAND_LINE_COMMENT=98
+MVEXPAND_MULTILINE_COMMENT=99
+MVEXPAND_WS=100
+INFO=101
+SHOW_LINE_COMMENT=102
+SHOW_MULTILINE_COMMENT=103
+SHOW_WS=104
+FUNCTIONS=105
+META_LINE_COMMENT=106
+META_MULTILINE_COMMENT=107
+META_WS=108
+COLON=109
+SETTING=110
+SETTING_LINE_COMMENT=111
+SETTTING_MULTILINE_COMMENT=112
+SETTING_WS=113
+LOOKUP_LINE_COMMENT=114
+LOOKUP_MULTILINE_COMMENT=115
+LOOKUP_WS=116
+LOOKUP_FIELD_LINE_COMMENT=117
+LOOKUP_FIELD_MULTILINE_COMMENT=118
+LOOKUP_FIELD_WS=119
+METRICS_LINE_COMMENT=120
+METRICS_MULTILINE_COMMENT=121
+METRICS_WS=122
+CLOSING_METRICS_LINE_COMMENT=123
+CLOSING_METRICS_MULTILINE_COMMENT=124
+CLOSING_METRICS_WS=125
 'dissect'=1
 'drop'=2
 'enrich'=3
@@ -129,60 +130,57 @@ CLOSING_METRICS_WS=124
 'explain'=5
 'from'=6
 'grok'=7
-'inlinestats'=8
-'keep'=9
-'limit'=10
-'lookup'=11
-'meta'=12
-'metrics'=13
-'mv_expand'=14
-'rename'=15
-'row'=16
-'show'=17
-'sort'=18
-'stats'=19
-'where'=20
-'|'=29
-'by'=33
-'and'=34
-'asc'=35
-'='=36
-'::'=37
-','=38
-'desc'=39
-'.'=40
-'false'=41
-'first'=42
-'last'=43
+'keep'=8
+'limit'=9
+'meta'=10
+'mv_expand'=11
+'rename'=12
+'row'=13
+'show'=14
+'sort'=15
+'stats'=16
+'where'=17
+'|'=26
+'by'=30
+'and'=31
+'asc'=32
+'='=33
+'::'=34
+','=35
+'desc'=36
+'.'=37
+'false'=38
+'first'=39
+'in'=40
+'is'=41
+'last'=42
+'like'=43
 '('=44
-'in'=45
-'is'=46
-'like'=47
-'not'=48
-'null'=49
-'nulls'=50
-'or'=51
-'?'=52
-'rlike'=53
-')'=54
-'true'=55
-'=='=56
-'=~'=57
-'!='=58
-'<'=59
-'<='=60
-'>'=61
-'>='=62
-'+'=63
-'-'=64
-'*'=65
-'/'=66
-'%'=67
-']'=70
+'not'=45
+'null'=46
+'nulls'=47
+'or'=48
+'?'=49
+'rlike'=50
+')'=51
+'true'=52
+'=='=53
+'=~'=54
+'!='=55
+'<'=56
+'<='=57
+'>'=58
+'>='=59
+'+'=60
+'-'=61
+'*'=62
+'/'=63
+'%'=64
+']'=67
 'metadata'=76
-'as'=84
-'on'=88
-'with'=89
-'info'=106
-'functions'=110
-':'=114
+'as'=85
+'on'=89
+'with'=90
+'info'=101
+'functions'=105
+':'=109
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
index 89059822d367b..a3ef2471d4e56 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
@@ -1,14 +1,24 @@
-
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
 parser grammar EsqlBaseParser;
 
-options {tokenVocab=EsqlBaseLexer;}
+@header {
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+}
+
+options {
+  superClass=ParserConfig;
+  tokenVocab=EsqlBaseLexer;
+}
 
 singleStatement
     : query EOF
@@ -22,27 +32,30 @@ query
 sourceCommand
     : explainCommand
     | fromCommand
+    | metaCommand
     | rowCommand
-    | metricsCommand
     | showCommand
-    | metaCommand
+    // in development
+    | {this.isDevVersion()}? metricsCommand
     ;
 
 processingCommand
     : evalCommand
-    | inlinestatsCommand
-    | limitCommand
-    | lookupCommand
+    | whereCommand
     | keepCommand
-    | sortCommand
+    | limitCommand
     | statsCommand
-    | whereCommand
+    | sortCommand
     | dropCommand
     | renameCommand
     | dissectCommand
     | grokCommand
     | enrichCommand
     | mvExpandCommand
+    // in development
+    | {this.isDevVersion()}? inlinestatsCommand
+    | {this.isDevVersion()}? lookupCommand
+    | {this.isDevVersion()}? matchCommand
     ;
 
 whereCommand
@@ -57,6 +70,7 @@ booleanExpression
     | left=booleanExpression operator=OR right=booleanExpression                 #logicalBinary
     | valueExpression (NOT)? IN LP valueExpression (COMMA valueExpression)* RP   #logicalIn
     | valueExpression IS NOT? NULL                                               #isNull
+    | {this.isDevVersion()}? matchBooleanExpression                              #matchExpression
     ;
 
 regexBooleanExpression
@@ -64,6 +78,10 @@ regexBooleanExpression
     | valueExpression (NOT)? kind=RLIKE pattern=string
     ;
 
+matchBooleanExpression
+    : valueExpression DEV_MATCH queryString=string
+    ;
+
 valueExpression
     : operatorExpression                                                                      #valueExpressionDefault
     | left=operatorExpression comparisonOperator right=operatorExpression                     #comparison
@@ -137,7 +155,7 @@ deprecated_metadata
     ;
 
 metricsCommand
-    : METRICS indexPattern (COMMA indexPattern)* aggregates=fields? (BY grouping=fields)?
+    : DEV_METRICS indexPattern (COMMA indexPattern)* aggregates=fields? (BY grouping=fields)?
     ;
 
 evalCommand
@@ -148,11 +166,6 @@ statsCommand
     : STATS stats=fields? (BY grouping=fields)?
     ;
 
-inlinestatsCommand
-    : INLINESTATS stats=fields (BY grouping=fields)?
-    ;
-
-
 qualifiedName
     : identifier (DOT identifier)*
     ;
@@ -289,6 +302,21 @@ enrichWithClause
     : (newName=qualifiedNamePattern ASSIGN)? enrichField=qualifiedNamePattern
     ;
 
+//
+// In development
+//
 lookupCommand
-    : LOOKUP tableName=indexPattern ON matchFields=qualifiedNamePatterns
+    : DEV_LOOKUP tableName=indexPattern ON matchFields=qualifiedNamePatterns
+    ;
+
+inlinestatsCommand
+    : DEV_INLINESTATS stats=fields (BY grouping=fields)?
+    ;
+
+matchCommand
+    : DEV_MATCH matchQuery
+    ;
+
+matchQuery
+    : QUOTED_STRING
     ;
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
index 63eb3a86419a3..747fbbc64cf5f 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
@@ -5,123 +5,124 @@ EVAL=4
 EXPLAIN=5
 FROM=6
 GROK=7
-INLINESTATS=8
-KEEP=9
-LIMIT=10
-LOOKUP=11
-META=12
-METRICS=13
-MV_EXPAND=14
-RENAME=15
-ROW=16
-SHOW=17
-SORT=18
-STATS=19
-WHERE=20
-UNKNOWN_CMD=21
-LINE_COMMENT=22
-MULTILINE_COMMENT=23
-WS=24
-UNQUOTED_SOURCE=25
-EXPLAIN_WS=26
-EXPLAIN_LINE_COMMENT=27
-EXPLAIN_MULTILINE_COMMENT=28
-PIPE=29
-QUOTED_STRING=30
-INTEGER_LITERAL=31
-DECIMAL_LITERAL=32
-BY=33
-AND=34
-ASC=35
-ASSIGN=36
-CAST_OP=37
-COMMA=38
-DESC=39
-DOT=40
-FALSE=41
-FIRST=42
-LAST=43
+KEEP=8
+LIMIT=9
+META=10
+MV_EXPAND=11
+RENAME=12
+ROW=13
+SHOW=14
+SORT=15
+STATS=16
+WHERE=17
+DEV_INLINESTATS=18
+DEV_LOOKUP=19
+DEV_MATCH=20
+DEV_METRICS=21
+UNKNOWN_CMD=22
+LINE_COMMENT=23
+MULTILINE_COMMENT=24
+WS=25
+PIPE=26
+QUOTED_STRING=27
+INTEGER_LITERAL=28
+DECIMAL_LITERAL=29
+BY=30
+AND=31
+ASC=32
+ASSIGN=33
+CAST_OP=34
+COMMA=35
+DESC=36
+DOT=37
+FALSE=38
+FIRST=39
+IN=40
+IS=41
+LAST=42
+LIKE=43
 LP=44
-IN=45
-IS=46
-LIKE=47
-NOT=48
-NULL=49
-NULLS=50
-OR=51
-PARAM=52
-RLIKE=53
-RP=54
-TRUE=55
-EQ=56
-CIEQ=57
-NEQ=58
-LT=59
-LTE=60
-GT=61
-GTE=62
-PLUS=63
-MINUS=64
-ASTERISK=65
-SLASH=66
-PERCENT=67
-NAMED_OR_POSITIONAL_PARAM=68
-OPENING_BRACKET=69
-CLOSING_BRACKET=70
-UNQUOTED_IDENTIFIER=71
-QUOTED_IDENTIFIER=72
-EXPR_LINE_COMMENT=73
-EXPR_MULTILINE_COMMENT=74
-EXPR_WS=75
+NOT=45
+NULL=46
+NULLS=47
+OR=48
+PARAM=49
+RLIKE=50
+RP=51
+TRUE=52
+EQ=53
+CIEQ=54
+NEQ=55
+LT=56
+LTE=57
+GT=58
+GTE=59
+PLUS=60
+MINUS=61
+ASTERISK=62
+SLASH=63
+PERCENT=64
+NAMED_OR_POSITIONAL_PARAM=65
+OPENING_BRACKET=66
+CLOSING_BRACKET=67
+UNQUOTED_IDENTIFIER=68
+QUOTED_IDENTIFIER=69
+EXPR_LINE_COMMENT=70
+EXPR_MULTILINE_COMMENT=71
+EXPR_WS=72
+EXPLAIN_WS=73
+EXPLAIN_LINE_COMMENT=74
+EXPLAIN_MULTILINE_COMMENT=75
 METADATA=76
-FROM_LINE_COMMENT=77
-FROM_MULTILINE_COMMENT=78
-FROM_WS=79
-ID_PATTERN=80
-PROJECT_LINE_COMMENT=81
-PROJECT_MULTILINE_COMMENT=82
-PROJECT_WS=83
-AS=84
-RENAME_LINE_COMMENT=85
-RENAME_MULTILINE_COMMENT=86
-RENAME_WS=87
-ON=88
-WITH=89
-ENRICH_POLICY_NAME=90
-ENRICH_LINE_COMMENT=91
-ENRICH_MULTILINE_COMMENT=92
-ENRICH_WS=93
-ENRICH_FIELD_LINE_COMMENT=94
-ENRICH_FIELD_MULTILINE_COMMENT=95
-ENRICH_FIELD_WS=96
-LOOKUP_LINE_COMMENT=97
-LOOKUP_MULTILINE_COMMENT=98
-LOOKUP_WS=99
-LOOKUP_FIELD_LINE_COMMENT=100
-LOOKUP_FIELD_MULTILINE_COMMENT=101
-LOOKUP_FIELD_WS=102
-MVEXPAND_LINE_COMMENT=103
-MVEXPAND_MULTILINE_COMMENT=104
-MVEXPAND_WS=105
-INFO=106
-SHOW_LINE_COMMENT=107
-SHOW_MULTILINE_COMMENT=108
-SHOW_WS=109
-FUNCTIONS=110
-META_LINE_COMMENT=111
-META_MULTILINE_COMMENT=112
-META_WS=113
-COLON=114
-SETTING=115
-SETTING_LINE_COMMENT=116
-SETTTING_MULTILINE_COMMENT=117
-SETTING_WS=118
-METRICS_LINE_COMMENT=119
-METRICS_MULTILINE_COMMENT=120
-METRICS_WS=121
-CLOSING_METRICS_LINE_COMMENT=122
-CLOSING_METRICS_MULTILINE_COMMENT=123
-CLOSING_METRICS_WS=124
+UNQUOTED_SOURCE=77
+FROM_LINE_COMMENT=78
+FROM_MULTILINE_COMMENT=79
+FROM_WS=80
+ID_PATTERN=81
+PROJECT_LINE_COMMENT=82
+PROJECT_MULTILINE_COMMENT=83
+PROJECT_WS=84
+AS=85
+RENAME_LINE_COMMENT=86
+RENAME_MULTILINE_COMMENT=87
+RENAME_WS=88
+ON=89
+WITH=90
+ENRICH_POLICY_NAME=91
+ENRICH_LINE_COMMENT=92
+ENRICH_MULTILINE_COMMENT=93
+ENRICH_WS=94
+ENRICH_FIELD_LINE_COMMENT=95
+ENRICH_FIELD_MULTILINE_COMMENT=96
+ENRICH_FIELD_WS=97
+MVEXPAND_LINE_COMMENT=98
+MVEXPAND_MULTILINE_COMMENT=99
+MVEXPAND_WS=100
+INFO=101
+SHOW_LINE_COMMENT=102
+SHOW_MULTILINE_COMMENT=103
+SHOW_WS=104
+FUNCTIONS=105
+META_LINE_COMMENT=106
+META_MULTILINE_COMMENT=107
+META_WS=108
+COLON=109
+SETTING=110
+SETTING_LINE_COMMENT=111
+SETTTING_MULTILINE_COMMENT=112
+SETTING_WS=113
+LOOKUP_LINE_COMMENT=114
+LOOKUP_MULTILINE_COMMENT=115
+LOOKUP_WS=116
+LOOKUP_FIELD_LINE_COMMENT=117
+LOOKUP_FIELD_MULTILINE_COMMENT=118
+LOOKUP_FIELD_WS=119
+METRICS_LINE_COMMENT=120
+METRICS_MULTILINE_COMMENT=121
+METRICS_WS=122
+CLOSING_METRICS_LINE_COMMENT=123
+CLOSING_METRICS_MULTILINE_COMMENT=124
+CLOSING_METRICS_WS=125
 'dissect'=1
 'drop'=2
 'enrich'=3
@@ -129,60 +130,57 @@ CLOSING_METRICS_WS=124
 'explain'=5
 'from'=6
 'grok'=7
-'inlinestats'=8
-'keep'=9
-'limit'=10
-'lookup'=11
-'meta'=12
-'metrics'=13
-'mv_expand'=14
-'rename'=15
-'row'=16
-'show'=17
-'sort'=18
-'stats'=19
-'where'=20
-'|'=29
-'by'=33
-'and'=34
-'asc'=35
-'='=36
-'::'=37
-','=38
-'desc'=39
-'.'=40
-'false'=41
-'first'=42
-'last'=43
+'keep'=8
+'limit'=9
+'meta'=10
+'mv_expand'=11
+'rename'=12
+'row'=13
+'show'=14
+'sort'=15
+'stats'=16
+'where'=17
+'|'=26
+'by'=30
+'and'=31
+'asc'=32
+'='=33
+'::'=34
+','=35
+'desc'=36
+'.'=37
+'false'=38
+'first'=39
+'in'=40
+'is'=41
+'last'=42
+'like'=43
 '('=44
-'in'=45
-'is'=46
-'like'=47
-'not'=48
-'null'=49
-'nulls'=50
-'or'=51
-'?'=52
-'rlike'=53
-')'=54
-'true'=55
-'=='=56
-'=~'=57
-'!='=58
-'<'=59
-'<='=60
-'>'=61
-'>='=62
-'+'=63
-'-'=64
-'*'=65
-'/'=66
-'%'=67
-']'=70
+'not'=45
+'null'=46
+'nulls'=47
+'or'=48
+'?'=49
+'rlike'=50
+')'=51
+'true'=52
+'=='=53
+'=~'=54
+'!='=55
+'<'=56
+'<='=57
+'>'=58
+'>='=59
+'+'=60
+'-'=61
+'*'=62
+'/'=63
+'%'=64
+']'=67
 'metadata'=76
-'as'=84
-'on'=88
-'with'=89
-'info'=106
-'functions'=110
-':'=114
+'as'=85
+'on'=89
+'with'=90
+'info'=101
+'functions'=105
+':'=109
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Evaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Evaluator.java
index f4704dc7c7e27..6ae51ca82d9a1 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Evaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Evaluator.java
@@ -67,7 +67,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock fieldBlock) {
           result.appendNull();
           continue position;
         }
-        result.appendBytesRef(FromBase64.process(fieldBlock.getBytesRef(fieldBlock.getFirstValueIndex(p), fieldScratch), oScratch));
+        result.appendBytesRef(FromBase64.process(fieldBlock.getBytesRef(fieldBlock.getFirstValueIndex(p), fieldScratch), this.oScratch));
       }
       return result.build();
     }
@@ -77,7 +77,7 @@ public BytesRefVector eval(int positionCount, BytesRefVector fieldVector) {
     try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
       BytesRef fieldScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendBytesRef(FromBase64.process(fieldVector.getBytesRef(p, fieldScratch), oScratch));
+        result.appendBytesRef(FromBase64.process(fieldVector.getBytesRef(p, fieldScratch), this.oScratch));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Evaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Evaluator.java
index eb0c483c7485d..3c102b655d235 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Evaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Evaluator.java
@@ -69,7 +69,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock fieldBlock) {
           continue position;
         }
         try {
-          result.appendBytesRef(ToBase64.process(fieldBlock.getBytesRef(fieldBlock.getFirstValueIndex(p), fieldScratch), oScratch));
+          result.appendBytesRef(ToBase64.process(fieldBlock.getBytesRef(fieldBlock.getFirstValueIndex(p), fieldScratch), this.oScratch));
         } catch (ArithmeticException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -84,7 +84,7 @@ public BytesRefBlock eval(int positionCount, BytesRefVector fieldVector) {
       BytesRef fieldScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBytesRef(ToBase64.process(fieldVector.getBytesRef(p, fieldScratch), oScratch));
+          result.appendBytesRef(ToBase64.process(fieldVector.getBytesRef(p, fieldScratch), this.oScratch));
         } catch (ArithmeticException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeFromDateNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeFromDateNanosEvaluator.java
new file mode 100644
index 0000000000000..92b629657b95b
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeFromDateNanosEvaluator.java
@@ -0,0 +1,122 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.Vector;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDatetime}.
+ * This class is generated. Do not edit it.
+ */
+public final class ToDatetimeFromDateNanosEvaluator extends AbstractConvertFunction.AbstractEvaluator {
+  public ToDatetimeFromDateNanosEvaluator(EvalOperator.ExpressionEvaluator field, Source source,
+      DriverContext driverContext) {
+    super(driverContext, field, source);
+  }
+
+  @Override
+  public String name() {
+    return "ToDatetimeFromDateNanos";
+  }
+
+  @Override
+  public Block evalVector(Vector v) {
+    LongVector vector = (LongVector) v;
+    int positionCount = v.getPositionCount();
+    if (vector.isConstant()) {
+      try {
+        return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount);
+      } catch (IllegalArgumentException  e) {
+        registerException(e);
+        return driverContext.blockFactory().newConstantNullBlock(positionCount);
+      }
+    }
+    try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) {
+      for (int p = 0; p < positionCount; p++) {
+        try {
+          builder.appendLong(evalValue(vector, p));
+        } catch (IllegalArgumentException  e) {
+          registerException(e);
+          builder.appendNull();
+        }
+      }
+      return builder.build();
+    }
+  }
+
+  private static long evalValue(LongVector container, int index) {
+    long value = container.getLong(index);
+    return ToDatetime.fromDatenanos(value);
+  }
+
+  @Override
+  public Block evalBlock(Block b) {
+    LongBlock block = (LongBlock) b;
+    int positionCount = block.getPositionCount();
+    try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) {
+      for (int p = 0; p < positionCount; p++) {
+        int valueCount = block.getValueCount(p);
+        int start = block.getFirstValueIndex(p);
+        int end = start + valueCount;
+        boolean positionOpened = false;
+        boolean valuesAppended = false;
+        for (int i = start; i < end; i++) {
+          try {
+            long value = evalValue(block, i);
+            if (positionOpened == false && valueCount > 1) {
+              builder.beginPositionEntry();
+              positionOpened = true;
+            }
+            builder.appendLong(value);
+            valuesAppended = true;
+          } catch (IllegalArgumentException  e) {
+            registerException(e);
+          }
+        }
+        if (valuesAppended == false) {
+          builder.appendNull();
+        } else if (positionOpened) {
+          builder.endPositionEntry();
+        }
+      }
+      return builder.build();
+    }
+  }
+
+  private static long evalValue(LongBlock container, int index) {
+    long value = container.getLong(index);
+    return ToDatetime.fromDatenanos(value);
+  }
+
+  public static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field;
+
+    public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) {
+      this.field = field;
+      this.source = source;
+    }
+
+    @Override
+    public ToDatetimeFromDateNanosEvaluator get(DriverContext context) {
+      return new ToDatetimeFromDateNanosEvaluator(field.get(context), source, context);
+    }
+
+    @Override
+    public String toString() {
+      return "ToDatetimeFromDateNanosEvaluator[field=" + field + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDateNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDateNanosEvaluator.java
new file mode 100644
index 0000000000000..37f13ea340a26
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDateNanosEvaluator.java
@@ -0,0 +1,109 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import java.lang.Override;
+import java.lang.String;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.Vector;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToString}.
+ * This class is generated. Do not edit it.
+ */
+public final class ToStringFromDateNanosEvaluator extends AbstractConvertFunction.AbstractEvaluator {
+  public ToStringFromDateNanosEvaluator(EvalOperator.ExpressionEvaluator field, Source source,
+      DriverContext driverContext) {
+    super(driverContext, field, source);
+  }
+
+  @Override
+  public String name() {
+    return "ToStringFromDateNanos";
+  }
+
+  @Override
+  public Block evalVector(Vector v) {
+    LongVector vector = (LongVector) v;
+    int positionCount = v.getPositionCount();
+    if (vector.isConstant()) {
+      return driverContext.blockFactory().newConstantBytesRefBlockWith(evalValue(vector, 0), positionCount);
+    }
+    try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      for (int p = 0; p < positionCount; p++) {
+        builder.appendBytesRef(evalValue(vector, p));
+      }
+      return builder.build();
+    }
+  }
+
+  private static BytesRef evalValue(LongVector container, int index) {
+    long value = container.getLong(index);
+    return ToString.fromDateNanos(value);
+  }
+
+  @Override
+  public Block evalBlock(Block b) {
+    LongBlock block = (LongBlock) b;
+    int positionCount = block.getPositionCount();
+    try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      for (int p = 0; p < positionCount; p++) {
+        int valueCount = block.getValueCount(p);
+        int start = block.getFirstValueIndex(p);
+        int end = start + valueCount;
+        boolean positionOpened = false;
+        boolean valuesAppended = false;
+        for (int i = start; i < end; i++) {
+          BytesRef value = evalValue(block, i);
+          if (positionOpened == false && valueCount > 1) {
+            builder.beginPositionEntry();
+            positionOpened = true;
+          }
+          builder.appendBytesRef(value);
+          valuesAppended = true;
+        }
+        if (valuesAppended == false) {
+          builder.appendNull();
+        } else if (positionOpened) {
+          builder.endPositionEntry();
+        }
+      }
+      return builder.build();
+    }
+  }
+
+  private static BytesRef evalValue(LongBlock container, int index) {
+    long value = container.getLong(index);
+    return ToString.fromDateNanos(value);
+  }
+
+  public static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field;
+
+    public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) {
+      this.field = field;
+      this.source = source;
+    }
+
+    @Override
+    public ToStringFromDateNanosEvaluator get(DriverContext context) {
+      return new ToStringFromDateNanosEvaluator(field.get(context), source, context);
+    }
+
+    @Override
+    public String toString() {
+      return "ToStringFromDateNanosEvaluator[field=" + field + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffConstantEvaluator.java
index fe54f8f5f9e12..dfef24cb556ea 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffConstantEvaluator.java
@@ -88,7 +88,7 @@ public IntBlock eval(int positionCount, LongBlock startTimestampBlock,
           continue position;
         }
         try {
-          result.appendInt(DateDiff.process(datePartFieldUnit, startTimestampBlock.getLong(startTimestampBlock.getFirstValueIndex(p)), endTimestampBlock.getLong(endTimestampBlock.getFirstValueIndex(p))));
+          result.appendInt(DateDiff.process(this.datePartFieldUnit, startTimestampBlock.getLong(startTimestampBlock.getFirstValueIndex(p)), endTimestampBlock.getLong(endTimestampBlock.getFirstValueIndex(p))));
         } catch (IllegalArgumentException | InvalidArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -103,7 +103,7 @@ public IntBlock eval(int positionCount, LongVector startTimestampVector,
     try(IntBlock.Builder result = driverContext.blockFactory().newIntBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendInt(DateDiff.process(datePartFieldUnit, startTimestampVector.getLong(p), endTimestampVector.getLong(p)));
+          result.appendInt(DateDiff.process(this.datePartFieldUnit, startTimestampVector.getLong(p), endTimestampVector.getLong(p)));
         } catch (IllegalArgumentException | InvalidArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java
index abff711e5c19a..d72ffe77ac914 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java
@@ -68,7 +68,7 @@ public LongBlock eval(int positionCount, LongBlock valueBlock) {
           result.appendNull();
           continue position;
         }
-        result.appendLong(DateExtract.process(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), chronoField, zone));
+        result.appendLong(DateExtract.process(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), this.chronoField, this.zone));
       }
       return result.build();
     }
@@ -77,7 +77,7 @@ public LongBlock eval(int positionCount, LongBlock valueBlock) {
   public LongVector eval(int positionCount, LongVector valueVector) {
     try(LongVector.FixedBuilder result = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendLong(p, DateExtract.process(valueVector.getLong(p), chronoField, zone));
+        result.appendLong(p, DateExtract.process(valueVector.getLong(p), this.chronoField, this.zone));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java
index e2c77cd2718c4..8812eba051336 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java
@@ -89,7 +89,7 @@ public LongBlock eval(int positionCount, LongBlock valueBlock, BytesRefBlock chr
           continue position;
         }
         try {
-          result.appendLong(DateExtract.process(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), chronoFieldBlock.getBytesRef(chronoFieldBlock.getFirstValueIndex(p), chronoFieldScratch), zone));
+          result.appendLong(DateExtract.process(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), chronoFieldBlock.getBytesRef(chronoFieldBlock.getFirstValueIndex(p), chronoFieldScratch), this.zone));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -105,7 +105,7 @@ public LongBlock eval(int positionCount, LongVector valueVector,
       BytesRef chronoFieldScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendLong(DateExtract.process(valueVector.getLong(p), chronoFieldVector.getBytesRef(p, chronoFieldScratch), zone));
+          result.appendLong(DateExtract.process(valueVector.getLong(p), chronoFieldVector.getBytesRef(p, chronoFieldScratch), this.zone));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java
index 770230e3a5a71..4731c888cef77 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java
@@ -66,7 +66,7 @@ public BytesRefBlock eval(int positionCount, LongBlock valBlock) {
           result.appendNull();
           continue position;
         }
-        result.appendBytesRef(DateFormat.process(valBlock.getLong(valBlock.getFirstValueIndex(p)), formatter));
+        result.appendBytesRef(DateFormat.process(valBlock.getLong(valBlock.getFirstValueIndex(p)), this.formatter));
       }
       return result.build();
     }
@@ -75,7 +75,7 @@ public BytesRefBlock eval(int positionCount, LongBlock valBlock) {
   public BytesRefVector eval(int positionCount, LongVector valVector) {
     try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendBytesRef(DateFormat.process(valVector.getLong(p), formatter));
+        result.appendBytesRef(DateFormat.process(valVector.getLong(p), this.formatter));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java
index 0ac3f5c327169..77633797de02d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java
@@ -88,7 +88,7 @@ public BytesRefBlock eval(int positionCount, LongBlock valBlock, BytesRefBlock f
           result.appendNull();
           continue position;
         }
-        result.appendBytesRef(DateFormat.process(valBlock.getLong(valBlock.getFirstValueIndex(p)), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch), locale));
+        result.appendBytesRef(DateFormat.process(valBlock.getLong(valBlock.getFirstValueIndex(p)), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch), this.locale));
       }
       return result.build();
     }
@@ -99,7 +99,7 @@ public BytesRefVector eval(int positionCount, LongVector valVector,
     try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
       BytesRef formatterScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendBytesRef(DateFormat.process(valVector.getLong(p), formatterVector.getBytesRef(p, formatterScratch), locale));
+        result.appendBytesRef(DateFormat.process(valVector.getLong(p), formatterVector.getBytesRef(p, formatterScratch), this.locale));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java
index c08c1a54f90ba..470ffab52920a 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java
@@ -68,7 +68,7 @@ public LongBlock eval(int positionCount, BytesRefBlock valBlock) {
           continue position;
         }
         try {
-          result.appendLong(DateParse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), formatter));
+          result.appendLong(DateParse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), this.formatter));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -83,7 +83,7 @@ public LongBlock eval(int positionCount, BytesRefVector valVector) {
       BytesRef valScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendLong(DateParse.process(valVector.getBytesRef(p, valScratch), formatter));
+          result.appendLong(DateParse.process(valVector.getBytesRef(p, valScratch), this.formatter));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java
index a28a3feb1c9b6..edfe6c39949c3 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java
@@ -89,7 +89,7 @@ public LongBlock eval(int positionCount, BytesRefBlock valBlock, BytesRefBlock f
           continue position;
         }
         try {
-          result.appendLong(DateParse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch), zoneId));
+          result.appendLong(DateParse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch), this.zoneId));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -106,7 +106,7 @@ public LongBlock eval(int positionCount, BytesRefVector valVector,
       BytesRef formatterScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendLong(DateParse.process(valVector.getBytesRef(p, valScratch), formatterVector.getBytesRef(p, formatterScratch), zoneId));
+          result.appendLong(DateParse.process(valVector.getBytesRef(p, valScratch), formatterVector.getBytesRef(p, formatterScratch), this.zoneId));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java
index b72203ce0de35..b8bc45296a60a 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java
@@ -64,7 +64,7 @@ public LongBlock eval(int positionCount, LongBlock fieldValBlock) {
           result.appendNull();
           continue position;
         }
-        result.appendLong(DateTrunc.process(fieldValBlock.getLong(fieldValBlock.getFirstValueIndex(p)), rounding));
+        result.appendLong(DateTrunc.process(fieldValBlock.getLong(fieldValBlock.getFirstValueIndex(p)), this.rounding));
       }
       return result.build();
     }
@@ -73,7 +73,7 @@ public LongBlock eval(int positionCount, LongBlock fieldValBlock) {
   public LongVector eval(int positionCount, LongVector fieldValVector) {
     try(LongVector.FixedBuilder result = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendLong(p, DateTrunc.process(fieldValVector.getLong(p), rounding));
+        result.appendLong(p, DateTrunc.process(fieldValVector.getLong(p), this.rounding));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java
index 1894d19d7b082..db1d5f8fe01ca 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java
@@ -39,7 +39,7 @@ public Block eval(Page page) {
   public LongVector eval(int positionCount) {
     try(LongVector.FixedBuilder result = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendLong(p, Now.process(now));
+        result.appendLong(p, Now.process(this.now));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java
index 57427f87b76f7..0347fd2e1065c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java
@@ -112,7 +112,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock ipBlock, IntBlock pre
           continue position;
         }
         try {
-          result.appendBytesRef(IpPrefix.process(ipBlock.getBytesRef(ipBlock.getFirstValueIndex(p), ipScratch), prefixLengthV4Block.getInt(prefixLengthV4Block.getFirstValueIndex(p)), prefixLengthV6Block.getInt(prefixLengthV6Block.getFirstValueIndex(p)), scratch));
+          result.appendBytesRef(IpPrefix.process(ipBlock.getBytesRef(ipBlock.getFirstValueIndex(p), ipScratch), prefixLengthV4Block.getInt(prefixLengthV4Block.getFirstValueIndex(p)), prefixLengthV6Block.getInt(prefixLengthV6Block.getFirstValueIndex(p)), this.scratch));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -128,7 +128,7 @@ public BytesRefBlock eval(int positionCount, BytesRefVector ipVector,
       BytesRef ipScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBytesRef(IpPrefix.process(ipVector.getBytesRef(p, ipScratch), prefixLengthV4Vector.getInt(p), prefixLengthV6Vector.getInt(p), scratch));
+          result.appendBytesRef(IpPrefix.process(ipVector.getBytesRef(p, ipScratch), prefixLengthV4Vector.getInt(p), prefixLengthV6Vector.getInt(p), this.scratch));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixOnlyV4Evaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixOnlyV4Evaluator.java
deleted file mode 100644
index a6cb7c7f9b687..0000000000000
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixOnlyV4Evaluator.java
+++ /dev/null
@@ -1,148 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License
-// 2.0; you may not use this file except in compliance with the Elastic License
-// 2.0.
-package org.elasticsearch.xpack.esql.expression.function.scalar.ip;
-
-import java.lang.IllegalArgumentException;
-import java.lang.Override;
-import java.lang.String;
-import java.util.function.Function;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.compute.data.Block;
-import org.elasticsearch.compute.data.BytesRefBlock;
-import org.elasticsearch.compute.data.BytesRefVector;
-import org.elasticsearch.compute.data.IntBlock;
-import org.elasticsearch.compute.data.IntVector;
-import org.elasticsearch.compute.data.Page;
-import org.elasticsearch.compute.operator.DriverContext;
-import org.elasticsearch.compute.operator.EvalOperator;
-import org.elasticsearch.core.Releasables;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.expression.function.Warnings;
-
-/**
- * {@link EvalOperator.ExpressionEvaluator} implementation for {@link IpPrefix}.
- * This class is generated. Do not edit it.
- */
-public final class IpPrefixOnlyV4Evaluator implements EvalOperator.ExpressionEvaluator {
-  private final Warnings warnings;
-
-  private final EvalOperator.ExpressionEvaluator ip;
-
-  private final EvalOperator.ExpressionEvaluator prefixLengthV4;
-
-  private final BytesRef scratch;
-
-  private final DriverContext driverContext;
-
-  public IpPrefixOnlyV4Evaluator(Source source, EvalOperator.ExpressionEvaluator ip,
-      EvalOperator.ExpressionEvaluator prefixLengthV4, BytesRef scratch,
-      DriverContext driverContext) {
-    this.warnings = new Warnings(source);
-    this.ip = ip;
-    this.prefixLengthV4 = prefixLengthV4;
-    this.scratch = scratch;
-    this.driverContext = driverContext;
-  }
-
-  @Override
-  public Block eval(Page page) {
-    try (BytesRefBlock ipBlock = (BytesRefBlock) ip.eval(page)) {
-      try (IntBlock prefixLengthV4Block = (IntBlock) prefixLengthV4.eval(page)) {
-        BytesRefVector ipVector = ipBlock.asVector();
-        if (ipVector == null) {
-          return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block);
-        }
-        IntVector prefixLengthV4Vector = prefixLengthV4Block.asVector();
-        if (prefixLengthV4Vector == null) {
-          return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block);
-        }
-        return eval(page.getPositionCount(), ipVector, prefixLengthV4Vector).asBlock();
-      }
-    }
-  }
-
-  public BytesRefBlock eval(int positionCount, BytesRefBlock ipBlock,
-      IntBlock prefixLengthV4Block) {
-    try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
-      BytesRef ipScratch = new BytesRef();
-      position: for (int p = 0; p < positionCount; p++) {
-        if (ipBlock.isNull(p)) {
-          result.appendNull();
-          continue position;
-        }
-        if (ipBlock.getValueCount(p) != 1) {
-          if (ipBlock.getValueCount(p) > 1) {
-            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
-          result.appendNull();
-          continue position;
-        }
-        if (prefixLengthV4Block.isNull(p)) {
-          result.appendNull();
-          continue position;
-        }
-        if (prefixLengthV4Block.getValueCount(p) != 1) {
-          if (prefixLengthV4Block.getValueCount(p) > 1) {
-            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
-          }
-          result.appendNull();
-          continue position;
-        }
-        result.appendBytesRef(IpPrefix.process(ipBlock.getBytesRef(ipBlock.getFirstValueIndex(p), ipScratch), prefixLengthV4Block.getInt(prefixLengthV4Block.getFirstValueIndex(p)), scratch));
-      }
-      return result.build();
-    }
-  }
-
-  public BytesRefVector eval(int positionCount, BytesRefVector ipVector,
-      IntVector prefixLengthV4Vector) {
-    try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
-      BytesRef ipScratch = new BytesRef();
-      position: for (int p = 0; p < positionCount; p++) {
-        result.appendBytesRef(IpPrefix.process(ipVector.getBytesRef(p, ipScratch), prefixLengthV4Vector.getInt(p), scratch));
-      }
-      return result.build();
-    }
-  }
-
-  @Override
-  public String toString() {
-    return "IpPrefixOnlyV4Evaluator[" + "ip=" + ip + ", prefixLengthV4=" + prefixLengthV4 + "]";
-  }
-
-  @Override
-  public void close() {
-    Releasables.closeExpectNoException(ip, prefixLengthV4);
-  }
-
-  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
-    private final Source source;
-
-    private final EvalOperator.ExpressionEvaluator.Factory ip;
-
-    private final EvalOperator.ExpressionEvaluator.Factory prefixLengthV4;
-
-    private final Function<DriverContext, BytesRef> scratch;
-
-    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory ip,
-        EvalOperator.ExpressionEvaluator.Factory prefixLengthV4,
-        Function<DriverContext, BytesRef> scratch) {
-      this.source = source;
-      this.ip = ip;
-      this.prefixLengthV4 = prefixLengthV4;
-      this.scratch = scratch;
-    }
-
-    @Override
-    public IpPrefixOnlyV4Evaluator get(DriverContext context) {
-      return new IpPrefixOnlyV4Evaluator(source, ip.get(context), prefixLengthV4.get(context), scratch.apply(context), context);
-    }
-
-    @Override
-    public String toString() {
-      return "IpPrefixOnlyV4Evaluator[" + "ip=" + ip + ", prefixLengthV4=" + prefixLengthV4 + "]";
-    }
-  }
-}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpDoubleEvaluator.java
new file mode 100644
index 0000000000000..37114256774df
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpDoubleEvaluator.java
@@ -0,0 +1,108 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.math;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.DoubleVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Exp}.
+ * This class is generated. Do not edit it.
+ */
+public final class ExpDoubleEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator val;
+
+  private final DriverContext driverContext;
+
+  public ExpDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
+      DriverContext driverContext) {
+    this.val = val;
+    this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) {
+      DoubleVector valVector = valBlock.asVector();
+      if (valVector == null) {
+        return eval(page.getPositionCount(), valBlock);
+      }
+      return eval(page.getPositionCount(), valVector).asBlock();
+    }
+  }
+
+  public DoubleBlock eval(int positionCount, DoubleBlock valBlock) {
+    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        if (valBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (valBlock.getValueCount(p) != 1) {
+          if (valBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        result.appendDouble(Exp.process(valBlock.getDouble(valBlock.getFirstValueIndex(p))));
+      }
+      return result.build();
+    }
+  }
+
+  public DoubleVector eval(int positionCount, DoubleVector valVector) {
+    try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        result.appendDouble(p, Exp.process(valVector.getDouble(p)));
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "ExpDoubleEvaluator[" + "val=" + val + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(val);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory val;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val) {
+      this.source = source;
+      this.val = val;
+    }
+
+    @Override
+    public ExpDoubleEvaluator get(DriverContext context) {
+      return new ExpDoubleEvaluator(source, val.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "ExpDoubleEvaluator[" + "val=" + val + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpIntEvaluator.java
new file mode 100644
index 0000000000000..5559453f97f39
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpIntEvaluator.java
@@ -0,0 +1,110 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.math;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.DoubleVector;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Exp}.
+ * This class is generated. Do not edit it.
+ */
+public final class ExpIntEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator val;
+
+  private final DriverContext driverContext;
+
+  public ExpIntEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
+      DriverContext driverContext) {
+    this.val = val;
+    this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (IntBlock valBlock = (IntBlock) val.eval(page)) {
+      IntVector valVector = valBlock.asVector();
+      if (valVector == null) {
+        return eval(page.getPositionCount(), valBlock);
+      }
+      return eval(page.getPositionCount(), valVector).asBlock();
+    }
+  }
+
+  public DoubleBlock eval(int positionCount, IntBlock valBlock) {
+    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        if (valBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (valBlock.getValueCount(p) != 1) {
+          if (valBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        result.appendDouble(Exp.process(valBlock.getInt(valBlock.getFirstValueIndex(p))));
+      }
+      return result.build();
+    }
+  }
+
+  public DoubleVector eval(int positionCount, IntVector valVector) {
+    try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        result.appendDouble(p, Exp.process(valVector.getInt(p)));
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "ExpIntEvaluator[" + "val=" + val + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(val);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory val;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val) {
+      this.source = source;
+      this.val = val;
+    }
+
+    @Override
+    public ExpIntEvaluator get(DriverContext context) {
+      return new ExpIntEvaluator(source, val.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "ExpIntEvaluator[" + "val=" + val + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpLongEvaluator.java
new file mode 100644
index 0000000000000..2642628088a27
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpLongEvaluator.java
@@ -0,0 +1,110 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.math;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.DoubleVector;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Exp}.
+ * This class is generated. Do not edit it.
+ */
+public final class ExpLongEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator val;
+
+  private final DriverContext driverContext;
+
+  public ExpLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
+      DriverContext driverContext) {
+    this.val = val;
+    this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (LongBlock valBlock = (LongBlock) val.eval(page)) {
+      LongVector valVector = valBlock.asVector();
+      if (valVector == null) {
+        return eval(page.getPositionCount(), valBlock);
+      }
+      return eval(page.getPositionCount(), valVector).asBlock();
+    }
+  }
+
+  public DoubleBlock eval(int positionCount, LongBlock valBlock) {
+    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        if (valBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (valBlock.getValueCount(p) != 1) {
+          if (valBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        result.appendDouble(Exp.process(valBlock.getLong(valBlock.getFirstValueIndex(p))));
+      }
+      return result.build();
+    }
+  }
+
+  public DoubleVector eval(int positionCount, LongVector valVector) {
+    try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        result.appendDouble(p, Exp.process(valVector.getLong(p)));
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "ExpLongEvaluator[" + "val=" + val + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(val);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory val;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val) {
+      this.source = source;
+      this.val = val;
+    }
+
+    @Override
+    public ExpLongEvaluator get(DriverContext context) {
+      return new ExpLongEvaluator(source, val.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "ExpLongEvaluator[" + "val=" + val + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpUnsignedLongEvaluator.java
new file mode 100644
index 0000000000000..52506d5cd30e8
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpUnsignedLongEvaluator.java
@@ -0,0 +1,110 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.math;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.DoubleVector;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Exp}.
+ * This class is generated. Do not edit it.
+ */
+public final class ExpUnsignedLongEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator val;
+
+  private final DriverContext driverContext;
+
+  public ExpUnsignedLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
+      DriverContext driverContext) {
+    this.val = val;
+    this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (LongBlock valBlock = (LongBlock) val.eval(page)) {
+      LongVector valVector = valBlock.asVector();
+      if (valVector == null) {
+        return eval(page.getPositionCount(), valBlock);
+      }
+      return eval(page.getPositionCount(), valVector).asBlock();
+    }
+  }
+
+  public DoubleBlock eval(int positionCount, LongBlock valBlock) {
+    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        if (valBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (valBlock.getValueCount(p) != 1) {
+          if (valBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        result.appendDouble(Exp.processUnsignedLong(valBlock.getLong(valBlock.getFirstValueIndex(p))));
+      }
+      return result.build();
+    }
+  }
+
+  public DoubleVector eval(int positionCount, LongVector valVector) {
+    try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        result.appendDouble(p, Exp.processUnsignedLong(valVector.getLong(p)));
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "ExpUnsignedLongEvaluator[" + "val=" + val + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(val);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory val;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val) {
+      this.source = source;
+      this.val = val;
+    }
+
+    @Override
+    public ExpUnsignedLongEvaluator get(DriverContext context) {
+      return new ExpUnsignedLongEvaluator(source, val.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "ExpUnsignedLongEvaluator[" + "val=" + val + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumDoubleEvaluator.java
new file mode 100644
index 0000000000000..5f6bc8361c1bb
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumDoubleEvaluator.java
@@ -0,0 +1,111 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import java.lang.ArithmeticException;
+import java.lang.Override;
+import java.lang.String;
+import java.util.function.Function;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.search.aggregations.metrics.CompensatedSum;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvPSeriesWeightedSum}.
+ * This class is generated. Do not edit it.
+ */
+public final class MvPSeriesWeightedSumDoubleEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator block;
+
+  private final CompensatedSum sum;
+
+  private final double p;
+
+  private final DriverContext driverContext;
+
+  public MvPSeriesWeightedSumDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator block,
+      CompensatedSum sum, double p, DriverContext driverContext) {
+    this.block = block;
+    this.sum = sum;
+    this.p = p;
+    this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (DoubleBlock blockBlock = (DoubleBlock) block.eval(page)) {
+      return eval(page.getPositionCount(), blockBlock);
+    }
+  }
+
+  public DoubleBlock eval(int positionCount, DoubleBlock blockBlock) {
+    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        boolean allBlocksAreNulls = true;
+        if (!blockBlock.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (allBlocksAreNulls) {
+          result.appendNull();
+          continue position;
+        }
+        try {
+          MvPSeriesWeightedSum.process(result, p, blockBlock, this.sum, this.p);
+        } catch (ArithmeticException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "MvPSeriesWeightedSumDoubleEvaluator[" + "block=" + block + ", p=" + p + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(block);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory block;
+
+    private final Function<DriverContext, CompensatedSum> sum;
+
+    private final double p;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory block,
+        Function<DriverContext, CompensatedSum> sum, double p) {
+      this.source = source;
+      this.block = block;
+      this.sum = sum;
+      this.p = p;
+    }
+
+    @Override
+    public MvPSeriesWeightedSumDoubleEvaluator get(DriverContext context) {
+      return new MvPSeriesWeightedSumDoubleEvaluator(source, block.get(context), sum.apply(context), p, context);
+    }
+
+    @Override
+    public String toString() {
+      return "MvPSeriesWeightedSumDoubleEvaluator[" + "block=" + block + ", p=" + p + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileDoubleEvaluator.java
new file mode 100644
index 0000000000000..dd370e90b2c86
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileDoubleEvaluator.java
@@ -0,0 +1,125 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import java.util.function.Function;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvPercentile}.
+ * This class is generated. Do not edit it.
+ */
+public final class MvPercentileDoubleEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator values;
+
+  private final EvalOperator.ExpressionEvaluator percentile;
+
+  private final MvPercentile.DoubleSortingScratch scratch;
+
+  private final DriverContext driverContext;
+
+  public MvPercentileDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator values,
+      EvalOperator.ExpressionEvaluator percentile, MvPercentile.DoubleSortingScratch scratch,
+      DriverContext driverContext) {
+    this.values = values;
+    this.percentile = percentile;
+    this.scratch = scratch;
+    this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (DoubleBlock valuesBlock = (DoubleBlock) values.eval(page)) {
+      try (DoubleBlock percentileBlock = (DoubleBlock) percentile.eval(page)) {
+        return eval(page.getPositionCount(), valuesBlock, percentileBlock);
+      }
+    }
+  }
+
+  public DoubleBlock eval(int positionCount, DoubleBlock valuesBlock, DoubleBlock percentileBlock) {
+    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        boolean allBlocksAreNulls = true;
+        if (!valuesBlock.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (percentileBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (percentileBlock.getValueCount(p) != 1) {
+          if (percentileBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        if (allBlocksAreNulls) {
+          result.appendNull();
+          continue position;
+        }
+        try {
+          MvPercentile.process(result, p, valuesBlock, percentileBlock.getDouble(percentileBlock.getFirstValueIndex(p)), this.scratch);
+        } catch (IllegalArgumentException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "MvPercentileDoubleEvaluator[" + "values=" + values + ", percentile=" + percentile + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(values, percentile);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory values;
+
+    private final EvalOperator.ExpressionEvaluator.Factory percentile;
+
+    private final Function<DriverContext, MvPercentile.DoubleSortingScratch> scratch;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory values,
+        EvalOperator.ExpressionEvaluator.Factory percentile,
+        Function<DriverContext, MvPercentile.DoubleSortingScratch> scratch) {
+      this.source = source;
+      this.values = values;
+      this.percentile = percentile;
+      this.scratch = scratch;
+    }
+
+    @Override
+    public MvPercentileDoubleEvaluator get(DriverContext context) {
+      return new MvPercentileDoubleEvaluator(source, values.get(context), percentile.get(context), scratch.apply(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "MvPercentileDoubleEvaluator[" + "values=" + values + ", percentile=" + percentile + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileIntegerEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileIntegerEvaluator.java
new file mode 100644
index 0000000000000..93dda414c7b33
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileIntegerEvaluator.java
@@ -0,0 +1,126 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import java.util.function.Function;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvPercentile}.
+ * This class is generated. Do not edit it.
+ */
+public final class MvPercentileIntegerEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator values;
+
+  private final EvalOperator.ExpressionEvaluator percentile;
+
+  private final MvPercentile.IntSortingScratch scratch;
+
+  private final DriverContext driverContext;
+
+  public MvPercentileIntegerEvaluator(Source source, EvalOperator.ExpressionEvaluator values,
+      EvalOperator.ExpressionEvaluator percentile, MvPercentile.IntSortingScratch scratch,
+      DriverContext driverContext) {
+    this.values = values;
+    this.percentile = percentile;
+    this.scratch = scratch;
+    this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (IntBlock valuesBlock = (IntBlock) values.eval(page)) {
+      try (DoubleBlock percentileBlock = (DoubleBlock) percentile.eval(page)) {
+        return eval(page.getPositionCount(), valuesBlock, percentileBlock);
+      }
+    }
+  }
+
+  public IntBlock eval(int positionCount, IntBlock valuesBlock, DoubleBlock percentileBlock) {
+    try(IntBlock.Builder result = driverContext.blockFactory().newIntBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        boolean allBlocksAreNulls = true;
+        if (!valuesBlock.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (percentileBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (percentileBlock.getValueCount(p) != 1) {
+          if (percentileBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        if (allBlocksAreNulls) {
+          result.appendNull();
+          continue position;
+        }
+        try {
+          MvPercentile.process(result, p, valuesBlock, percentileBlock.getDouble(percentileBlock.getFirstValueIndex(p)), this.scratch);
+        } catch (IllegalArgumentException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "MvPercentileIntegerEvaluator[" + "values=" + values + ", percentile=" + percentile + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(values, percentile);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory values;
+
+    private final EvalOperator.ExpressionEvaluator.Factory percentile;
+
+    private final Function<DriverContext, MvPercentile.IntSortingScratch> scratch;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory values,
+        EvalOperator.ExpressionEvaluator.Factory percentile,
+        Function<DriverContext, MvPercentile.IntSortingScratch> scratch) {
+      this.source = source;
+      this.values = values;
+      this.percentile = percentile;
+      this.scratch = scratch;
+    }
+
+    @Override
+    public MvPercentileIntegerEvaluator get(DriverContext context) {
+      return new MvPercentileIntegerEvaluator(source, values.get(context), percentile.get(context), scratch.apply(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "MvPercentileIntegerEvaluator[" + "values=" + values + ", percentile=" + percentile + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileLongEvaluator.java
new file mode 100644
index 0000000000000..10d0b7c3283b2
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileLongEvaluator.java
@@ -0,0 +1,126 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import java.util.function.Function;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvPercentile}.
+ * This class is generated. Do not edit it.
+ */
+public final class MvPercentileLongEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator values;
+
+  private final EvalOperator.ExpressionEvaluator percentile;
+
+  private final MvPercentile.LongSortingScratch scratch;
+
+  private final DriverContext driverContext;
+
+  public MvPercentileLongEvaluator(Source source, EvalOperator.ExpressionEvaluator values,
+      EvalOperator.ExpressionEvaluator percentile, MvPercentile.LongSortingScratch scratch,
+      DriverContext driverContext) {
+    this.values = values;
+    this.percentile = percentile;
+    this.scratch = scratch;
+    this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (LongBlock valuesBlock = (LongBlock) values.eval(page)) {
+      try (DoubleBlock percentileBlock = (DoubleBlock) percentile.eval(page)) {
+        return eval(page.getPositionCount(), valuesBlock, percentileBlock);
+      }
+    }
+  }
+
+  public LongBlock eval(int positionCount, LongBlock valuesBlock, DoubleBlock percentileBlock) {
+    try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        boolean allBlocksAreNulls = true;
+        if (!valuesBlock.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (percentileBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (percentileBlock.getValueCount(p) != 1) {
+          if (percentileBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        if (allBlocksAreNulls) {
+          result.appendNull();
+          continue position;
+        }
+        try {
+          MvPercentile.process(result, p, valuesBlock, percentileBlock.getDouble(percentileBlock.getFirstValueIndex(p)), this.scratch);
+        } catch (IllegalArgumentException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "MvPercentileLongEvaluator[" + "values=" + values + ", percentile=" + percentile + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(values, percentile);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory values;
+
+    private final EvalOperator.ExpressionEvaluator.Factory percentile;
+
+    private final Function<DriverContext, MvPercentile.LongSortingScratch> scratch;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory values,
+        EvalOperator.ExpressionEvaluator.Factory percentile,
+        Function<DriverContext, MvPercentile.LongSortingScratch> scratch) {
+      this.source = source;
+      this.values = values;
+      this.percentile = percentile;
+      this.scratch = scratch;
+    }
+
+    @Override
+    public MvPercentileLongEvaluator get(DriverContext context) {
+      return new MvPercentileLongEvaluator(source, values.get(context), percentile.get(context), scratch.apply(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "MvPercentileLongEvaluator[" + "values=" + values + ", percentile=" + percentile + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndConstantEvaluator.java
index b5a5634bc0fd1..30f5ef487f612 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndConstantEvaluator.java
@@ -67,7 +67,7 @@ public BooleanBlock eval(int positionCount, LongBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialContains.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue));
+          result.appendBoolean(SpatialContains.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -81,7 +81,7 @@ public BooleanBlock eval(int positionCount, LongVector leftValueVector) {
     try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialContains.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue));
+          result.appendBoolean(SpatialContains.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndConstantEvaluator.java
index 3e2de0ebd397a..9205f1faaa3d0 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndConstantEvaluator.java
@@ -70,7 +70,7 @@ public BooleanBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialContains.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue));
+          result.appendBoolean(SpatialContains.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -85,7 +85,7 @@ public BooleanBlock eval(int positionCount, BytesRefVector leftValueVector) {
       BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialContains.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue));
+          result.appendBoolean(SpatialContains.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndConstantEvaluator.java
index f345c135747e7..a76ccaadfc4bf 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndConstantEvaluator.java
@@ -67,7 +67,7 @@ public BooleanBlock eval(int positionCount, LongBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialContains.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue));
+          result.appendBoolean(SpatialContains.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -81,7 +81,7 @@ public BooleanBlock eval(int positionCount, LongVector leftValueVector) {
     try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialContains.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue));
+          result.appendBoolean(SpatialContains.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndConstantEvaluator.java
index f2316c17db1ec..adb2e2ea0ced2 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndConstantEvaluator.java
@@ -70,7 +70,7 @@ public BooleanBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialContains.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue));
+          result.appendBoolean(SpatialContains.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -85,7 +85,7 @@ public BooleanBlock eval(int positionCount, BytesRefVector leftValueVector) {
       BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialContains.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue));
+          result.appendBoolean(SpatialContains.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndConstantEvaluator.java
index 3c46f859c80f8..3a50c1cc1f717 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndConstantEvaluator.java
@@ -67,7 +67,7 @@ public BooleanBlock eval(int positionCount, LongBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialDisjoint.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue));
+          result.appendBoolean(SpatialDisjoint.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -81,7 +81,7 @@ public BooleanBlock eval(int positionCount, LongVector leftValueVector) {
     try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialDisjoint.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue));
+          result.appendBoolean(SpatialDisjoint.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndConstantEvaluator.java
index 6e5becc402135..64dcf096e0dd8 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndConstantEvaluator.java
@@ -70,7 +70,7 @@ public BooleanBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialDisjoint.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue));
+          result.appendBoolean(SpatialDisjoint.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -85,7 +85,7 @@ public BooleanBlock eval(int positionCount, BytesRefVector leftValueVector) {
       BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialDisjoint.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue));
+          result.appendBoolean(SpatialDisjoint.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndConstantEvaluator.java
index 6fa2d7a6dd639..73912f2f4c5de 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndConstantEvaluator.java
@@ -67,7 +67,7 @@ public BooleanBlock eval(int positionCount, LongBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialDisjoint.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue));
+          result.appendBoolean(SpatialDisjoint.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -81,7 +81,7 @@ public BooleanBlock eval(int positionCount, LongVector leftValueVector) {
     try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialDisjoint.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue));
+          result.appendBoolean(SpatialDisjoint.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndConstantEvaluator.java
index c3930168ae594..1cd5e8504c7cf 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndConstantEvaluator.java
@@ -70,7 +70,7 @@ public BooleanBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialDisjoint.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue));
+          result.appendBoolean(SpatialDisjoint.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -85,7 +85,7 @@ public BooleanBlock eval(int positionCount, BytesRefVector leftValueVector) {
       BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialDisjoint.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue));
+          result.appendBoolean(SpatialDisjoint.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndConstantEvaluator.java
index 56912e3233a4c..38386aa7d51f1 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndConstantEvaluator.java
@@ -67,7 +67,7 @@ public BooleanBlock eval(int positionCount, LongBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialIntersects.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue));
+          result.appendBoolean(SpatialIntersects.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -81,7 +81,7 @@ public BooleanBlock eval(int positionCount, LongVector leftValueVector) {
     try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialIntersects.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue));
+          result.appendBoolean(SpatialIntersects.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndConstantEvaluator.java
index 26c4abdc51ecf..70fd91ed0a1ab 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndConstantEvaluator.java
@@ -70,7 +70,7 @@ public BooleanBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialIntersects.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue));
+          result.appendBoolean(SpatialIntersects.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -85,7 +85,7 @@ public BooleanBlock eval(int positionCount, BytesRefVector leftValueVector) {
       BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialIntersects.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue));
+          result.appendBoolean(SpatialIntersects.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndConstantEvaluator.java
index 405d013a77f5a..f6c16997bedc3 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndConstantEvaluator.java
@@ -67,7 +67,7 @@ public BooleanBlock eval(int positionCount, LongBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialIntersects.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue));
+          result.appendBoolean(SpatialIntersects.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -81,7 +81,7 @@ public BooleanBlock eval(int positionCount, LongVector leftValueVector) {
     try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialIntersects.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue));
+          result.appendBoolean(SpatialIntersects.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndConstantEvaluator.java
index dea6989a830ab..94ecf64f7252e 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndConstantEvaluator.java
@@ -70,7 +70,7 @@ public BooleanBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialIntersects.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue));
+          result.appendBoolean(SpatialIntersects.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -85,7 +85,7 @@ public BooleanBlock eval(int positionCount, BytesRefVector leftValueVector) {
       BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialIntersects.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue));
+          result.appendBoolean(SpatialIntersects.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndConstantEvaluator.java
index 5c31df936236b..0619d401154da 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndConstantEvaluator.java
@@ -67,7 +67,7 @@ public BooleanBlock eval(int positionCount, LongBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialWithin.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue));
+          result.appendBoolean(SpatialWithin.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -81,7 +81,7 @@ public BooleanBlock eval(int positionCount, LongVector leftValueVector) {
     try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialWithin.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue));
+          result.appendBoolean(SpatialWithin.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndConstantEvaluator.java
index 1a1bd4cd64535..82de0c349dc7d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndConstantEvaluator.java
@@ -70,7 +70,7 @@ public BooleanBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialWithin.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue));
+          result.appendBoolean(SpatialWithin.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -85,7 +85,7 @@ public BooleanBlock eval(int positionCount, BytesRefVector leftValueVector) {
       BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialWithin.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue));
+          result.appendBoolean(SpatialWithin.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndConstantEvaluator.java
index d19182ffb2341..b686e1aec1f4c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndConstantEvaluator.java
@@ -67,7 +67,7 @@ public BooleanBlock eval(int positionCount, LongBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialWithin.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue));
+          result.appendBoolean(SpatialWithin.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -81,7 +81,7 @@ public BooleanBlock eval(int positionCount, LongVector leftValueVector) {
     try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialWithin.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue));
+          result.appendBoolean(SpatialWithin.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndConstantEvaluator.java
index cca5ef92918d8..a5d3d48717531 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndConstantEvaluator.java
@@ -70,7 +70,7 @@ public BooleanBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendBoolean(SpatialWithin.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue));
+          result.appendBoolean(SpatialWithin.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -85,7 +85,7 @@ public BooleanBlock eval(int positionCount, BytesRefVector leftValueVector) {
       BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBoolean(SpatialWithin.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue));
+          result.appendBoolean(SpatialWithin.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java
index 21b987f830a2c..01d70b2a53366 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianPointDocValuesAndConstantEvaluator.java
@@ -66,7 +66,7 @@ public DoubleBlock eval(int positionCount, LongBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendDouble(StDistance.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue));
+          result.appendDouble(StDistance.processCartesianPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -80,7 +80,7 @@ public DoubleBlock eval(int positionCount, LongVector leftValueVector) {
     try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendDouble(StDistance.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue));
+          result.appendDouble(StDistance.processCartesianPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java
index 23416e56788b6..b4f3dbc3df326 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceCartesianSourceAndConstantEvaluator.java
@@ -69,7 +69,7 @@ public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendDouble(StDistance.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue));
+          result.appendDouble(StDistance.processCartesianSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -84,7 +84,7 @@ public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) {
       BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendDouble(StDistance.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue));
+          result.appendDouble(StDistance.processCartesianSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java
index 3f96c8bf20ab7..b0980697e9377 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoPointDocValuesAndConstantEvaluator.java
@@ -66,7 +66,7 @@ public DoubleBlock eval(int positionCount, LongBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendDouble(StDistance.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), rightValue));
+          result.appendDouble(StDistance.processGeoPointDocValuesAndConstant(leftValueBlock.getLong(leftValueBlock.getFirstValueIndex(p)), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -80,7 +80,7 @@ public DoubleBlock eval(int positionCount, LongVector leftValueVector) {
     try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendDouble(StDistance.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), rightValue));
+          result.appendDouble(StDistance.processGeoPointDocValuesAndConstant(leftValueVector.getLong(p), this.rightValue));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java
index 556444ac8d740..70bcc64e58834 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistanceGeoSourceAndConstantEvaluator.java
@@ -69,7 +69,7 @@ public DoubleBlock eval(int positionCount, BytesRefBlock leftValueBlock) {
           continue position;
         }
         try {
-          result.appendDouble(StDistance.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), rightValue));
+          result.appendDouble(StDistance.processGeoSourceAndConstant(leftValueBlock.getBytesRef(leftValueBlock.getFirstValueIndex(p), leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -84,7 +84,7 @@ public DoubleBlock eval(int positionCount, BytesRefVector leftValueVector) {
       BytesRef leftValueScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendDouble(StDistance.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), rightValue));
+          result.appendDouble(StDistance.processGeoSourceAndConstant(leftValueVector.getBytesRef(p, leftValueScratch), this.rightValue));
         } catch (IllegalArgumentException | IOException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatchEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatchEvaluator.java
index 21491b4272ea1..96726b310a654 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatchEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatchEvaluator.java
@@ -71,7 +71,7 @@ public BooleanBlock eval(int positionCount, BytesRefBlock inputBlock) {
           result.appendNull();
           continue position;
         }
-        result.appendBoolean(AutomataMatch.process(inputBlock.getBytesRef(inputBlock.getFirstValueIndex(p), inputScratch), automaton, pattern));
+        result.appendBoolean(AutomataMatch.process(inputBlock.getBytesRef(inputBlock.getFirstValueIndex(p), inputScratch), this.automaton, this.pattern));
       }
       return result.build();
     }
@@ -81,7 +81,7 @@ public BooleanVector eval(int positionCount, BytesRefVector inputVector) {
     try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) {
       BytesRef inputScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendBoolean(p, AutomataMatch.process(inputVector.getBytesRef(p, inputScratch), automaton, pattern));
+        result.appendBoolean(p, AutomataMatch.process(inputVector.getBytesRef(p, inputScratch), this.automaton, this.pattern));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java
index e73cc58590fc1..955e05ae7445f 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java
@@ -87,7 +87,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock[] valuesBlocks) {
           int o = valuesBlocks[i].getFirstValueIndex(p);
           valuesValues[i] = valuesBlocks[i].getBytesRef(o, valuesScratch[i]);
         }
-        result.appendBytesRef(Concat.process(scratch, valuesValues));
+        result.appendBytesRef(Concat.process(this.scratch, valuesValues));
       }
       return result.build();
     }
@@ -105,7 +105,7 @@ public BytesRefVector eval(int positionCount, BytesRefVector[] valuesVectors) {
         for (int i = 0; i < valuesVectors.length; i++) {
           valuesValues[i] = valuesVectors[i].getBytesRef(p, valuesScratch[i]);
         }
-        result.appendBytesRef(Concat.process(scratch, valuesValues));
+        result.appendBytesRef(Concat.process(this.scratch, valuesValues));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java
index 8effe5e6d72cd..d932f4f1ea9b0 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java
@@ -93,7 +93,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock, IntBlock le
           result.appendNull();
           continue position;
         }
-        result.appendBytesRef(Left.process(out, cp, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), lengthBlock.getInt(lengthBlock.getFirstValueIndex(p))));
+        result.appendBytesRef(Left.process(this.out, this.cp, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), lengthBlock.getInt(lengthBlock.getFirstValueIndex(p))));
       }
       return result.build();
     }
@@ -103,7 +103,7 @@ public BytesRefVector eval(int positionCount, BytesRefVector strVector, IntVecto
     try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
       BytesRef strScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendBytesRef(Left.process(out, cp, strVector.getBytesRef(p, strScratch), lengthVector.getInt(p)));
+        result.appendBytesRef(Left.process(this.out, this.cp, strVector.getBytesRef(p, strScratch), lengthVector.getInt(p)));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatConstantEvaluator.java
index e83c7c7720828..3ba3f40a8ed5d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatConstantEvaluator.java
@@ -71,7 +71,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock) {
           continue position;
         }
         try {
-          result.appendBytesRef(Repeat.processConstantNumber(scratch, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), number));
+          result.appendBytesRef(Repeat.processConstantNumber(this.scratch, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), this.number));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -86,7 +86,7 @@ public BytesRefBlock eval(int positionCount, BytesRefVector strVector) {
       BytesRef strScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBytesRef(Repeat.processConstantNumber(scratch, strVector.getBytesRef(p, strScratch), number));
+          result.appendBytesRef(Repeat.processConstantNumber(this.scratch, strVector.getBytesRef(p, strScratch), this.number));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatEvaluator.java
index 3723a35283c4b..313c99ad7c454 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatEvaluator.java
@@ -91,7 +91,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock, IntBlock nu
           continue position;
         }
         try {
-          result.appendBytesRef(Repeat.process(scratch, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), numberBlock.getInt(numberBlock.getFirstValueIndex(p))));
+          result.appendBytesRef(Repeat.process(this.scratch, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), numberBlock.getInt(numberBlock.getFirstValueIndex(p))));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -106,7 +106,7 @@ public BytesRefBlock eval(int positionCount, BytesRefVector strVector, IntVector
       BytesRef strScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBytesRef(Repeat.process(scratch, strVector.getBytesRef(p, strScratch), numberVector.getInt(p)));
+          result.appendBytesRef(Repeat.process(this.scratch, strVector.getBytesRef(p, strScratch), numberVector.getInt(p)));
         } catch (IllegalArgumentException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java
index 2b898377f59f6..63623b8428fab 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java
@@ -89,7 +89,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock, BytesRefBlo
           continue position;
         }
         try {
-          result.appendBytesRef(Replace.process(strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), regex, newStrBlock.getBytesRef(newStrBlock.getFirstValueIndex(p), newStrScratch)));
+          result.appendBytesRef(Replace.process(strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), this.regex, newStrBlock.getBytesRef(newStrBlock.getFirstValueIndex(p), newStrScratch)));
         } catch (PatternSyntaxException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -106,7 +106,7 @@ public BytesRefBlock eval(int positionCount, BytesRefVector strVector,
       BytesRef newStrScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendBytesRef(Replace.process(strVector.getBytesRef(p, strScratch), regex, newStrVector.getBytesRef(p, newStrScratch)));
+          result.appendBytesRef(Replace.process(strVector.getBytesRef(p, strScratch), this.regex, newStrVector.getBytesRef(p, newStrScratch)));
         } catch (PatternSyntaxException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java
index 57cad6c63242d..b100cd592495e 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java
@@ -93,7 +93,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock, IntBlock le
           result.appendNull();
           continue position;
         }
-        result.appendBytesRef(Right.process(out, cp, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), lengthBlock.getInt(lengthBlock.getFirstValueIndex(p))));
+        result.appendBytesRef(Right.process(this.out, this.cp, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), lengthBlock.getInt(lengthBlock.getFirstValueIndex(p))));
       }
       return result.build();
     }
@@ -103,7 +103,7 @@ public BytesRefVector eval(int positionCount, BytesRefVector strVector, IntVecto
     try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
       BytesRef strScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendBytesRef(Right.process(out, cp, strVector.getBytesRef(p, strScratch), lengthVector.getInt(p)));
+        result.appendBytesRef(Right.process(this.out, this.cp, strVector.getBytesRef(p, strScratch), lengthVector.getInt(p)));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java
index d58b1aee0ee9d..31639b981f84d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java
@@ -69,7 +69,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock) {
           result.appendNull();
           continue position;
         }
-        Split.process(result, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), delim, scratch);
+        Split.process(result, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), this.delim, this.scratch);
       }
       return result.build();
     }
@@ -79,7 +79,7 @@ public BytesRefBlock eval(int positionCount, BytesRefVector strVector) {
     try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
       BytesRef strScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        Split.process(result, strVector.getBytesRef(p, strScratch), delim, scratch);
+        Split.process(result, strVector.getBytesRef(p, strScratch), this.delim, this.scratch);
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java
index 753febd88ca58..bccfa9ab680ea 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java
@@ -87,7 +87,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock, BytesRefBlo
           result.appendNull();
           continue position;
         }
-        Split.process(result, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), delimBlock.getBytesRef(delimBlock.getFirstValueIndex(p), delimScratch), scratch);
+        Split.process(result, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), delimBlock.getBytesRef(delimBlock.getFirstValueIndex(p), delimScratch), this.scratch);
       }
       return result.build();
     }
@@ -99,7 +99,7 @@ public BytesRefBlock eval(int positionCount, BytesRefVector strVector,
       BytesRef strScratch = new BytesRef();
       BytesRef delimScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        Split.process(result, strVector.getBytesRef(p, strScratch), delimVector.getBytesRef(p, delimScratch), scratch);
+        Split.process(result, strVector.getBytesRef(p, strScratch), delimVector.getBytesRef(p, delimScratch), this.scratch);
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerEvaluator.java
index ee30b2b282162..703fb1cae5d8f 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerEvaluator.java
@@ -66,7 +66,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock valBlock) {
           result.appendNull();
           continue position;
         }
-        result.appendBytesRef(ToLower.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), locale));
+        result.appendBytesRef(ToLower.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), this.locale));
       }
       return result.build();
     }
@@ -76,7 +76,7 @@ public BytesRefVector eval(int positionCount, BytesRefVector valVector) {
     try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
       BytesRef valScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendBytesRef(ToLower.process(valVector.getBytesRef(p, valScratch), locale));
+        result.appendBytesRef(ToLower.process(valVector.getBytesRef(p, valScratch), this.locale));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperEvaluator.java
index cf72804b7e354..2a99af3033b0d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperEvaluator.java
@@ -66,7 +66,7 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock valBlock) {
           result.appendNull();
           continue position;
         }
-        result.appendBytesRef(ToUpper.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), locale));
+        result.appendBytesRef(ToUpper.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), this.locale));
       }
       return result.build();
     }
@@ -76,7 +76,7 @@ public BytesRefVector eval(int positionCount, BytesRefVector valVector) {
     try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
       BytesRef valScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendBytesRef(ToUpper.process(valVector.getBytesRef(p, valScratch), locale));
+        result.appendBytesRef(ToUpper.process(valVector.getBytesRef(p, valScratch), this.locale));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java
index 44ed1ebebd817..962fb4e2e6819 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java
@@ -67,7 +67,7 @@ public LongBlock eval(int positionCount, LongBlock datetimeBlock) {
           continue position;
         }
         try {
-          result.appendLong(Add.processDatetimes(datetimeBlock.getLong(datetimeBlock.getFirstValueIndex(p)), temporalAmount));
+          result.appendLong(Add.processDatetimes(datetimeBlock.getLong(datetimeBlock.getFirstValueIndex(p)), this.temporalAmount));
         } catch (ArithmeticException | DateTimeException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -81,7 +81,7 @@ public LongBlock eval(int positionCount, LongVector datetimeVector) {
     try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendLong(Add.processDatetimes(datetimeVector.getLong(p), temporalAmount));
+          result.appendLong(Add.processDatetimes(datetimeVector.getLong(p), this.temporalAmount));
         } catch (ArithmeticException | DateTimeException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java
index fbf25c5fec393..d6a81f2873e3a 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java
@@ -4,6 +4,7 @@
 // 2.0.
 package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic;
 
+import java.lang.ArithmeticException;
 import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
@@ -50,7 +51,7 @@ public Block eval(Page page) {
         if (rhsVector == null) {
           return eval(page.getPositionCount(), lhsBlock, rhsBlock);
         }
-        return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock();
+        return eval(page.getPositionCount(), lhsVector, rhsVector);
       }
     }
   }
@@ -80,16 +81,26 @@ public DoubleBlock eval(int positionCount, DoubleBlock lhsBlock, DoubleBlock rhs
           result.appendNull();
           continue position;
         }
-        result.appendDouble(Add.processDoubles(lhsBlock.getDouble(lhsBlock.getFirstValueIndex(p)), rhsBlock.getDouble(rhsBlock.getFirstValueIndex(p))));
+        try {
+          result.appendDouble(Add.processDoubles(lhsBlock.getDouble(lhsBlock.getFirstValueIndex(p)), rhsBlock.getDouble(rhsBlock.getFirstValueIndex(p))));
+        } catch (ArithmeticException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
       }
       return result.build();
     }
   }
 
-  public DoubleVector eval(int positionCount, DoubleVector lhsVector, DoubleVector rhsVector) {
-    try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) {
+  public DoubleBlock eval(int positionCount, DoubleVector lhsVector, DoubleVector rhsVector) {
+    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendDouble(p, Add.processDoubles(lhsVector.getDouble(p), rhsVector.getDouble(p)));
+        try {
+          result.appendDouble(Add.processDoubles(lhsVector.getDouble(p), rhsVector.getDouble(p)));
+        } catch (ArithmeticException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java
index 9f3d5aa6d8b10..9104b9b920a93 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java
@@ -4,6 +4,7 @@
 // 2.0.
 package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic;
 
+import java.lang.ArithmeticException;
 import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
@@ -50,7 +51,7 @@ public Block eval(Page page) {
         if (rhsVector == null) {
           return eval(page.getPositionCount(), lhsBlock, rhsBlock);
         }
-        return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock();
+        return eval(page.getPositionCount(), lhsVector, rhsVector);
       }
     }
   }
@@ -80,16 +81,26 @@ public DoubleBlock eval(int positionCount, DoubleBlock lhsBlock, DoubleBlock rhs
           result.appendNull();
           continue position;
         }
-        result.appendDouble(Mul.processDoubles(lhsBlock.getDouble(lhsBlock.getFirstValueIndex(p)), rhsBlock.getDouble(rhsBlock.getFirstValueIndex(p))));
+        try {
+          result.appendDouble(Mul.processDoubles(lhsBlock.getDouble(lhsBlock.getFirstValueIndex(p)), rhsBlock.getDouble(rhsBlock.getFirstValueIndex(p))));
+        } catch (ArithmeticException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
       }
       return result.build();
     }
   }
 
-  public DoubleVector eval(int positionCount, DoubleVector lhsVector, DoubleVector rhsVector) {
-    try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) {
+  public DoubleBlock eval(int positionCount, DoubleVector lhsVector, DoubleVector rhsVector) {
+    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendDouble(p, Mul.processDoubles(lhsVector.getDouble(p), rhsVector.getDouble(p)));
+        try {
+          result.appendDouble(Mul.processDoubles(lhsVector.getDouble(p), rhsVector.getDouble(p)));
+        } catch (ArithmeticException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java
index f87f3c217e16e..e9b540468fb3c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java
@@ -67,7 +67,7 @@ public LongBlock eval(int positionCount, LongBlock datetimeBlock) {
           continue position;
         }
         try {
-          result.appendLong(Sub.processDatetimes(datetimeBlock.getLong(datetimeBlock.getFirstValueIndex(p)), temporalAmount));
+          result.appendLong(Sub.processDatetimes(datetimeBlock.getLong(datetimeBlock.getFirstValueIndex(p)), this.temporalAmount));
         } catch (ArithmeticException | DateTimeException e) {
           warnings.registerException(e);
           result.appendNull();
@@ -81,7 +81,7 @@ public LongBlock eval(int positionCount, LongVector datetimeVector) {
     try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
         try {
-          result.appendLong(Sub.processDatetimes(datetimeVector.getLong(p), temporalAmount));
+          result.appendLong(Sub.processDatetimes(datetimeVector.getLong(p), this.temporalAmount));
         } catch (ArithmeticException | DateTimeException e) {
           warnings.registerException(e);
           result.appendNull();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java
index 291cb5648e213..c9097145df61f 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java
@@ -4,6 +4,7 @@
 // 2.0.
 package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic;
 
+import java.lang.ArithmeticException;
 import java.lang.IllegalArgumentException;
 import java.lang.Override;
 import java.lang.String;
@@ -50,7 +51,7 @@ public Block eval(Page page) {
         if (rhsVector == null) {
           return eval(page.getPositionCount(), lhsBlock, rhsBlock);
         }
-        return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock();
+        return eval(page.getPositionCount(), lhsVector, rhsVector);
       }
     }
   }
@@ -80,16 +81,26 @@ public DoubleBlock eval(int positionCount, DoubleBlock lhsBlock, DoubleBlock rhs
           result.appendNull();
           continue position;
         }
-        result.appendDouble(Sub.processDoubles(lhsBlock.getDouble(lhsBlock.getFirstValueIndex(p)), rhsBlock.getDouble(rhsBlock.getFirstValueIndex(p))));
+        try {
+          result.appendDouble(Sub.processDoubles(lhsBlock.getDouble(lhsBlock.getFirstValueIndex(p)), rhsBlock.getDouble(rhsBlock.getFirstValueIndex(p))));
+        } catch (ArithmeticException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
       }
       return result.build();
     }
   }
 
-  public DoubleVector eval(int positionCount, DoubleVector lhsVector, DoubleVector rhsVector) {
-    try(DoubleVector.FixedBuilder result = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) {
+  public DoubleBlock eval(int positionCount, DoubleVector lhsVector, DoubleVector rhsVector) {
+    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendDouble(p, Sub.processDoubles(lhsVector.getDouble(p), rhsVector.getDouble(p)));
+        try {
+          result.appendDouble(Sub.processDoubles(lhsVector.getDouble(p), rhsVector.getDouble(p)));
+        } catch (ArithmeticException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsConstantEvaluator.java
index 4a1737f01a245..b2d177f61ef59 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsConstantEvaluator.java
@@ -68,7 +68,7 @@ public BooleanBlock eval(int positionCount, BytesRefBlock lhsBlock) {
           result.appendNull();
           continue position;
         }
-        result.appendBoolean(InsensitiveEquals.processConstant(lhsBlock.getBytesRef(lhsBlock.getFirstValueIndex(p), lhsScratch), rhs));
+        result.appendBoolean(InsensitiveEquals.processConstant(lhsBlock.getBytesRef(lhsBlock.getFirstValueIndex(p), lhsScratch), this.rhs));
       }
       return result.build();
     }
@@ -78,7 +78,7 @@ public BooleanVector eval(int positionCount, BytesRefVector lhsVector) {
     try(BooleanVector.FixedBuilder result = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) {
       BytesRef lhsScratch = new BytesRef();
       position: for (int p = 0; p < positionCount; p++) {
-        result.appendBoolean(p, InsensitiveEquals.processConstant(lhsVector.getBytesRef(p, lhsScratch), rhs));
+        result.appendBoolean(p, InsensitiveEquals.processConstant(lhsVector.getBytesRef(p, lhsScratch), this.rhs));
       }
       return result.build();
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/EsqlInfoTransportAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/EsqlInfoTransportAction.java
index b454bc197c90e..ded6c7ee3501b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/EsqlInfoTransportAction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/EsqlInfoTransportAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.esql;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/EsqlUsageTransportAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/EsqlUsageTransportAction.java
index c258e8758f2ba..4c08abd0a23fc 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/EsqlUsageTransportAction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/EsqlUsageTransportAction.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index cb09dbe1f2e6d..120323ebeb7a6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -8,9 +8,12 @@
 package org.elasticsearch.xpack.esql.action;
 
 import org.elasticsearch.Build;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction;
+import org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin;
 import org.elasticsearch.xpack.esql.plugin.EsqlFeatures;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -34,6 +37,11 @@ public enum Cap {
          */
         FN_MV_APPEND,
 
+        /**
+         * Support for {@code MV_PERCENTILE} function.
+         */
+        FN_MV_PERCENTILE,
+
         /**
          * Support for function {@code IP_PREFIX}.
          */
@@ -44,6 +52,16 @@ public enum Cap {
          */
         FN_SUBSTRING_EMPTY_NULL,
 
+        /**
+         * Support for the {@code INLINESTATS} syntax.
+         */
+        INLINESTATS(EsqlPlugin.INLINESTATS_FEATURE_FLAG),
+
+        /**
+         * Support for the expressions in grouping in {@code INLINESTATS} syntax.
+         */
+        INLINESTATS_V2(EsqlPlugin.INLINESTATS_FEATURE_FLAG),
+
         /**
          * Support for aggregation function {@code TOP}.
          */
@@ -59,11 +77,21 @@ public enum Cap {
          */
         AGG_MAX_MIN_IP_SUPPORT,
 
+        /**
+         * Support for strings in aggregations {@code MAX} and {@code MIN}.
+         */
+        AGG_MAX_MIN_STRING_SUPPORT,
+
         /**
          * Support for booleans in {@code TOP} aggregation.
          */
         AGG_TOP_BOOLEAN_SUPPORT,
 
+        /**
+         * Support for ips in {@code TOP} aggregation.
+         */
+        AGG_TOP_IP_SUPPORT,
+
         /**
          * Optimization for ST_CENTROID changed some results in cartesian data. #108713
          */
@@ -107,6 +135,11 @@ public enum Cap {
          */
         ST_DISTANCE,
 
+        /**
+         * Fix determination of CRS types in spatial functions when folding.
+         */
+        SPATIAL_FUNCTIONS_FIX_CRSTYPE_FOLDING,
+
         /**
          * Fix to GROK and DISSECT that allows extracting attributes with the same name as the input
          * https://github.com/elastic/elasticsearch/issues/110184
@@ -144,6 +177,17 @@ public enum Cap {
          */
         UNION_TYPES_REMOVE_FIELDS,
 
+        /**
+         * Fix for union-types when renaming unrelated columns.
+         * https://github.com/elastic/elasticsearch/issues/111452
+         */
+        UNION_TYPES_FIX_RENAME_RESOLUTION,
+
+        /**
+         * Fix for union-types when some indexes are missing the required field. Done in #111932.
+         */
+        UNION_TYPES_MISSING_FIELD,
+
         /**
          * Fix a parsing issue where numbers below Long.MIN_VALUE threw an exception instead of parsing as doubles.
          * see <a href="https://github.com/elastic/elasticsearch/issues/104323"> Parsing large numbers is inconsistent #104323 </a>
@@ -165,24 +209,103 @@ public enum Cap {
          * Fix for non-unique attribute names in ROW and logical plans.
          * https://github.com/elastic/elasticsearch/issues/110541
          */
-        UNIQUE_NAMES;
+        UNIQUE_NAMES,
+
+        /**
+         * Make attributes of GROK/DISSECT adjustable and fix a shadowing bug when pushing them down past PROJECT.
+         * https://github.com/elastic/elasticsearch/issues/108008
+         */
+        FIXED_PUSHDOWN_PAST_PROJECT,
+
+        /**
+         * Adds the {@code MV_PSERIES_WEIGHTED_SUM} function for converting sorted lists of numbers into
+         * a bounded score. This is a generalization of the
+         * <a href="https://en.wikipedia.org/wiki/Riemann_zeta_function">riemann zeta function</a> but we
+         * don't name it that because we don't support complex numbers and don't want to make folks think
+         * of mystical number theory things. This is just a weighted sum that is adjacent to magic.
+         */
+        MV_PSERIES_WEIGHTED_SUM,
+
+        /**
+         * Support for match operator
+         */
+        MATCH_OPERATOR(true),
+
+        /**
+         * Add CombineBinaryComparisons rule.
+         */
+        COMBINE_BINARY_COMPARISONS,
+
+        /**
+         * MATCH command support
+         */
+        MATCH_COMMAND(true),
+
+        /**
+         * Support for nanosecond dates as a data type
+         */
+        DATE_NANOS_TYPE(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
+
+        /**
+         * Support CIDRMatch in CombineDisjunctions rule.
+         */
+        COMBINE_DISJUNCTIVE_CIDRMATCHES,
+
+        /**
+         * Support sending HTTP headers about the status of an async query.
+         */
+        ASYNC_QUERY_STATUS_HEADERS,
+
+        /**
+         * Consider the upper bound when computing the interval in BUCKET auto mode.
+         */
+        BUCKET_INCLUSIVE_UPPER_BOUND,
+
+        /**
+         * Changed error messages for fields with conflicting types in different indices.
+         */
+        SHORT_ERROR_MESSAGES_FOR_UNSUPPORTED_FIELDS,
+
+        /**
+         * Support for the whole number spans in BUCKET function.
+         */
+        BUCKET_WHOLE_NUMBER_AS_SPAN,
+
+        /**
+         * Allow mixed numeric types in coalesce
+         */
+        MIXED_NUMERIC_TYPES_IN_COALESCE;
 
         private final boolean snapshotOnly;
+        private final FeatureFlag featureFlag;
 
         Cap() {
-            snapshotOnly = false;
+            this(false, null);
         };
 
         Cap(boolean snapshotOnly) {
-            this.snapshotOnly = snapshotOnly;
+            this(snapshotOnly, null);
         };
 
-        public String capabilityName() {
-            return name().toLowerCase(Locale.ROOT);
+        Cap(FeatureFlag featureFlag) {
+            this(false, featureFlag);
         }
 
-        public boolean snapshotOnly() {
-            return snapshotOnly;
+        Cap(boolean snapshotOnly, FeatureFlag featureFlag) {
+            assert featureFlag == null || snapshotOnly == false;
+            this.snapshotOnly = snapshotOnly;
+            this.featureFlag = featureFlag;
+        }
+
+        public boolean isEnabled() {
+            if (featureFlag == null) {
+                return Build.current().isSnapshot() || this.snapshotOnly == false;
+            }
+            return featureFlag.isEnabled();
+        }
+
+        public String capabilityName() {
+            return name().toLowerCase(Locale.ROOT);
         }
     }
 
@@ -191,7 +314,7 @@ public boolean snapshotOnly() {
     private static Set<String> capabilities() {
         List<String> caps = new ArrayList<>();
         for (Cap cap : Cap.values()) {
-            if (Build.current().isSnapshot() || cap.snapshotOnly == false) {
+            if (cap.isEnabled()) {
                 caps.add(cap.capabilityName());
             }
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java
index 5c9b4244ec0ca..4ab310863c61d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java
@@ -50,6 +50,7 @@ public class EsqlQueryRequest extends org.elasticsearch.xpack.core.esql.action.E
     private TimeValue keepAlive = DEFAULT_KEEP_ALIVE;
     private boolean keepOnCompletion;
     private boolean onSnapshotBuild = Build.current().isSnapshot();
+    private boolean acceptedPragmaRisks = false;
 
     /**
      * "Tables" provided in the request for use with things like {@code LOOKUP}.
@@ -78,8 +79,9 @@ public ActionRequestValidationException validate() {
         if (Strings.hasText(query) == false) {
             validationException = addValidationError("[" + RequestXContent.QUERY_FIELD + "] is required", validationException);
         }
+
         if (onSnapshotBuild == false) {
-            if (pragmas.isEmpty() == false) {
+            if (pragmas.isEmpty() == false && acceptedPragmaRisks == false) {
                 validationException = addValidationError(
                     "[" + RequestXContent.PRAGMA_FIELD + "] only allowed in snapshot builds",
                     validationException
@@ -230,4 +232,8 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
     void onSnapshotBuild(boolean onSnapshotBuild) {
         this.onSnapshotBuild = onSnapshotBuild;
     }
+
+    void acceptedPragmaRisks(boolean accepted) {
+        this.acceptedPragmaRisks = accepted;
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java
index 6bb52fc8c0aaf..2161efca1d2b4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveFieldsAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.action.fieldcaps.TransportFieldCapabilitiesAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlSearchShardsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlSearchShardsAction.java
new file mode 100644
index 0000000000000..4ccbb47650a92
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlSearchShardsAction.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.search.SearchShardsRequest;
+import org.elasticsearch.action.search.SearchShardsResponse;
+import org.elasticsearch.action.search.TransportSearchShardsAction;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+/**
+ * A fork of the search shards API for ES|QL. This fork allows us to gradually introduce features and optimizations to this
+ * internal API without risking breaking the search-shards API, which is used by the search API. For now, this API delegates
+ * to the search-shards API, but gradually, we will decouple this API completely from the search-shards API.
+ */
+public class EsqlSearchShardsAction extends HandledTransportAction<SearchShardsRequest, SearchShardsResponse> {
+    public static final String NAME = "indices:data/read/esql/search_shards";
+    public static final ActionType<SearchShardsResponse> TYPE = new ActionType<>(NAME);
+    private final TransportSearchShardsAction searchShardsAction;
+
+    @Inject
+    public EsqlSearchShardsAction(
+        TransportService transportService,
+        ActionFilters actionFilters,
+        TransportSearchShardsAction searchShardsAction
+    ) {
+        super(
+            NAME,
+            transportService,
+            actionFilters,
+            SearchShardsRequest::new,
+            // TODO replace DIRECT_EXECUTOR_SERVICE when removing workaround for https://github.com/elastic/elasticsearch/issues/97916
+            transportService.getThreadPool().executor(ThreadPool.Names.SEARCH_COORDINATION)
+        );
+        this.searchShardsAction = searchShardsAction;
+    }
+
+    @Override
+    protected void doExecute(Task task, SearchShardsRequest request, ActionListener<SearchShardsResponse> listener) {
+        searchShardsAction.searchShards(task, request, listener);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java
index 0bc1eb46abefe..0b1bafdab1a99 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
-import org.elasticsearch.compute.lucene.UnsupportedValueSource;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -28,6 +27,7 @@
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.spatialToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString;
 
@@ -118,6 +118,14 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa
                     return builder.value(dateTimeToString(longVal));
                 }
             };
+            case DATE_NANOS -> new PositionToXContent(block) {
+                @Override
+                protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex)
+                    throws IOException {
+                    long longVal = ((LongBlock) block).getLong(valueIndex);
+                    return builder.value(nanoTimeToString(longVal));
+                }
+            };
             case GEO_POINT, GEO_SHAPE, CARTESIAN_POINT, CARTESIAN_SHAPE -> new PositionToXContent(block) {
                 @Override
                 protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex)
@@ -151,7 +159,7 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa
                 @Override
                 protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex)
                     throws IOException {
-                    return builder.value(UnsupportedValueSource.UNSUPPORTED_OUTPUT);
+                    return builder.value((String) null);
                 }
             };
             case SOURCE -> new PositionToXContent(block) {
@@ -165,7 +173,7 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa
                     }
                 }
             };
-            case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, SHORT, BYTE, OBJECT, NESTED, FLOAT, HALF_FLOAT, SCALED_FLOAT,
+            case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, SHORT, BYTE, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT,
                 PARTIAL_AGG -> throw new IllegalArgumentException("can't convert values of type [" + columnInfo.type() + "]");
         };
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java
index 4c511a4450bc8..76573a8f4cc1d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java
@@ -69,6 +69,7 @@ String fields() {
     private static final ParseField PARAMS_FIELD = new ParseField("params");
     private static final ParseField LOCALE_FIELD = new ParseField("locale");
     private static final ParseField PROFILE_FIELD = new ParseField("profile");
+    private static final ParseField ACCEPT_PRAGMA_RISKS = new ParseField("accept_pragma_risks");
     static final ParseField TABLES_FIELD = new ParseField("tables");
 
     static final ParseField WAIT_FOR_COMPLETION_TIMEOUT = new ParseField("wait_for_completion_timeout");
@@ -92,6 +93,7 @@ private static void objectParserCommon(ObjectParser<EsqlQueryRequest, ?> parser)
         parser.declareString(EsqlQueryRequest::query, QUERY_FIELD);
         parser.declareBoolean(EsqlQueryRequest::columnar, COLUMNAR_FIELD);
         parser.declareObject(EsqlQueryRequest::filter, (p, c) -> AbstractQueryBuilder.parseTopLevelQuery(p), FILTER_FIELD);
+        parser.declareBoolean(EsqlQueryRequest::acceptedPragmaRisks, ACCEPT_PRAGMA_RISKS);
         parser.declareObject(
             EsqlQueryRequest::pragmas,
             (p, c) -> new QueryPragmas(Settings.builder().loadFromMap(p.map()).build()),
@@ -153,14 +155,16 @@ private static QueryParams parseParams(XContentParser p) throws IOException {
                         );
                     }
                     for (Map.Entry<String, Object> entry : param.fields.entrySet()) {
-                        if (isValidParamName(entry.getKey()) == false) {
+                        String name = entry.getKey();
+                        if (isValidParamName(name) == false) {
                             errors.add(
                                 new XContentParseException(
                                     loc,
                                     "["
-                                        + entry.getKey()
+                                        + name
                                         + "] is not a valid parameter name, "
-                                        + "a valid parameter name starts with a letter and contains letters, digits and underscores only"
+                                        + "a valid parameter name starts with a letter or underscore, "
+                                        + "and contains letters, digits and underscores only"
                                 )
                             );
                         }
@@ -168,7 +172,7 @@ private static QueryParams parseParams(XContentParser p) throws IOException {
                         if (type == null) {
                             errors.add(new XContentParseException(loc, entry + " is not supported as a parameter"));
                         }
-                        currentParam = new QueryParam(entry.getKey(), entry.getValue(), type);
+                        currentParam = new QueryParam(name, entry.getValue(), type);
                         namedParams.add(currentParam);
                     }
                 } else {
@@ -201,6 +205,7 @@ private static QueryParams parseParams(XContentParser p) throws IOException {
                 }
             }
         }
+        // don't allow mixed named and unnamed parameters
         if (namedParams.isEmpty() == false && unNamedParams.isEmpty() == false) {
             errors.add(
                 new XContentParseException(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java
index 290a816275a29..3b18bda120e2e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java
@@ -9,42 +9,31 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.compute.data.Block;
-import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.Page;
-import org.elasticsearch.compute.lucene.UnsupportedValueSource;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
-import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.planner.PlannerUtils;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.longToUnsignedLong;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.spatialToString;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToIP;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToVersion;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString;
 
 /**
@@ -134,12 +123,16 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR
                 long longVal = ((LongBlock) block).getLong(offset);
                 yield dateTimeToString(longVal);
             }
+            case DATE_NANOS -> {
+                long longVal = ((LongBlock) block).getLong(offset);
+                yield nanoTimeToString(longVal);
+            }
             case BOOLEAN -> ((BooleanBlock) block).getBoolean(offset);
             case VERSION -> versionToString(((BytesRefBlock) block).getBytesRef(offset, scratch));
             case GEO_POINT, GEO_SHAPE, CARTESIAN_POINT, CARTESIAN_SHAPE -> spatialToString(
                 ((BytesRefBlock) block).getBytesRef(offset, scratch)
             );
-            case UNSUPPORTED -> UnsupportedValueSource.UNSUPPORTED_OUTPUT;
+            case UNSUPPORTED -> (String) null;
             case SOURCE -> {
                 BytesRef val = ((BytesRefBlock) block).getBytesRef(offset, scratch);
                 try {
@@ -151,59 +144,8 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR
                     throw new UncheckedIOException(e);
                 }
             }
-            case SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, NESTED, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE,
-                NULL, PARTIAL_AGG -> throw EsqlIllegalArgumentException.illegalDataType(dataType);
+            case SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, NULL,
+                PARTIAL_AGG -> throw EsqlIllegalArgumentException.illegalDataType(dataType);
         };
     }
-
-    /**
-     * Converts a list of values to Pages so that we can parse from xcontent. It's not
-     * super efficient, but it doesn't really have to be.
-     */
-    static Page valuesToPage(BlockFactory blockFactory, List<ColumnInfoImpl> columns, List<List<Object>> values) {
-        List<DataType> dataTypes = columns.stream().map(ColumnInfoImpl::type).toList();
-        List<Block.Builder> results = dataTypes.stream()
-            .map(c -> PlannerUtils.toElementType(c).newBlockBuilder(values.size(), blockFactory))
-            .toList();
-
-        for (List<Object> row : values) {
-            for (int c = 0; c < row.size(); c++) {
-                var builder = results.get(c);
-                var value = row.get(c);
-                switch (dataTypes.get(c)) {
-                    case UNSIGNED_LONG -> ((LongBlock.Builder) builder).appendLong(longToUnsignedLong(((Number) value).longValue(), true));
-                    case LONG, COUNTER_LONG -> ((LongBlock.Builder) builder).appendLong(((Number) value).longValue());
-                    case INTEGER, COUNTER_INTEGER -> ((IntBlock.Builder) builder).appendInt(((Number) value).intValue());
-                    case DOUBLE, COUNTER_DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble(((Number) value).doubleValue());
-                    case KEYWORD, TEXT, UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(value.toString()));
-                    case IP -> ((BytesRefBlock.Builder) builder).appendBytesRef(stringToIP(value.toString()));
-                    case DATETIME -> {
-                        long longVal = dateTimeToLong(value.toString());
-                        ((LongBlock.Builder) builder).appendLong(longVal);
-                    }
-                    case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean(((Boolean) value));
-                    case NULL -> builder.appendNull();
-                    case VERSION -> ((BytesRefBlock.Builder) builder).appendBytesRef(stringToVersion(new BytesRef(value.toString())));
-                    case SOURCE -> {
-                        @SuppressWarnings("unchecked")
-                        Map<String, ?> o = (Map<String, ?>) value;
-                        try {
-                            try (XContentBuilder sourceBuilder = JsonXContent.contentBuilder()) {
-                                sourceBuilder.map(o);
-                                ((BytesRefBlock.Builder) builder).appendBytesRef(BytesReference.bytes(sourceBuilder).toBytesRef());
-                            }
-                        } catch (IOException e) {
-                            throw new UncheckedIOException(e);
-                        }
-                    }
-                    case GEO_POINT, GEO_SHAPE, CARTESIAN_POINT, CARTESIAN_SHAPE -> {
-                        // This just converts WKT to WKB, so does not need CRS knowledge, we could merge GEO and CARTESIAN here
-                        BytesRef wkb = stringToSpatial(value.toString());
-                        ((BytesRefBlock.Builder) builder).appendBytesRef(wkb);
-                    }
-                }
-            }
-        }
-        return new Page(results.stream().map(Block.Builder::build).toArray(Block[]::new));
-    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
index a43be3b8c2c3a..5baced5bc93f2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
-import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
 import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
@@ -36,17 +35,15 @@
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
 import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.core.rule.ParameterizedRule;
 import org.elasticsearch.xpack.esql.core.rule.ParameterizedRuleExecutor;
 import org.elasticsearch.xpack.esql.core.rule.Rule;
 import org.elasticsearch.xpack.esql.core.rule.RuleExecutor;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
+import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField;
 import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.core.util.Holder;
@@ -59,11 +56,17 @@
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToUnsignedLong;
+import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.DateTimeArithmeticOperation;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
-import org.elasticsearch.xpack.esql.optimizer.rules.SubstituteSurrogates;
-import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
+import org.elasticsearch.xpack.esql.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.plan.logical.Drop;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
@@ -75,14 +78,14 @@
 import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.Rename;
+import org.elasticsearch.xpack.esql.plan.logical.Stats;
 import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation;
 import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.stats.FeatureMetric;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
-import org.elasticsearch.xpack.esql.type.MultiTypeEsField;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -113,11 +116,11 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED;
 import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount;
 import static org.elasticsearch.xpack.esql.stats.FeatureMetric.LIMIT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount;
 
 /**
  * This class is part of the planner. Resolves references (such as variable and index names) and performs implicit casting.
@@ -126,7 +129,7 @@ public class Analyzer extends ParameterizedRuleExecutor<LogicalPlan, AnalyzerCon
     // marker list of attributes for plans that do not have any concrete fields to return, but have other computed columns to return
     // ie from test | stats c = count(*)
     public static final List<Attribute> NO_FIELDS = List.of(
-        new ReferenceAttribute(Source.EMPTY, "<no-fields>", DataType.NULL, null, Nullability.TRUE, null, true)
+        new ReferenceAttribute(Source.EMPTY, "<no-fields>", DataType.NULL, Nullability.TRUE, null, true)
     );
     private static final Iterable<RuleExecutor.Batch<LogicalPlan>> rules;
 
@@ -215,11 +218,14 @@ protected LogicalPlan rule(UnresolvedRelation plan, AnalyzerContext context) {
      * Specific flattening method, different from the default EsRelation that:
      * 1. takes care of data type widening (for certain types)
      * 2. drops the object and keyword hierarchy
+     * <p>
+     *     Public for testing.
+     * </p>
      */
-    private static List<Attribute> mappingAsAttributes(Source source, Map<String, EsField> mapping) {
+    public static List<Attribute> mappingAsAttributes(Source source, Map<String, EsField> mapping) {
         var list = new ArrayList<Attribute>();
         mappingAsAttributes(list, source, null, mapping);
-        list.sort(Comparator.comparing(Attribute::qualifiedName));
+        list.sort(Comparator.comparing(Attribute::name));
         return list;
     }
 
@@ -242,11 +248,11 @@ private static void mappingAsAttributes(List<Attribute> list, Source source, Fie
                     ? new UnsupportedAttribute(source, name, uef)
                     : new FieldAttribute(source, parent, name, t);
                 // primitive branch
-                if (EsqlDataTypes.isPrimitive(type)) {
+                if (DataType.isPrimitive(type)) {
                     list.add(attribute);
                 }
-                // allow compound object even if they are unknown (but not NESTED)
-                if (type != NESTED && fieldProperties.isEmpty() == false) {
+                // allow compound object even if they are unknown
+                if (fieldProperties.isEmpty() == false) {
                     mappingAsAttributes(list, source, attribute, fieldProperties);
                 }
             }
@@ -287,7 +293,7 @@ protected LogicalPlan rule(Enrich plan, AnalyzerContext context) {
                 );
             } else {
                 String error = context.enrichResolution().getError(policyName, plan.mode());
-                var policyNameExp = new UnresolvedAttribute(plan.policyName().source(), policyName, null, error);
+                var policyNameExp = new UnresolvedAttribute(plan.policyName().source(), policyName, error);
                 return new Enrich(plan.source(), plan.child(), plan.mode(), policyNameExp, plan.matchField(), null, Map.of(), List.of());
             }
         }
@@ -332,9 +338,9 @@ private static NamedExpression createEnrichFieldExpression(
                 if (CollectionUtils.isEmpty(similar) == false) {
                     msg += ", did you mean " + (similar.size() == 1 ? "[" + similar.get(0) + "]" : "any of " + similar) + "?";
                 }
-                return new UnresolvedAttribute(source, enrichFieldName, null, msg);
+                return new UnresolvedAttribute(source, enrichFieldName, msg);
             } else {
-                return new ReferenceAttribute(source, enrichFieldName, mappedField.dataType(), null, Nullability.TRUE, null, false);
+                return new ReferenceAttribute(source, enrichFieldName, mappedField.dataType(), Nullability.TRUE, null, false);
             }
         }
     }
@@ -357,7 +363,7 @@ protected LogicalPlan rule(Lookup lookup, AnalyzerContext context) {
                 if (CollectionUtils.isEmpty(potentialMatches) == false) {
                     message = UnresolvedAttribute.errorMessage(tableName, potentialMatches).replace("column", "table");
                 }
-                tableNameExpression = new UnresolvedAttribute(tableNameExpression.source(), tableName, null, message);
+                tableNameExpression = new UnresolvedAttribute(tableNameExpression.source(), tableName, message);
             }
             // wrap the table in a local relationship for idiomatic field resolution
             else {
@@ -400,8 +406,8 @@ protected LogicalPlan doRule(LogicalPlan plan) {
                 childrenOutput.addAll(output);
             }
 
-            if (plan instanceof Aggregate agg) {
-                return resolveAggregate(agg, childrenOutput);
+            if (plan instanceof Stats stats) {
+                return resolveStats(stats, childrenOutput);
             }
 
             if (plan instanceof Drop d) {
@@ -435,11 +441,12 @@ protected LogicalPlan doRule(LogicalPlan plan) {
             return plan.transformExpressionsOnly(UnresolvedAttribute.class, ua -> maybeResolveAttribute(ua, childrenOutput));
         }
 
-        private LogicalPlan resolveAggregate(Aggregate a, List<Attribute> childrenOutput) {
+        private LogicalPlan resolveStats(Stats stats, List<Attribute> childrenOutput) {
             // if the grouping is resolved but the aggs are not, use the former to resolve the latter
             // e.g. STATS a ... GROUP BY a = x + 1
             Holder<Boolean> changed = new Holder<>(false);
-            List<Expression> groupings = a.groupings();
+            List<Expression> groupings = stats.groupings();
+            List<? extends NamedExpression> aggregates = stats.aggregates();
             // first resolve groupings since the aggs might refer to them
             // trying to globally resolve unresolved attributes will lead to some being marked as unresolvable
             if (Resolvables.resolved(groupings) == false) {
@@ -453,23 +460,23 @@ private LogicalPlan resolveAggregate(Aggregate a, List<Attribute> childrenOutput
                 }
                 groupings = newGroupings;
                 if (changed.get()) {
-                    a = new Aggregate(a.source(), a.child(), a.aggregateType(), newGroupings, a.aggregates());
+                    stats = stats.with(stats.child(), newGroupings, stats.aggregates());
                     changed.set(false);
                 }
             }
 
-            if (a.expressionsResolved() == false) {
-                AttributeMap<Expression> resolved = new AttributeMap<>();
+            if (Resolvables.resolved(groupings) == false || (Resolvables.resolved(aggregates) == false)) {
+                ArrayList<Attribute> resolved = new ArrayList<>();
                 for (Expression e : groupings) {
                     Attribute attr = Expressions.attribute(e);
                     if (attr != null && attr.resolved()) {
-                        resolved.put(attr, attr);
+                        resolved.add(attr);
                     }
                 }
-                List<Attribute> resolvedList = NamedExpressions.mergeOutputAttributes(new ArrayList<>(resolved.keySet()), childrenOutput);
-                List<NamedExpression> newAggregates = new ArrayList<>();
+                List<Attribute> resolvedList = NamedExpressions.mergeOutputAttributes(resolved, childrenOutput);
 
-                for (NamedExpression aggregate : a.aggregates()) {
+                List<NamedExpression> newAggregates = new ArrayList<>();
+                for (NamedExpression aggregate : stats.aggregates()) {
                     var agg = (NamedExpression) aggregate.transformUp(UnresolvedAttribute.class, ua -> {
                         Expression ne = ua;
                         Attribute maybeResolved = maybeResolveAttribute(ua, resolvedList);
@@ -482,10 +489,10 @@ private LogicalPlan resolveAggregate(Aggregate a, List<Attribute> childrenOutput
                     newAggregates.add(agg);
                 }
 
-                a = changed.get() ? new Aggregate(a.source(), a.child(), a.aggregateType(), groupings, newAggregates) : a;
+                stats = changed.get() ? stats.with(stats.child(), groupings, newAggregates) : stats;
             }
 
-            return a;
+            return (LogicalPlan) stats;
         }
 
         private LogicalPlan resolveMvExpand(MvExpand p, List<Attribute> childrenOutput) {
@@ -499,15 +506,7 @@ private LogicalPlan resolveMvExpand(MvExpand p, List<Attribute> childrenOutput)
                     p.child(),
                     resolved,
                     resolved.resolved()
-                        ? new ReferenceAttribute(
-                            resolved.source(),
-                            resolved.name(),
-                            resolved.dataType(),
-                            null,
-                            resolved.nullable(),
-                            null,
-                            false
-                        )
+                        ? new ReferenceAttribute(resolved.source(), resolved.name(), resolved.dataType(), resolved.nullable(), null, false)
                         : resolved
                 );
             }
@@ -556,7 +555,6 @@ private LogicalPlan resolveLookup(Lookup l, List<Attribute> childrenOutput) {
                                 matchFieldChildReference = new UnresolvedAttribute(
                                     attr.source(),
                                     attr.name(),
-                                    attr.qualifier(),
                                     attr.id(),
                                     "column type mismatch, table column was ["
                                         + joinedAttribute.dataType().typeName()
@@ -812,9 +810,18 @@ private LogicalPlan resolveEnrich(Enrich enrich, List<Attribute> childrenOutput)
                     String matchType = enrich.policy().getType();
                     DataType[] allowed = allowedEnrichTypes(matchType);
                     if (Arrays.asList(allowed).contains(dataType) == false) {
-                        String suffix = "only " + Arrays.toString(allowed) + " allowed for type [" + matchType + "]";
+                        String suffix = "only ["
+                            + Arrays.stream(allowed).map(DataType::typeName).collect(Collectors.joining(", "))
+                            + "] allowed for type ["
+                            + matchType
+                            + "]";
                         resolved = ua.withUnresolvedMessage(
-                            "Unsupported type [" + resolved.dataType() + "] for enrich matching field [" + ua.name() + "]; " + suffix
+                            "Unsupported type ["
+                                + resolved.dataType().typeName()
+                                + "] for enrich matching field ["
+                                + ua.name()
+                                + "]; "
+                                + suffix
                         );
                     }
                 }
@@ -841,8 +848,8 @@ private DataType[] allowedEnrichTypes(String matchType) {
     }
 
     private static List<Attribute> resolveAgainstList(UnresolvedNamePattern up, Collection<Attribute> attrList) {
-        UnresolvedAttribute ua = new UnresolvedAttribute(up.source(), up.pattern(), null);
-        Predicate<Attribute> matcher = a -> up.match(a.name()) || up.match(a.qualifiedName());
+        UnresolvedAttribute ua = new UnresolvedAttribute(up.source(), up.pattern());
+        Predicate<Attribute> matcher = a -> up.match(a.name());
         var matches = AnalyzerRules.maybeResolveAgainstList(matcher, () -> ua, attrList, true, a -> Analyzer.handleSpecialFields(ua, a));
         return potentialCandidatesIfNoMatchesFound(ua, matches, attrList, list -> UnresolvedNamePattern.errorMessage(up.pattern(), list));
     }
@@ -858,12 +865,15 @@ private static List<Attribute> potentialCandidatesIfNoMatchesFound(
         Collection<Attribute> attrList,
         java.util.function.Function<List<String>, String> messageProducer
     ) {
+        if (ua.customMessage()) {
+            return List.of();
+        }
         // none found - add error message
         if (matches.isEmpty()) {
             Set<String> names = new HashSet<>(attrList.size());
             for (var a : attrList) {
                 String nameCandidate = a.name();
-                if (EsqlDataTypes.isPrimitive(a.dataType())) {
+                if (DataType.isPrimitive(a.dataType())) {
                     names.add(nameCandidate);
                 }
             }
@@ -939,6 +949,24 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) {
         return b;
     }
 
+    /**
+     * Cast string literals in ScalarFunction, EsqlArithmeticOperation, BinaryComparison and In to desired data types.
+     * For example, the string literals in the following expressions will be cast implicitly to the field data type on the left hand side.
+     * date > "2024-08-21"
+     * date in ("2024-08-21", "2024-08-22", "2024-08-23")
+     * date = "2024-08-21" + 3 days
+     * ip == "127.0.0.1"
+     * version != "1.0"
+     *
+     * If the inputs to Coalesce are mixed numeric types, cast the rest of the numeric field or value to the first numeric data type if
+     * applicable. For example, implicit casting converts:
+     * Coalesce(Long, Int) to Coalesce(Long, Long)
+     * Coalesce(null, Long, Int) to Coalesce(null, Long, Long)
+     * Coalesce(Double, Long, Int) to Coalesce(Double, Double, Double)
+     * Coalesce(null, Double, Long, Int) to Coalesce(null, Double, Double, Double)
+     *
+     * Coalesce(Int, Long) will NOT be converted to Coalesce(Long, Long) or Coalesce(Int, Int).
+     */
     private static class ImplicitCasting extends ParameterizedRule<LogicalPlan, LogicalPlan, AnalyzerContext> {
         @Override
         public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) {
@@ -968,22 +996,38 @@ private static Expression processScalarFunction(EsqlScalarFunction f, EsqlFuncti
             boolean childrenChanged = false;
             DataType targetDataType = DataType.NULL;
             Expression arg;
+            DataType targetNumericType = null;
+            boolean castNumericArgs = true;
             for (int i = 0; i < args.size(); i++) {
                 arg = args.get(i);
-                if (arg.resolved() && arg.dataType() == KEYWORD && arg.foldable() && ((arg instanceof EsqlScalarFunction) == false)) {
-                    if (i < targetDataTypes.size()) {
-                        targetDataType = targetDataTypes.get(i);
-                    }
-                    if (targetDataType != DataType.NULL && targetDataType != DataType.UNSUPPORTED) {
-                        Expression e = castStringLiteral(arg, targetDataType);
-                        childrenChanged = true;
-                        newChildren.add(e);
-                        continue;
+                if (arg.resolved()) {
+                    var dataType = arg.dataType();
+                    if (dataType == KEYWORD) {
+                        if (arg.foldable() && ((arg instanceof EsqlScalarFunction) == false)) {
+                            if (i < targetDataTypes.size()) {
+                                targetDataType = targetDataTypes.get(i);
+                            }
+                            if (targetDataType != DataType.NULL && targetDataType != DataType.UNSUPPORTED) {
+                                Expression e = castStringLiteral(arg, targetDataType);
+                                childrenChanged = true;
+                                newChildren.add(e);
+                                continue;
+                            }
+                        }
+                    } else if (dataType.isNumeric() && canCastMixedNumericTypes(f) && castNumericArgs) {
+                        if (targetNumericType == null) {
+                            targetNumericType = dataType;  // target data type is the first numeric data type
+                        } else if (dataType != targetNumericType) {
+                            castNumericArgs = canCastNumeric(dataType, targetNumericType);
+                        }
                     }
                 }
                 newChildren.add(args.get(i));
             }
-            return childrenChanged ? f.replaceChildren(newChildren) : f;
+            Expression resultF = childrenChanged ? f.replaceChildren(newChildren) : f;
+            return targetNumericType != null && castNumericArgs
+                ? castMixedNumericTypes((EsqlScalarFunction) resultF, targetNumericType)
+                : resultF;
         }
 
         private static Expression processBinaryOperator(BinaryOperator<?, ?, ?, ?> o) {
@@ -998,7 +1042,7 @@ private static Expression processBinaryOperator(BinaryOperator<?, ?, ?, ?> o) {
             Expression from = Literal.NULL;
 
             if (left.dataType() == KEYWORD && left.foldable() && (left instanceof EsqlScalarFunction == false)) {
-                if (supportsImplicitCasting(right.dataType())) {
+                if (supportsStringImplicitCasting(right.dataType())) {
                     targetDataType = right.dataType();
                     from = left;
                 } else if (supportsImplicitTemporalCasting(right, o)) {
@@ -1007,7 +1051,7 @@ private static Expression processBinaryOperator(BinaryOperator<?, ?, ?, ?> o) {
                 }
             }
             if (right.dataType() == KEYWORD && right.foldable() && (right instanceof EsqlScalarFunction == false)) {
-                if (supportsImplicitCasting(left.dataType())) {
+                if (supportsStringImplicitCasting(left.dataType())) {
                     targetDataType = left.dataType();
                     from = right;
                 } else if (supportsImplicitTemporalCasting(left, o)) {
@@ -1027,16 +1071,18 @@ private static Expression processBinaryOperator(BinaryOperator<?, ?, ?, ?> o) {
         private static Expression processIn(In in) {
             Expression left = in.value();
             List<Expression> right = in.list();
+            DataType targetDataType = left.dataType();
 
-            if (left.resolved() == false || supportsImplicitCasting(left.dataType()) == false) {
+            if (left.resolved() == false || supportsStringImplicitCasting(targetDataType) == false) {
                 return in;
             }
+
             List<Expression> newChildren = new ArrayList<>(right.size() + 1);
             boolean childrenChanged = false;
 
             for (Expression value : right) {
-                if (value.dataType() == KEYWORD && value.foldable()) {
-                    Expression e = castStringLiteral(value, left.dataType());
+                if (value.resolved() && value.dataType() == KEYWORD && value.foldable()) {
+                    Expression e = castStringLiteral(value, targetDataType);
                     newChildren.add(e);
                     childrenChanged = true;
                 } else {
@@ -1047,11 +1093,47 @@ private static Expression processIn(In in) {
             return childrenChanged ? in.replaceChildren(newChildren) : in;
         }
 
+        private static boolean canCastMixedNumericTypes(EsqlScalarFunction f) {
+            return f instanceof Coalesce;
+        }
+
+        private static boolean canCastNumeric(DataType from, DataType to) {
+            DataType commonType = EsqlDataTypeConverter.commonType(from, to);
+            return commonType == to;
+        }
+
+        private static Expression castMixedNumericTypes(EsqlScalarFunction f, DataType targetNumericType) {
+            List<Expression> newChildren = new ArrayList<>(f.children().size());
+            boolean childrenChanged = false;
+            DataType childDataType;
+
+            for (Expression e : f.children()) {
+                childDataType = e.dataType();
+                if (childDataType.isNumeric() == false
+                    || childDataType == targetNumericType
+                    || canCastNumeric(childDataType, targetNumericType) == false) {
+                    newChildren.add(e);
+                    continue;
+                }
+                childrenChanged = true;
+                // add a casting function
+                switch (targetNumericType) {
+                    case INTEGER -> newChildren.add(new ToInteger(e.source(), e));
+                    case LONG -> newChildren.add(new ToLong(e.source(), e));
+                    case DOUBLE -> newChildren.add(new ToDouble(e.source(), e));
+                    case UNSIGNED_LONG -> newChildren.add(new ToUnsignedLong(e.source(), e));
+                    default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetNumericType);
+                }
+
+            }
+            return childrenChanged ? f.replaceChildren(newChildren) : f;
+        }
+
         private static boolean supportsImplicitTemporalCasting(Expression e, BinaryOperator<?, ?, ?, ?> o) {
             return isTemporalAmount(e.dataType()) && (o instanceof DateTimeArithmeticOperation);
         }
 
-        private static boolean supportsImplicitCasting(DataType type) {
+        private static boolean supportsStringImplicitCasting(DataType type) {
             return type == DATETIME || type == IP || type == VERSION || type == BOOLEAN;
         }
 
@@ -1067,24 +1149,19 @@ public static Expression castStringLiteral(Expression from, DataType target) {
                     target,
                     e.getMessage()
                 );
-                return new UnsupportedAttribute(
-                    from.source(),
-                    String.valueOf(from.fold()),
-                    new UnsupportedEsField(String.valueOf(from.fold()), from.dataType().typeName()),
-                    message
-                );
+                return new UnresolvedAttribute(from.source(), String.valueOf(from.fold()), message);
             }
         }
     }
 
     /**
      * The EsqlIndexResolver will create InvalidMappedField instances for fields that are ambiguous (i.e. have multiple mappings).
-     * During ResolveRefs we do not convert these to UnresolvedAttribute instances, as we want to first determine if they can
+     * During {@link ResolveRefs} we do not convert these to UnresolvedAttribute instances, as we want to first determine if they can
      * instead be handled by conversion functions within the query. This rule looks for matching conversion functions and converts
      * those fields into MultiTypeEsField, which encapsulates the knowledge of how to convert these into a single type.
      * This knowledge will be used later in generating the FieldExtractExec with built-in type conversion.
      * Any fields which could not be resolved by conversion functions will be converted to UnresolvedAttribute instances in a later rule
-     * (See UnresolveUnionTypes below).
+     * (See {@link UnionTypesCleanup} below).
      */
     private static class ResolveUnionTypes extends Rule<LogicalPlan, LogicalPlan> {
 
@@ -1104,7 +1181,7 @@ public LogicalPlan apply(LogicalPlan plan) {
                 }
             });
 
-            return plan.transformUp(LogicalPlan.class, p -> p.resolved() || p.childrenResolved() == false ? p : doRule(p));
+            return plan.transformUp(LogicalPlan.class, p -> p.childrenResolved() == false ? p : doRule(p));
         }
 
         private LogicalPlan doRule(LogicalPlan plan) {
@@ -1120,23 +1197,6 @@ private LogicalPlan doRule(LogicalPlan plan) {
                 return plan;
             }
 
-            // In ResolveRefs the aggregates are resolved from the groupings, which might have an unresolved MultiTypeEsField.
-            // Now that we have resolved those, we need to re-resolve the aggregates.
-            if (plan instanceof Aggregate agg) {
-                // If the union-types resolution occurred in a child of the aggregate, we need to check the groupings
-                plan = agg.transformExpressionsOnly(FieldAttribute.class, UnionTypesCleanup::checkUnresolved);
-
-                // Aggregates where the grouping key comes from a union-type field need to be resolved against the grouping key
-                Map<Attribute, Expression> resolved = new HashMap<>();
-                for (Expression e : agg.groupings()) {
-                    Attribute attr = Expressions.attribute(e);
-                    if (attr != null && attr.resolved()) {
-                        resolved.put(attr, e);
-                    }
-                }
-                plan = plan.transformExpressionsOnly(UnresolvedAttribute.class, ua -> resolveAttribute(ua, resolved));
-            }
-
             // And add generated fields to EsRelation, so these new attributes will appear in the OutputExec of the Fragment
             // and thereby get used in FieldExtractExec
             plan = plan.transformDown(EsRelation.class, esr -> {
@@ -1158,21 +1218,12 @@ private LogicalPlan doRule(LogicalPlan plan) {
             return plan;
         }
 
-        private Expression resolveAttribute(UnresolvedAttribute ua, Map<Attribute, Expression> resolved) {
-            var named = resolveAgainstList(ua, resolved.keySet());
-            return switch (named.size()) {
-                case 0 -> ua;
-                case 1 -> named.get(0).equals(ua) ? ua : resolved.get(named.get(0));
-                default -> ua.withUnresolvedMessage("Resolved [" + ua + "] unexpectedly to multiple attributes " + named);
-            };
-        }
-
         private Expression resolveConvertFunction(AbstractConvertFunction convert, List<FieldAttribute> unionFieldAttributes) {
             if (convert.field() instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf) {
                 HashMap<TypeResolutionKey, Expression> typeResolutions = new HashMap<>();
                 Set<DataType> supportedTypes = convert.supportedTypes();
-                imf.getTypesToIndices().keySet().forEach(typeName -> {
-                    DataType type = DataType.fromTypeName(typeName);
+                imf.types().forEach(type -> {
+                    // TODO: Shouldn't we perform widening of small numerical types here?
                     if (supportedTypes.contains(type)) {
                         TypeResolutionKey key = new TypeResolutionKey(fa.name(), type);
                         var concreteConvert = typeSpecificConvert(convert, fa.source(), type, imf);
@@ -1196,7 +1247,7 @@ private Expression createIfDoesNotAlreadyExist(
             List<FieldAttribute> unionFieldAttributes
         ) {
             // Generate new ID for the field and suffix it with the data type to maintain unique attribute names.
-            String unionTypedFieldName = SubstituteSurrogates.rawTemporaryName(
+            String unionTypedFieldName = LogicalPlanOptimizer.rawTemporaryName(
                 fa.name(),
                 "converted_to",
                 resolvedField.getDataType().typeName()
@@ -1228,7 +1279,7 @@ private MultiTypeEsField resolvedMultiTypeEsField(FieldAttribute fa, HashMap<Typ
         private Expression typeSpecificConvert(AbstractConvertFunction convert, Source source, DataType type, InvalidMappedField mtf) {
             EsField field = new EsField(mtf.getName(), type, mtf.getProperties(), mtf.isAggregatable());
             NameId id = ((FieldAttribute) convert.field()).id();
-            FieldAttribute resolvedAttr = new FieldAttribute(source, null, field.getName(), field, null, Nullability.TRUE, id, false);
+            FieldAttribute resolvedAttr = new FieldAttribute(source, null, field.getName(), field, Nullability.TRUE, id, false);
             return convert.replaceChildren(Collections.singletonList(resolvedAttr));
         }
     }
@@ -1245,13 +1296,10 @@ private Expression typeSpecificConvert(AbstractConvertFunction convert, Source s
      */
     private static class UnionTypesCleanup extends Rule<LogicalPlan, LogicalPlan> {
         public LogicalPlan apply(LogicalPlan plan) {
-            LogicalPlan planWithCheckedUnionTypes = plan.transformUp(LogicalPlan.class, p -> {
-                if (p instanceof EsRelation esRelation) {
-                    // Leave esRelation as InvalidMappedField so that UNSUPPORTED fields can still pass through
-                    return esRelation;
-                }
-                return p.transformExpressionsOnly(FieldAttribute.class, UnionTypesCleanup::checkUnresolved);
-            });
+            LogicalPlan planWithCheckedUnionTypes = plan.transformUp(
+                LogicalPlan.class,
+                p -> p.transformExpressionsOnly(FieldAttribute.class, UnionTypesCleanup::checkUnresolved)
+            );
 
             // To drop synthetic attributes at the end, we need to compute the plan's output.
             // This is only legal to do if the plan is resolved.
@@ -1263,7 +1311,14 @@ public LogicalPlan apply(LogicalPlan plan) {
         static Attribute checkUnresolved(FieldAttribute fa) {
             if (fa.field() instanceof InvalidMappedField imf) {
                 String unresolvedMessage = "Cannot use field [" + fa.name() + "] due to ambiguities being " + imf.errorMessage();
-                return new UnresolvedAttribute(fa.source(), fa.name(), fa.qualifier(), fa.id(), unresolvedMessage, null);
+                String types = imf.getTypesToIndices().keySet().stream().collect(Collectors.joining(","));
+                return new UnsupportedAttribute(
+                    fa.source(),
+                    fa.name(),
+                    new UnsupportedEsField(imf.getName(), types),
+                    unresolvedMessage,
+                    fa.id()
+                );
             }
             return fa;
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java
index 5585a3f117d2f..6eb572ae96f4d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java
@@ -7,12 +7,12 @@
 
 package org.elasticsearch.xpack.esql.analysis;
 
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 public record AnalyzerContext(
-    EsqlConfiguration configuration,
+    Configuration configuration,
     EsqlFunctionRegistry functionRegistry,
     IndexResolution indexResolution,
     EnrichResolution enrichResolution
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerRules.java
index 242c947e56de9..89f9d694e3a16 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerRules.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerRules.java
@@ -74,20 +74,9 @@ public static List<Attribute> maybeResolveAgainstList(
         Collection<Attribute> attrList,
         java.util.function.Function<Attribute, Attribute> fieldInspector
     ) {
-        // first take into account the qualified version
-        final String qualifier = u.qualifier();
         final String name = u.name();
-        final boolean qualified = u.qualifier() != null;
 
-        Predicate<Attribute> predicate = a -> {
-            return qualified ? Objects.equals(qualifier, a.qualifiedName()) :
-            // if the field is unqualified
-            // first check the names directly
-                (Objects.equals(name, a.name()))
-                    // but also if the qualifier might not be quoted and if there's any ambiguity with nested fields
-                    || Objects.equals(name, a.qualifiedName());
-
-        };
+        Predicate<Attribute> predicate = a -> Objects.equals(name, a.name());
         return maybeResolveAgainstList(predicate, () -> u, attrList, false, fieldInspector);
     }
 
@@ -117,7 +106,7 @@ public static List<Attribute> maybeResolveAgainstList(
         // found exact match or multiple if pattern
         if (matches.size() == 1 || isPattern) {
             // NB: only add the location if the match is univocal; b/c otherwise adding the location will overwrite any preexisting one
-            matches.replaceAll(e -> fieldInspector.apply(e));
+            matches.replaceAll(fieldInspector::apply);
             return matches;
         }
 
@@ -125,17 +114,9 @@ public static List<Attribute> maybeResolveAgainstList(
         List<String> refs = matches.stream().sorted((a, b) -> {
             int lineDiff = a.sourceLocation().getLineNumber() - b.sourceLocation().getLineNumber();
             int colDiff = a.sourceLocation().getColumnNumber() - b.sourceLocation().getColumnNumber();
-            return lineDiff != 0 ? lineDiff : (colDiff != 0 ? colDiff : a.qualifiedName().compareTo(b.qualifiedName()));
-        })
-            .map(
-                a -> "line "
-                    + a.sourceLocation().toString().substring(1)
-                    + " ["
-                    + (a.qualifier() != null ? "\"" + a.qualifier() + "\".\"" + a.name() + "\"" : a.name())
-                    + "]"
-            )
-            .toList();
-
-        throw new IllegalStateException("Reference [" + ua.qualifiedName() + "] is ambiguous; " + "matches any of " + refs);
+            return lineDiff != 0 ? lineDiff : (colDiff != 0 ? colDiff : a.name().compareTo(b.name()));
+        }).map(a -> "line " + a.sourceLocation().toString().substring(1) + " [" + a.name() + "]").toList();
+
+        throw new IllegalStateException("Reference [" + ua.name() + "] is ambiguous; " + "matches any of " + refs);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java
index bda222c2b3e3f..9d7c5e141a2b1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/PreAnalyzer.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.esql.analysis;
 
-import org.elasticsearch.xpack.esql.core.analyzer.TableInfo;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation;
@@ -46,7 +45,7 @@ protected PreAnalysis doPreAnalyze(LogicalPlan plan) {
         List<TableInfo> indices = new ArrayList<>();
         List<Enrich> unresolvedEnriches = new ArrayList<>();
 
-        plan.forEachUp(UnresolvedRelation.class, p -> indices.add(new TableInfo(p.table(), p.frozen())));
+        plan.forEachUp(UnresolvedRelation.class, p -> indices.add(new TableInfo(p.table())));
         plan.forEachUp(Enrich.class, unresolvedEnriches::add);
 
         // mark plan as preAnalyzed (if it were marked, there would be no analysis)
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/TableInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/TableInfo.java
new file mode 100644
index 0000000000000..eff658e8997b0
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/TableInfo.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.analysis;
+
+import org.elasticsearch.xpack.esql.plan.TableIdentifier;
+
+public class TableInfo {
+
+    private final TableIdentifier id;
+
+    public TableInfo(TableIdentifier id) {
+        this.id = id;
+    }
+
+    public TableIdentifier id() {
+        return id;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
index 99dd012d860d7..374cc17b7b902 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.analysis;
 
+import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.xpack.esql.common.Failure;
 import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
@@ -19,8 +20,11 @@
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
+import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate;
+import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.StringQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate;
@@ -30,6 +34,7 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
+import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.plan.logical.Limit;
@@ -42,13 +47,13 @@
 import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan;
 import org.elasticsearch.xpack.esql.stats.FeatureMetric;
 import org.elasticsearch.xpack.esql.stats.Metrics;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.Collection;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 import java.util.function.Consumer;
 import java.util.stream.Stream;
@@ -56,6 +61,7 @@
 import static org.elasticsearch.xpack.esql.common.Failure.fail;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizer.PushFiltersToSource.canPushToSource;
 
 /**
  * This class is part of the planner. Responsible for failing impossible queries with a human-readable error message.  In particular, this
@@ -105,16 +111,23 @@ else if (p.resolved()) {
                 }
 
                 e.forEachUp(ae -> {
-                    // we're only interested in the children
+                    // Special handling for Project and unsupported/union types: disallow renaming them but pass them through otherwise.
+                    if (p instanceof Project) {
+                        if (ae instanceof Alias as && as.child() instanceof UnsupportedAttribute ua) {
+                            failures.add(fail(ae, ua.unresolvedMessage()));
+                        }
+                        if (ae instanceof UnsupportedAttribute) {
+                            return;
+                        }
+                    }
+
+                    // Do not fail multiple times in case the children are already unresolved.
                     if (ae.childrenResolved() == false) {
                         return;
                     }
 
                     if (ae instanceof Unresolvable u) {
-                        // special handling for Project and unsupported types
-                        if (p instanceof Project == false || u instanceof UnsupportedAttribute == false) {
-                            failures.add(fail(ae, u.unresolvedMessage()));
-                        }
+                        failures.add(fail(ae, u.unresolvedMessage()));
                     }
                     if (ae.typeResolved().unresolved()) {
                         failures.add(fail(ae, ae.typeResolved().message()));
@@ -171,6 +184,9 @@ else if (p instanceof Lookup lookup) {
             checkOperationsOnUnsignedLong(p, failures);
             checkBinaryComparison(p, failures);
             checkForSortOnSpatialTypes(p, failures);
+
+            checkFilterMatchConditions(p, failures);
+            checkMatchCommand(p, failures);
         });
         checkRemoteEnrich(plan, failures);
 
@@ -364,7 +380,7 @@ private static void checkRegexExtractOnlyOnStrings(LogicalPlan p, Set<Failure> f
     private static void checkRow(LogicalPlan p, Set<Failure> failures) {
         if (p instanceof Row row) {
             row.fields().forEach(a -> {
-                if (EsqlDataTypes.isRepresentable(a.dataType()) == false) {
+                if (DataType.isRepresentable(a.dataType()) == false) {
                     failures.add(fail(a, "cannot use [{}] directly in a row assignment", a.child().sourceText()));
                 }
             });
@@ -376,7 +392,7 @@ private static void checkEvalFields(LogicalPlan p, Set<Failure> failures) {
             eval.fields().forEach(field -> {
                 // check supported types
                 DataType dataType = field.dataType();
-                if (EsqlDataTypes.isRepresentable(dataType) == false) {
+                if (DataType.isRepresentable(dataType) == false) {
                     failures.add(
                         fail(field, "EVAL does not support type [{}] in expression [{}]", dataType.typeName(), field.child().sourceText())
                     );
@@ -389,6 +405,11 @@ private static void checkEvalFields(LogicalPlan p, Set<Failure> failures) {
                         failures.add(fail(af, "aggregate function [{}] not allowed outside STATS command", af.sourceText()));
                     }
                 });
+                // check no MATCH expressions are used
+                field.forEachDown(
+                    MatchQueryPredicate.class,
+                    mqp -> { failures.add(fail(mqp, "EVAL does not support MATCH expressions")); }
+                );
             });
         }
     }
@@ -528,7 +549,7 @@ private static void checkForSortOnSpatialTypes(LogicalPlan p, Set<Failure> local
         if (p instanceof OrderBy ob) {
             ob.forEachExpression(Attribute.class, attr -> {
                 DataType dataType = attr.dataType();
-                if (EsqlDataTypes.isSpatial(dataType)) {
+                if (DataType.isSpatial(dataType)) {
                     localFailures.add(fail(attr, "cannot sort on " + dataType.typeName()));
                 }
             });
@@ -574,4 +595,61 @@ private static void checkRemoteEnrich(LogicalPlan plan, Set<Failure> failures) {
             }
         });
     }
+
+    /**
+     * Currently any filter condition using MATCH needs to be pushed down to the Lucene query.
+     * Conditions that use a combination of MATCH and ES|QL functions (e.g. `title MATCH "anna" OR DATE_EXTRACT("year", date) > 2010)
+     * cannot be pushed down to Lucene.
+     * Another condition is for MATCH to use index fields that have been mapped as text or keyword.
+     * We are using canPushToSource at the Verifier level because we want to detect any condition that cannot be pushed down
+     * early in the execution, rather than fail at the compute engine level.
+     * In the future we will be able to handle MATCH at the compute and we will no longer need these checks.
+     */
+    private static void checkFilterMatchConditions(LogicalPlan plan, Set<Failure> failures) {
+        if (plan instanceof Filter f) {
+            Expression condition = f.condition();
+
+            Holder<Boolean> hasMatch = new Holder<>(false);
+            condition.forEachDown(MatchQueryPredicate.class, mqp -> {
+                hasMatch.set(true);
+                var field = mqp.field();
+                if (field instanceof FieldAttribute == false) {
+                    failures.add(fail(mqp, "MATCH requires a mapped index field, found [" + field.sourceText() + "]"));
+                }
+
+                if (DataType.isString(field.dataType()) == false) {
+                    var message = LoggerMessageFormat.format(
+                        null,
+                        "MATCH requires a text or keyword field, but [{}] has type [{}]",
+                        field.sourceText(),
+                        field.dataType().esType()
+                    );
+                    failures.add(fail(mqp, message));
+                }
+            });
+
+            if (canPushToSource(condition, x -> false)) {
+                return;
+            }
+            if (hasMatch.get()) {
+                failures.add(fail(condition, "Invalid condition using MATCH"));
+            }
+        }
+    }
+
+    private static void checkMatchCommand(LogicalPlan plan, Set<Failure> failures) {
+        if (plan instanceof Filter f) {
+            Expression condition = f.condition();
+            if (condition instanceof StringQueryPredicate) {
+                // Similar to cases present in org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineFilters -
+                // we can't check if it can be pushed down as we don't have yet information about the fields present in the
+                // StringQueryPredicate
+                plan.forEachDown(LogicalPlan.class, lp -> {
+                    if ((lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation) == false) {
+                        failures.add(fail(plan, "MATCH cannot be used after {}", lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT)));
+                    }
+                });
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
index 0f80fb809fa0b..373bf5a99056d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
@@ -463,9 +463,7 @@ private static class LookupRequest extends TransportRequest implements IndicesRe
             super(in);
             this.sessionId = in.readString();
             this.shardId = new ShardId(in);
-            String inputDataType = (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_EXTENDED_ENRICH_INPUT_TYPE))
-                ? in.readString()
-                : "unknown";
+            String inputDataType = (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) ? in.readString() : "unknown";
             this.inputDataType = DataType.fromTypeName(inputDataType);
             this.matchType = in.readString();
             this.matchField = in.readString();
@@ -482,7 +480,7 @@ public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(sessionId);
             out.writeWriteable(shardId);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_EXTENDED_ENRICH_INPUT_TYPE)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
                 out.writeString(inputDataType.typeName());
             }
             out.writeString(matchType);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java
index 2d42241d77ada..f77bfa6d3f862 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java
@@ -36,10 +36,13 @@
 import org.elasticsearch.xpack.core.enrich.EnrichMetadata;
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.session.IndexResolver;
 
@@ -326,14 +329,16 @@ private static class LookupResponse extends TransportResponse {
         }
 
         LookupResponse(StreamInput in) throws IOException {
-            this.policies = in.readMap(StreamInput::readString, ResolvedEnrichPolicy::new);
-            this.failures = in.readMap(StreamInput::readString, StreamInput::readString);
+            PlanStreamInput planIn = new PlanStreamInput(in, PlanNameRegistry.INSTANCE, in.namedWriteableRegistry(), null);
+            this.policies = planIn.readMap(StreamInput::readString, ResolvedEnrichPolicy::new);
+            this.failures = planIn.readMap(StreamInput::readString, StreamInput::readString);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeMap(policies, (o, v) -> v.writeTo(o));
-            out.writeMap(failures, StreamOutput::writeString);
+            PlanStreamOutput pso = new PlanStreamOutput(out, new PlanNameRegistry(), null);
+            pso.writeMap(policies, StreamOutput::writeWriteable);
+            pso.writeMap(failures, StreamOutput::writeString);
         }
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java
index 44443973764e6..63f22bd40ac39 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ResolvedEnrichPolicy.java
@@ -29,7 +29,7 @@ public ResolvedEnrichPolicy(StreamInput in) throws IOException {
             in.readString(),
             in.readStringCollectionAsList(),
             in.readMap(StreamInput::readString),
-            in.readMap(EsField::new)
+            in.readMap(EsField::readFrom)
         );
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
index 4e07c3084ab7b..88dfb7a377aa9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.planner.Mapper;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.session.EsqlSession;
 import org.elasticsearch.xpack.esql.session.IndexResolver;
 import org.elasticsearch.xpack.esql.session.Result;
@@ -49,7 +49,7 @@ public PlanExecutor(IndexResolver indexResolver) {
     public void esql(
         EsqlQueryRequest request,
         String sessionId,
-        EsqlConfiguration cfg,
+        Configuration cfg,
         EnrichPolicyResolver enrichPolicyResolver,
         BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase,
         ActionListener<Result> listener
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java
index 8f7fcef0ff07e..b97374d179a44 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.Locale;
 
@@ -66,21 +65,21 @@ public static Expression.TypeResolution isExact(Expression e, String operationNa
     private static final String[] POINT_TYPE_NAMES = new String[] { GEO_POINT.typeName(), CARTESIAN_POINT.typeName() };
     private static final String[] NON_SPATIAL_TYPE_NAMES = DataType.types()
         .stream()
-        .filter(EsqlDataTypes::isRepresentable)
-        .filter(t -> EsqlDataTypes.isSpatial(t) == false)
+        .filter(DataType::isRepresentable)
+        .filter(t -> DataType.isSpatial(t) == false)
         .map(DataType::esType)
         .toArray(String[]::new);
 
     public static Expression.TypeResolution isSpatialPoint(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) {
-        return isType(e, EsqlDataTypes::isSpatialPoint, operationName, paramOrd, POINT_TYPE_NAMES);
+        return isType(e, DataType::isSpatialPoint, operationName, paramOrd, POINT_TYPE_NAMES);
     }
 
     public static Expression.TypeResolution isSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) {
-        return isType(e, EsqlDataTypes::isSpatial, operationName, paramOrd, SPATIAL_TYPE_NAMES);
+        return isType(e, DataType::isSpatial, operationName, paramOrd, SPATIAL_TYPE_NAMES);
     }
 
     public static Expression.TypeResolution isNotSpatial(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) {
-        return isType(e, t -> EsqlDataTypes.isSpatial(t) == false, operationName, paramOrd, NON_SPATIAL_TYPE_NAMES);
+        return isType(e, t -> DataType.isSpatial(t) == false, operationName, paramOrd, NON_SPATIAL_TYPE_NAMES);
     }
 
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java
index d0c8adfd3c858..624ea9a030208 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java
@@ -33,7 +33,8 @@ public static List<Attribute> mergeOutputAttributes(
     /**
      * Merges output expressions of a command given the new attributes plus the existing inputs that are emitted as outputs.
      * As a general rule, child output will come first in the list, followed by the new fields.
-     * In case of name collisions, only last entry is preserved (previous expressions with the same name are discarded)
+     * In case of name collisions, only the last entry is preserved (previous expressions with the same name are discarded)
+     * and the new attributes have precedence over the child output.
      * @param fields the fields added by the command
      * @param childOutput the command input that has to be propagated as output
      * @return
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java
index dd51c5ba6473a..97bb1dbcd3e4a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -18,29 +19,39 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Objects;
 
-public class Order extends org.elasticsearch.xpack.esql.core.expression.Order {
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isExact;
+
+public class Order extends Expression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Order", Order::new);
 
+    private final Expression child;
+    private final OrderDirection direction;
+    private final NullsPosition nulls;
+
     public Order(Source source, Expression child, OrderDirection direction, NullsPosition nulls) {
-        super(source, child, direction, nulls);
+        super(source, List.of(child));
+        this.child = child;
+        this.direction = direction;
+        this.nulls = nulls == null ? NullsPosition.ANY : nulls;
     }
 
     public Order(StreamInput in) throws IOException {
         this(
             Source.readFrom((PlanStreamInput) in),
             in.readNamedWriteable(Expression.class),
-            in.readEnum(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.class),
-            in.readEnum(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.class)
+            in.readEnum(OrderDirection.class),
+            in.readEnum(NullsPosition.class)
         );
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         Source.EMPTY.writeTo(out);
-        out.writeNamedWriteable(child());
-        out.writeEnum(direction());
-        out.writeEnum(nullsPosition());
+        out.writeNamedWriteable(child);
+        out.writeEnum(direction);
+        out.writeEnum(nulls);
     }
 
     @Override
@@ -50,19 +61,78 @@ public String getWriteableName() {
 
     @Override
     protected TypeResolution resolveType() {
-        if (DataType.isString(child().dataType())) {
+        if (DataType.isString(child.dataType())) {
             return TypeResolution.TYPE_RESOLVED;
         }
-        return super.resolveType();
+        return isExact(child, "ORDER BY cannot be applied to field of data type [{}]: {}");
+    }
+
+    @Override
+    public DataType dataType() {
+        return child.dataType();
     }
 
     @Override
     public Order replaceChildren(List<Expression> newChildren) {
-        return new Order(source(), newChildren.get(0), direction(), nullsPosition());
+        return new Order(source(), newChildren.get(0), direction, nulls);
+    }
+
+    @Override
+    protected NodeInfo<Order> info() {
+        return NodeInfo.create(this, Order::new, child, direction, nulls);
     }
 
     @Override
-    protected NodeInfo<org.elasticsearch.xpack.esql.core.expression.Order> info() {
-        return NodeInfo.create(this, Order::new, child(), direction(), nullsPosition());
+    public Nullability nullable() {
+        return Nullability.FALSE;
     }
+
+    public Expression child() {
+        return child;
+    }
+
+    public OrderDirection direction() {
+        return direction;
+    }
+
+    public NullsPosition nullsPosition() {
+        return nulls;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(child, direction, nulls);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        Order other = (Order) obj;
+        return Objects.equals(direction, other.direction) && Objects.equals(nulls, other.nulls) && Objects.equals(child, other.child);
+    }
+
+    public enum OrderDirection {
+        ASC,
+        DESC
+    }
+
+    public enum NullsPosition {
+        FIRST,
+        LAST,
+        /**
+         * Nulls position has not been specified by the user and an appropriate default will be used.
+         *
+         * The default values are chosen such that it stays compatible with previous behavior. Unfortunately, this results in
+         * inconsistencies across different types of queries (see https://github.com/elastic/elasticsearch/issues/77068).
+         */
+        ANY;
+    }
+
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
index de7e63e16d53e..0d50623fe77eb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -10,11 +10,11 @@
 import org.elasticsearch.Build;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.xpack.esql.core.ParsingException;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.Check;
@@ -95,6 +95,8 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMax;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMedian;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
+import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvPSeriesWeightedSum;
+import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvPercentile;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSlice;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSort;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum;
@@ -124,6 +126,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Trim;
 import org.elasticsearch.xpack.esql.plan.logical.meta.MetaFunctions;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.lang.reflect.Constructor;
 import java.util.ArrayList;
@@ -196,6 +199,7 @@ public class EsqlFunctionRegistry {
 
     private SnapshotFunctionRegistry snapshotRegistry = null;
 
+    @SuppressWarnings("this-escape")
     public EsqlFunctionRegistry() {
         register(functions());
         buildDataTypesForStringLiteralConversion(functions());
@@ -360,11 +364,14 @@ private FunctionDefinition[][] functions() {
                 def(MvMax.class, MvMax::new, "mv_max"),
                 def(MvMedian.class, MvMedian::new, "mv_median"),
                 def(MvMin.class, MvMin::new, "mv_min"),
+                def(MvPercentile.class, MvPercentile::new, "mv_percentile"),
+                def(MvPSeriesWeightedSum.class, MvPSeriesWeightedSum::new, "mv_pseries_weighted_sum"),
                 def(MvSort.class, MvSort::new, "mv_sort"),
                 def(MvSlice.class, MvSlice::new, "mv_slice"),
                 def(MvZip.class, MvZip::new, "mv_zip"),
                 def(MvSum.class, MvSum::new, "mv_sum"),
                 def(Split.class, Split::new, "split") } };
+
     }
 
     private static FunctionDefinition[][] snapshotFunctions() {
@@ -475,7 +482,7 @@ public static FunctionDescription description(FunctionDefinition def) {
         Constructor<?> constructor = constructors[0];
         FunctionInfo functionInfo = functionInfo(def);
         String functionDescription = functionInfo == null ? "" : functionInfo.description().replace('\n', ' ');
-        String[] returnType = functionInfo == null ? new String[] { "?" } : functionInfo.returnType();
+        String[] returnType = functionInfo == null ? new String[] { "?" } : removeUnderConstruction(functionInfo.returnType());
         var params = constructor.getParameters(); // no multiple c'tors supported
 
         List<EsqlFunctionRegistry.ArgSignature> args = new ArrayList<>(params.length);
@@ -486,7 +493,7 @@ public static FunctionDescription description(FunctionDefinition def) {
                 Param paramInfo = params[i].getAnnotation(Param.class);
                 String name = paramInfo == null ? params[i].getName() : paramInfo.name();
                 variadic |= List.class.isAssignableFrom(params[i].getType());
-                String[] type = paramInfo == null ? new String[] { "?" } : paramInfo.type();
+                String[] type = paramInfo == null ? new String[] { "?" } : removeUnderConstruction(paramInfo.type());
                 String desc = paramInfo == null ? "" : paramInfo.description().replace('\n', ' ');
                 boolean optional = paramInfo == null ? false : paramInfo.optional();
                 DataType targetDataType = getTargetType(type);
@@ -496,6 +503,18 @@ public static FunctionDescription description(FunctionDefinition def) {
         return new FunctionDescription(def.name(), args, returnType, functionDescription, variadic, isAggregation);
     }
 
+    /**
+     * Remove types that are being actively built.
+     */
+    private static String[] removeUnderConstruction(String[] types) {
+        for (Map.Entry<DataType, FeatureFlag> underConstruction : DataType.UNDER_CONSTRUCTION.entrySet()) {
+            if (underConstruction.getValue().isEnabled() == false) {
+                types = Arrays.stream(types).filter(t -> underConstruction.getKey().typeName().equals(t) == false).toArray(String[]::new);
+            }
+        }
+        return types;
+    }
+
     public static FunctionInfo functionInfo(FunctionDefinition def) {
         var constructors = def.clazz().getConstructors();
         if (constructors.length == 0) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionDefinition.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionDefinition.java
index d93fc077dece4..208c2e8b97e16 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionDefinition.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionDefinition.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.esql.expression.function;
 
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.List;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java
index 94e3aa4e1dd68..f275496c6787a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java
@@ -23,6 +23,11 @@
      */
     String[] returnType();
 
+    /**
+     * Whether this function is a preview (Not ready for production environments) or not.
+     */
+    boolean preview() default false;
+
     /**
      * The description of the function rendered in {@code META FUNCTIONS}
      * and the docs. These should be complete sentences.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionResolutionStrategy.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionResolutionStrategy.java
index 4e7f47db0b252..b06424ce73e1f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionResolutionStrategy.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionResolutionStrategy.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.esql.expression.function;
 
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 /**
  * Strategy indicating the type of resolution to apply for resolving the actual function definition in a pluggable way.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnresolvedFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnresolvedFunction.java
index ab3475635ddbd..e35ca7e518532 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnresolvedFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnresolvedFunction.java
@@ -12,11 +12,11 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
 import java.util.LinkedHashSet;
@@ -132,6 +132,11 @@ public boolean analyzed() {
         return analyzed;
     }
 
+    @Override
+    protected TypeResolution resolveType() {
+        return new TypeResolution("unresolved function [" + name + "]");
+    }
+
     @Override
     public DataType dataType() {
         throw new UnresolvedException("dataType", this);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
index a553361f60a18..5961d1c21bb02 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.expression.function;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -20,7 +21,9 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
 import java.io.IOException;
@@ -36,17 +39,17 @@ public final class UnsupportedAttribute extends FieldAttribute implements Unreso
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         Attribute.class,
         "UnsupportedAttribute",
-        UnsupportedAttribute::new
+        UnsupportedAttribute::readFrom
     );
     public static final NamedWriteableRegistry.Entry NAMED_EXPRESSION_ENTRY = new NamedWriteableRegistry.Entry(
         NamedExpression.class,
         ENTRY.name,
-        UnsupportedAttribute::new
+        UnsupportedAttribute::readFrom
     );
     public static final NamedWriteableRegistry.Entry EXPRESSION_ENTRY = new NamedWriteableRegistry.Entry(
         Expression.class,
         ENTRY.name,
-        UnsupportedAttribute::new
+        UnsupportedAttribute::readFrom
     );
 
     private final String message;
@@ -65,16 +68,18 @@ public UnsupportedAttribute(Source source, String name, UnsupportedEsField field
     }
 
     public UnsupportedAttribute(Source source, String name, UnsupportedEsField field, String customMessage, NameId id) {
-        super(source, null, name, field, null, Nullability.TRUE, id, false);
+        super(source, null, name, field, Nullability.TRUE, id, false);
         this.hasCustomMessage = customMessage != null;
-        this.message = customMessage == null ? errorMessage(qualifiedName(), field) : customMessage;
+        this.message = customMessage == null ? errorMessage(name(), field) : customMessage;
     }
 
-    public UnsupportedAttribute(StreamInput in) throws IOException {
+    private UnsupportedAttribute(StreamInput in) throws IOException {
         this(
             Source.readFrom((PlanStreamInput) in),
             in.readString(),
-            new UnsupportedEsField(in),
+            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION)
+                ? EsField.readFrom(in)
+                : new UnsupportedEsField(in),
             in.readOptionalString(),
             NameId.readFrom((PlanStreamInput) in)
         );
@@ -82,11 +87,21 @@ public UnsupportedAttribute(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeString(name());
-        field().writeTo(out);
-        out.writeOptionalString(hasCustomMessage ? message : null);
-        id().writeTo(out);
+        if (((PlanStreamOutput) out).writeAttributeCacheHeader(this)) {
+            Source.EMPTY.writeTo(out);
+            out.writeString(name());
+            if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION)) {
+                field().writeTo(out);
+            } else {
+                field().writeContent(out);
+            }
+            out.writeOptionalString(hasCustomMessage ? message : null);
+            id().writeTo(out);
+        }
+    }
+
+    public static UnsupportedAttribute readFrom(StreamInput in) throws IOException {
+        return ((PlanStreamInput) in).readAttributeWithCache(UnsupportedAttribute::new);
     }
 
     @Override
@@ -117,15 +132,7 @@ protected NodeInfo<FieldAttribute> info() {
     }
 
     @Override
-    protected Attribute clone(
-        Source source,
-        String name,
-        DataType type,
-        String qualifier,
-        Nullability nullability,
-        NameId id,
-        boolean synthetic
-    ) {
+    protected Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) {
         return new UnsupportedAttribute(source, name, field(), hasCustomMessage ? message : null, id);
     }
 
@@ -135,7 +142,7 @@ protected String label() {
 
     @Override
     public String toString() {
-        return "!" + qualifiedName();
+        return "!" + name();
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java
index 52e053f843e14..9b6190408dbd4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
+import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvCount;
@@ -35,7 +36,32 @@
 public class Count extends AggregateFunction implements EnclosedAgg, ToAggregator, SurrogateExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Count", Count::new);
 
-    @FunctionInfo(returnType = "long", description = "Returns the total number (count) of input values.", isAggregation = true)
+    @FunctionInfo(
+        returnType = "long",
+        description = "Returns the total number (count) of input values.",
+        isAggregation = true,
+        examples = {
+            @Example(file = "stats", tag = "count"),
+            @Example(description = "To count the number of rows, use `COUNT()` or `COUNT(*)`", file = "docs", tag = "countAll"),
+            @Example(
+                description = "The expression can use inline functions. This example splits a string into "
+                    + "multiple values using the `SPLIT` function and counts the values",
+                file = "stats",
+                tag = "docsCountWithExpression"
+            ),
+            @Example(
+                description = "To count the number of times an expression returns `TRUE` use "
+                    + "a <<esql-where>> command to remove rows that shouldn't be included",
+                file = "stats",
+                tag = "count-where"
+            ),
+            @Example(
+                description = "To count the same stream of data based on two different expressions "
+                    + "use the pattern `COUNT(<expression> OR NULL)`",
+                file = "stats",
+                tag = "count-or-null"
+            ) }
+    )
     public Count(
         Source source,
         @Param(
@@ -54,7 +80,7 @@ public Count(
                 "text",
                 "unsigned_long",
                 "version" },
-            description = "Column or literal for which to count the number of values."
+            description = "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)."
         ) Expression field
     ) {
         super(source, field);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
index 7686d10a03d9e..858c6e659449c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
+import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.OptionalArgument;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -53,15 +54,63 @@ public class CountDistinct extends AggregateFunction implements OptionalArgument
     private static final int DEFAULT_PRECISION = 3000;
     private final Expression precision;
 
-    @FunctionInfo(returnType = "long", description = "Returns the approximate number of distinct values.", isAggregation = true)
+    @FunctionInfo(
+        returnType = "long",
+        description = "Returns the approximate number of distinct values.",
+        appendix = """
+            [discrete]
+            [[esql-agg-count-distinct-approximate]]
+            ==== Counts are approximate
+
+            Computing exact counts requires loading values into a set and returning its
+            size. This doesn't scale when working on high-cardinality sets and/or large
+            values as the required memory usage and the need to communicate those
+            per-shard sets between nodes would utilize too many resources of the cluster.
+
+            This `COUNT_DISTINCT` function is based on the
+            https://static.googleusercontent.com/media/research.google.com/fr//pubs/archive/40671.pdf[HyperLogLog++]
+            algorithm, which counts based on the hashes of the values with some interesting
+            properties:
+
+            include::../../../aggregations/metrics/cardinality-aggregation.asciidoc[tag=explanation]
+
+            The `COUNT_DISTINCT` function takes an optional second parameter to configure
+            the precision threshold. The precision_threshold options allows to trade memory
+            for accuracy, and defines a unique count below which counts are expected to be
+            close to accurate. Above this value, counts might become a bit more fuzzy. The
+            maximum supported value is 40000, thresholds above this number will have the
+            same effect as a threshold of 40000. The default value is `3000`.
+            """,
+        isAggregation = true,
+        examples = {
+            @Example(file = "stats_count_distinct", tag = "count-distinct"),
+            @Example(
+                description = "With the optional second parameter to configure the precision threshold",
+                file = "stats_count_distinct",
+                tag = "count-distinct-precision"
+            ),
+            @Example(
+                description = "The expression can use inline functions. This example splits a string into "
+                    + "multiple values using the `SPLIT` function and counts the unique values",
+                file = "stats_count_distinct",
+                tag = "docsCountDistinctWithExpression"
+            ) }
+    )
     public CountDistinct(
         Source source,
         @Param(
             name = "field",
-            type = { "boolean", "cartesian_point", "date", "double", "geo_point", "integer", "ip", "keyword", "long", "text", "version" },
+            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
             description = "Column or literal for which to count the number of distinct values."
         ) Expression field,
-        @Param(optional = true, name = "precision", type = { "integer" }) Expression precision
+        @Param(
+            optional = true,
+            name = "precision",
+            type = { "integer", "long", "unsigned_long" },
+            description = "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. "
+                + "The maximum supported value is 40000. Thresholds above this number will have the "
+                + "same effect as a threshold of 40000. The default value is 3000."
+        ) Expression precision
     ) {
         super(source, field, precision != null ? List.of(precision) : List.of());
         this.precision = precision;
@@ -108,19 +157,17 @@ protected TypeResolution resolveType() {
             return new TypeResolution("Unresolved children");
         }
 
-        TypeResolution resolution = EsqlTypeResolutions.isExact(field(), sourceText(), DEFAULT);
-        if (resolution.unresolved()) {
-            return resolution;
-        }
+        TypeResolution resolution = EsqlTypeResolutions.isExact(field(), sourceText(), DEFAULT)
+            .and(
+                isType(
+                    field(),
+                    dt -> dt != DataType.UNSIGNED_LONG && dt != DataType.SOURCE,
+                    sourceText(),
+                    DEFAULT,
+                    "any exact type except unsigned_long, _source, or counter types"
+                )
+            );
 
-        boolean resolved = resolution.resolved();
-        resolution = isType(
-            field(),
-            dt -> resolved && dt != DataType.UNSIGNED_LONG && dt != DataType.SOURCE,
-            sourceText(),
-            DEFAULT,
-            "any exact type except unsigned_long, _source, or counter types"
-        );
         if (resolution.unresolved() || precision == null) {
             return resolution;
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java
index 4438ccec04c4c..e7f790f90803a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.MaxBooleanAggregatorFunctionSupplier;
+import org.elasticsearch.compute.aggregation.MaxBytesRefAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.MaxDoubleAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.MaxIntAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.MaxIpAggregatorFunctionSupplier;
@@ -32,12 +33,15 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial;
 
 public class Max extends AggregateFunction implements ToAggregator, SurrogateExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Max", Max::new);
 
     @FunctionInfo(
-        returnType = { "boolean", "double", "integer", "long", "date", "ip" },
+        returnType = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" },
         description = "The maximum value of a field.",
         isAggregation = true,
         examples = {
@@ -50,7 +54,13 @@ public class Max extends AggregateFunction implements ToAggregator, SurrogateExp
                 tag = "docsStatsMaxNestedExpression"
             ) }
     )
-    public Max(Source source, @Param(name = "field", type = { "boolean", "double", "integer", "long", "date", "ip" }) Expression field) {
+    public Max(
+        Source source,
+        @Param(
+            name = "field",
+            type = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" }
+        ) Expression field
+    ) {
         super(source, field);
     }
 
@@ -76,14 +86,11 @@ public Max replaceChildren(List<Expression> newChildren) {
     @Override
     protected TypeResolution resolveType() {
         return TypeResolutions.isType(
-            this,
-            e -> e == DataType.BOOLEAN || e == DataType.DATETIME || e == DataType.IP || (e.isNumeric() && e != DataType.UNSIGNED_LONG),
+            field(),
+            t -> isRepresentable(t) && t != UNSIGNED_LONG && isSpatial(t) == false,
             sourceText(),
             DEFAULT,
-            "boolean",
-            "datetime",
-            "ip",
-            "numeric except unsigned_long or counter types"
+            "representable except unsigned_long and spatial types"
         );
     }
 
@@ -110,6 +117,9 @@ public final AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         if (type == DataType.IP) {
             return new MaxIpAggregatorFunctionSupplier(inputChannels);
         }
+        if (type == DataType.VERSION || type == DataType.KEYWORD || type == DataType.TEXT) {
+            return new MaxBytesRefAggregatorFunctionSupplier(inputChannels);
+        }
         throw EsqlIllegalArgumentException.illegalDataType(type);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java
index 36207df331e47..348fef577c934 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
+import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble;
@@ -32,9 +33,26 @@ public class Median extends AggregateFunction implements SurrogateExpression {
 
     // TODO: Add the compression parameter
     @FunctionInfo(
-        returnType = { "double", "integer", "long" },
-        description = "The value that is greater than half of all values and less than half of all values.",
-        isAggregation = true
+        returnType = "double",
+        description = "The value that is greater than half of all values and less than half of all values, "
+            + "also known as the 50% <<esql-percentile>>.",
+        note = "Like <<esql-percentile>>, `MEDIAN` is <<esql-percentile-approximate,usually approximate>>.",
+        appendix = """
+            [WARNING]
+            ====
+            `MEDIAN` is also {wikipedia}/Nondeterministic_algorithm[non-deterministic].
+            This means you can get slightly different results using the same data.
+            ====""",
+        isAggregation = true,
+        examples = {
+            @Example(file = "stats_percentile", tag = "median"),
+            @Example(
+                description = "The expression can use inline functions. For example, to calculate the median of "
+                    + "the maximum values of a multivalued column, first use `MV_MAX` to get the "
+                    + "maximum value per row, and use the result with the `MEDIAN` function",
+                file = "stats_percentile",
+                tag = "docsStatsMedianNestedExpression"
+            ), }
     )
     public Median(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) {
         super(source, field);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java
index 23d55942cc72f..46661e96b1d48 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviation.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 
@@ -31,9 +32,33 @@ public class MedianAbsoluteDeviation extends NumericAggregate {
 
     // TODO: Add parameter
     @FunctionInfo(
-        returnType = { "double", "integer", "long" },
-        description = "The median absolute deviation, a measure of variability.",
-        isAggregation = true
+        returnType = "double",
+        description = "Returns the median absolute deviation, a measure of variability. It is a robust "
+            + "statistic, meaning that it is useful for describing data that may have outliers, "
+            + "or may not be normally distributed. For such data it can be more descriptive "
+            + "than standard deviation."
+            + "\n\n"
+            + "It is calculated as the median of each data point's deviation from the median of "
+            + "the entire sample. That is, for a random variable `X`, the median absolute "
+            + "deviation is `median(|median(X) - X|)`.",
+        note = "Like <<esql-percentile>>, `MEDIAN_ABSOLUTE_DEVIATION` is <<esql-percentile-approximate,usually approximate>>.",
+        appendix = """
+            [WARNING]
+            ====
+            `MEDIAN_ABSOLUTE_DEVIATION` is also {wikipedia}/Nondeterministic_algorithm[non-deterministic].
+            This means you can get slightly different results using the same data.
+            ====""",
+        isAggregation = true,
+        examples = {
+            @Example(file = "stats_percentile", tag = "median-absolute-deviation"),
+            @Example(
+                description = "The expression can use inline functions. For example, to calculate the the "
+                    + "median absolute deviation of the maximum values of a multivalued column, first "
+                    + "use `MV_MAX` to get the maximum value per row, and use the result with the "
+                    + "`MEDIAN_ABSOLUTE_DEVIATION` function",
+                file = "stats_percentile",
+                tag = "docsStatsMADNestedExpression"
+            ), }
     )
     public MedianAbsoluteDeviation(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) {
         super(source, field);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java
index 490d227206e06..6866811995059 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.MinBooleanAggregatorFunctionSupplier;
+import org.elasticsearch.compute.aggregation.MinBytesRefAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.MinDoubleAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.MinIntAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.MinIpAggregatorFunctionSupplier;
@@ -32,12 +33,15 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial;
 
 public class Min extends AggregateFunction implements ToAggregator, SurrogateExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Min", Min::new);
 
     @FunctionInfo(
-        returnType = { "boolean", "double", "integer", "long", "date", "ip" },
+        returnType = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" },
         description = "The minimum value of a field.",
         isAggregation = true,
         examples = {
@@ -50,7 +54,13 @@ public class Min extends AggregateFunction implements ToAggregator, SurrogateExp
                 tag = "docsStatsMinNestedExpression"
             ) }
     )
-    public Min(Source source, @Param(name = "field", type = { "boolean", "double", "integer", "long", "date", "ip" }) Expression field) {
+    public Min(
+        Source source,
+        @Param(
+            name = "field",
+            type = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" }
+        ) Expression field
+    ) {
         super(source, field);
     }
 
@@ -76,14 +86,11 @@ public Min replaceChildren(List<Expression> newChildren) {
     @Override
     protected TypeResolution resolveType() {
         return TypeResolutions.isType(
-            this,
-            e -> e == DataType.BOOLEAN || e == DataType.DATETIME || e == DataType.IP || (e.isNumeric() && e != DataType.UNSIGNED_LONG),
+            field(),
+            t -> isRepresentable(t) && t != UNSIGNED_LONG && isSpatial(t) == false,
             sourceText(),
             DEFAULT,
-            "boolean",
-            "datetime",
-            "ip",
-            "numeric except unsigned_long or counter types"
+            "representable except unsigned_long and spatial types"
         );
     }
 
@@ -110,6 +117,9 @@ public final AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         if (type == DataType.IP) {
             return new MinIpAggregatorFunctionSupplier(inputChannels);
         }
+        if (type == DataType.VERSION || type == DataType.KEYWORD || type == DataType.TEXT) {
+            return new MinBytesRefAggregatorFunctionSupplier(inputChannels);
+        }
         throw EsqlIllegalArgumentException.illegalDataType(type);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
index 54cebc7daad5d..0d5dd4b66501c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
@@ -18,9 +18,12 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble;
+import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvPercentile;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
 import java.io.IOException;
@@ -31,7 +34,7 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
 
-public class Percentile extends NumericAggregate {
+public class Percentile extends NumericAggregate implements SurrogateExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
         Expression.class,
         "Percentile",
@@ -152,4 +155,15 @@ protected AggregatorFunctionSupplier doubleSupplier(List<Integer> inputChannels)
     private int percentileValue() {
         return ((Number) percentile.fold()).intValue();
     }
+
+    @Override
+    public Expression surrogate() {
+        var field = field();
+
+        if (field.foldable()) {
+            return new MvPercentile(source(), new ToDouble(source(), field), percentile());
+        }
+
+        return null;
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java
index 682590bb7e857..f5597b7d64e81 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.planner.ToAggregator;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.time.Duration;
@@ -125,7 +124,7 @@ protected TypeResolution resolveType() {
         );
         if (unit != null) {
             resolution = resolution.and(
-                isType(unit, dt -> dt.isWholeNumber() || EsqlDataTypes.isTemporalAmount(dt), sourceText(), SECOND, "time_duration")
+                isType(unit, dt -> dt.isWholeNumber() || DataType.isTemporalAmount(dt), sourceText(), SECOND, "time_duration")
             );
         }
         return resolution;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
index d5681ba8d856e..b9cd99f8eb7f0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.planner.ToAggregator;
@@ -38,7 +39,12 @@ public class SpatialCentroid extends SpatialAggregateFunction implements ToAggre
         SpatialCentroid::new
     );
 
-    @FunctionInfo(returnType = { "geo_point", "cartesian_point" }, description = "The centroid of a spatial field.", isAggregation = true)
+    @FunctionInfo(
+        returnType = { "geo_point", "cartesian_point" },
+        description = "Calculate the spatial centroid over a field with spatial point geometry type.",
+        isAggregation = true,
+        examples = @Example(file = "spatial", tag = "st_centroid_agg-airports")
+    )
     public SpatialCentroid(Source source, @Param(name = "field", type = { "geo_point", "cartesian_point" }) Expression field) {
         super(source, field, false);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java
index e15cf774c3c3f..4f85a15732a6f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
+import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum;
@@ -37,7 +38,20 @@
 public class Sum extends NumericAggregate implements SurrogateExpression {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Sum", Sum::new);
 
-    @FunctionInfo(returnType = "long", description = "The sum of a numeric field.", isAggregation = true)
+    @FunctionInfo(
+        returnType = { "long", "double" },
+        description = "The sum of a numeric expression.",
+        isAggregation = true,
+        examples = {
+            @Example(file = "stats", tag = "sum"),
+            @Example(
+                description = "The expression can use inline functions. For example, to calculate "
+                    + "the sum of each employee's maximum salary changes, apply the "
+                    + "`MV_MAX` function to each row and then sum the results",
+                file = "stats",
+                tag = "docsStatsSumNestedExpression"
+            ) }
+    )
     public Sum(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) {
         super(source, field);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java
index 84fcd9d1f7caf..4927acc3e1cd9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.compute.aggregation.TopBooleanAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.TopDoubleAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.TopIntAggregatorFunctionSupplier;
+import org.elasticsearch.compute.aggregation.TopIpAggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.TopLongAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
@@ -47,7 +48,7 @@ public class Top extends AggregateFunction implements ToAggregator, SurrogateExp
     private static final String ORDER_DESC = "DESC";
 
     @FunctionInfo(
-        returnType = { "boolean", "double", "integer", "long", "date" },
+        returnType = { "boolean", "double", "integer", "long", "date", "ip" },
         description = "Collects the top values for a field. Includes repeated values.",
         isAggregation = true,
         examples = @Example(file = "stats_top", tag = "top")
@@ -56,7 +57,7 @@ public Top(
         Source source,
         @Param(
             name = "field",
-            type = { "boolean", "double", "integer", "long", "date" },
+            type = { "boolean", "double", "integer", "long", "date", "ip" },
             description = "The field to collect the top values for."
         ) Expression field,
         @Param(name = "limit", type = { "integer" }, description = "The maximum number of values to collect.") Expression limit,
@@ -121,9 +122,15 @@ protected TypeResolution resolveType() {
 
         var typeResolution = isType(
             field(),
-            dt -> dt == DataType.BOOLEAN || dt == DataType.DATETIME || dt.isNumeric() && dt != DataType.UNSIGNED_LONG,
+            dt -> dt == DataType.BOOLEAN
+                || dt == DataType.DATETIME
+                || dt == DataType.IP
+                || (dt.isNumeric() && dt != DataType.UNSIGNED_LONG),
             sourceText(),
             FIRST,
+            "boolean",
+            "date",
+            "ip",
             "numeric except unsigned_long or counter types"
         ).and(isNotNullAndFoldable(limitField(), sourceText(), SECOND))
             .and(isType(limitField(), dt -> dt == DataType.INTEGER, sourceText(), SECOND, "integer"))
@@ -180,6 +187,9 @@ public AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         if (type == DataType.BOOLEAN) {
             return new TopBooleanAggregatorFunctionSupplier(inputChannels, limitValue(), orderValue());
         }
+        if (type == DataType.IP) {
+            return new TopIpAggregatorFunctionSupplier(inputChannels, limitValue(), orderValue());
+        }
         throw EsqlIllegalArgumentException.illegalDataType(type);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
index 7d2fbcddb113b..136e1233601f9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
@@ -17,10 +17,11 @@
 import org.elasticsearch.compute.aggregation.ValuesLongAggregatorFunctionSupplier;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions;
+import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.planner.ToAggregator;
@@ -29,18 +30,31 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 
 public class Values extends AggregateFunction implements ToAggregator {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Values", Values::new);
 
     @FunctionInfo(
-        returnType = { "boolean|date|double|integer|ip|keyword|long|text|version" },
-        description = "Collect values for a field.",
-        isAggregation = true
+        returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+        preview = true,
+        description = "Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed. "
+            + "If you need the values returned in order use <<esql-mv_sort>>.",
+        appendix = """
+            [WARNING]
+            ====
+            This can use a significant amount of memory and ES|QL doesn't yet
+            grow aggregations beyond memory. So this aggregation will work until
+            it is used to collect more values than can fit into memory. Once it
+            collects too many values it will fail the query with
+            a <<circuit-breaker-errors, Circuit Breaker Error>>.
+            ====""",
+        isAggregation = true,
+        examples = @Example(file = "string", tag = "values-grouped")
     )
     public Values(
         Source source,
-        @Param(name = "field", type = { "boolean|date|double|integer|ip|keyword|long|text|version" }) Expression v
+        @Param(name = "field", type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }) Expression v
     ) {
         super(source, v);
     }
@@ -71,7 +85,13 @@ public DataType dataType() {
 
     @Override
     protected TypeResolution resolveType() {
-        return EsqlTypeResolutions.isNotSpatial(field(), sourceText(), DEFAULT);
+        return TypeResolutions.isType(
+            field(),
+            dt -> DataType.isSpatial(dt) == false && dt != UNSIGNED_LONG,
+            sourceText(),
+            DEFAULT,
+            "any type except unsigned_long and spatial types"
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java
index 75315c48b3f45..23a20d9897e72 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
+import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAvg;
@@ -42,7 +43,12 @@ public class WeightedAvg extends AggregateFunction implements SurrogateExpressio
 
     private static final String invalidWeightError = "{} argument of [{}] cannot be null or 0, received [{}]";
 
-    @FunctionInfo(returnType = "double", description = "The weighted average of a numeric field.", isAggregation = true)
+    @FunctionInfo(
+        returnType = "double",
+        description = "The weighted average of a numeric expression.",
+        isAggregation = true,
+        examples = @Example(file = "stats", tag = "weighted-avg")
+    )
     public WeightedAvg(
         Source source,
         @Param(name = "number", type = { "double", "integer", "long" }, description = "A numeric value.") Expression field,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java
index 055e34ad5a633..1c10c7d2fa9ef 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java
@@ -68,18 +68,18 @@
  *                 {@code dataType}: This will return the datatype of your function.
  *                 May be based on its current parameters.
  *             </li>
- *         </ul>
- *
- *         Finally, you may want to implement some interfaces.
- *         Check their JavaDocs to see if they are suitable for your function:
- *         <ul>
- *             <li>
- *                 {@link org.elasticsearch.xpack.esql.planner.ToAggregator}: (More information about aggregators below)
- *             </li>
  *             <li>
- *                 {@link org.elasticsearch.xpack.esql.expression.SurrogateExpression}
+ *                 Implement {@link org.elasticsearch.xpack.esql.expression.SurrogateExpression}, and its required
+ *                 {@link org.elasticsearch.xpack.esql.expression.SurrogateExpression#surrogate()} method.
+ *                 <p>
+ *                     It's used to be able to fold the aggregation when it receives only literals,
+ *                     or when the aggregation can be simplified.
+ *                 </p>
  *             </li>
  *         </ul>
+ *
+ *         Finally, implement {@link org.elasticsearch.xpack.esql.planner.ToAggregator} (More information about aggregators below).
+ *         The only case when this interface is not required is when it always returns another function in its surrogate.
  *     </li>
  *     <li>
  *         To introduce your aggregation to the engine:
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
index 3ce51b8086dd0..5fabfe0e03d89 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
@@ -33,11 +33,11 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Function;
 
@@ -145,9 +145,7 @@ another in which the bucket size is provided directly (two parameters).
             ),
             @Example(description = """
                 The range can be omitted if the desired bucket size is known in advance. Simply
-                provide it as the second argument:""", file = "bucket", tag = "docsBucketNumericWithSpan", explanation = """
-                NOTE: When providing the bucket size as the second parameter, it must be
-                of a floating point type."""),
+                provide it as the second argument:""", file = "bucket", tag = "docsBucketNumericWithSpan"),
             @Example(
                 description = "Create hourly buckets for the last 24 hours, and calculate the number of events per hour:",
                 file = "bucket",
@@ -177,23 +175,23 @@ public Bucket(
         ) Expression field,
         @Param(
             name = "buckets",
-            type = { "integer", "double", "date_period", "time_duration" },
-            description = "Target number of buckets."
+            type = { "integer", "long", "double", "date_period", "time_duration" },
+            description = "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted."
         ) Expression buckets,
         @Param(
             name = "from",
-            type = { "integer", "long", "double", "date" },
+            type = { "integer", "long", "double", "date", "keyword", "text" },
             optional = true,
-            description = "Start of the range. Can be a number or a date expressed as a string."
+            description = "Start of the range. Can be a number, a date or a date expressed as a string."
         ) Expression from,
         @Param(
             name = "to",
-            type = { "integer", "long", "double", "date" },
+            type = { "integer", "long", "double", "date", "keyword", "text" },
             optional = true,
-            description = "End of the range. Can be a number or a date expressed as a string."
+            description = "End of the range. Can be a number, a date or a date expressed as a string."
         ) Expression to
     ) {
-        super(source, from != null && to != null ? List.of(field, buckets, from, to) : List.of(field, buckets));
+        super(source, fields(field, buckets, from, to));
         this.field = field;
         this.buckets = buckets;
         this.from = from;
@@ -210,6 +208,19 @@ private Bucket(StreamInput in) throws IOException {
         );
     }
 
+    private static List<Expression> fields(Expression field, Expression buckets, Expression from, Expression to) {
+        List<Expression> list = new ArrayList<>(4);
+        list.add(field);
+        list.add(buckets);
+        if (from != null) {
+            list.add(from);
+            if (to != null) {
+                list.add(to);
+            }
+        }
+        return list;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         source().writeTo(out);
@@ -239,7 +250,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEv
                 long t = foldToLong(to);
                 preparedRounding = new DateRoundingPicker(b, f, t).pickRounding().prepareForUnknown();
             } else {
-                assert EsqlDataTypes.isTemporalAmount(buckets.dataType()) : "Unexpected span data type [" + buckets.dataType() + "]";
+                assert DataType.isTemporalAmount(buckets.dataType()) : "Unexpected span data type [" + buckets.dataType() + "]";
                 preparedRounding = DateTrunc.createRounding(buckets.fold(), DEFAULT_TZ);
             }
             return DateTrunc.evaluator(source(), toEvaluator.apply(field), preparedRounding);
@@ -252,7 +263,6 @@ public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEv
                 double t = ((Number) to.fold()).doubleValue();
                 roundTo = pickRounding(b, f, t);
             } else {
-                assert buckets.dataType().isRationalNumber() : "Unexpected rounding data type [" + buckets.dataType() + "]";
                 roundTo = ((Number) buckets.fold()).doubleValue();
             }
             Literal rounding = new Literal(source(), roundTo, DataType.DOUBLE);
@@ -289,7 +299,7 @@ boolean roundingIsOk(Rounding rounding) {
             while (used < buckets) {
                 bucket = r.nextRoundingValue(bucket);
                 used++;
-                if (bucket > to) {
+                if (bucket >= to) {
                     return true;
                 }
             }
@@ -308,7 +318,7 @@ private double pickRounding(int buckets, double from, double to) {
     // datetime, integer, string/datetime, string/datetime
     // datetime, rounding/duration, -, -
     // numeric, integer, numeric, numeric
-    // numeric, double, -, -
+    // numeric, numeric, -, -
     @Override
     protected TypeResolution resolveType() {
         if (childrenResolved() == false) {
@@ -323,7 +333,7 @@ protected TypeResolution resolveType() {
         if (fieldType == DataType.DATETIME) {
             TypeResolution resolution = isType(
                 buckets,
-                dt -> dt.isWholeNumber() || EsqlDataTypes.isTemporalAmount(dt),
+                dt -> dt.isWholeNumber() || DataType.isTemporalAmount(dt),
                 sourceText(),
                 SECOND,
                 "integral",
@@ -337,9 +347,18 @@ protected TypeResolution resolveType() {
                 : resolution.and(checkArgsCount(2)); // temporal amount
         }
         if (fieldType.isNumeric()) {
-            return bucketsType.isWholeNumber()
-                ? checkArgsCount(4).and(() -> isNumeric(from, sourceText(), THIRD)).and(() -> isNumeric(to, sourceText(), FOURTH))
-                : isNumeric(buckets, sourceText(), SECOND).and(checkArgsCount(2));
+            return isNumeric(buckets, sourceText(), SECOND).and(() -> {
+                if (bucketsType.isRationalNumber()) {
+                    return checkArgsCount(2);
+                } else { // second arg is a whole number: either a span, but as a whole, or count, and we must expect a range
+                    var resolution = checkArgsCount(2);
+                    if (resolution.resolved() == false) {
+                        resolution = checkArgsCount(4).and(() -> isNumeric(from, sourceText(), THIRD))
+                            .and(() -> isNumeric(to, sourceText(), FOURTH));
+                    }
+                    return resolution;
+                }
+            });
         }
         return isType(field, e -> false, sourceText(), FIRST, "datetime", "numeric");
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlConfigurationFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlConfigurationFunction.java
index 3b092ddc29de8..4b5fcddcf0dfa 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlConfigurationFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlConfigurationFunction.java
@@ -8,8 +8,8 @@
 package org.elasticsearch.xpack.esql.expression.function.scalar;
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.List;
 import java.util.Objects;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
index b731a400deba3..cf97558cd2676 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
@@ -36,7 +36,7 @@
 import java.util.Set;
 import java.util.function.Function;
 
-import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isTypeOrUnionType;
 
 /**
  * Base class for functions that converts a field into a function-specific type.
@@ -76,14 +76,14 @@ protected final TypeResolution resolveType() {
         if (childrenResolved() == false) {
             return new TypeResolution("Unresolved children");
         }
-        return isType(field(), factories()::containsKey, sourceText(), null, supportedTypesNames(supportedTypes()));
+        return isTypeOrUnionType(field(), factories()::containsKey, sourceText(), null, supportedTypesNames(supportedTypes()));
     }
 
     public Set<DataType> supportedTypes() {
         return factories().keySet();
     }
 
-    public static String supportedTypesNames(Set<DataType> types) {
+    private static String supportedTypesNames(Set<DataType> types) {
         List<String> supportedTypesNames = new ArrayList<>(types.size());
         HashSet<DataType> supportTypes = new HashSet<>(types);
         if (supportTypes.containsAll(NUMERIC_TYPES)) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java
index 917abc9d77168..c66ba7f87a1c5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java
@@ -10,6 +10,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.compute.ann.ConvertEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -24,6 +25,7 @@
 import java.util.Map;
 
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
@@ -41,6 +43,7 @@ public class ToDatetime extends AbstractConvertFunction {
 
     private static final Map<DataType, BuildFactory> EVALUATORS = Map.ofEntries(
         Map.entry(DATETIME, (field, source) -> field),
+        Map.entry(DATE_NANOS, ToDatetimeFromDateNanosEvaluator.Factory::new),
         Map.entry(LONG, (field, source) -> field),
         Map.entry(KEYWORD, ToDatetimeFromStringEvaluator.Factory::new),
         Map.entry(TEXT, ToDatetimeFromStringEvaluator.Factory::new),
@@ -55,6 +58,8 @@ public class ToDatetime extends AbstractConvertFunction {
             Converts an input value to a date value.
             A string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`.
             To convert dates in other formats, use <<esql-date_parse>>.""",
+        note = "Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is "
+            + "truncated, not rounded.",
         examples = {
             @Example(file = "date", tag = "to_datetime-str", explanation = """
                 Note that in this example, the last value in the source multi-valued field has not been converted.
@@ -81,7 +86,7 @@ public ToDatetime(
         Source source,
         @Param(
             name = "field",
-            type = { "date", "keyword", "text", "double", "long", "unsigned_long", "integer" },
+            type = { "date", "date_nanos", "keyword", "text", "double", "long", "unsigned_long", "integer" },
             description = "Input value. The input can be a single- or multi-valued column or an expression."
         ) Expression field
     ) {
@@ -121,4 +126,9 @@ protected NodeInfo<? extends Expression> info() {
     static long fromKeyword(BytesRef in) {
         return dateTimeToLong(in.utf8ToString());
     }
+
+    @ConvertEvaluator(extraName = "FromDateNanos", warnExceptions = { IllegalArgumentException.class })
+    static long fromDatenanos(long in) {
+        return DateUtils.toMilliSeconds(in);
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java
index 4811051c3f488..e5f138df159cd 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java
@@ -26,6 +26,7 @@
 
 import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
@@ -42,6 +43,7 @@ public class ToLong extends AbstractConvertFunction {
     private static final Map<DataType, BuildFactory> EVALUATORS = Map.ofEntries(
         Map.entry(LONG, (fieldEval, source) -> fieldEval),
         Map.entry(DATETIME, (fieldEval, source) -> fieldEval),
+        Map.entry(DATE_NANOS, (fieldEval, source) -> fieldEval),
         Map.entry(BOOLEAN, ToLongFromBooleanEvaluator.Factory::new),
         Map.entry(KEYWORD, ToLongFromStringEvaluator.Factory::new),
         Map.entry(TEXT, ToLongFromStringEvaluator.Factory::new),
@@ -76,6 +78,7 @@ public ToLong(
             type = {
                 "boolean",
                 "date",
+                "date_nanos",
                 "keyword",
                 "text",
                 "double",
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java
index cb9eae6b5f435..f9bc15c4d6903 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java
@@ -28,6 +28,7 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
@@ -40,6 +41,7 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.numericBooleanToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.spatialToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.unsignedLongToString;
@@ -52,6 +54,7 @@ public class ToString extends AbstractConvertFunction implements EvaluatorMapper
         Map.entry(KEYWORD, (fieldEval, source) -> fieldEval),
         Map.entry(BOOLEAN, ToStringFromBooleanEvaluator.Factory::new),
         Map.entry(DATETIME, ToStringFromDatetimeEvaluator.Factory::new),
+        Map.entry(DATE_NANOS, ToStringFromDateNanosEvaluator.Factory::new),
         Map.entry(IP, ToStringFromIPEvaluator.Factory::new),
         Map.entry(DOUBLE, ToStringFromDoubleEvaluator.Factory::new),
         Map.entry(LONG, ToStringFromLongEvaluator.Factory::new),
@@ -81,6 +84,7 @@ public ToString(
                 "cartesian_point",
                 "cartesian_shape",
                 "date",
+                "date_nanos",
                 "double",
                 "geo_point",
                 "geo_shape",
@@ -141,6 +145,11 @@ static BytesRef fromDatetime(long datetime) {
         return new BytesRef(dateTimeToString(datetime));
     }
 
+    @ConvertEvaluator(extraName = "FromDateNanos")
+    static BytesRef fromDateNanos(long datetime) {
+        return new BytesRef(nanoTimeToString(datetime));
+    }
+
     @ConvertEvaluator(extraName = "FromDouble")
     static BytesRef fromDouble(double dbl) {
         return numericBooleanToString(dbl);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java
index f9dcdeb342cb5..bbb19ca0eecab 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -26,6 +25,7 @@
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
 import java.time.ZoneId;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java
index bfc1bbaa5101d..bfca72a563c05 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.compute.ann.Fixed;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -26,7 +25,7 @@
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
 import java.util.List;
@@ -149,11 +148,11 @@ public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEv
             throw new IllegalArgumentException("unsupported data type for format [" + format.dataType() + "]");
         }
         if (format.foldable()) {
-            DateFormatter formatter = toFormatter(format.fold(), ((EsqlConfiguration) configuration()).locale());
+            DateFormatter formatter = toFormatter(format.fold(), configuration().locale());
             return new DateFormatConstantEvaluator.Factory(source(), fieldEvaluator, formatter);
         }
         var formatEvaluator = toEvaluator.apply(format);
-        return new DateFormatEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, ((EsqlConfiguration) configuration()).locale());
+        return new DateFormatEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, configuration().locale());
     }
 
     private static DateFormatter toFormatter(Object format, Locale locale) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
index c39905f261d88..d5ec3d1d96fae 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.time.Duration;
@@ -110,7 +109,7 @@ protected TypeResolution resolveType() {
             return new TypeResolution("Unresolved children");
         }
 
-        return isType(interval, EsqlDataTypes::isTemporalAmount, sourceText(), FIRST, "dateperiod", "timeduration").and(
+        return isType(interval, DataType::isTemporalAmount, sourceText(), FIRST, "dateperiod", "timeduration").and(
             isDate(timestampField, sourceText(), SECOND)
         );
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java
index f6625c690230d..0654aec3a0522 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.compute.ann.Fixed;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -22,6 +21,7 @@
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
 import java.util.List;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java
index cffb208940aa5..cb0f9fdd8d5db 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java
@@ -44,6 +44,8 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             MvMax.ENTRY,
             MvMedian.ENTRY,
             MvMin.ENTRY,
+            MvPercentile.ENTRY,
+            MvPSeriesWeightedSum.ENTRY,
             MvSlice.ENTRY,
             MvSort.ENTRY,
             MvSum.ENTRY,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java
index dc4b78d980c28..deb170d9e569c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -132,14 +131,14 @@ protected TypeResolution resolveType() {
             return new TypeResolution("Unresolved children");
         }
 
-        TypeResolution resolution = isType(field1, EsqlDataTypes::isRepresentable, sourceText(), FIRST, "representable");
+        TypeResolution resolution = isType(field1, DataType::isRepresentable, sourceText(), FIRST, "representable");
         if (resolution.unresolved()) {
             return resolution;
         }
         dataType = field1.dataType();
         if (dataType == DataType.NULL) {
             dataType = field2.dataType();
-            return isType(field2, EsqlDataTypes::isRepresentable, sourceText(), SECOND, "representable");
+            return isType(field2, DataType::isRepresentable, sourceText(), SECOND, "representable");
         }
         return isType(field2, t -> t == dataType, sourceText(), SECOND, dataType.typeName());
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java
index 01f24365be225..9f678641060ff 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java
@@ -27,8 +27,8 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.unsignedLongToDouble;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable;
 
 /**
  * Reduce a multivalued field to a single valued field containing the average value.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java
index faf7d36e4a24c..f36d121ef104f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.List;
@@ -47,6 +46,7 @@ public MvCount(
                 "cartesian_point",
                 "cartesian_shape",
                 "date",
+                "date_nanos",
                 "double",
                 "geo_point",
                 "geo_shape",
@@ -74,7 +74,7 @@ public String getWriteableName() {
 
     @Override
     protected TypeResolution resolveFieldType() {
-        return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable");
+        return isType(field(), DataType::isRepresentable, sourceText(), null, "representable");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java
index d17bc26ab808b..b17ddddb422ce 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java
@@ -14,11 +14,11 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.List;
@@ -86,7 +86,7 @@ public String getWriteableName() {
 
     @Override
     protected TypeResolution resolveFieldType() {
-        return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable");
+        return isType(field(), DataType::isRepresentable, sourceText(), null, "representable");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java
index 25e6a85a485c1..6e76888f72b1d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java
@@ -22,11 +22,11 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.List;
@@ -45,6 +45,7 @@ public class MvFirst extends AbstractMultivalueFunction {
             "cartesian_point",
             "cartesian_shape",
             "date",
+            "date_nanos",
             "double",
             "geo_point",
             "geo_shape",
@@ -76,6 +77,7 @@ public MvFirst(
                 "cartesian_point",
                 "cartesian_shape",
                 "date",
+                "date_nanos",
                 "double",
                 "geo_point",
                 "geo_shape",
@@ -103,7 +105,7 @@ public String getWriteableName() {
 
     @Override
     protected TypeResolution resolveFieldType() {
-        return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable");
+        return isType(field(), DataType::isRepresentable, sourceText(), null, "representable");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java
index 2a9a498ecf9d3..198731ca601f4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java
@@ -22,11 +22,11 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.List;
@@ -45,6 +45,7 @@ public class MvLast extends AbstractMultivalueFunction {
             "cartesian_point",
             "cartesian_shape",
             "date",
+            "date_nanos",
             "double",
             "geo_point",
             "geo_shape",
@@ -76,6 +77,7 @@ public MvLast(
                 "cartesian_point",
                 "cartesian_shape",
                 "date",
+                "date_nanos",
                 "double",
                 "geo_point",
                 "geo_shape",
@@ -103,7 +105,7 @@ public String getWriteableName() {
 
     @Override
     protected TypeResolution resolveFieldType() {
-        return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable");
+        return isType(field(), DataType::isRepresentable, sourceText(), null, "representable");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java
index 24873cc1da2e9..6a53c652d3420 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java
@@ -26,8 +26,8 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatial;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial;
 
 /**
  * Reduce a multivalued field to a single valued field containing the maximum value.
@@ -36,7 +36,7 @@ public class MvMax extends AbstractMultivalueFunction {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvMax", MvMax::new);
 
     @FunctionInfo(
-        returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+        returnType = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
         description = "Converts a multivalued expression into a single valued column containing the maximum value.",
         examples = {
             @Example(file = "math", tag = "mv_max"),
@@ -51,7 +51,7 @@ public MvMax(
         Source source,
         @Param(
             name = "field",
-            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
             description = "Multivalue expression."
         ) Expression v
     ) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java
index 4e7d6dd4e29b2..e9e6899117805 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java
@@ -31,9 +31,9 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.bigIntegerToUnsignedLong;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.unsignedLongToBigInteger;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable;
 
 /**
  * Reduce a multivalued field to a single valued field containing the average value.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java
index 205a09953fde3..4cc83c99b2c08 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java
@@ -26,8 +26,8 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatial;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial;
 
 /**
  * Reduce a multivalued field to a single valued field containing the minimum value.
@@ -36,7 +36,7 @@ public class MvMin extends AbstractMultivalueFunction {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvMin", MvMin::new);
 
     @FunctionInfo(
-        returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+        returnType = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
         description = "Converts a multivalued expression into a single valued column containing the minimum value.",
         examples = {
             @Example(file = "math", tag = "mv_min"),
@@ -51,7 +51,7 @@ public MvMin(
         Source source,
         @Param(
             name = "field",
-            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
             description = "Multivalue expression."
         ) Expression field
     ) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java
new file mode 100644
index 0000000000000..212f626090789
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.compute.ann.Evaluator;
+import org.elasticsearch.compute.ann.Fixed;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
+import org.elasticsearch.search.aggregations.metrics.CompensatedSum;
+import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.planner.PlannerUtils;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
+
+/**
+ * Reduce a multivalued field to a single valued field containing the weighted sum of all element applying the P series function.
+ */
+public class MvPSeriesWeightedSum extends EsqlScalarFunction implements EvaluatorMapper {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        Expression.class,
+        "MvPSeriesWeightedSum",
+        MvPSeriesWeightedSum::new
+    );
+
+    private final Expression field, p;
+
+    @FunctionInfo(
+        returnType = { "double" },
+
+        description = "Converts a multivalued expression into a single-valued column by multiplying every "
+            + "element on the input list by its corresponding term in P-Series and computing the sum.",
+        examples = @Example(file = "mv_pseries_weighted_sum", tag = "example")
+    )
+    public MvPSeriesWeightedSum(
+        Source source,
+        @Param(name = "number", type = { "double" }, description = "Multivalue expression.") Expression field,
+        @Param(
+            name = "p",
+            type = { "double" },
+            description = "It is a constant number that represents the 'p' parameter in the P-Series. "
+                + "It impacts every element's contribution to the weighted sum."
+        ) Expression p
+    ) {
+        super(source, Arrays.asList(field, p));
+        this.field = field;
+        this.p = p;
+    }
+
+    private MvPSeriesWeightedSum(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class));
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution resolution = TypeResolutions.isType(field, dt -> dt == DOUBLE, sourceText(), FIRST, "double");
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+
+        resolution = TypeResolutions.isType(p, dt -> dt == DOUBLE, sourceText(), SECOND, "double");
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+
+        if (p.dataType() == NULL) {
+            // If the type is `null` this parameter doesn't have to be foldable. It's effectively foldable anyway.
+            // TODO figure out if the tests are wrong here, or if null is really different from foldable null
+            return resolution;
+        }
+
+        return isFoldable(p, sourceText(), SECOND);
+    }
+
+    @Override
+    public boolean foldable() {
+        return field.foldable() && p.foldable();
+    }
+
+    @Override
+    public EvalOperator.ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
+        return switch (PlannerUtils.toElementType(field.dataType())) {
+            case DOUBLE -> new MvPSeriesWeightedSumDoubleEvaluator.Factory(
+                source(),
+                toEvaluator.apply(field),
+                ctx -> new CompensatedSum(),
+                (Double) p.fold()
+            );
+            case NULL -> EvalOperator.CONSTANT_NULL_FACTORY;
+            default -> throw EsqlIllegalArgumentException.illegalDataType(field.dataType());
+        };
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new MvPSeriesWeightedSum(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, MvPSeriesWeightedSum::new, field, p);
+    }
+
+    @Override
+    public DataType dataType() {
+        if (p.dataType() == NULL) {
+            return NULL;
+        }
+        return field.dataType();
+    }
+
+    @Evaluator(extraName = "Double", warnExceptions = ArithmeticException.class)
+    static void process(
+        DoubleBlock.Builder builder,
+        int position,
+        DoubleBlock block,
+        @Fixed(includeInToString = false, build = true) CompensatedSum sum,
+        @Fixed double p
+    ) {
+        sum.reset(0, 0);
+        int start = block.getFirstValueIndex(position);
+        int end = block.getValueCount(position) + start;
+
+        for (int i = start; i < end; i++) {
+            double current_score = block.getDouble(i) / Math.pow(i - start + 1, p);
+            sum.add(current_score);
+        }
+        if (Double.isFinite(sum.value())) {
+            builder.appendDouble(sum.value());
+        } else {
+            throw new ArithmeticException("double overflow");
+        }
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeNamedWriteable(field);
+        out.writeNamedWriteable(p);
+    }
+
+    Expression field() {
+        return field;
+    }
+
+    Expression p() {
+        return p;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java
new file mode 100644
index 0000000000000..1eb0c70a7b08e
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java
@@ -0,0 +1,454 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.compute.ann.Evaluator;
+import org.elasticsearch.compute.ann.Fixed;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
+import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
+import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.planner.PlannerUtils;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+
+public class MvPercentile extends EsqlScalarFunction {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        Expression.class,
+        "MvPercentile",
+        MvPercentile::new
+    );
+
+    /**
+     * 2^52 is the smallest integer where it and all smaller integers can be represented exactly as double
+     */
+    private static final double MAX_SAFE_LONG_DOUBLE = Double.longBitsToDouble(0x4330000000000000L);
+
+    private final Expression field;
+    private final Expression percentile;
+
+    @FunctionInfo(
+        returnType = { "double", "integer", "long" },
+        description = "Converts a multivalued field into a single valued field containing "
+            + "the value at which a certain percentage of observed values occur.",
+        examples = @Example(file = "mv_percentile", tag = "example")
+    )
+    public MvPercentile(
+        Source source,
+        @Param(name = "number", type = { "double", "integer", "long" }, description = "Multivalue expression.") Expression field,
+        @Param(
+            name = "percentile",
+            type = { "double", "integer", "long" },
+            description = "The percentile to calculate. Must be a number between 0 and 100. "
+                + "Numbers out of range will return a null instead."
+        ) Expression percentile
+    ) {
+        super(source, List.of(field, percentile));
+        this.field = field;
+        this.percentile = percentile;
+    }
+
+    private MvPercentile(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class));
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeNamedWriteable(field);
+        out.writeNamedWriteable(percentile);
+    }
+
+    @Override
+    protected Expression.TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        return isType(field, dt -> dt.isNumeric() && dt != UNSIGNED_LONG, sourceText(), FIRST, "numeric except unsigned_long").and(
+            isType(percentile, dt -> dt.isNumeric() && dt != UNSIGNED_LONG, sourceText(), SECOND, "numeric except unsigned_long")
+        );
+    }
+
+    @Override
+    public boolean foldable() {
+        return field.foldable() && percentile.foldable();
+    }
+
+    public final Expression field() {
+        return field;
+    }
+
+    @Override
+    public DataType dataType() {
+        return field.dataType();
+    }
+
+    @Override
+    public final ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
+        var fieldEval = toEvaluator.apply(field);
+        var percentileEval = Cast.cast(source(), percentile.dataType(), DOUBLE, toEvaluator.apply(percentile));
+
+        return switch (PlannerUtils.toElementType(field.dataType())) {
+            case INT -> new MvPercentileIntegerEvaluator.Factory(source(), fieldEval, percentileEval, (d) -> new IntSortingScratch());
+            case LONG -> new MvPercentileLongEvaluator.Factory(source(), fieldEval, percentileEval, (d) -> new LongSortingScratch());
+            case DOUBLE -> new MvPercentileDoubleEvaluator.Factory(source(), fieldEval, percentileEval, (d) -> new DoubleSortingScratch());
+            default -> throw EsqlIllegalArgumentException.illegalDataType(field.dataType());
+        };
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new MvPercentile(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, MvPercentile::new, field, percentile);
+    }
+
+    static class DoubleSortingScratch {
+        private static final double[] EMPTY = new double[0];
+
+        public double[] values = EMPTY;
+    }
+
+    static class IntSortingScratch {
+        private static final int[] EMPTY = new int[0];
+
+        public int[] values = EMPTY;
+    }
+
+    static class LongSortingScratch {
+        private static final long[] EMPTY = new long[0];
+
+        public long[] values = EMPTY;
+    }
+
+    // Evaluators
+
+    @Evaluator(extraName = "Double", warnExceptions = IllegalArgumentException.class)
+    static void process(
+        DoubleBlock.Builder builder,
+        int position,
+        DoubleBlock values,
+        double percentile,
+        @Fixed(includeInToString = false, build = true) DoubleSortingScratch scratch
+    ) {
+        int valueCount = values.getValueCount(position);
+        int firstValueIndex = values.getFirstValueIndex(position);
+
+        if (valueCount == 0) {
+            builder.appendNull();
+            return;
+        }
+
+        if (percentile < 0 || percentile > 100) {
+            throw new IllegalArgumentException("Percentile parameter must be a number between 0 and 100, found [" + percentile + "]");
+        }
+
+        builder.appendDouble(calculateDoublePercentile(values, firstValueIndex, valueCount, percentile, scratch));
+    }
+
+    @Evaluator(extraName = "Integer", warnExceptions = IllegalArgumentException.class)
+    static void process(
+        IntBlock.Builder builder,
+        int position,
+        IntBlock values,
+        double percentile,
+        @Fixed(includeInToString = false, build = true) IntSortingScratch scratch
+    ) {
+        int valueCount = values.getValueCount(position);
+        int firstValueIndex = values.getFirstValueIndex(position);
+
+        if (valueCount == 0) {
+            builder.appendNull();
+            return;
+        }
+
+        if (percentile < 0 || percentile > 100) {
+            throw new IllegalArgumentException("Percentile parameter must be a number between 0 and 100, found [" + percentile + "]");
+        }
+
+        builder.appendInt(calculateIntPercentile(values, firstValueIndex, valueCount, percentile, scratch));
+    }
+
+    @Evaluator(extraName = "Long", warnExceptions = IllegalArgumentException.class)
+    static void process(
+        LongBlock.Builder builder,
+        int position,
+        LongBlock values,
+        double percentile,
+        @Fixed(includeInToString = false, build = true) LongSortingScratch scratch
+    ) {
+        int valueCount = values.getValueCount(position);
+        int firstValueIndex = values.getFirstValueIndex(position);
+
+        if (valueCount == 0) {
+            builder.appendNull();
+            return;
+        }
+
+        if (percentile < 0 || percentile > 100) {
+            throw new IllegalArgumentException("Percentile parameter must be a number between 0 and 100, found [" + percentile + "]");
+        }
+
+        builder.appendLong(calculateLongPercentile(values, firstValueIndex, valueCount, percentile, scratch));
+    }
+
+    // Percentile calculators
+
+    static double calculateDoublePercentile(
+        DoubleBlock valuesBlock,
+        int firstValueIndex,
+        int valueCount,
+        double percentile,
+        DoubleSortingScratch scratch
+    ) {
+        if (valueCount == 1) {
+            return valuesBlock.getDouble(firstValueIndex);
+        }
+
+        var p = percentile / 100.0;
+        var index = p * (valueCount - 1);
+        var lowerIndex = (int) index;
+        var upperIndex = lowerIndex + 1;
+        var fraction = index - lowerIndex;
+
+        if (valuesBlock.mvSortedAscending()) {
+            if (percentile == 0) {
+                return valuesBlock.getDouble(0);
+            } else if (percentile == 100) {
+                return valuesBlock.getDouble(valueCount - 1);
+            } else {
+                assert lowerIndex >= 0 && upperIndex < valueCount;
+                return calculateDoublePercentile(
+                    fraction,
+                    valuesBlock.getDouble(firstValueIndex + lowerIndex),
+                    valuesBlock.getDouble(firstValueIndex + upperIndex)
+                );
+            }
+        }
+
+        if (percentile == 0) {
+            double min = Double.POSITIVE_INFINITY;
+            for (int i = 0; i < valueCount; i++) {
+                min = Math.min(min, valuesBlock.getDouble(firstValueIndex + i));
+            }
+            return min;
+        } else if (percentile == 100) {
+            double max = Double.NEGATIVE_INFINITY;
+            for (int i = 0; i < valueCount; i++) {
+                max = Math.max(max, valuesBlock.getDouble(firstValueIndex + i));
+            }
+            return max;
+        }
+
+        if (scratch.values.length < valueCount) {
+            scratch.values = new double[ArrayUtil.oversize(valueCount, Double.BYTES)];
+        }
+
+        for (int i = 0; i < valueCount; i++) {
+            scratch.values[i] = valuesBlock.getDouble(firstValueIndex + i);
+        }
+
+        Arrays.sort(scratch.values, 0, valueCount);
+
+        assert lowerIndex >= 0 && upperIndex < valueCount;
+        return calculateDoublePercentile(fraction, scratch.values[lowerIndex], scratch.values[upperIndex]);
+    }
+
+    static int calculateIntPercentile(
+        IntBlock valuesBlock,
+        int firstValueIndex,
+        int valueCount,
+        double percentile,
+        IntSortingScratch scratch
+    ) {
+        if (valueCount == 1) {
+            return valuesBlock.getInt(firstValueIndex);
+        }
+
+        var p = percentile / 100.0;
+        var index = p * (valueCount - 1);
+        var lowerIndex = (int) index;
+        var upperIndex = lowerIndex + 1;
+        var fraction = index - lowerIndex;
+
+        if (valuesBlock.mvSortedAscending()) {
+            if (percentile == 0) {
+                return valuesBlock.getInt(0);
+            } else if (percentile == 100) {
+                return valuesBlock.getInt(valueCount - 1);
+            } else {
+                assert lowerIndex >= 0 && upperIndex < valueCount;
+                var lowerValue = valuesBlock.getInt(firstValueIndex + lowerIndex);
+                var upperValue = valuesBlock.getInt(firstValueIndex + upperIndex);
+                var difference = (long) upperValue - lowerValue;
+                return lowerValue + (int) (fraction * difference);
+            }
+        }
+
+        if (percentile == 0) {
+            int min = Integer.MAX_VALUE;
+            for (int i = 0; i < valueCount; i++) {
+                min = Math.min(min, valuesBlock.getInt(firstValueIndex + i));
+            }
+            return min;
+        } else if (percentile == 100) {
+            int max = Integer.MIN_VALUE;
+            for (int i = 0; i < valueCount; i++) {
+                max = Math.max(max, valuesBlock.getInt(firstValueIndex + i));
+            }
+            return max;
+        }
+
+        if (scratch.values.length < valueCount) {
+            scratch.values = new int[ArrayUtil.oversize(valueCount, Integer.BYTES)];
+        }
+
+        for (int i = 0; i < valueCount; i++) {
+            scratch.values[i] = valuesBlock.getInt(firstValueIndex + i);
+        }
+
+        Arrays.sort(scratch.values, 0, valueCount);
+
+        assert lowerIndex >= 0 && upperIndex < valueCount;
+        var lowerValue = scratch.values[lowerIndex];
+        var upperValue = scratch.values[upperIndex];
+        var difference = (long) upperValue - lowerValue;
+        return lowerValue + (int) (fraction * difference);
+    }
+
+    static long calculateLongPercentile(
+        LongBlock valuesBlock,
+        int firstValueIndex,
+        int valueCount,
+        double percentile,
+        LongSortingScratch scratch
+    ) {
+        if (valueCount == 1) {
+            return valuesBlock.getLong(firstValueIndex);
+        }
+
+        var p = percentile / 100.0;
+        var index = p * (valueCount - 1);
+        var lowerIndex = (int) index;
+        var upperIndex = lowerIndex + 1;
+        var fraction = index - lowerIndex;
+
+        if (valuesBlock.mvSortedAscending()) {
+            if (percentile == 0) {
+                return valuesBlock.getLong(0);
+            } else if (percentile == 100) {
+                return valuesBlock.getLong(valueCount - 1);
+            } else {
+                assert lowerIndex >= 0 && upperIndex < valueCount;
+                return calculateLongPercentile(
+                    fraction,
+                    valuesBlock.getLong(firstValueIndex + lowerIndex),
+                    valuesBlock.getLong(firstValueIndex + upperIndex)
+                );
+            }
+        }
+
+        if (percentile == 0) {
+            long min = Long.MAX_VALUE;
+            for (int i = 0; i < valueCount; i++) {
+                min = Math.min(min, valuesBlock.getLong(firstValueIndex + i));
+            }
+            return min;
+        } else if (percentile == 100) {
+            long max = Long.MIN_VALUE;
+            for (int i = 0; i < valueCount; i++) {
+                max = Math.max(max, valuesBlock.getLong(firstValueIndex + i));
+            }
+            return max;
+        }
+
+        if (scratch.values.length < valueCount) {
+            scratch.values = new long[ArrayUtil.oversize(valueCount, Long.BYTES)];
+        }
+
+        for (int i = 0; i < valueCount; i++) {
+            scratch.values[i] = valuesBlock.getLong(firstValueIndex + i);
+        }
+
+        Arrays.sort(scratch.values, 0, valueCount);
+
+        assert lowerIndex >= 0 && upperIndex < valueCount;
+        return calculateLongPercentile(fraction, scratch.values[lowerIndex], scratch.values[upperIndex]);
+    }
+
+    /**
+     * Calculates a percentile for a long avoiding overflows and double precision issues.
+     * <p>
+     *     To do that, if the values are over the limit of the representable double integers,
+     *     it uses instead BigDecimals for the calculations.
+     * </p>
+     */
+    private static long calculateLongPercentile(double fraction, long lowerValue, long upperValue) {
+        if (upperValue < MAX_SAFE_LONG_DOUBLE && lowerValue > -MAX_SAFE_LONG_DOUBLE) {
+            var difference = upperValue - lowerValue;
+            return lowerValue + (long) (fraction * difference);
+        }
+
+        var lowerValueBigDecimal = new BigDecimal(lowerValue);
+        var upperValueBigDecimal = new BigDecimal(upperValue);
+        var difference = upperValueBigDecimal.subtract(lowerValueBigDecimal);
+        var fractionBigDecimal = new BigDecimal(fraction);
+        return lowerValueBigDecimal.add(fractionBigDecimal.multiply(difference)).longValue();
+    }
+
+    /**
+     * Calculates a percentile for a double avoiding overflows.
+     * <p>
+     *     If the values are too separated (negative + positive), it uses a slightly different approach.
+     *     This approach would fail if the values are big but not separated, so it's only used in this case.
+     * </p>
+     */
+    private static double calculateDoublePercentile(double fraction, double lowerValue, double upperValue) {
+        if (lowerValue < 0 && upperValue > 0) {
+            // Order is required to avoid `upper - lower` overflows
+            return (lowerValue + fraction * upperValue) - fraction * lowerValue;
+        }
+
+        var difference = upperValue - lowerValue;
+        return lowerValue + fraction * difference;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
index 3728f4305d5c7..c332e94b20049 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -153,7 +152,7 @@ protected TypeResolution resolveType() {
             return new TypeResolution("Unresolved children");
         }
 
-        TypeResolution resolution = isType(field, EsqlDataTypes::isRepresentable, sourceText(), FIRST, "representable");
+        TypeResolution resolution = isType(field, DataType::isRepresentable, sourceText(), FIRST, "representable");
         if (resolution.unresolved()) {
             return resolution;
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
index ee83236ac6a63..4ed9a01c29797 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
@@ -44,7 +44,6 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -128,7 +127,7 @@ protected TypeResolution resolveType() {
             return new TypeResolution("Unresolved children");
         }
 
-        TypeResolution resolution = isType(field, EsqlDataTypes::isRepresentable, sourceText(), FIRST, "representable");
+        TypeResolution resolution = isType(field, DataType::isRepresentable, sourceText(), FIRST, "representable");
 
         if (resolution.unresolved()) {
             return resolution;
@@ -277,7 +276,7 @@ public EvalOperator.ExpressionEvaluator get(DriverContext context) {
 
         @Override
         public String toString() {
-            return "MvSort" + dataType + "[field=" + field + ", order=" + order + "]";
+            return "MvSort" + dataType.pascalCaseName() + "[field=" + field + ", order=" + order + "]";
         }
     }
 
@@ -311,7 +310,7 @@ public Block eval(Page page) {
 
         @Override
         public String toString() {
-            return "MvSort" + dataType + "[field=" + field + ", order=" + order + "]";
+            return "MvSort" + dataType.pascalCaseName() + "[field=" + field + ", order=" + order + "]";
         }
 
         @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java
index eabf5e20ad1b0..cec4ce007d994 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java
@@ -27,8 +27,8 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAddExact;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable;
 
 /**
  * Reduce a multivalued field to a single valued field containing the sum of all values.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java
index 4f9219247d5c2..b4781c7e41f98 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java
@@ -184,6 +184,16 @@
  *         looks ok.
  *     </li>
  *     <li>
+ *         Let's finish up the code by making the tests backwards compatible. Since this is a new
+ *         feature we just have to convince the tests not to run in a cluster that includes older
+ *         versions of Elasticsearch. We do that with a {@link org.elasticsearch.rest.RestHandler#supportedCapabilities capability}
+ *         on the REST handler. ESQL has a <strong>ton</strong> of capabilities so we list them
+ *         all in {@link org.elasticsearch.xpack.esql.action.EsqlCapabilities}. Add a new one
+ *         for your function. Now add something like {@code required_capability: my_function}
+ *         to all of your csv-spec tests. Run those csv-spec tests as integration tests to double
+ *         check that they run on the main branch.
+ *     </li>
+ *     <li>
  *         Open the PR. The subject and description of the PR are important because those'll turn
  *         into the commit message we see in the commit history. Good PR descriptions make me very
  *         happy. But functions don't need an essay.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java
index 1beef40ce0c42..84d776888c7ae 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
 import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.List;
@@ -43,7 +42,7 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
     }
 
     private final SpatialTypeResolver spatialTypeResolver;
-    protected SpatialCrsType crsType;
+    private SpatialCrsType crsType;
     protected final boolean leftDocValues;
     protected final boolean rightDocValues;
 
@@ -163,7 +162,7 @@ protected TypeResolution isSameSpatialType(
                 ? isType(expression, dt -> dt == spatialDataType, operationName, paramOrd, compatibleTypeNames(spatialDataType))
                 : isType(
                     expression,
-                    dt -> EsqlDataTypes.isSpatial(dt) && spatialCRSCompatible(spatialDataType, dt),
+                    dt -> DataType.isSpatial(dt) && spatialCRSCompatible(spatialDataType, dt),
                     operationName,
                     paramOrd,
                     compatibleTypeNames(spatialDataType)
@@ -180,12 +179,12 @@ public void setCrsType(DataType dataType) {
     private static final String[] CARTESIAN_TYPE_NAMES = new String[] { GEO_POINT.typeName(), GEO_SHAPE.typeName() };
 
     protected static boolean spatialCRSCompatible(DataType spatialDataType, DataType otherDataType) {
-        return EsqlDataTypes.isSpatialGeo(spatialDataType) && EsqlDataTypes.isSpatialGeo(otherDataType)
-            || EsqlDataTypes.isSpatialGeo(spatialDataType) == false && EsqlDataTypes.isSpatialGeo(otherDataType) == false;
+        return DataType.isSpatialGeo(spatialDataType) && DataType.isSpatialGeo(otherDataType)
+            || DataType.isSpatialGeo(spatialDataType) == false && DataType.isSpatialGeo(otherDataType) == false;
     }
 
     static String[] compatibleTypeNames(DataType spatialDataType) {
-        return EsqlDataTypes.isSpatialGeo(spatialDataType) ? GEO_TYPE_NAMES : CARTESIAN_TYPE_NAMES;
+        return DataType.isSpatialGeo(spatialDataType) ? GEO_TYPE_NAMES : CARTESIAN_TYPE_NAMES;
     }
 
     @Override
@@ -214,8 +213,8 @@ public enum SpatialCrsType {
         UNSPECIFIED;
 
         public static SpatialCrsType fromDataType(DataType dataType) {
-            return EsqlDataTypes.isSpatialGeo(dataType) ? SpatialCrsType.GEO
-                : EsqlDataTypes.isSpatial(dataType) ? SpatialCrsType.CARTESIAN
+            return DataType.isSpatialGeo(dataType) ? SpatialCrsType.GEO
+                : DataType.isSpatial(dataType) ? SpatialCrsType.CARTESIAN
                 : SpatialCrsType.UNSPECIFIED;
         }
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
index 6c2d11ab0ad16..6788d13cf345e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.geo.Component2D;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.compute.ann.Evaluator;
@@ -31,7 +32,6 @@
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -152,8 +152,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public ShapeField.QueryRelation queryRelation() {
-        return ShapeField.QueryRelation.CONTAINS;
+    public ShapeRelation queryRelation() {
+        return ShapeRelation.CONTAINS;
     }
 
     @Override
@@ -177,10 +177,10 @@ protected NodeInfo<? extends Expression> info() {
     @Override
     public Object fold() {
         try {
-            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType, left());
+            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left());
             Geometry rightGeom = makeGeometryFromLiteral(right());
-            Component2D[] components = asLuceneComponent2Ds(crsType, rightGeom);
-            return (crsType == SpatialCrsType.GEO)
+            Component2D[] components = asLuceneComponent2Ds(crsType(), rightGeom);
+            return (crsType() == SpatialCrsType.GEO)
                 ? GEO.geometryRelatesGeometries(docValueReader, components)
                 : CARTESIAN.geometryRelatesGeometries(docValueReader, components);
         } catch (IOException e) {
@@ -221,7 +221,7 @@ public SpatialRelatesFunction surrogate() {
                         SpatialContainsGeoSourceAndConstantEvaluator.Factory::new
                     )
                 );
-                if (EsqlDataTypes.isSpatialPoint(spatialType)) {
+                if (DataType.isSpatialPoint(spatialType)) {
                     evaluatorMap.put(
                         SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(),
                         new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields(
@@ -253,7 +253,7 @@ public SpatialRelatesFunction surrogate() {
                         SpatialContainsCartesianSourceAndConstantEvaluator.Factory::new
                     )
                 );
-                if (EsqlDataTypes.isSpatialPoint(spatialType)) {
+                if (DataType.isSpatialPoint(spatialType)) {
                     evaluatorMap.put(
                         SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(),
                         new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
index e5520079e1b10..eac50f84bd12d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.geo.Component2D;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.compute.ann.Evaluator;
@@ -29,7 +30,6 @@
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -107,8 +107,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public ShapeField.QueryRelation queryRelation() {
-        return ShapeField.QueryRelation.DISJOINT;
+    public ShapeRelation queryRelation() {
+        return ShapeRelation.DISJOINT;
     }
 
     @Override
@@ -132,9 +132,9 @@ protected NodeInfo<? extends Expression> info() {
     @Override
     public Object fold() {
         try {
-            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType, left());
-            Component2D component2D = asLuceneComponent2D(crsType, right());
-            return (crsType == SpatialCrsType.GEO)
+            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left());
+            Component2D component2D = asLuceneComponent2D(crsType(), right());
+            return (crsType() == SpatialCrsType.GEO)
                 ? GEO.geometryRelatesGeometry(docValueReader, component2D)
                 : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D);
         } catch (IOException e) {
@@ -163,7 +163,7 @@ public Object fold() {
                         SpatialDisjointGeoSourceAndConstantEvaluator.Factory::new
                     )
                 );
-                if (EsqlDataTypes.isSpatialPoint(spatialType)) {
+                if (DataType.isSpatialPoint(spatialType)) {
                     evaluatorMap.put(
                         SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(),
                         new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields(
@@ -195,7 +195,7 @@ public Object fold() {
                         SpatialDisjointCartesianSourceAndConstantEvaluator.Factory::new
                     )
                 );
-                if (EsqlDataTypes.isSpatialPoint(spatialType)) {
+                if (DataType.isSpatialPoint(spatialType)) {
                     evaluatorMap.put(
                         SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(),
                         new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
index 045690340f6ac..886551d1f3154 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.geo.Component2D;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.compute.ann.Evaluator;
@@ -29,7 +30,6 @@
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -105,8 +105,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public ShapeField.QueryRelation queryRelation() {
-        return ShapeField.QueryRelation.INTERSECTS;
+    public ShapeRelation queryRelation() {
+        return ShapeRelation.INTERSECTS;
     }
 
     @Override
@@ -130,9 +130,9 @@ protected NodeInfo<? extends Expression> info() {
     @Override
     public Object fold() {
         try {
-            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType, left());
-            Component2D component2D = asLuceneComponent2D(crsType, right());
-            return (crsType == SpatialCrsType.GEO)
+            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left());
+            Component2D component2D = asLuceneComponent2D(crsType(), right());
+            return (crsType() == SpatialCrsType.GEO)
                 ? GEO.geometryRelatesGeometry(docValueReader, component2D)
                 : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D);
         } catch (IOException e) {
@@ -161,7 +161,7 @@ public Object fold() {
                         SpatialIntersectsGeoSourceAndConstantEvaluator.Factory::new
                     )
                 );
-                if (EsqlDataTypes.isSpatialPoint(spatialType)) {
+                if (DataType.isSpatialPoint(spatialType)) {
                     evaluatorMap.put(
                         SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(),
                         new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields(
@@ -193,7 +193,7 @@ public Object fold() {
                         SpatialIntersectsCartesianSourceAndConstantEvaluator.Factory::new
                     )
                 );
-                if (EsqlDataTypes.isSpatialPoint(spatialType)) {
+                if (DataType.isSpatialPoint(spatialType)) {
                     evaluatorMap.put(
                         SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(),
                         new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
index 68005ecbfed47..36e98984d2303 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
@@ -10,6 +10,7 @@
 import org.apache.lucene.document.ShapeField;
 import org.apache.lucene.geo.Component2D;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.geometry.Geometry;
@@ -24,7 +25,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.Map;
@@ -51,7 +51,7 @@ protected SpatialRelatesFunction(StreamInput in, boolean leftDocValues, boolean
         super(in, leftDocValues, rightDocValues, false);
     }
 
-    public abstract ShapeField.QueryRelation queryRelation();
+    public abstract ShapeRelation queryRelation();
 
     @Override
     public DataType dataType() {
@@ -78,7 +78,7 @@ private static boolean isPushableFieldAttribute(Expression exp, Predicate<FieldA
         return exp instanceof FieldAttribute fa
             && fa.getExactInfo().hasExact()
             && isAggregatable.test(fa)
-            && EsqlDataTypes.isSpatial(fa.dataType());
+            && DataType.isSpatial(fa.dataType());
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java
index 600c3529acd13..64f991dd1a08f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java
@@ -28,7 +28,7 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.xpack.esql.core.planner.ExpressionTranslators.valueOf;
+import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf;
 
 public class SpatialRelatesUtils {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
index f72571a4b5250..0b210f07a02f4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.geo.Component2D;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.compute.ann.Evaluator;
@@ -30,7 +31,6 @@
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -107,8 +107,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public ShapeField.QueryRelation queryRelation() {
-        return ShapeField.QueryRelation.WITHIN;
+    public ShapeRelation queryRelation() {
+        return ShapeRelation.WITHIN;
     }
 
     @Override
@@ -132,9 +132,9 @@ protected NodeInfo<? extends Expression> info() {
     @Override
     public Object fold() {
         try {
-            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType, left());
-            Component2D component2D = asLuceneComponent2D(crsType, right());
-            return (crsType == SpatialCrsType.GEO)
+            GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left());
+            Component2D component2D = asLuceneComponent2D(crsType(), right());
+            return (crsType() == SpatialCrsType.GEO)
                 ? GEO.geometryRelatesGeometry(docValueReader, component2D)
                 : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D);
         } catch (IOException e) {
@@ -173,7 +173,7 @@ public SpatialRelatesFunction surrogate() {
                     SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSourceAndConstant(spatialType, otherType),
                     new SpatialEvaluatorFactory.SpatialEvaluatorWithConstantFactory(SpatialWithinGeoSourceAndConstantEvaluator.Factory::new)
                 );
-                if (EsqlDataTypes.isSpatialPoint(spatialType)) {
+                if (DataType.isSpatialPoint(spatialType)) {
                     evaluatorMap.put(
                         SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(),
                         new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields(
@@ -205,7 +205,7 @@ public SpatialRelatesFunction surrogate() {
                         SpatialWithinCartesianSourceAndConstantEvaluator.Factory::new
                     )
                 );
-                if (EsqlDataTypes.isSpatialPoint(spatialType)) {
+                if (DataType.isSpatialPoint(spatialType)) {
                     evaluatorMap.put(
                         SpatialEvaluatorFactory.SpatialEvaluatorKey.fromSources(spatialType, otherType).withLeftDocValues(),
                         new SpatialEvaluatorFactory.SpatialEvaluatorFactoryWithFields(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java
index 1fdd4241aa222..14bded51aa55f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java
@@ -173,7 +173,7 @@ protected NodeInfo<? extends Expression> info() {
     public Object fold() {
         var leftGeom = makeGeometryFromLiteral(left());
         var rightGeom = makeGeometryFromLiteral(right());
-        return (crsType == SpatialCrsType.GEO) ? GEO.distance(leftGeom, rightGeom) : CARTESIAN.distance(leftGeom, rightGeom);
+        return (crsType() == SpatialCrsType.GEO) ? GEO.distance(leftGeom, rightGeom) : CARTESIAN.distance(leftGeom, rightGeom);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java
index 56090b0fb9e32..bb923ec924d31 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java
@@ -16,6 +16,9 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
 import java.io.IOException;
@@ -27,7 +30,16 @@
 public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike implements EvaluatorMapper {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "RLike", RLike::new);
 
-    public RLike(Source source, Expression value, RLikePattern pattern) {
+    @FunctionInfo(returnType = "boolean", description = """
+        Use `RLIKE` to filter data based on string patterns using using
+        <<regexp-syntax,regular expressions>>. `RLIKE` usually acts on a field placed on
+        the left-hand side of the operator, but it can also act on a constant (literal)
+        expression. The right-hand side of the operator represents the pattern.""", examples = @Example(file = "docs", tag = "rlike"))
+    public RLike(
+        Source source,
+        @Param(name = "str", type = { "keyword", "text" }, description = "A literal value.") Expression value,
+        @Param(name = "pattern", type = { "keyword", "text" }, description = "A regular expression.") RLikePattern pattern
+    ) {
         super(source, value, pattern);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java
index 9cea1cebe926c..62255a0a31ea6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.compute.ann.Fixed;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -25,7 +24,7 @@
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
 import java.util.List;
@@ -99,7 +98,7 @@ static BytesRef process(BytesRef val, @Fixed Locale locale) {
     @Override
     public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
         var fieldEvaluator = toEvaluator.apply(field);
-        return new ToLowerEvaluator.Factory(source(), fieldEvaluator, ((EsqlConfiguration) configuration()).locale());
+        return new ToLowerEvaluator.Factory(source(), fieldEvaluator, configuration().locale());
     }
 
     public Expression field() {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java
index 4703bd5b2ba91..e6eba0d01e4da 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.compute.ann.Fixed;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -25,7 +24,7 @@
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
 import java.util.List;
@@ -99,7 +98,7 @@ static BytesRef process(BytesRef val, @Fixed Locale locale) {
     @Override
     public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
         var fieldEvaluator = toEvaluator.apply(field);
-        return new ToUpperEvaluator.Factory(source(), fieldEvaluator, ((EsqlConfiguration) configuration()).locale());
+        return new ToUpperEvaluator.Factory(source(), fieldEvaluator, configuration().locale());
     }
 
     public Expression field() {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java
index 325cc0aea4461..15470bb56b29f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java
@@ -47,8 +47,8 @@ also act on a constant (literal) expression. The right-hand side of the operator
         * `?` matches one character.""", examples = @Example(file = "docs", tag = "like"))
     public WildcardLike(
         Source source,
-        @Param(name = "str", type = { "keyword", "text" }) Expression left,
-        @Param(name = "pattern", type = { "keyword", "text" }) WildcardPattern pattern
+        @Param(name = "str", type = { "keyword", "text" }, description = "A literal expression.") Expression left,
+        @Param(name = "pattern", type = { "keyword", "text" }, description = "Pattern.") WildcardPattern pattern
     ) {
         super(source, left, pattern, false);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java
index a92741229aaa9..8f8d885ee379b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java
@@ -12,10 +12,11 @@
 import org.elasticsearch.compute.ann.Evaluator;
 import org.elasticsearch.compute.ann.Fixed;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 
 import java.io.IOException;
 import java.time.DateTimeException;
@@ -23,15 +24,31 @@
 import java.time.Period;
 import java.time.temporal.TemporalAmount;
 
-import static org.elasticsearch.xpack.esql.core.type.DateUtils.asDateTime;
-import static org.elasticsearch.xpack.esql.core.type.DateUtils.asMillis;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.asDateTime;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.asMillis;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAddExact;
 import static org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation.OperationSymbol.ADD;
 
 public class Add extends DateTimeArithmeticOperation implements BinaryComparisonInversible {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Add", Add::new);
 
-    public Add(Source source, Expression left, Expression right) {
+    @FunctionInfo(
+        returnType = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" },
+        description = "Add two numbers together. " + "If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`."
+    )
+    public Add(
+        Source source,
+        @Param(
+            name = "lhs",
+            description = "A numeric value or a date time value.",
+            type = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" }
+        ) Expression left,
+        @Param(
+            name = "rhs",
+            description = "A numeric value or a date time value.",
+            type = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" }
+        ) Expression right
+    ) {
         super(
             source,
             left,
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/ArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ArithmeticOperation.java
similarity index 80%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/ArithmeticOperation.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ArithmeticOperation.java
index 8dc0f58083179..cb7e7c4643fb9 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/ArithmeticOperation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ArithmeticOperation.java
@@ -4,16 +4,17 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic;
+package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic;
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
+import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryArithmeticOperation;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypeConverter;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.commonType;
 
 public abstract class ArithmeticOperation extends BinaryOperator<Object, Object, Object, BinaryArithmeticOperation> {
 
@@ -36,7 +37,7 @@ public ArithmeticOperation swapLeftAndRight() {
     @Override
     public DataType dataType() {
         if (dataType == null) {
-            dataType = DataTypeConverter.commonType(left().dataType(), right().dataType());
+            dataType = commonType(left().dataType(), right().dataType());
         }
         return dataType;
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/BinaryComparisonInversible.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/BinaryComparisonInversible.java
similarity index 91%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/BinaryComparisonInversible.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/BinaryComparisonInversible.java
index 358ad59ec6356..b0ab4c48d970e 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/BinaryComparisonInversible.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/BinaryComparisonInversible.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic;
+package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic;
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
index 45cc5b9bdc5c0..5b7cc74faed86 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.time.Duration;
@@ -27,9 +26,9 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
 import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTimeOrTemporal;
 import static org.elasticsearch.xpack.esql.core.type.DataType.isNull;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isDateTimeOrTemporal;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount;
 
 public abstract class DateTimeArithmeticOperation extends EsqlArithmeticOperation {
     /** Arithmetic (quad) function. */
@@ -71,7 +70,7 @@ interface DatetimeArithmeticEvaluator {
     protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) {
         return TypeResolutions.isType(
             e,
-            t -> t.isNumeric() || EsqlDataTypes.isDateTimeOrTemporal(t) || DataType.isNull(t),
+            t -> t.isNumeric() || DataType.isDateTimeOrTemporal(t) || DataType.isNull(t),
             sourceText(),
             paramOrdinal,
             "datetime",
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java
index 6d84ce3558571..f1e197cf350b6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java
@@ -11,11 +11,12 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.compute.ann.Evaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 
 import java.io.IOException;
 
@@ -27,7 +28,18 @@ public class Div extends EsqlArithmeticOperation implements BinaryComparisonInve
 
     private DataType type;
 
-    public Div(Source source, Expression left, Expression right) {
+    @FunctionInfo(
+        returnType = { "double", "integer", "long", "unsigned_long" },
+        description = "Divide one number by another. "
+            + "If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+        note = "Division of two integer types will yield an integer result, rounding towards 0. "
+            + "If you need floating point division, <<esql-cast-operator>> one of the arguments to a `DOUBLE`."
+    )
+    public Div(
+        Source source,
+        @Param(name = "lhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression left,
+        @Param(name = "rhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression right
+    ) {
         this(source, left, right, null);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java
index 647071c44cfd3..400e70b641111 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java
@@ -13,14 +13,12 @@
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.ArithmeticOperation;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryArithmeticOperation;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
 
 import java.io.IOException;
 import java.util.List;
@@ -31,6 +29,7 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.commonType;
 
 public abstract class EsqlArithmeticOperation extends ArithmeticOperation implements EvaluatorMapper {
     public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
@@ -133,7 +132,7 @@ public Object fold() {
 
     public DataType dataType() {
         if (dataType == null) {
-            dataType = EsqlDataTypeRegistry.INSTANCE.commonType(left().dataType(), right().dataType());
+            dataType = commonType(left().dataType(), right().dataType());
         }
         return dataType;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java
index 151c886bfdf8d..2381e3df9b617 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 
 import java.io.IOException;
 
@@ -23,7 +25,16 @@
 public class Mod extends EsqlArithmeticOperation {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Mod", Mod::new);
 
-    public Mod(Source source, Expression left, Expression right) {
+    @FunctionInfo(
+        returnType = { "double", "integer", "long", "unsigned_long" },
+        description = "Divide one number by another and return the remainder. "
+            + "If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`."
+    )
+    public Mod(
+        Source source,
+        @Param(name = "lhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression left,
+        @Param(name = "rhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression right
+    ) {
         super(
             source,
             left,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java
index 4a13a458cadc8..03981a821f52d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java
@@ -11,10 +11,11 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.compute.ann.Evaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 
 import java.io.IOException;
 
@@ -24,7 +25,16 @@
 public class Mul extends EsqlArithmeticOperation implements BinaryComparisonInversible {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Mul", Mul::new);
 
-    public Mul(Source source, Expression left, Expression right) {
+    @FunctionInfo(
+        returnType = { "double", "integer", "long", "unsigned_long" },
+        description = "Multiply two numbers together. "
+            + "If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`."
+    )
+    public Mul(
+        Source source,
+        @Param(name = "lhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression left,
+        @Param(name = "rhs", description = "A numeric value.", type = { "double", "integer", "long", "unsigned_long" }) Expression right
+    ) {
         super(
             source,
             left,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
index d1ed5579c4485..67b770d14339e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
@@ -17,6 +17,8 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction;
 
 import java.io.IOException;
@@ -29,12 +31,23 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
 import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount;
 
 public class Neg extends UnaryScalarFunction {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Neg", Neg::new);
 
-    public Neg(Source source, Expression field) {
+    @FunctionInfo(
+        returnType = { "double", "integer", "long", "date_period", "time_duration" },
+        description = "Returns the negation of the argument."
+    )
+    public Neg(
+        Source source,
+        @Param(
+            name = "field",
+            description = "A numeric value or a date time interval.",
+            type = { "double", "integer", "long", "date_period", "time_duration" }
+        ) Expression field
+    ) {
         super(source, field);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java
index 92330b537e761..27f5579129cc9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java
@@ -12,12 +12,12 @@
 import org.elasticsearch.compute.ann.Evaluator;
 import org.elasticsearch.compute.ann.Fixed;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 
 import java.io.IOException;
 import java.time.DateTimeException;
@@ -26,15 +26,32 @@
 import java.time.temporal.TemporalAmount;
 
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
-import static org.elasticsearch.xpack.esql.core.type.DateUtils.asDateTime;
-import static org.elasticsearch.xpack.esql.core.type.DateUtils.asMillis;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.asDateTime;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.asMillis;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongSubtractExact;
 import static org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation.OperationSymbol.SUB;
 
 public class Sub extends DateTimeArithmeticOperation implements BinaryComparisonInversible {
     public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Sub", Sub::new);
 
-    public Sub(Source source, Expression left, Expression right) {
+    @FunctionInfo(
+        returnType = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" },
+        description = "Subtract one number from another. "
+            + "If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`."
+    )
+    public Sub(
+        Source source,
+        @Param(
+            name = "lhs",
+            description = "A numeric value or a date time value.",
+            type = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" }
+        ) Expression left,
+        @Param(
+            name = "rhs",
+            description = "A numeric value or a date time value.",
+            type = { "double", "integer", "long", "date_period", "datetime", "time_duration", "unsigned_long" }
+        ) Expression right
+    ) {
         super(
             source,
             left,
@@ -69,7 +86,7 @@ public String getWriteableName() {
     protected TypeResolution resolveType() {
         TypeResolution resolution = super.resolveType();
         // As opposed to general date time arithmetics, we cannot subtract a datetime from something else.
-        if (resolution.resolved() && EsqlDataTypes.isDateTimeOrTemporal(dataType()) && DataType.isDateTime(right().dataType())) {
+        if (resolution.resolved() && DataType.isDateTimeOrTemporal(dataType()) && DataType.isDateTime(right().dataType())) {
             return new TypeResolution(
                 format(
                     null,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java
index 26a74e7bdb03c..614d9aa3ec920 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -33,6 +35,7 @@ public class Equals extends EsqlBinaryComparison implements Negatable<EsqlBinary
         Map.entry(DataType.LONG, EqualsLongsEvaluator.Factory::new),
         Map.entry(DataType.UNSIGNED_LONG, EqualsLongsEvaluator.Factory::new),
         Map.entry(DataType.DATETIME, EqualsLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATE_NANOS, EqualsLongsEvaluator.Factory::new),
         Map.entry(DataType.GEO_POINT, EqualsGeometriesEvaluator.Factory::new),
         Map.entry(DataType.CARTESIAN_POINT, EqualsGeometriesEvaluator.Factory::new),
         Map.entry(DataType.GEO_SHAPE, EqualsGeometriesEvaluator.Factory::new),
@@ -43,7 +46,54 @@ public class Equals extends EsqlBinaryComparison implements Negatable<EsqlBinary
         Map.entry(DataType.IP, EqualsKeywordsEvaluator.Factory::new)
     );
 
-    public Equals(Source source, Expression left, Expression right) {
+    @FunctionInfo(
+        returnType = { "boolean" },
+        description = "Check if two fields are equal. "
+            + "If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+        note = "This is pushed to the underlying search index if one side of the comparison is constant "
+            + "and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>."
+    )
+    public Equals(
+        Source source,
+        @Param(
+            name = "lhs",
+            type = {
+                "boolean",
+                "cartesian_point",
+                "cartesian_shape",
+                "date",
+                "double",
+                "geo_point",
+                "geo_shape",
+                "integer",
+                "ip",
+                "keyword",
+                "long",
+                "text",
+                "unsigned_long",
+                "version" },
+            description = "An expression."
+        ) Expression left,
+        @Param(
+            name = "rhs",
+            type = {
+                "boolean",
+                "cartesian_point",
+                "cartesian_shape",
+                "date",
+                "double",
+                "geo_point",
+                "geo_shape",
+                "integer",
+                "ip",
+                "keyword",
+                "long",
+                "text",
+                "unsigned_long",
+                "version" },
+            description = "An expression."
+        ) Expression right
+    ) {
         super(source, left, right, BinaryComparisonOperation.EQ, evaluatorMap);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java
index 52d4c111b2eae..b50d70e69819d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
 
 import java.io.IOException;
 import java.time.ZoneId;
@@ -32,6 +31,7 @@
 
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.commonType;
 
 public abstract class EsqlBinaryComparison extends BinaryComparison implements EvaluatorMapper {
     public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
@@ -172,7 +172,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(
         Function<Expression, EvalOperator.ExpressionEvaluator.Factory> toEvaluator
     ) {
         // Our type is always boolean, so figure out the evaluator type from the inputs
-        DataType commonType = EsqlDataTypeRegistry.INSTANCE.commonType(left().dataType(), right().dataType());
+        DataType commonType = commonType(left().dataType(), right().dataType());
         EvalOperator.ExpressionEvaluator.Factory lhs;
         EvalOperator.ExpressionEvaluator.Factory rhs;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java
index 8ce8bf30ef617..6aef5c76f7385 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -32,13 +34,33 @@ public class GreaterThan extends EsqlBinaryComparison implements Negatable<EsqlB
         Map.entry(DataType.LONG, GreaterThanLongsEvaluator.Factory::new),
         Map.entry(DataType.UNSIGNED_LONG, GreaterThanLongsEvaluator.Factory::new),
         Map.entry(DataType.DATETIME, GreaterThanLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATE_NANOS, GreaterThanLongsEvaluator.Factory::new),
         Map.entry(DataType.KEYWORD, GreaterThanKeywordsEvaluator.Factory::new),
         Map.entry(DataType.TEXT, GreaterThanKeywordsEvaluator.Factory::new),
         Map.entry(DataType.VERSION, GreaterThanKeywordsEvaluator.Factory::new),
         Map.entry(DataType.IP, GreaterThanKeywordsEvaluator.Factory::new)
     );
 
-    public GreaterThan(Source source, Expression left, Expression right) {
+    @FunctionInfo(
+        returnType = { "boolean" },
+        description = "Check if one field is greater than another. "
+            + "If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+        note = "This is pushed to the underlying search index if one side of the comparison is constant "
+            + "and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>."
+    )
+    public GreaterThan(
+        Source source,
+        @Param(
+            name = "lhs",
+            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            description = "An expression."
+        ) Expression left,
+        @Param(
+            name = "rhs",
+            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            description = "An expression."
+        ) Expression right
+    ) {
         super(source, left, right, BinaryComparisonOperation.GT, evaluatorMap);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java
index d7bfec75dabfc..1b1d301428d73 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -32,13 +34,33 @@ public class GreaterThanOrEqual extends EsqlBinaryComparison implements Negatabl
         Map.entry(DataType.LONG, GreaterThanOrEqualLongsEvaluator.Factory::new),
         Map.entry(DataType.UNSIGNED_LONG, GreaterThanOrEqualLongsEvaluator.Factory::new),
         Map.entry(DataType.DATETIME, GreaterThanOrEqualLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATE_NANOS, GreaterThanOrEqualLongsEvaluator.Factory::new),
         Map.entry(DataType.KEYWORD, GreaterThanOrEqualKeywordsEvaluator.Factory::new),
         Map.entry(DataType.TEXT, GreaterThanOrEqualKeywordsEvaluator.Factory::new),
         Map.entry(DataType.VERSION, GreaterThanOrEqualKeywordsEvaluator.Factory::new),
         Map.entry(DataType.IP, GreaterThanOrEqualKeywordsEvaluator.Factory::new)
     );
 
-    public GreaterThanOrEqual(Source source, Expression left, Expression right) {
+    @FunctionInfo(
+        returnType = { "boolean" },
+        description = "Check if one field is greater than or equal to another. "
+            + "If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+        note = "This is pushed to the underlying search index if one side of the comparison is constant "
+            + "and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>."
+    )
+    public GreaterThanOrEqual(
+        Source source,
+        @Param(
+            name = "lhs",
+            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            description = "An expression."
+        ) Expression left,
+        @Param(
+            name = "rhs",
+            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            description = "An expression."
+        ) Expression right
+    ) {
         super(source, left, right, BinaryComparisonOperation.GTE, evaluatorMap);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
index 8dde4596a09fc..333f32e82c579 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
@@ -23,11 +23,11 @@
 import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
+import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
 
 import java.io.IOException;
 import java.util.BitSet;
@@ -60,10 +60,46 @@ public class In extends EsqlScalarFunction {
     @FunctionInfo(
         returnType = "boolean",
         description = "The `IN` operator allows testing whether a field or expression equals an element in a list of literals, "
-            + "fields or expressions:",
+            + "fields or expressions.",
         examples = @Example(file = "row", tag = "in-with-expressions")
     )
-    public In(Source source, Expression value, List<Expression> list) {
+    public In(
+        Source source,
+        @Param(
+            name = "field",
+            type = {
+                "boolean",
+                "cartesian_point",
+                "cartesian_shape",
+                "double",
+                "geo_point",
+                "geo_shape",
+                "integer",
+                "ip",
+                "keyword",
+                "long",
+                "text",
+                "version" },
+            description = "An expression."
+        ) Expression value,
+        @Param(
+            name = "inlist",
+            type = {
+                "boolean",
+                "cartesian_point",
+                "cartesian_shape",
+                "double",
+                "geo_point",
+                "geo_shape",
+                "integer",
+                "ip",
+                "keyword",
+                "long",
+                "text",
+                "version" },
+            description = "A list of items."
+        ) List<Expression> list
+    ) {
         super(source, CollectionUtils.combine(list, value));
         this.value = value;
         this.list = list;
@@ -134,10 +170,10 @@ protected boolean areCompatible(DataType left, DataType right) {
             // automatic numerical conversions not applicable for UNSIGNED_LONG, see Verifier#validateUnsignedLongOperator().
             return left == right;
         }
-        if (EsqlDataTypes.isSpatial(left) && EsqlDataTypes.isSpatial(right)) {
+        if (DataType.isSpatial(left) && DataType.isSpatial(right)) {
             return left == right;
         }
-        return EsqlDataTypes.areCompatible(left, right);
+        return DataType.areCompatible(left, right);
     }
 
     @Override
@@ -210,7 +246,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(
             || commonType == IP
             || commonType == VERSION
             || commonType == UNSUPPORTED
-            || EsqlDataTypes.isSpatial(commonType)) {
+            || DataType.isSpatial(commonType)) {
             return new InBytesRefEvaluator.Factory(source(), toEvaluator.apply(value), factories);
         }
         if (commonType == NULL) {
@@ -225,7 +261,7 @@ private DataType commonType() {
             if (e.dataType() == NULL && value.dataType() != NULL) {
                 continue;
             }
-            if (EsqlDataTypes.isSpatial(commonType)) {
+            if (DataType.isSpatial(commonType)) {
                 if (e.dataType() == commonType) {
                     continue;
                 } else {
@@ -233,7 +269,7 @@ private DataType commonType() {
                     break;
                 }
             }
-            commonType = EsqlDataTypeRegistry.INSTANCE.commonType(commonType, e.dataType());
+            commonType = EsqlDataTypeConverter.commonType(commonType, e.dataType());
         }
         return commonType;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java
index b1562b6dc2be4..333b18e21ab1b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -32,13 +34,33 @@ public class LessThan extends EsqlBinaryComparison implements Negatable<EsqlBina
         Map.entry(DataType.LONG, LessThanLongsEvaluator.Factory::new),
         Map.entry(DataType.UNSIGNED_LONG, LessThanLongsEvaluator.Factory::new),
         Map.entry(DataType.DATETIME, LessThanLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATE_NANOS, LessThanLongsEvaluator.Factory::new),
         Map.entry(DataType.KEYWORD, LessThanKeywordsEvaluator.Factory::new),
         Map.entry(DataType.TEXT, LessThanKeywordsEvaluator.Factory::new),
         Map.entry(DataType.VERSION, LessThanKeywordsEvaluator.Factory::new),
         Map.entry(DataType.IP, LessThanKeywordsEvaluator.Factory::new)
     );
 
-    public LessThan(Source source, Expression left, Expression right) {
+    @FunctionInfo(
+        returnType = { "boolean" },
+        description = "Check if one field is less than another. "
+            + "If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+        note = "This is pushed to the underlying search index if one side of the comparison is constant "
+            + "and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>."
+    )
+    public LessThan(
+        Source source,
+        @Param(
+            name = "lhs",
+            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            description = "An expression."
+        ) Expression left,
+        @Param(
+            name = "rhs",
+            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            description = "An expression."
+        ) Expression right
+    ) {
         this(source, left, right, null);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java
index c31e055c8dd1a..94c3911a02f64 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -32,13 +34,33 @@ public class LessThanOrEqual extends EsqlBinaryComparison implements Negatable<E
         Map.entry(DataType.LONG, LessThanOrEqualLongsEvaluator.Factory::new),
         Map.entry(DataType.UNSIGNED_LONG, LessThanOrEqualLongsEvaluator.Factory::new),
         Map.entry(DataType.DATETIME, LessThanOrEqualLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATE_NANOS, LessThanOrEqualLongsEvaluator.Factory::new),
         Map.entry(DataType.KEYWORD, LessThanOrEqualKeywordsEvaluator.Factory::new),
         Map.entry(DataType.TEXT, LessThanOrEqualKeywordsEvaluator.Factory::new),
         Map.entry(DataType.VERSION, LessThanOrEqualKeywordsEvaluator.Factory::new),
         Map.entry(DataType.IP, LessThanOrEqualKeywordsEvaluator.Factory::new)
     );
 
-    public LessThanOrEqual(Source source, Expression left, Expression right) {
+    @FunctionInfo(
+        returnType = { "boolean" },
+        description = "Check if one field is less than or equal to another. "
+            + "If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+        note = "This is pushed to the underlying search index if one side of the comparison is constant "
+            + "and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>."
+    )
+    public LessThanOrEqual(
+        Source source,
+        @Param(
+            name = "lhs",
+            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            description = "An expression."
+        ) Expression left,
+        @Param(
+            name = "rhs",
+            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            description = "An expression."
+        ) Expression right
+    ) {
         this(source, left, right, null);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java
index 179ff61d9c017..abca00212b848 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -33,6 +35,7 @@ public class NotEquals extends EsqlBinaryComparison implements Negatable<EsqlBin
         Map.entry(DataType.LONG, NotEqualsLongsEvaluator.Factory::new),
         Map.entry(DataType.UNSIGNED_LONG, NotEqualsLongsEvaluator.Factory::new),
         Map.entry(DataType.DATETIME, NotEqualsLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATE_NANOS, NotEqualsLongsEvaluator.Factory::new),
         Map.entry(DataType.GEO_POINT, NotEqualsGeometriesEvaluator.Factory::new),
         Map.entry(DataType.CARTESIAN_POINT, NotEqualsGeometriesEvaluator.Factory::new),
         Map.entry(DataType.GEO_SHAPE, NotEqualsGeometriesEvaluator.Factory::new),
@@ -43,7 +46,54 @@ public class NotEquals extends EsqlBinaryComparison implements Negatable<EsqlBin
         Map.entry(DataType.IP, NotEqualsKeywordsEvaluator.Factory::new)
     );
 
-    public NotEquals(Source source, Expression left, Expression right) {
+    @FunctionInfo(
+        returnType = { "boolean" },
+        description = "Check if two fields are unequal. "
+            + "If either field is <<esql-multivalued-fields,multivalued>> then the result is `null`.",
+        note = "This is pushed to the underlying search index if one side of the comparison is constant "
+            + "and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>."
+    )
+    public NotEquals(
+        Source source,
+        @Param(
+            name = "lhs",
+            type = {
+                "boolean",
+                "cartesian_point",
+                "cartesian_shape",
+                "date",
+                "double",
+                "geo_point",
+                "geo_shape",
+                "integer",
+                "ip",
+                "keyword",
+                "long",
+                "text",
+                "unsigned_long",
+                "version" },
+            description = "An expression."
+        ) Expression left,
+        @Param(
+            name = "rhs",
+            type = {
+                "boolean",
+                "cartesian_point",
+                "cartesian_shape",
+                "date",
+                "double",
+                "geo_point",
+                "geo_shape",
+                "integer",
+                "ip",
+                "keyword",
+                "long",
+                "text",
+                "unsigned_long",
+                "version" },
+            description = "An expression."
+        ) Expression right
+    ) {
         super(source, left, right, BinaryComparisonOperation.NEQ, evaluatorMap);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java
new file mode 100644
index 0000000000000..d368c570a3f76
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.index;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+public class EsIndex implements Writeable {
+
+    private final String name;
+    private final Map<String, EsField> mapping;
+    private final Set<String> concreteIndices;
+
+    public EsIndex(String name, Map<String, EsField> mapping) {
+        this(name, mapping, Set.of());
+    }
+
+    public EsIndex(String name, Map<String, EsField> mapping, Set<String> concreteIndices) {
+        assert name != null;
+        assert mapping != null;
+        this.name = name;
+        this.mapping = mapping;
+        this.concreteIndices = concreteIndices;
+    }
+
+    @SuppressWarnings("unchecked")
+    public EsIndex(StreamInput in) throws IOException {
+        this(in.readString(), in.readImmutableMap(StreamInput::readString, EsField::readFrom), (Set<String>) in.readGenericValue());
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name());
+        out.writeMap(mapping(), (o, x) -> x.writeTo(out));
+        out.writeGenericValue(concreteIndices());
+    }
+
+    public String name() {
+        return name;
+    }
+
+    public Map<String, EsField> mapping() {
+        return mapping;
+    }
+
+    public Set<String> concreteIndices() {
+        return concreteIndices;
+    }
+
+    @Override
+    public String toString() {
+        return name;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, mapping, concreteIndices);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        EsIndex other = (EsIndex) obj;
+        return Objects.equals(name, other.name)
+            && Objects.equals(mapping, other.mapping)
+            && Objects.equals(concreteIndices, other.concreteIndices);
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexResolution.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java
similarity index 97%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexResolution.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java
index 2e42f7d998f4f..725b6412afc77 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexResolution.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java
@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.index;
+package org.elasticsearch.xpack.esql.index;
 
 import org.elasticsearch.core.Nullable;
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/MappingException.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/MappingException.java
similarity index 90%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/MappingException.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/MappingException.java
index 16a450f5b849f..f4109d6ac0ead 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/MappingException.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/MappingException.java
@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.index;
+package org.elasticsearch.xpack.esql.index;
 
 import org.elasticsearch.xpack.esql.core.QlClientException;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index e4051523c7a5e..56eab73cd042b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -7,45 +7,22 @@
 
 package org.elasticsearch.xpack.esql.io.stream;
 
-import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.util.iterable.Iterables;
-import org.elasticsearch.dissect.DissectParser;
-import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.transport.RemoteClusterAware;
-import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.expression.Order;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.EsField;
-import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
-import org.elasticsearch.xpack.esql.plan.logical.Dissect;
-import org.elasticsearch.xpack.esql.plan.logical.Dissect.Parser;
+import org.elasticsearch.xpack.esql.index.EsIndex;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
-import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
-import org.elasticsearch.xpack.esql.plan.logical.Eval;
-import org.elasticsearch.xpack.esql.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.plan.logical.Grok;
-import org.elasticsearch.xpack.esql.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.plan.logical.Lookup;
-import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
-import org.elasticsearch.xpack.esql.plan.logical.OrderBy;
-import org.elasticsearch.xpack.esql.plan.logical.Project;
-import org.elasticsearch.xpack.esql.plan.logical.TopN;
-import org.elasticsearch.xpack.esql.plan.logical.join.Join;
-import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
-import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.esql.plan.physical.DissectExec;
 import org.elasticsearch.xpack.esql.plan.physical.EnrichExec;
@@ -76,8 +53,6 @@
 import java.util.Set;
 
 import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.Entry.of;
-import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader;
-import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter;
 
 /**
  * A utility class that consists solely of static methods that describe how to serialize and
@@ -111,20 +86,15 @@ public static String name(Class<?> cls) {
     public static List<PlanNameRegistry.Entry> namedTypeEntries() {
         List<PlanNameRegistry.Entry> declared = List.of(
             // Physical Plan Nodes
-            of(PhysicalPlan.class, AggregateExec.class, PlanNamedTypes::writeAggregateExec, PlanNamedTypes::readAggregateExec),
-            of(PhysicalPlan.class, DissectExec.class, PlanNamedTypes::writeDissectExec, PlanNamedTypes::readDissectExec),
-            of(PhysicalPlan.class, EsQueryExec.class, PlanNamedTypes::writeEsQueryExec, PlanNamedTypes::readEsQueryExec),
-            of(PhysicalPlan.class, EsSourceExec.class, PlanNamedTypes::writeEsSourceExec, PlanNamedTypes::readEsSourceExec),
-            of(PhysicalPlan.class, EvalExec.class, PlanNamedTypes::writeEvalExec, PlanNamedTypes::readEvalExec),
+            of(PhysicalPlan.class, AggregateExec.ENTRY),
+            of(PhysicalPlan.class, DissectExec.ENTRY),
+            of(PhysicalPlan.class, EsQueryExec.ENTRY),
+            of(PhysicalPlan.class, EsSourceExec.ENTRY),
+            of(PhysicalPlan.class, EvalExec.ENTRY),
             of(PhysicalPlan.class, EnrichExec.class, PlanNamedTypes::writeEnrichExec, PlanNamedTypes::readEnrichExec),
-            of(PhysicalPlan.class, ExchangeExec.class, PlanNamedTypes::writeExchangeExec, PlanNamedTypes::readExchangeExec),
-            of(PhysicalPlan.class, ExchangeSinkExec.class, PlanNamedTypes::writeExchangeSinkExec, PlanNamedTypes::readExchangeSinkExec),
-            of(
-                PhysicalPlan.class,
-                ExchangeSourceExec.class,
-                PlanNamedTypes::writeExchangeSourceExec,
-                PlanNamedTypes::readExchangeSourceExec
-            ),
+            of(PhysicalPlan.class, ExchangeExec.ENTRY),
+            of(PhysicalPlan.class, ExchangeSinkExec.ENTRY),
+            of(PhysicalPlan.class, ExchangeSourceExec.ENTRY),
             of(PhysicalPlan.class, FieldExtractExec.class, PlanNamedTypes::writeFieldExtractExec, PlanNamedTypes::readFieldExtractExec),
             of(PhysicalPlan.class, FilterExec.class, PlanNamedTypes::writeFilterExec, PlanNamedTypes::readFilterExec),
             of(PhysicalPlan.class, FragmentExec.class, PlanNamedTypes::writeFragmentExec, PlanNamedTypes::readFragmentExec),
@@ -137,144 +107,18 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
             of(PhysicalPlan.class, ProjectExec.class, PlanNamedTypes::writeProjectExec, PlanNamedTypes::readProjectExec),
             of(PhysicalPlan.class, RowExec.class, PlanNamedTypes::writeRowExec, PlanNamedTypes::readRowExec),
             of(PhysicalPlan.class, ShowExec.class, PlanNamedTypes::writeShowExec, PlanNamedTypes::readShowExec),
-            of(PhysicalPlan.class, TopNExec.class, PlanNamedTypes::writeTopNExec, PlanNamedTypes::readTopNExec),
-            // Logical Plan Nodes - a subset of plans that end up being actually serialized
-            of(LogicalPlan.class, Aggregate.class, Aggregate::writeAggregate, Aggregate::new),
-            of(LogicalPlan.class, Dissect.class, PlanNamedTypes::writeDissect, PlanNamedTypes::readDissect),
-            of(LogicalPlan.class, EsRelation.class, PlanNamedTypes::writeEsRelation, PlanNamedTypes::readEsRelation),
-            of(LogicalPlan.class, Eval.class, PlanNamedTypes::writeEval, PlanNamedTypes::readEval),
-            of(LogicalPlan.class, Enrich.class, PlanNamedTypes::writeEnrich, PlanNamedTypes::readEnrich),
-            of(LogicalPlan.class, EsqlProject.class, PlanNamedTypes::writeEsqlProject, PlanNamedTypes::readEsqlProject),
-            of(LogicalPlan.class, Filter.class, PlanNamedTypes::writeFilter, PlanNamedTypes::readFilter),
-            of(LogicalPlan.class, Grok.class, PlanNamedTypes::writeGrok, PlanNamedTypes::readGrok),
-            of(LogicalPlan.class, Join.class, (out, p) -> p.writeTo(out), Join::new),
-            of(LogicalPlan.class, Limit.class, PlanNamedTypes::writeLimit, PlanNamedTypes::readLimit),
-            of(LogicalPlan.class, LocalRelation.class, (out, p) -> p.writeTo(out), LocalRelation::new),
-            of(LogicalPlan.class, Lookup.class, (out, p) -> p.writeTo(out), Lookup::new),
-            of(LogicalPlan.class, MvExpand.class, PlanNamedTypes::writeMvExpand, PlanNamedTypes::readMvExpand),
-            of(LogicalPlan.class, OrderBy.class, PlanNamedTypes::writeOrderBy, PlanNamedTypes::readOrderBy),
-            of(LogicalPlan.class, Project.class, PlanNamedTypes::writeProject, PlanNamedTypes::readProject),
-            of(LogicalPlan.class, TopN.class, PlanNamedTypes::writeTopN, PlanNamedTypes::readTopN)
+            of(PhysicalPlan.class, TopNExec.class, PlanNamedTypes::writeTopNExec, PlanNamedTypes::readTopNExec)
         );
         return declared;
     }
 
     // -- physical plan nodes
-    static AggregateExec readAggregateExec(PlanStreamInput in) throws IOException {
-        return new AggregateExec(
-            Source.readFrom(in),
-            in.readPhysicalPlanNode(),
-            in.readNamedWriteableCollectionAsList(Expression.class),
-            in.readNamedWriteableCollectionAsList(NamedExpression.class),
-            in.readEnum(AggregateExec.Mode.class),
-            in.readOptionalVInt()
-        );
-    }
-
-    static void writeAggregateExec(PlanStreamOutput out, AggregateExec aggregateExec) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writePhysicalPlanNode(aggregateExec.child());
-        out.writeNamedWriteableCollection(aggregateExec.groupings());
-        out.writeNamedWriteableCollection(aggregateExec.aggregates());
-        out.writeEnum(aggregateExec.getMode());
-        out.writeOptionalVInt(aggregateExec.estimatedRowSize());
-    }
-
-    static DissectExec readDissectExec(PlanStreamInput in) throws IOException {
-        return new DissectExec(
-            Source.readFrom(in),
-            in.readPhysicalPlanNode(),
-            in.readNamedWriteable(Expression.class),
-            readDissectParser(in),
-            in.readNamedWriteableCollectionAsList(Attribute.class)
-        );
-    }
-
-    static void writeDissectExec(PlanStreamOutput out, DissectExec dissectExec) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writePhysicalPlanNode(dissectExec.child());
-        out.writeNamedWriteable(dissectExec.inputExpression());
-        writeDissectParser(out, dissectExec.parser());
-        out.writeNamedWriteableCollection(dissectExec.extractedFields());
-    }
-
-    static EsQueryExec readEsQueryExec(PlanStreamInput in) throws IOException {
-        return new EsQueryExec(
-            Source.readFrom(in),
-            readEsIndex(in),
-            readIndexMode(in),
-            in.readNamedWriteableCollectionAsList(Attribute.class),
-            in.readOptionalNamedWriteable(QueryBuilder.class),
-            in.readOptionalNamed(Expression.class),
-            in.readOptionalCollectionAsList(readerFromPlanReader(PlanNamedTypes::readFieldSort)),
-            in.readOptionalVInt()
-        );
-    }
-
-    static void writeEsQueryExec(PlanStreamOutput out, EsQueryExec esQueryExec) throws IOException {
-        assert esQueryExec.children().size() == 0;
-        Source.EMPTY.writeTo(out);
-        writeEsIndex(out, esQueryExec.index());
-        writeIndexMode(out, esQueryExec.indexMode());
-        out.writeNamedWriteableCollection(esQueryExec.output());
-        out.writeOptionalNamedWriteable(esQueryExec.query());
-        out.writeOptionalNamedWriteable(esQueryExec.limit());
-        out.writeOptionalCollection(esQueryExec.sorts(), writerFromPlanWriter(PlanNamedTypes::writeFieldSort));
-        out.writeOptionalInt(esQueryExec.estimatedRowSize());
-    }
-
-    static EsSourceExec readEsSourceExec(PlanStreamInput in) throws IOException {
-        return new EsSourceExec(
-            Source.readFrom(in),
-            readEsIndex(in),
-            in.readNamedWriteableCollectionAsList(Attribute.class),
-            in.readOptionalNamedWriteable(QueryBuilder.class),
-            readIndexMode(in)
-        );
-    }
-
-    static void writeEsSourceExec(PlanStreamOutput out, EsSourceExec esSourceExec) throws IOException {
-        Source.EMPTY.writeTo(out);
-        writeEsIndex(out, esSourceExec.index());
-        out.writeNamedWriteableCollection(esSourceExec.output());
-        out.writeOptionalNamedWriteable(esSourceExec.query());
-        writeIndexMode(out, esSourceExec.indexMode());
-    }
-
-    static IndexMode readIndexMode(StreamInput in) throws IOException {
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_TO_SOURCE)) {
-            return IndexMode.fromString(in.readString());
-        } else {
-            return IndexMode.STANDARD;
-        }
-    }
-
-    static void writeIndexMode(StreamOutput out, IndexMode indexMode) throws IOException {
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_TO_SOURCE)) {
-            out.writeString(indexMode.getName());
-        } else if (indexMode != IndexMode.STANDARD) {
-            throw new IllegalStateException("not ready to support index mode [" + indexMode + "]");
-        }
-    }
-
-    static EvalExec readEvalExec(PlanStreamInput in) throws IOException {
-        return new EvalExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readCollectionAsList(Alias::new));
-    }
-
-    static void writeEvalExec(PlanStreamOutput out, EvalExec evalExec) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writePhysicalPlanNode(evalExec.child());
-        out.writeCollection(evalExec.fields());
-    }
-
     static EnrichExec readEnrichExec(PlanStreamInput in) throws IOException {
         final Source source = Source.readFrom(in);
         final PhysicalPlan child = in.readPhysicalPlanNode();
         final NamedExpression matchField = in.readNamedWriteable(NamedExpression.class);
         final String policyName = in.readString();
-        final String matchType = (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_EXTENDED_ENRICH_TYPES))
-            ? in.readString()
-            : "match";
+        final String matchType = (in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) ? in.readString() : "match";
         final String policyMatchField = in.readString();
         final Map<String, String> concreteIndices;
         final Enrich.Mode mode;
@@ -283,7 +127,7 @@ static EnrichExec readEnrichExec(PlanStreamInput in) throws IOException {
             concreteIndices = in.readMap(StreamInput::readString, StreamInput::readString);
         } else {
             mode = Enrich.Mode.ANY;
-            EsIndex esIndex = readEsIndex(in);
+            EsIndex esIndex = new EsIndex(in);
             if (esIndex.concreteIndices().size() != 1) {
                 throw new IllegalStateException("expected a single concrete enrich index; got " + esIndex.concreteIndices());
             }
@@ -307,7 +151,7 @@ static void writeEnrichExec(PlanStreamOutput out, EnrichExec enrich) throws IOEx
         out.writePhysicalPlanNode(enrich.child());
         out.writeNamedWriteable(enrich.matchField());
         out.writeString(enrich.policyName());
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_EXTENDED_ENRICH_TYPES)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeString(enrich.matchType());
         }
         out.writeString(enrich.policyMatchField());
@@ -317,7 +161,7 @@ static void writeEnrichExec(PlanStreamOutput out, EnrichExec enrich) throws IOEx
         } else {
             if (enrich.concreteIndices().keySet().equals(Set.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY))) {
                 String concreteIndex = enrich.concreteIndices().get(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
-                writeEsIndex(out, new EsIndex(concreteIndex, Map.of(), Set.of(concreteIndex)));
+                new EsIndex(concreteIndex, Map.of(), Set.of(concreteIndex)).writeTo(out);
             } else {
                 throw new IllegalStateException("expected a single concrete enrich index; got " + enrich.concreteIndices());
             }
@@ -325,47 +169,6 @@ static void writeEnrichExec(PlanStreamOutput out, EnrichExec enrich) throws IOEx
         out.writeNamedWriteableCollection(enrich.enrichFields());
     }
 
-    static ExchangeExec readExchangeExec(PlanStreamInput in) throws IOException {
-        return new ExchangeExec(
-            Source.readFrom(in),
-            in.readNamedWriteableCollectionAsList(Attribute.class),
-            in.readBoolean(),
-            in.readPhysicalPlanNode()
-        );
-    }
-
-    static void writeExchangeExec(PlanStreamOutput out, ExchangeExec exchangeExec) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeNamedWriteableCollection(exchangeExec.output());
-        out.writeBoolean(exchangeExec.isInBetweenAggs());
-        out.writePhysicalPlanNode(exchangeExec.child());
-    }
-
-    static ExchangeSinkExec readExchangeSinkExec(PlanStreamInput in) throws IOException {
-        return new ExchangeSinkExec(
-            Source.readFrom(in),
-            in.readNamedWriteableCollectionAsList(Attribute.class),
-            in.readBoolean(),
-            in.readPhysicalPlanNode()
-        );
-    }
-
-    static void writeExchangeSinkExec(PlanStreamOutput out, ExchangeSinkExec exchangeSinkExec) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeNamedWriteableCollection(exchangeSinkExec.output());
-        out.writeBoolean(exchangeSinkExec.isIntermediateAgg());
-        out.writePhysicalPlanNode(exchangeSinkExec.child());
-    }
-
-    static ExchangeSourceExec readExchangeSourceExec(PlanStreamInput in) throws IOException {
-        return new ExchangeSourceExec(Source.readFrom(in), in.readNamedWriteableCollectionAsList(Attribute.class), in.readBoolean());
-    }
-
-    static void writeExchangeSourceExec(PlanStreamOutput out, ExchangeSourceExec exchangeSourceExec) throws IOException {
-        out.writeNamedWriteableCollection(exchangeSourceExec.output());
-        out.writeBoolean(exchangeSourceExec.isIntermediateAgg());
-    }
-
     static FieldExtractExec readFieldExtractExec(PlanStreamInput in) throws IOException {
         return new FieldExtractExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readNamedWriteableCollectionAsList(Attribute.class));
     }
@@ -389,19 +192,19 @@ static void writeFilterExec(PlanStreamOutput out, FilterExec filterExec) throws
     static FragmentExec readFragmentExec(PlanStreamInput in) throws IOException {
         return new FragmentExec(
             Source.readFrom(in),
-            in.readLogicalPlanNode(),
+            in.readNamedWriteable(LogicalPlan.class),
             in.readOptionalNamedWriteable(QueryBuilder.class),
             in.readOptionalVInt(),
-            in.getTransportVersion().onOrAfter(TransportVersions.ESQL_REDUCER_NODE_FRAGMENT) ? in.readOptionalPhysicalPlanNode() : null
+            in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0) ? in.readOptionalPhysicalPlanNode() : null
         );
     }
 
     static void writeFragmentExec(PlanStreamOutput out, FragmentExec fragmentExec) throws IOException {
         Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(fragmentExec.fragment());
+        out.writeNamedWriteable(fragmentExec.fragment());
         out.writeOptionalNamedWriteable(fragmentExec.esFilter());
         out.writeOptionalVInt(fragmentExec.estimatedRowSize());
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_REDUCER_NODE_FRAGMENT)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
             out.writeOptionalPhysicalPlanNode(fragmentExec.reducer());
         }
     }
@@ -521,289 +324,4 @@ static void writeTopNExec(PlanStreamOutput out, TopNExec topNExec) throws IOExce
         out.writeNamedWriteable(topNExec.limit());
         out.writeOptionalVInt(topNExec.estimatedRowSize());
     }
-
-    static Dissect readDissect(PlanStreamInput in) throws IOException {
-        return new Dissect(
-            Source.readFrom(in),
-            in.readLogicalPlanNode(),
-            in.readNamedWriteable(Expression.class),
-            readDissectParser(in),
-            in.readNamedWriteableCollectionAsList(Attribute.class)
-        );
-    }
-
-    static void writeDissect(PlanStreamOutput out, Dissect dissect) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(dissect.child());
-        out.writeNamedWriteable(dissect.input());
-        writeDissectParser(out, dissect.parser());
-        out.writeNamedWriteableCollection(dissect.extractedFields());
-    }
-
-    static EsRelation readEsRelation(PlanStreamInput in) throws IOException {
-        Source source = Source.readFrom(in);
-        EsIndex esIndex = readEsIndex(in);
-        List<Attribute> attributes = in.readNamedWriteableCollectionAsList(Attribute.class);
-        if (supportingEsSourceOptions(in.getTransportVersion())) {
-            readEsSourceOptions(in); // consume optional strings sent by remote
-        }
-        final IndexMode indexMode = readIndexMode(in);
-        boolean frozen = in.readBoolean();
-        return new EsRelation(source, esIndex, attributes, indexMode, frozen);
-    }
-
-    static void writeEsRelation(PlanStreamOutput out, EsRelation relation) throws IOException {
-        assert relation.children().size() == 0;
-        Source.EMPTY.writeTo(out);
-        writeEsIndex(out, relation.index());
-        out.writeNamedWriteableCollection(relation.output());
-        if (supportingEsSourceOptions(out.getTransportVersion())) {
-            writeEsSourceOptions(out); // write (null) string fillers expected by remote
-        }
-        writeIndexMode(out, relation.indexMode());
-        out.writeBoolean(relation.frozen());
-    }
-
-    private static boolean supportingEsSourceOptions(TransportVersion version) {
-        return version.onOrAfter(TransportVersions.ESQL_ES_SOURCE_OPTIONS)
-            && version.before(TransportVersions.ESQL_REMOVE_ES_SOURCE_OPTIONS);
-    }
-
-    private static void readEsSourceOptions(PlanStreamInput in) throws IOException {
-        // allowNoIndices
-        in.readOptionalString();
-        // ignoreUnavailable
-        in.readOptionalString();
-        // preference
-        in.readOptionalString();
-    }
-
-    private static void writeEsSourceOptions(PlanStreamOutput out) throws IOException {
-        // allowNoIndices
-        out.writeOptionalString(null);
-        // ignoreUnavailable
-        out.writeOptionalString(null);
-        // preference
-        out.writeOptionalString(null);
-    }
-
-    static Eval readEval(PlanStreamInput in) throws IOException {
-        return new Eval(Source.readFrom(in), in.readLogicalPlanNode(), in.readCollectionAsList(Alias::new));
-    }
-
-    static void writeEval(PlanStreamOutput out, Eval eval) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(eval.child());
-        out.writeCollection(eval.fields());
-    }
-
-    static Enrich readEnrich(PlanStreamInput in) throws IOException {
-        Enrich.Mode mode = Enrich.Mode.ANY;
-        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
-            mode = in.readEnum(Enrich.Mode.class);
-        }
-        final Source source = Source.readFrom(in);
-        final LogicalPlan child = in.readLogicalPlanNode();
-        final Expression policyName = in.readNamedWriteable(Expression.class);
-        final NamedExpression matchField = in.readNamedWriteable(NamedExpression.class);
-        if (in.getTransportVersion().before(TransportVersions.V_8_13_0)) {
-            in.readString(); // discard the old policy name
-        }
-        final EnrichPolicy policy = new EnrichPolicy(in);
-        final Map<String, String> concreteIndices;
-        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
-            concreteIndices = in.readMap(StreamInput::readString, StreamInput::readString);
-        } else {
-            EsIndex esIndex = readEsIndex(in);
-            if (esIndex.concreteIndices().size() > 1) {
-                throw new IllegalStateException("expected a single enrich index; got " + esIndex);
-            }
-            concreteIndices = Map.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, Iterables.get(esIndex.concreteIndices(), 0));
-        }
-        return new Enrich(
-            source,
-            child,
-            mode,
-            policyName,
-            matchField,
-            policy,
-            concreteIndices,
-            in.readNamedWriteableCollectionAsList(NamedExpression.class)
-        );
-    }
-
-    static void writeEnrich(PlanStreamOutput out, Enrich enrich) throws IOException {
-        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
-            out.writeEnum(enrich.mode());
-        }
-
-        Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(enrich.child());
-        out.writeNamedWriteable(enrich.policyName());
-        out.writeNamedWriteable(enrich.matchField());
-        if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) {
-            out.writeString(BytesRefs.toString(enrich.policyName().fold())); // old policy name
-        }
-        enrich.policy().writeTo(out);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
-            out.writeMap(enrich.concreteIndices(), StreamOutput::writeString, StreamOutput::writeString);
-        } else {
-            Map<String, String> concreteIndices = enrich.concreteIndices();
-            if (concreteIndices.keySet().equals(Set.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY))) {
-                String enrichIndex = concreteIndices.get(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
-                EsIndex esIndex = new EsIndex(enrichIndex, Map.of(), Set.of(enrichIndex));
-                writeEsIndex(out, esIndex);
-            } else {
-                throw new IllegalStateException("expected a single enrich index; got " + concreteIndices);
-            }
-        }
-        out.writeNamedWriteableCollection(enrich.enrichFields());
-    }
-
-    static EsqlProject readEsqlProject(PlanStreamInput in) throws IOException {
-        return new EsqlProject(Source.readFrom(in), in.readLogicalPlanNode(), in.readNamedWriteableCollectionAsList(NamedExpression.class));
-    }
-
-    static void writeEsqlProject(PlanStreamOutput out, EsqlProject project) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(project.child());
-        out.writeNamedWriteableCollection(project.projections());
-    }
-
-    static Filter readFilter(PlanStreamInput in) throws IOException {
-        return new Filter(Source.readFrom(in), in.readLogicalPlanNode(), in.readNamedWriteable(Expression.class));
-    }
-
-    static void writeFilter(PlanStreamOutput out, Filter filter) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(filter.child());
-        out.writeNamedWriteable(filter.condition());
-    }
-
-    static Grok readGrok(PlanStreamInput in) throws IOException {
-        Source source;
-        return new Grok(
-            source = Source.readFrom(in),
-            in.readLogicalPlanNode(),
-            in.readNamedWriteable(Expression.class),
-            Grok.pattern(source, in.readString()),
-            in.readNamedWriteableCollectionAsList(Attribute.class)
-        );
-    }
-
-    static void writeGrok(PlanStreamOutput out, Grok grok) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(grok.child());
-        out.writeNamedWriteable(grok.input());
-        out.writeString(grok.parser().pattern());
-        out.writeNamedWriteableCollection(grok.extractedFields());
-    }
-
-    static Limit readLimit(PlanStreamInput in) throws IOException {
-        return new Limit(Source.readFrom(in), in.readNamedWriteable(Expression.class), in.readLogicalPlanNode());
-    }
-
-    static void writeLimit(PlanStreamOutput out, Limit limit) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeNamedWriteable(limit.limit());
-        out.writeLogicalPlanNode(limit.child());
-    }
-
-    static MvExpand readMvExpand(PlanStreamInput in) throws IOException {
-        return new MvExpand(
-            Source.readFrom(in),
-            in.readLogicalPlanNode(),
-            in.readNamedWriteable(NamedExpression.class),
-            in.readNamedWriteable(Attribute.class)
-        );
-    }
-
-    static void writeMvExpand(PlanStreamOutput out, MvExpand mvExpand) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(mvExpand.child());
-        out.writeNamedWriteable(mvExpand.target());
-        out.writeNamedWriteable(mvExpand.expanded());
-    }
-
-    static OrderBy readOrderBy(PlanStreamInput in) throws IOException {
-        return new OrderBy(
-            Source.readFrom(in),
-            in.readLogicalPlanNode(),
-            in.readCollectionAsList(org.elasticsearch.xpack.esql.expression.Order::new)
-        );
-    }
-
-    static void writeOrderBy(PlanStreamOutput out, OrderBy order) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(order.child());
-        out.writeCollection(order.order());
-    }
-
-    static Project readProject(PlanStreamInput in) throws IOException {
-        return new Project(Source.readFrom(in), in.readLogicalPlanNode(), in.readNamedWriteableCollectionAsList(NamedExpression.class));
-    }
-
-    static void writeProject(PlanStreamOutput out, Project project) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(project.child());
-        out.writeNamedWriteableCollection(project.projections());
-    }
-
-    static TopN readTopN(PlanStreamInput in) throws IOException {
-        return new TopN(
-            Source.readFrom(in),
-            in.readLogicalPlanNode(),
-            in.readCollectionAsList(org.elasticsearch.xpack.esql.expression.Order::new),
-            in.readNamedWriteable(Expression.class)
-        );
-    }
-
-    static void writeTopN(PlanStreamOutput out, TopN topN) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(topN.child());
-        out.writeCollection(topN.order());
-        out.writeNamedWriteable(topN.limit());
-    }
-
-    // -- ancillary supporting classes of plan nodes, etc
-
-    static EsQueryExec.FieldSort readFieldSort(PlanStreamInput in) throws IOException {
-        return new EsQueryExec.FieldSort(
-            new FieldAttribute(in),
-            in.readEnum(Order.OrderDirection.class),
-            in.readEnum(Order.NullsPosition.class)
-        );
-    }
-
-    static void writeFieldSort(PlanStreamOutput out, EsQueryExec.FieldSort fieldSort) throws IOException {
-        fieldSort.field().writeTo(out);
-        out.writeEnum(fieldSort.direction());
-        out.writeEnum(fieldSort.nulls());
-    }
-
-    @SuppressWarnings("unchecked")
-    static EsIndex readEsIndex(PlanStreamInput in) throws IOException {
-        return new EsIndex(
-            in.readString(),
-            in.readImmutableMap(StreamInput::readString, i -> i.readNamedWriteable(EsField.class)),
-            (Set<String>) in.readGenericValue()
-        );
-    }
-
-    static void writeEsIndex(PlanStreamOutput out, EsIndex esIndex) throws IOException {
-        out.writeString(esIndex.name());
-        out.writeMap(esIndex.mapping(), StreamOutput::writeNamedWriteable);
-        out.writeGenericValue(esIndex.concreteIndices());
-    }
-
-    static Parser readDissectParser(PlanStreamInput in) throws IOException {
-        String pattern = in.readString();
-        String appendSeparator = in.readString();
-        return new Parser(pattern, appendSeparator, new DissectParser(pattern, appendSeparator));
-    }
-
-    static void writeDissectParser(PlanStreamOutput out, Parser dissectParser) throws IOException {
-        out.writeString(dissectParser.pattern());
-        out.writeString(dissectParser.appendSeparator());
-    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
index 0633595a5796d..ad66378da5d9e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
@@ -7,11 +7,14 @@
 
 package org.elasticsearch.xpack.esql.io.stream;
 
+import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
@@ -21,14 +24,16 @@
 import org.elasticsearch.compute.data.DoubleBigArrayBlock;
 import org.elasticsearch.compute.data.IntBigArrayBlock;
 import org.elasticsearch.compute.data.LongBigArrayBlock;
+import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.xpack.esql.Column;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
+import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanNamedReader;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader;
-import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -60,18 +65,22 @@ public NameId apply(long streamNameId) {
 
     private final Map<Integer, Block> cachedBlocks = new HashMap<>();
 
+    private Attribute[] attributesCache = new Attribute[64];
+
+    private EsField[] esFieldsCache = new EsField[64];
+
     private final PlanNameRegistry registry;
 
     // hook for nameId, where can cache and map, for now just return a NameId of the same long value.
     private final LongFunction<NameId> nameIdFunction;
 
-    private final EsqlConfiguration configuration;
+    private final Configuration configuration;
 
     public PlanStreamInput(
         StreamInput streamInput,
         PlanNameRegistry registry,
         NamedWriteableRegistry namedWriteableRegistry,
-        EsqlConfiguration configuration
+        Configuration configuration
     ) {
         super(streamInput, namedWriteableRegistry);
         this.registry = registry;
@@ -79,10 +88,6 @@ public PlanStreamInput(
         this.nameIdFunction = new NameIdMapper();
     }
 
-    public LogicalPlan readLogicalPlanNode() throws IOException {
-        return readNamed(LogicalPlan.class);
-    }
-
     public PhysicalPlan readPhysicalPlanNode() throws IOException {
         return readNamed(PhysicalPlan.class);
     }
@@ -114,7 +119,7 @@ public <T> T readOptionalNamed(Class<T> type) throws IOException {
         }
     }
 
-    public EsqlConfiguration configuration() throws IOException {
+    public Configuration configuration() throws IOException {
         return configuration;
     }
 
@@ -123,7 +128,7 @@ public EsqlConfiguration configuration() throws IOException {
      * <p>
      *     These {@link Block}s are not tracked by {@link BlockFactory} and closing them
      *     does nothing so they should be small. We do make sure not to send duplicates,
-     *     reusing blocks sent as part of the {@link EsqlConfiguration#tables()} if
+     *     reusing blocks sent as part of the {@link Configuration#tables()} if
      *     possible, otherwise sending a {@linkplain Block} inline.
      * </p>
      */
@@ -168,7 +173,7 @@ public Block readCachedBlock() throws IOException {
      * <p>
      *     These {@link Block}s are not tracked by {@link BlockFactory} and closing them
      *     does nothing so they should be small. We do make sure not to send duplicates,
-     *     reusing blocks sent as part of the {@link EsqlConfiguration#tables()} if
+     *     reusing blocks sent as part of the {@link Configuration#tables()} if
      *     possible, otherwise sending a {@linkplain Block} inline.
      * </p>
      */
@@ -206,4 +211,90 @@ static void throwOnNullOptionalRead(Class<?> type) throws IOException {
     public NameId mapNameId(long l) {
         return nameIdFunction.apply(l);
     }
+
+    /**
+     * @param constructor the constructor needed to build the actual attribute when read from the wire
+     * @throws IOException
+     */
+    @Override
+    @SuppressWarnings("unchecked")
+    public <A extends Attribute> A readAttributeWithCache(CheckedFunction<StreamInput, A, IOException> constructor) throws IOException {
+        if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION)) {
+            // it's safe to cast to int, since the max value for this is {@link PlanStreamOutput#MAX_SERIALIZED_ATTRIBUTES}
+            int cacheId = Math.toIntExact(readZLong());
+            if (cacheId < 0) {
+                cacheId = -1 - cacheId;
+                Attribute result = constructor.apply(this);
+                cacheAttribute(cacheId, result);
+                return (A) result;
+            } else {
+                return (A) attributeFromCache(cacheId);
+            }
+        } else {
+            return constructor.apply(this);
+        }
+    }
+
+    private Attribute attributeFromCache(int id) throws IOException {
+        if (attributesCache[id] == null) {
+            throw new IOException("Attribute ID not found in serialization cache [" + id + "]");
+        }
+        return attributesCache[id];
+    }
+
+    /**
+     * Add an attribute to the cache, based on the serialization ID generated by {@link PlanStreamOutput}
+     * @param id The ID that will reference the attribute. Generated  at serialization time
+     * @param attr The attribute to cache
+     */
+    private void cacheAttribute(int id, Attribute attr) {
+        assert id >= 0;
+        if (id >= attributesCache.length) {
+            attributesCache = ArrayUtil.grow(attributesCache);
+        }
+        attributesCache[id] = attr;
+    }
+
+    @SuppressWarnings("unchecked")
+    public <A extends EsField> A readEsFieldWithCache() throws IOException {
+        if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION)) {
+            // it's safe to cast to int, since the max value for this is {@link PlanStreamOutput#MAX_SERIALIZED_ATTRIBUTES}
+            int cacheId = Math.toIntExact(readZLong());
+            if (cacheId < 0) {
+                String className = readString();
+                Writeable.Reader<? extends EsField> reader = EsField.getReader(className);
+                cacheId = -1 - cacheId;
+                EsField result = reader.read(this);
+                cacheEsField(cacheId, result);
+                return (A) result;
+            } else {
+                return (A) esFieldFromCache(cacheId);
+            }
+        } else {
+            String className = readString();
+            Writeable.Reader<? extends EsField> reader = EsField.getReader(className);
+            return (A) reader.read(this);
+        }
+    }
+
+    private EsField esFieldFromCache(int id) throws IOException {
+        if (esFieldsCache[id] == null) {
+            throw new IOException("Attribute ID not found in serialization cache [" + id + "]");
+        }
+        return esFieldsCache[id];
+    }
+
+    /**
+     * Add an EsField to the cache, based on the serialization ID generated by {@link PlanStreamOutput}
+     * @param id The ID that will reference the field. Generated  at serialization time
+     * @param field The EsField to cache
+     */
+    private void cacheEsField(int id, EsField field) {
+        assert id >= 0;
+        if (id >= esFieldsCache.length) {
+            esFieldsCache = ArrayUtil.grow(esFieldsCache);
+        }
+        esFieldsCache[id] = field;
+    }
+
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
index 674476ec4f736..d76c61eac05d1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.io.stream;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -19,11 +20,12 @@
 import org.elasticsearch.compute.data.LongBigArrayBlock;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.Column;
+import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter;
-import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.plan.logical.join.Join;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
 import java.util.IdentityHashMap;
@@ -34,7 +36,14 @@
  * A customized stream output used to serialize ESQL physical plan fragments. Complements stream
  * output with methods that write plan nodes, Attributes, Expressions, etc.
  */
-public final class PlanStreamOutput extends StreamOutput {
+public final class PlanStreamOutput extends StreamOutput implements org.elasticsearch.xpack.esql.core.util.PlanStreamOutput {
+
+    /**
+     * max number of attributes that can be cached for serialization
+     * <p>
+     * TODO should this be a cluster setting...?
+     */
+    protected static final int MAX_SERIALIZED_ATTRIBUTES = 1_000_000;
 
     /**
      * Cache of written blocks. We use an {@link IdentityHashMap} for this
@@ -44,6 +53,21 @@ public final class PlanStreamOutput extends StreamOutput {
      */
     private final Map<Block, BytesReference> cachedBlocks = new IdentityHashMap<>();
 
+    /**
+     * Cache for field attributes.
+     * Field attributes can be a significant part of the query execution plan, especially
+     * for queries like `from *`, that can have thousands of output columns.
+     * Attributes can be shared by many plan nodes (eg. ExcahngeSink output, Project output, EsRelation fields);
+     * in addition, multiple Attributes can share the same parent field.
+     * This cache allows to send each attribute only once; from the second occurrence, only an id will be sent
+     */
+    protected final Map<Attribute, Integer> cachedAttributes = new IdentityHashMap<>();
+
+    /**
+     * Cache for EsFields.
+     */
+    protected final Map<EsField, Integer> cachedEsFields = new IdentityHashMap<>();
+
     private final StreamOutput delegate;
     private final PlanNameRegistry registry;
 
@@ -51,16 +75,18 @@ public final class PlanStreamOutput extends StreamOutput {
 
     private int nextCachedBlock = 0;
 
-    public PlanStreamOutput(StreamOutput delegate, PlanNameRegistry registry, @Nullable EsqlConfiguration configuration)
-        throws IOException {
-        this(delegate, registry, configuration, PlanNamedTypes::name);
+    private int maxSerializedAttributes;
+
+    public PlanStreamOutput(StreamOutput delegate, PlanNameRegistry registry, @Nullable Configuration configuration) throws IOException {
+        this(delegate, registry, configuration, PlanNamedTypes::name, MAX_SERIALIZED_ATTRIBUTES);
     }
 
     public PlanStreamOutput(
         StreamOutput delegate,
         PlanNameRegistry registry,
-        @Nullable EsqlConfiguration configuration,
-        Function<Class<?>, String> nameSupplier
+        @Nullable Configuration configuration,
+        Function<Class<?>, String> nameSupplier,
+        int maxSerializedAttributes
     ) throws IOException {
         this.delegate = delegate;
         this.registry = registry;
@@ -72,11 +98,7 @@ public PlanStreamOutput(
                 }
             }
         }
-    }
-
-    public void writeLogicalPlanNode(LogicalPlan logicalPlan) throws IOException {
-        assert logicalPlan.children().size() <= 1 || (logicalPlan instanceof Join && logicalPlan.children().size() == 2);
-        writeNamed(LogicalPlan.class, logicalPlan);
+        this.maxSerializedAttributes = maxSerializedAttributes;
     }
 
     public void writePhysicalPlanNode(PhysicalPlan physicalPlan) throws IOException {
@@ -137,7 +159,7 @@ public void setTransportVersion(TransportVersion version) {
      * <p>
      *     These {@link Block}s are not tracked by {@link BlockFactory} and closing them
      *     does nothing so they should be small. We do make sure not to send duplicates,
-     *     reusing blocks sent as part of the {@link EsqlConfiguration#tables()} if
+     *     reusing blocks sent as part of the {@link Configuration#tables()} if
      *     possible, otherwise sending a {@linkplain Block} inline.
      * </p>
      */
@@ -158,6 +180,69 @@ public void writeCachedBlock(Block block) throws IOException {
         nextCachedBlock++;
     }
 
+    @Override
+    public boolean writeAttributeCacheHeader(Attribute attribute) throws IOException {
+        if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ATTRIBUTE_CACHED_SERIALIZATION)) {
+            Integer cacheId = attributeIdFromCache(attribute);
+            if (cacheId != null) {
+                writeZLong(cacheId);
+                return false;
+            }
+
+            cacheId = cacheAttribute(attribute);
+            writeZLong(-1 - cacheId);
+        }
+        return true;
+    }
+
+    private Integer attributeIdFromCache(Attribute attr) {
+        return cachedAttributes.get(attr);
+    }
+
+    private int cacheAttribute(Attribute attr) {
+        if (cachedAttributes.containsKey(attr)) {
+            throw new IllegalArgumentException("Attribute already present in the serialization cache [" + attr + "]");
+        }
+        int id = cachedAttributes.size();
+        if (id >= maxSerializedAttributes) {
+            throw new InvalidArgumentException("Limit of the number of serialized attributes exceeded [{}]", maxSerializedAttributes);
+        }
+        cachedAttributes.put(attr, id);
+        return id;
+    }
+
+    @Override
+    public boolean writeEsFieldCacheHeader(EsField field) throws IOException {
+        if (getTransportVersion().onOrAfter(TransportVersions.ESQL_ES_FIELD_CACHED_SERIALIZATION)) {
+            Integer cacheId = esFieldIdFromCache(field);
+            if (cacheId != null) {
+                writeZLong(cacheId);
+                return false;
+            }
+
+            cacheId = cacheEsField(field);
+            writeZLong(-1 - cacheId);
+        }
+        writeString(field.getWriteableName());
+        return true;
+    }
+
+    private Integer esFieldIdFromCache(EsField field) {
+        return cachedEsFields.get(field);
+    }
+
+    private int cacheEsField(EsField attr) {
+        if (cachedEsFields.containsKey(attr)) {
+            throw new IllegalArgumentException("EsField already present in the serialization cache [" + attr + "]");
+        }
+        int id = cachedEsFields.size();
+        if (id >= maxSerializedAttributes) {
+            throw new InvalidArgumentException("Limit of the number of serialized EsFields exceeded [{}]", maxSerializedAttributes);
+        }
+        cachedEsFields.put(attr, id);
+        return id;
+    }
+
     /**
      * The byte representing a {@link Block} sent for the first time. The byte
      * will be followed by a {@link StreamOutput#writeVInt} encoded identifier
@@ -175,7 +260,7 @@ public void writeCachedBlock(Block block) throws IOException {
 
     /**
      * The byte representing a {@link Block} that was part of the
-     * {@link EsqlConfiguration#tables()} map. It is followed a string for
+     * {@link Configuration#tables()} map. It is followed a string for
      * the table name and then a string for the column name.
      */
     static final byte FROM_CONFIG_KEY = 2;
@@ -193,7 +278,7 @@ static BytesReference fromPreviousKey(int id) throws IOException {
     }
 
     /**
-     * Build the key for reading a {@link Block} from the {@link EsqlConfiguration}.
+     * Build the key for reading a {@link Block} from the {@link Configuration}.
      * This is important because some operations like {@code LOOKUP} frequently read
      * {@linkplain Block}s directly from the configuration.
      * <p>
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java
index f0bc5697d4002..ef5cf50c76541 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.esql.optimizer;
 
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
 
 import java.util.Objects;
@@ -15,7 +15,7 @@
 public final class LocalLogicalOptimizerContext extends LogicalOptimizerContext {
     private final SearchStats searchStats;
 
-    public LocalLogicalOptimizerContext(EsqlConfiguration configuration, SearchStats searchStats) {
+    public LocalLogicalOptimizerContext(Configuration configuration, SearchStats searchStats) {
         super(configuration);
         this.searchStats = searchStats;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
index edb8c0cde6052..69134923e6f47 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
@@ -153,14 +153,14 @@ else if (plan instanceof Project project) {
                         Alias nullAlias = nullLiteral.get(f.dataType());
                         // save the first field as null (per datatype)
                         if (nullAlias == null) {
-                            Alias alias = new Alias(f.source(), f.name(), null, Literal.of(f, null), f.id());
+                            Alias alias = new Alias(f.source(), f.name(), Literal.of(f, null), f.id());
                             nullLiteral.put(dt, alias);
                             projection = alias.toAttribute();
                         }
                         // otherwise point to it
                         else {
                             // since avoids creating field copies
-                            projection = new Alias(f.source(), f.name(), f.qualifier(), nullAlias.toAttribute(), f.id());
+                            projection = new Alias(f.source(), f.name(), nullAlias.toAttribute(), f.id());
                         }
                     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java
index a6b86fcdd783f..c11e1a4ec49e4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.esql.optimizer;
 
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
 
-public record LocalPhysicalOptimizerContext(EsqlConfiguration configuration, SearchStats searchStats) {}
+public record LocalPhysicalOptimizerContext(Configuration configuration, SearchStats searchStats) {}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
index b9be9b7ad029e..ccab93df187fe 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
@@ -27,10 +27,12 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.expression.TypedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
+import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
+import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate;
+import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.StringQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
@@ -44,9 +46,11 @@
 import org.elasticsearch.xpack.esql.core.rule.Rule;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.core.util.Queries;
 import org.elasticsearch.xpack.esql.core.util.Queries.Clause;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
+import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialAggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
@@ -57,8 +61,12 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveBinaryComparison;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerRules.OptimizerRule;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
@@ -250,8 +258,10 @@ protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext
                 for (Expression exp : splitAnd(filterExec.condition())) {
                     (canPushToSource(exp, x -> hasIdenticalDelegate(x, ctx.searchStats())) ? pushable : nonPushable).add(exp);
                 }
-                if (pushable.size() > 0) { // update the executable with pushable conditions
-                    Query queryDSL = TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(pushable));
+                // Combine GT, GTE, LT and LTE in pushable to Range if possible
+                List<Expression> newPushable = combineEligiblePushableToRange(pushable);
+                if (newPushable.size() > 0) { // update the executable with pushable conditions
+                    Query queryDSL = TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(newPushable));
                     QueryBuilder planQuery = queryDSL.asBuilder();
                     var query = Queries.combine(Clause.FILTER, asList(queryExec.query(), planQuery));
                     queryExec = new EsQueryExec(
@@ -275,6 +285,79 @@ protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext
             return plan;
         }
 
+        private static List<Expression> combineEligiblePushableToRange(List<Expression> pushable) {
+            List<EsqlBinaryComparison> bcs = new ArrayList<>();
+            List<Range> ranges = new ArrayList<>();
+            List<Expression> others = new ArrayList<>();
+            boolean changed = false;
+
+            pushable.forEach(e -> {
+                if (e instanceof GreaterThan || e instanceof GreaterThanOrEqual || e instanceof LessThan || e instanceof LessThanOrEqual) {
+                    if (((EsqlBinaryComparison) e).right().foldable()) {
+                        bcs.add((EsqlBinaryComparison) e);
+                    } else {
+                        others.add(e);
+                    }
+                } else {
+                    others.add(e);
+                }
+            });
+
+            for (int i = 0, step = 1; i < bcs.size() - 1; i += step, step = 1) {
+                BinaryComparison main = bcs.get(i);
+                for (int j = i + 1; j < bcs.size(); j++) {
+                    BinaryComparison other = bcs.get(j);
+                    if (main.left().semanticEquals(other.left())) {
+                        // >/>= AND </<=
+                        if ((main instanceof GreaterThan || main instanceof GreaterThanOrEqual)
+                            && (other instanceof LessThan || other instanceof LessThanOrEqual)) {
+                            bcs.remove(j);
+                            bcs.remove(i);
+
+                            ranges.add(
+                                new Range(
+                                    main.source(),
+                                    main.left(),
+                                    main.right(),
+                                    main instanceof GreaterThanOrEqual,
+                                    other.right(),
+                                    other instanceof LessThanOrEqual,
+                                    main.zoneId()
+                                )
+                            );
+
+                            changed = true;
+                            step = 0;
+                            break;
+                        }
+                        // </<= AND >/>=
+                        else if ((other instanceof GreaterThan || other instanceof GreaterThanOrEqual)
+                            && (main instanceof LessThan || main instanceof LessThanOrEqual)) {
+                                bcs.remove(j);
+                                bcs.remove(i);
+
+                                ranges.add(
+                                    new Range(
+                                        main.source(),
+                                        main.left(),
+                                        other.right(),
+                                        other instanceof GreaterThanOrEqual,
+                                        main.right(),
+                                        main instanceof LessThanOrEqual,
+                                        main.zoneId()
+                                    )
+                                );
+
+                                changed = true;
+                                step = 0;
+                                break;
+                            }
+                    }
+                }
+            }
+            return changed ? CollectionUtils.combine(others, bcs, ranges) : pushable;
+        }
+
         public static boolean canPushToSource(Expression exp, Predicate<FieldAttribute> hasIdenticalDelegate) {
             if (exp instanceof BinaryComparison bc) {
                 return isAttributePushable(bc.left(), bc, hasIdenticalDelegate) && bc.right().foldable();
@@ -300,6 +383,10 @@ public static boolean canPushToSource(Expression exp, Predicate<FieldAttribute>
                     && Expressions.foldable(cidrMatch.matches());
             } else if (exp instanceof SpatialRelatesFunction bc) {
                 return bc.canPushToSource(LocalPhysicalPlanOptimizer::isAggregatable);
+            } else if (exp instanceof MatchQueryPredicate mqp) {
+                return mqp.field() instanceof FieldAttribute && DataType.isString(mqp.field().dataType());
+            } else if (exp instanceof StringQueryPredicate) {
+                return true;
             }
             return false;
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java
index bfd3a5569b0e9..67148e67cbc19 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java
@@ -7,18 +7,18 @@
 
 package org.elasticsearch.xpack.esql.optimizer;
 
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.Objects;
 
 public class LogicalOptimizerContext {
-    private final EsqlConfiguration configuration;
+    private final Configuration configuration;
 
-    public LogicalOptimizerContext(EsqlConfiguration configuration) {
+    public LogicalOptimizerContext(Configuration configuration) {
         this.configuration = configuration;
     }
 
-    public EsqlConfiguration configuration() {
+    public Configuration configuration() {
         return configuration;
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
index 1a95349353221..282f46e0de7bb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
@@ -15,14 +15,20 @@
 import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
-import org.elasticsearch.xpack.esql.core.expression.Order;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.NameId;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.rule.ParameterizedRule;
 import org.elasticsearch.xpack.esql.core.rule.ParameterizedRuleExecutor;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.Order;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.optimizer.rules.AddDefaultTopN;
 import org.elasticsearch.xpack.esql.optimizer.rules.BooleanFunctionEqualsElimination;
 import org.elasticsearch.xpack.esql.optimizer.rules.BooleanSimplification;
-import org.elasticsearch.xpack.esql.optimizer.rules.CombineDisjunctionsToIn;
+import org.elasticsearch.xpack.esql.optimizer.rules.CombineBinaryComparisons;
+import org.elasticsearch.xpack.esql.optimizer.rules.CombineDisjunctions;
 import org.elasticsearch.xpack.esql.optimizer.rules.CombineEvals;
 import org.elasticsearch.xpack.esql.optimizer.rules.CombineProjections;
 import org.elasticsearch.xpack.esql.optimizer.rules.ConstantFolding;
@@ -64,6 +70,7 @@
 import org.elasticsearch.xpack.esql.optimizer.rules.SubstituteSpatialSurrogates;
 import org.elasticsearch.xpack.esql.optimizer.rules.SubstituteSurrogates;
 import org.elasticsearch.xpack.esql.optimizer.rules.TranslateMetricsAggregate;
+import org.elasticsearch.xpack.esql.plan.GeneratingPlan;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.OrderBy;
@@ -71,11 +78,13 @@
 import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import static java.util.Arrays.asList;
@@ -111,6 +120,34 @@ public LogicalPlanOptimizer(LogicalOptimizerContext optimizerContext) {
         super(optimizerContext);
     }
 
+    public static String temporaryName(Expression inner, Expression outer, int suffix) {
+        String in = toString(inner);
+        String out = toString(outer);
+        return rawTemporaryName(in, out, String.valueOf(suffix));
+    }
+
+    public static String locallyUniqueTemporaryName(String inner, String outer) {
+        return FieldAttribute.SYNTHETIC_ATTRIBUTE_NAME_PREFIX + inner + "$" + outer + "$" + new NameId();
+    }
+
+    public static String rawTemporaryName(String inner, String outer, String suffix) {
+        return FieldAttribute.SYNTHETIC_ATTRIBUTE_NAME_PREFIX + inner + "$" + outer + "$" + suffix;
+    }
+
+    static String toString(Expression ex) {
+        return ex instanceof AggregateFunction af ? af.functionName() : extractString(ex);
+    }
+
+    static String extractString(Expression ex) {
+        return ex instanceof NamedExpression ne ? ne.name() : limitToString(ex.sourceText()).replace(' ', '_');
+    }
+
+    static int TO_STRING_LIMIT = 16;
+
+    static String limitToString(String string) {
+        return string.length() > TO_STRING_LIMIT ? string.substring(0, TO_STRING_LIMIT - 1) + ">" : string;
+    }
+
     public LogicalPlan optimize(LogicalPlan verified) {
         var optimized = execute(verified);
 
@@ -118,6 +155,7 @@ public LogicalPlan optimize(LogicalPlan verified) {
         if (failures.hasFailures()) {
             throw new VerificationException(failures);
         }
+        optimized.setOptimized();
         return optimized;
     }
 
@@ -171,8 +209,9 @@ protected static Batch<LogicalPlan> operators() {
             new PropagateEquals(),
             new PropagateNullable(),
             new BooleanFunctionEqualsElimination(),
-            new CombineDisjunctionsToIn(),
-            new SimplifyComparisonsArithmetics(EsqlDataTypes::areCompatible),
+            new CombineBinaryComparisons(),
+            new CombineDisjunctions(),
+            new SimplifyComparisonsArithmetics(DataType::areCompatible),
             // prune/elimination
             new PruneFilters(),
             new PruneColumns(),
@@ -211,35 +250,26 @@ public static LogicalPlan skipPlan(UnaryPlan plan, LocalSupplier supplier) {
 
     /**
      * Pushes LogicalPlans which generate new attributes (Eval, Grok/Dissect, Enrich), past OrderBys and Projections.
-     * Although it seems arbitrary whether the OrderBy or the Eval is executed first, this transformation ensures that OrderBys only
-     * separated by an eval can be combined by PushDownAndCombineOrderBy.
-     *
-     * E.g.:
-     *
-     * ... | sort a | eval x = b + 1 | sort x
-     *
-     * becomes
-     *
-     * ... | eval x = b + 1 | sort a | sort x
-     *
-     * Ordering the Evals before the OrderBys has the advantage that it's always possible to order the plans like this.
+     * Although it seems arbitrary whether the OrderBy or the generating plan is executed first, this transformation ensures that OrderBys
+     * only separated by e.g. an Eval can be combined by {@link PushDownAndCombineOrderBy}.
+     * <p>
+     * E.g. {@code ... | sort a | eval x = b + 1 | sort x} becomes {@code ... | eval x = b + 1 | sort a | sort x}
+     * <p>
+     * Ordering the generating plans before the OrderBys has the advantage that it's always possible to order the plans like this.
      * E.g., in the example above it would not be possible to put the eval after the two orderBys.
-     *
-     * In case one of the Eval's fields would shadow the orderBy's attributes, we rename the attribute first.
-     *
-     * E.g.
-     *
-     * ... | sort a | eval a = b + 1 | ...
-     *
-     * becomes
-     *
-     * ... | eval $$a = a | eval a = b + 1 | sort $$a | drop $$a
+     * <p>
+     * In case one of the generating plan's attributes would shadow the OrderBy's attributes, we alias the generated attribute first.
+     * <p>
+     * E.g. {@code ... | sort a | eval a = b + 1 | ...} becomes {@code ... | eval $$a = a | eval a = b + 1 | sort $$a | drop $$a ...}
+     * <p>
+     * In case the generating plan's attributes would shadow the Project's attributes, we rename the generated attributes in place.
+     * <p>
+     * E.g. {@code ... | rename a as z | eval a = b + 1 | ...} becomes {@code ... eval $$a = b + 1 | rename a as z, $$a as a ...}
      */
-    public static LogicalPlan pushGeneratingPlanPastProjectAndOrderBy(UnaryPlan generatingPlan, List<Attribute> generatedAttributes) {
+    public static <Plan extends UnaryPlan & GeneratingPlan<Plan>> LogicalPlan pushGeneratingPlanPastProjectAndOrderBy(Plan generatingPlan) {
         LogicalPlan child = generatingPlan.child();
-
         if (child instanceof OrderBy orderBy) {
-            Set<String> evalFieldNames = new LinkedHashSet<>(Expressions.names(generatedAttributes));
+            Set<String> evalFieldNames = new LinkedHashSet<>(Expressions.names(generatingPlan.generatedAttributes()));
 
             // Look for attributes in the OrderBy's expressions and create aliases with temporary names for them.
             AttributeReplacement nonShadowedOrders = renameAttributesInExpressions(evalFieldNames, orderBy.order());
@@ -260,9 +290,65 @@ public static LogicalPlan pushGeneratingPlanPastProjectAndOrderBy(UnaryPlan gene
             }
 
             return orderBy.replaceChild(generatingPlan.replaceChild(orderBy.child()));
-        } else if (child instanceof Project) {
-            var projectWithEvalChild = pushDownPastProject(generatingPlan);
-            return projectWithEvalChild.withProjections(mergeOutputExpressions(generatedAttributes, projectWithEvalChild.projections()));
+        } else if (child instanceof Project project) {
+            // We need to account for attribute shadowing: a rename might rely on a name generated in an Eval/Grok/Dissect/Enrich.
+            // E.g. in:
+            //
+            // Eval[[2 * x{f}#1 AS y]]
+            // \_Project[[x{f}#1, y{f}#2, y{f}#2 AS z]]
+            //
+            // Just moving the Eval down breaks z because we shadow y{f}#2.
+            // Instead, we use a different alias in the Eval, eventually renaming back to y:
+            //
+            // Project[[x{f}#1, y{f}#2 as z, $$y{r}#3 as y]]
+            // \_Eval[[2 * x{f}#1 as $$y]]
+
+            List<Attribute> generatedAttributes = generatingPlan.generatedAttributes();
+
+            @SuppressWarnings("unchecked")
+            Plan generatingPlanWithResolvedExpressions = (Plan) resolveRenamesFromProject(generatingPlan, project);
+
+            Set<String> namesReferencedInRenames = new HashSet<>();
+            for (NamedExpression ne : project.projections()) {
+                if (ne instanceof Alias as) {
+                    namesReferencedInRenames.addAll(as.child().references().names());
+                }
+            }
+            Map<String, String> renameGeneratedAttributeTo = newNamesForConflictingAttributes(
+                generatingPlan.generatedAttributes(),
+                namesReferencedInRenames
+            );
+            List<String> newNames = generatedAttributes.stream()
+                .map(attr -> renameGeneratedAttributeTo.getOrDefault(attr.name(), attr.name()))
+                .toList();
+            Plan generatingPlanWithRenamedAttributes = generatingPlanWithResolvedExpressions.withGeneratedNames(newNames);
+
+            // Put the project at the top, but include the generated attributes.
+            // Any generated attributes that had to be renamed need to be re-renamed to their original names.
+            List<NamedExpression> generatedAttributesRenamedToOriginal = new ArrayList<>(generatedAttributes.size());
+            List<Attribute> renamedGeneratedAttributes = generatingPlanWithRenamedAttributes.generatedAttributes();
+            for (int i = 0; i < generatedAttributes.size(); i++) {
+                Attribute originalAttribute = generatedAttributes.get(i);
+                Attribute renamedAttribute = renamedGeneratedAttributes.get(i);
+                if (originalAttribute.name().equals(renamedAttribute.name())) {
+                    generatedAttributesRenamedToOriginal.add(renamedAttribute);
+                } else {
+                    generatedAttributesRenamedToOriginal.add(
+                        new Alias(
+                            originalAttribute.source(),
+                            originalAttribute.name(),
+                            renamedAttribute,
+                            originalAttribute.id(),
+                            originalAttribute.synthetic()
+                        )
+                    );
+                }
+            }
+
+            Project projectWithGeneratingChild = project.replaceChild(generatingPlanWithRenamedAttributes.replaceChild(project.child()));
+            return projectWithGeneratingChild.withProjections(
+                mergeOutputExpressions(generatedAttributesRenamedToOriginal, projectWithGeneratingChild.projections())
+            );
         }
 
         return generatingPlan;
@@ -286,9 +372,10 @@ private static AttributeReplacement renameAttributesInExpressions(
             rewrittenExpressions.add(expr.transformUp(Attribute.class, attr -> {
                 if (attributeNamesToRename.contains(attr.name())) {
                     Alias renamedAttribute = aliasesForReplacedAttributes.computeIfAbsent(attr, a -> {
-                        String tempName = SubstituteSurrogates.rawTemporaryName(a.name(), "temp_name", a.id().toString());
+                        String tempName = locallyUniqueTemporaryName(a.name(), "temp_name");
                         // TODO: this should be synthetic
-                        return new Alias(a.source(), tempName, null, a, null, false);
+                        // blocked on https://github.com/elastic/elasticsearch/issues/98703
+                        return new Alias(a.source(), tempName, a, null, false);
                     });
                     return renamedAttribute.toAttribute();
                 }
@@ -300,16 +387,28 @@ private static AttributeReplacement renameAttributesInExpressions(
         return new AttributeReplacement(rewrittenExpressions, aliasesForReplacedAttributes);
     }
 
+    private static Map<String, String> newNamesForConflictingAttributes(
+        List<Attribute> potentiallyConflictingAttributes,
+        Set<String> reservedNames
+    ) {
+        if (reservedNames.isEmpty()) {
+            return Map.of();
+        }
+
+        Map<String, String> renameAttributeTo = new HashMap<>();
+        for (Attribute attr : potentiallyConflictingAttributes) {
+            String name = attr.name();
+            if (reservedNames.contains(name)) {
+                renameAttributeTo.putIfAbsent(name, locallyUniqueTemporaryName(name, "temp_name"));
+            }
+        }
+
+        return renameAttributeTo;
+    }
+
     public static Project pushDownPastProject(UnaryPlan parent) {
         if (parent.child() instanceof Project project) {
-            AttributeMap.Builder<Expression> aliasBuilder = AttributeMap.builder();
-            project.forEachExpression(Alias.class, a -> aliasBuilder.put(a.toAttribute(), a.child()));
-            var aliases = aliasBuilder.build();
-
-            var expressionsWithResolvedAliases = (UnaryPlan) parent.transformExpressionsOnly(
-                ReferenceAttribute.class,
-                r -> aliases.resolve(r, r)
-            );
+            UnaryPlan expressionsWithResolvedAliases = resolveRenamesFromProject(parent, project);
 
             return project.replaceChild(expressionsWithResolvedAliases.replaceChild(project.child()));
         } else {
@@ -317,6 +416,14 @@ public static Project pushDownPastProject(UnaryPlan parent) {
         }
     }
 
+    private static UnaryPlan resolveRenamesFromProject(UnaryPlan plan, Project project) {
+        AttributeMap.Builder<Expression> aliasBuilder = AttributeMap.builder();
+        project.forEachExpression(Alias.class, a -> aliasBuilder.put(a.toAttribute(), a.child()));
+        var aliases = aliasBuilder.build();
+
+        return (UnaryPlan) plan.transformExpressionsOnly(ReferenceAttribute.class, r -> aliases.resolve(r, r));
+    }
+
     public abstract static class ParameterizedOptimizerRule<SubPlan extends LogicalPlan, P> extends ParameterizedRule<
         SubPlan,
         LogicalPlan,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
index d9141d737c949..733fe2e8762bb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
@@ -12,14 +12,14 @@
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
-import org.elasticsearch.xpack.esql.core.plan.QueryPlan;
+import org.elasticsearch.xpack.esql.plan.GeneratingPlan;
+import org.elasticsearch.xpack.esql.plan.QueryPlan;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
-import org.elasticsearch.xpack.esql.plan.logical.Eval;
+import org.elasticsearch.xpack.esql.plan.logical.InlineStats;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
-import org.elasticsearch.xpack.esql.plan.logical.RegexExtract;
 import org.elasticsearch.xpack.esql.plan.logical.Row;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
@@ -100,21 +100,16 @@ protected AttributeSet generates(LogicalPlan logicalPlan) {
             if (logicalPlan instanceof EsRelation
                 || logicalPlan instanceof LocalRelation
                 || logicalPlan instanceof Row
-                || logicalPlan instanceof Aggregate) {
+                || logicalPlan instanceof Aggregate
+                || logicalPlan instanceof InlineStats) {
                 return logicalPlan.outputSet();
             }
-            if (logicalPlan instanceof Eval eval) {
-                return new AttributeSet(Expressions.asAttributes(eval.fields()));
-            }
-            if (logicalPlan instanceof RegexExtract extract) {
-                return new AttributeSet(extract.extractedFields());
+            if (logicalPlan instanceof GeneratingPlan<?> generating) {
+                return new AttributeSet(generating.generatedAttributes());
             }
             if (logicalPlan instanceof MvExpand mvExpand) {
                 return new AttributeSet(mvExpand.expanded());
             }
-            if (logicalPlan instanceof Enrich enrich) {
-                return new AttributeSet(Expressions.asAttributes(enrich.enrichFields()));
-            }
 
             return AttributeSet.EMPTY;
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalOptimizerContext.java
index 7e3ba256f4ffc..7c94935844d6b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalOptimizerContext.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalOptimizerContext.java
@@ -7,6 +7,6 @@
 
 package org.elasticsearch.xpack.esql.optimizer;
 
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
-public record PhysicalOptimizerContext(EsqlConfiguration configuration) {}
+public record PhysicalOptimizerContext(Configuration configuration) {}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java
index 75b05b7e5631c..2a410d6f386a4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java
@@ -122,7 +122,9 @@ public PhysicalPlan apply(PhysicalPlan plan) {
                     if (p instanceof HashJoinExec join) {
                         attributes.removeAll(join.addedFields());
                         for (Attribute rhs : join.rightFields()) {
-                            attributes.remove(rhs);
+                            if (join.leftFields().stream().anyMatch(x -> x.semanticEquals(rhs)) == false) {
+                                attributes.remove(rhs);
+                            }
                         }
                     }
                     if (p instanceof EnrichExec ee) {
@@ -148,7 +150,7 @@ public PhysicalPlan apply(PhysicalPlan plan) {
                             // add a synthetic field (so it doesn't clash with the user defined one) to return a constant
                             // to avoid the block from being trimmed
                             if (output.isEmpty()) {
-                                var alias = new Alias(logicalFragment.source(), "<all-fields-projected>", null, Literal.NULL, null, true);
+                                var alias = new Alias(logicalFragment.source(), "<all-fields-projected>", Literal.NULL, null, true);
                                 List<Alias> fields = singletonList(alias);
                                 logicalFragment = new Eval(logicalFragment.source(), logicalFragment, fields);
                                 output = Expressions.asAttributes(fields);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineBinaryComparisons.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineBinaryComparisons.java
new file mode 100644
index 0000000000000..0d1d5baf920d7
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineBinaryComparisons.java
@@ -0,0 +1,210 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.optimizer.rules;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
+import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public final class CombineBinaryComparisons extends OptimizerRules.OptimizerExpressionRule<BinaryLogic> {
+
+    public CombineBinaryComparisons() {
+        super(OptimizerRules.TransformDirection.DOWN);
+    }
+
+    @Override
+    public Expression rule(BinaryLogic e) {
+        if (e instanceof And and) {
+            return combine(and);
+        } else if (e instanceof Or or) {
+            return combine(or);
+        }
+        return e;
+    }
+
+    // combine conjunction
+    private static Expression combine(And and) {
+        List<BinaryComparison> bcs = new ArrayList<>();
+        List<Expression> exps = new ArrayList<>();
+        boolean changed = false;
+        List<Expression> andExps = Predicates.splitAnd(and);
+
+        andExps.sort((o1, o2) -> {
+            if (o1 instanceof NotEquals && o2 instanceof NotEquals) {
+                return 0; // keep NotEquals' order
+            } else if (o1 instanceof NotEquals || o2 instanceof NotEquals) {
+                return o1 instanceof NotEquals ? 1 : -1; // push NotEquals up
+            } else {
+                return 0; // keep non-Ranges' and non-NotEquals' order
+            }
+        });
+        for (Expression ex : andExps) {
+            if (ex instanceof BinaryComparison bc && (ex instanceof Equals || ex instanceof NotEquals) == false) {
+                if (bc.right().foldable() && (findExistingComparison(bc, bcs, true))) {
+                    changed = true;
+                } else {
+                    bcs.add(bc);
+                }
+            } else if (ex instanceof NotEquals neq) {
+                if (neq.right().foldable() && notEqualsIsRemovableFromConjunction(neq, bcs)) {
+                    // the non-equality can simply be dropped: either superfluous or has been merged with an updated range/inequality
+                    changed = true;
+                } else { // not foldable OR not overlapping
+                    exps.add(ex);
+                }
+            } else {
+                exps.add(ex);
+            }
+        }
+        return changed ? Predicates.combineAnd(CollectionUtils.combine(exps, bcs)) : and;
+    }
+
+    // combine disjunction
+    private static Expression combine(Or or) {
+        List<BinaryComparison> bcs = new ArrayList<>();
+        List<Expression> exps = new ArrayList<>();
+        boolean changed = false;
+        for (Expression ex : Predicates.splitOr(or)) {
+            if (ex instanceof BinaryComparison bc) {
+                if (bc.right().foldable() && findExistingComparison(bc, bcs, false)) {
+                    changed = true;
+                } else {
+                    bcs.add(bc);
+                }
+            } else {
+                exps.add(ex);
+            }
+        }
+        return changed ? Predicates.combineOr(CollectionUtils.combine(exps, bcs)) : or;
+    }
+
+    /**
+     * Find commonalities between the given comparison in the given list.
+     * The method can be applied both for conjunctive (AND) or disjunctive purposes (OR).
+     */
+    private static boolean findExistingComparison(BinaryComparison main, List<BinaryComparison> bcs, boolean conjunctive) {
+        Object value = main.right().fold();
+        // NB: the loop modifies the list (hence why the int is used)
+        for (int i = 0; i < bcs.size(); i++) {
+            BinaryComparison other = bcs.get(i);
+            // skip if cannot evaluate
+            if (other.right().foldable() == false) {
+                continue;
+            }
+            // if bc is a higher/lower value or gte vs gt, use it instead
+            if ((other instanceof GreaterThan || other instanceof GreaterThanOrEqual)
+                && (main instanceof GreaterThan || main instanceof GreaterThanOrEqual)) {
+                if (main.left().semanticEquals(other.left())) {
+                    Integer compare = BinaryComparison.compare(value, other.right().fold());
+                    if (compare != null) {
+                        // AND
+                        if ((conjunctive &&
+                        // a > 3 AND a > 2 -> a > 3
+                            (compare > 0 ||
+                            // a > 2 AND a >= 2 -> a > 2
+                                (compare == 0 && main instanceof GreaterThan && other instanceof GreaterThanOrEqual))) ||
+                        // OR
+                            (conjunctive == false &&
+                            // a > 2 OR a > 3 -> a > 2
+                                (compare < 0 ||
+                                // a >= 2 OR a > 2 -> a >= 2
+                                    (compare == 0 && main instanceof GreaterThanOrEqual && other instanceof GreaterThan)))) {
+                            bcs.remove(i);
+                            bcs.add(i, main);
+                        }
+                        // found a match
+                        return true;
+                    }
+                    return false;
+                }
+            }
+            // if bc is a lower/higher value or lte vs lt, use it instead
+            else if ((other instanceof LessThan || other instanceof LessThanOrEqual)
+                && (main instanceof LessThan || main instanceof LessThanOrEqual)) {
+                    if (main.left().semanticEquals(other.left())) {
+                        Integer compare = BinaryComparison.compare(value, other.right().fold());
+                        if (compare != null) {
+                            // AND
+                            if ((conjunctive &&
+                            // a < 2 AND a < 3 -> a < 2
+                                (compare < 0 ||
+                            // a < 2 AND a <= 2 -> a < 2
+                                    (compare == 0 && main instanceof LessThan && other instanceof LessThanOrEqual))) ||
+                            // OR
+                                (conjunctive == false &&
+                            // a < 2 OR a < 3 -> a < 3
+                                    (compare > 0 ||
+                            // a <= 2 OR a < 2 -> a <= 2
+                                        (compare == 0 && main instanceof LessThanOrEqual && other instanceof LessThan)))) {
+                                bcs.remove(i);
+                                bcs.add(i, main);
+                            }
+                            // found a match
+                            return true;
+                        }
+                        return false;
+                    }
+                }
+        }
+        return false;
+    }
+
+    private static boolean notEqualsIsRemovableFromConjunction(NotEquals notEquals, List<BinaryComparison> bcs) {
+        Object neqVal = notEquals.right().fold();
+        Integer comp;
+
+        // check on "condition-overlapping" inequalities:
+        // a != 2 AND a > 3 -> a > 3 (discard NotEquals)
+        // a != 2 AND a >= 2 -> a > 2 (discard NotEquals plus update inequality)
+        // a != 2 AND a > 1 -> nop (do nothing)
+        //
+        // a != 2 AND a < 3 -> nop
+        // a != 2 AND a <= 2 -> a < 2
+        // a != 2 AND a < 1 -> a < 1
+        for (int i = 0; i < bcs.size(); i++) {
+            BinaryComparison bc = bcs.get(i);
+            if (notEquals.left().semanticEquals(bc.left())) {
+                if (bc instanceof LessThan || bc instanceof LessThanOrEqual) {
+                    comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold()) : null;
+                    if (comp != null) {
+                        if (comp >= 0) {
+                            if (comp == 0 && bc instanceof LessThanOrEqual) { // a != 2 AND a <= 2 -> a < 2
+                                bcs.set(i, new LessThan(bc.source(), bc.left(), bc.right(), bc.zoneId()));
+                            } // else : comp > 0 (a != 2 AND a </<= 1 -> a </<= 1), or == 0 && bc i.of "<" (a != 2 AND a < 2 -> a < 2)
+                            return true;
+                        } // else: comp < 0 : a != 2 AND a </<= 3 -> nop
+                    } // else: non-comparable, nop
+                } else if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) {
+                    comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold()) : null;
+                    if (comp != null) {
+                        if (comp <= 0) {
+                            if (comp == 0 && bc instanceof GreaterThanOrEqual) { // a != 2 AND a >= 2 -> a > 2
+                                bcs.set(i, new GreaterThan(bc.source(), bc.left(), bc.right(), bc.zoneId()));
+                            } // else: comp < 0 (a != 2 AND a >/>= 3 -> a >/>= 3), or == 0 && bc i.of ">" (a != 2 AND a > 2 -> a > 2)
+                            return true;
+                        } // else: comp > 0 : a != 2 AND a >/>= 1 -> nop
+                    } // else: non-comparable, nop
+                } // else: other non-relevant type
+            }
+        }
+        return false;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctions.java
new file mode 100644
index 0000000000000..ceac1aa9ca75b
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctions.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.optimizer.rules;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.combineOr;
+import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitOr;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
+
+/**
+ * Combine disjunctive Equals, In or CIDRMatch expressions on the same field into an In or CIDRMatch expression.
+ * This rule looks for both simple equalities:
+ * 1. a == 1 OR a == 2 becomes a IN (1, 2)
+ * and combinations of In
+ * 2. a == 1 OR a IN (2) becomes a IN (1, 2)
+ * 3. a IN (1) OR a IN (2) becomes a IN (1, 2)
+ * and combinations of CIDRMatch
+ * 4. CIDRMatch(a, ip1) OR CIDRMatch(a, ip2) OR a == ip3 or a IN (ip4, ip5) becomes CIDRMatch(a, ip1, ip2, ip3, ip4, ip5)
+ * <p>
+ * This rule does NOT check for type compatibility as that phase has been
+ * already be verified in the analyzer.
+ */
+public final class CombineDisjunctions extends OptimizerRules.OptimizerExpressionRule<Or> {
+    public CombineDisjunctions() {
+        super(OptimizerRules.TransformDirection.UP);
+    }
+
+    protected static In createIn(Expression key, List<Expression> values, ZoneId zoneId) {
+        return new In(key.source(), key, values);
+    }
+
+    protected static Equals createEquals(Expression k, Set<Expression> v, ZoneId finalZoneId) {
+        return new Equals(k.source(), k, v.iterator().next(), finalZoneId);
+    }
+
+    protected static CIDRMatch createCIDRMatch(Expression k, List<Expression> v) {
+        return new CIDRMatch(k.source(), k, v);
+    }
+
+    @Override
+    public Expression rule(Or or) {
+        Expression e = or;
+        // look only at equals, In and CIDRMatch
+        List<Expression> exps = splitOr(e);
+
+        Map<Expression, Set<Expression>> ins = new LinkedHashMap<>();
+        Map<Expression, Set<Expression>> cidrs = new LinkedHashMap<>();
+        Map<Expression, Set<Expression>> ips = new LinkedHashMap<>();
+        ZoneId zoneId = null;
+        List<Expression> ors = new LinkedList<>();
+        boolean changed = false;
+        for (Expression exp : exps) {
+            if (exp instanceof Equals eq) {
+                // consider only equals against foldables
+                if (eq.right().foldable()) {
+                    ins.computeIfAbsent(eq.left(), k -> new LinkedHashSet<>()).add(eq.right());
+                    if (eq.left().dataType() == DataType.IP) {
+                        Object value = eq.right().fold();
+                        // ImplicitCasting and ConstantFolding(includes explicit casting) are applied before CombineDisjunctions.
+                        // They fold the input IP string to an internal IP format. These happen to Equals and IN, but not for CIDRMatch,
+                        // as CIDRMatch takes strings as input, ImplicitCasting does not apply to it, and the first input to CIDRMatch is a
+                        // field, ConstantFolding does not apply to it either.
+                        // If the data type is IP, convert the internal IP format in Equals and IN to the format that is compatible with
+                        // CIDRMatch, and store them in a separate map, so that they can be combined into existing CIDRMatch later.
+                        if (value instanceof BytesRef bytesRef) {
+                            value = ipToString(bytesRef);
+                        }
+                        ips.computeIfAbsent(eq.left(), k -> new LinkedHashSet<>()).add(new Literal(Source.EMPTY, value, DataType.IP));
+                    }
+                } else {
+                    ors.add(exp);
+                }
+                if (zoneId == null) {
+                    zoneId = eq.zoneId();
+                }
+            } else if (exp instanceof In in) {
+                ins.computeIfAbsent(in.value(), k -> new LinkedHashSet<>()).addAll(in.list());
+                if (in.value().dataType() == DataType.IP) {
+                    List<Expression> values = new ArrayList<>(in.list().size());
+                    for (Expression i : in.list()) {
+                        Object value = i.fold();
+                        // Same as Equals.
+                        if (value instanceof BytesRef bytesRef) {
+                            value = ipToString(bytesRef);
+                        }
+                        values.add(new Literal(Source.EMPTY, value, DataType.IP));
+                    }
+                    ips.computeIfAbsent(in.value(), k -> new LinkedHashSet<>()).addAll(values);
+                }
+            } else if (exp instanceof CIDRMatch cm) {
+                cidrs.computeIfAbsent(cm.ipField(), k -> new LinkedHashSet<>()).addAll(cm.matches());
+            } else {
+                ors.add(exp);
+            }
+        }
+
+        if (cidrs.isEmpty() == false) {
+            for (Expression f : ips.keySet()) {
+                cidrs.computeIfAbsent(f, k -> new LinkedHashSet<>()).addAll(ips.get(f));
+                ins.remove(f);
+            }
+        }
+
+        if (ins.isEmpty() == false) {
+            // combine equals alongside the existing ors
+            final ZoneId finalZoneId = zoneId;
+            ins.forEach(
+                (k, v) -> { ors.add(v.size() == 1 ? createEquals(k, v, finalZoneId) : createIn(k, new ArrayList<>(v), finalZoneId)); }
+            );
+
+            changed = true;
+        }
+
+        if (cidrs.isEmpty() == false) {
+            cidrs.forEach((k, v) -> { ors.add(createCIDRMatch(k, new ArrayList<>(v))); });
+            changed = true;
+        }
+
+        if (changed) {
+            // TODO: this makes a QL `or`, not an ESQL `or`
+            Expression combineOr = combineOr(ors);
+            // check the result semantically since the result might different in order
+            // but be actually the same which can trigger a loop
+            // e.g. a == 1 OR a == 2 OR null --> null OR a in (1,2) --> literalsOnTheRight --> cycle
+            if (e.semanticEquals(combineOr) == false) {
+                e = combineOr;
+            }
+        }
+
+        return e;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java
deleted file mode 100644
index 42d4bf730a644..0000000000000
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.optimizer.rules;
-
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
-
-import java.time.ZoneId;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.combineOr;
-import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitOr;
-
-/**
- * Combine disjunctions on the same field into an In expression.
- * This rule looks for both simple equalities:
- * 1. a == 1 OR a == 2 becomes a IN (1, 2)
- * and combinations of In
- * 2. a == 1 OR a IN (2) becomes a IN (1, 2)
- * 3. a IN (1) OR a IN (2) becomes a IN (1, 2)
- * <p>
- * This rule does NOT check for type compatibility as that phase has been
- * already be verified in the analyzer.
- */
-public final class CombineDisjunctionsToIn extends OptimizerRules.OptimizerExpressionRule<Or> {
-    public CombineDisjunctionsToIn() {
-        super(OptimizerRules.TransformDirection.UP);
-    }
-
-    protected In createIn(Expression key, List<Expression> values, ZoneId zoneId) {
-        return new In(key.source(), key, values);
-    }
-
-    protected Equals createEquals(Expression k, Set<Expression> v, ZoneId finalZoneId) {
-        return new Equals(k.source(), k, v.iterator().next(), finalZoneId);
-    }
-
-    @Override
-    public Expression rule(Or or) {
-        Expression e = or;
-        // look only at equals and In
-        List<Expression> exps = splitOr(e);
-
-        Map<Expression, Set<Expression>> found = new LinkedHashMap<>();
-        ZoneId zoneId = null;
-        List<Expression> ors = new LinkedList<>();
-
-        for (Expression exp : exps) {
-            if (exp instanceof Equals eq) {
-                // consider only equals against foldables
-                if (eq.right().foldable()) {
-                    found.computeIfAbsent(eq.left(), k -> new LinkedHashSet<>()).add(eq.right());
-                } else {
-                    ors.add(exp);
-                }
-                if (zoneId == null) {
-                    zoneId = eq.zoneId();
-                }
-            } else if (exp instanceof In in) {
-                found.computeIfAbsent(in.value(), k -> new LinkedHashSet<>()).addAll(in.list());
-            } else {
-                ors.add(exp);
-            }
-        }
-
-        if (found.isEmpty() == false) {
-            // combine equals alongside the existing ors
-            final ZoneId finalZoneId = zoneId;
-            found.forEach(
-                (k, v) -> { ors.add(v.size() == 1 ? createEquals(k, v, finalZoneId) : createIn(k, new ArrayList<>(v), finalZoneId)); }
-            );
-
-            // TODO: this makes a QL `or`, not an ESQL `or`
-            Expression combineOr = combineOr(ors);
-            // check the result semantically since the result might different in order
-            // but be actually the same which can trigger a loop
-            // e.g. a == 1 OR a == 2 OR null --> null OR a in (1,2) --> literalsOnTheRight --> cycle
-            if (e.semanticEquals(combineOr) == false) {
-                e = combineOr;
-            }
-        }
-
-        return e;
-    }
-}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneLiteralsInOrderBy.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneLiteralsInOrderBy.java
index 1fe67c2c435c2..73273cbf17d92 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneLiteralsInOrderBy.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneLiteralsInOrderBy.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.esql.optimizer.rules;
 
-import org.elasticsearch.xpack.esql.core.expression.Order;
+import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.OrderBy;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneRedundantSortClauses.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneRedundantSortClauses.java
index dc68ae5981429..33918d29d5ded 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneRedundantSortClauses.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneRedundantSortClauses.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.esql.optimizer.rules;
 
 import org.elasticsearch.xpack.esql.core.expression.ExpressionSet;
-import org.elasticsearch.xpack.esql.core.expression.Order;
+import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.OrderBy;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEnrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEnrich.java
index 7185f63964c34..5e6def37cbf04 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEnrich.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEnrich.java
@@ -11,11 +11,9 @@
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 
-import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes;
-
 public final class PushDownEnrich extends OptimizerRules.OptimizerRule<Enrich> {
     @Override
     protected LogicalPlan rule(Enrich en) {
-        return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(en, asAttributes(en.enrichFields()));
+        return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(en);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEval.java
index 92c25a60bba77..1f5fd072f267c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEval.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEval.java
@@ -11,11 +11,9 @@
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 
-import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes;
-
 public final class PushDownEval extends OptimizerRules.OptimizerRule<Eval> {
     @Override
     protected LogicalPlan rule(Eval eval) {
-        return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(eval, asAttributes(eval.fields()));
+        return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(eval);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownRegexExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownRegexExtract.java
index d24a61f89dd7f..3f64f47e11879 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownRegexExtract.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownRegexExtract.java
@@ -14,6 +14,6 @@
 public final class PushDownRegexExtract extends OptimizerRules.OptimizerRule<RegexExtract> {
     @Override
     protected LogicalPlan rule(RegexExtract re) {
-        return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(re, re.extractedFields());
+        return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(re);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/RemoveStatsOverride.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/RemoveStatsOverride.java
index 5592a04e2f813..0f8e0f450e585 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/RemoveStatsOverride.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/RemoveStatsOverride.java
@@ -11,26 +11,30 @@
 import org.elasticsearch.xpack.esql.analysis.AnalyzerRules;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
-import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.plan.logical.Stats;
 
 import java.util.ArrayList;
 import java.util.List;
 
 /**
- * Rule that removes Aggregate overrides in grouping, aggregates and across them inside.
- * The overrides appear when the same alias is used multiple times in aggregations and/or groupings:
- * STATS x = COUNT(*), x = MIN(a) BY x = b + 1, x = c + 10
+ * Removes {@link Stats} overrides in grouping, aggregates and across them inside.
+ * The overrides appear when the same alias is used multiple times in aggregations
+ * and/or groupings:
+ * {@code STATS x = COUNT(*), x = MIN(a) BY x = b + 1, x = c + 10}
  * becomes
- * STATS BY x = c + 10
- * That is the last declaration for a given alias, overrides all the other declarations, with
- * groups having priority vs aggregates.
+ * {@code STATS BY x = c + 10}
+ * and
+ * {@code INLINESTATS x = COUNT(*), x = MIN(a) BY x = b + 1, x = c + 10}
+ * becomes
+ * {@code INLINESTATS BY x = c + 10}
+ * This is "last one wins", with groups having priority over aggregates.
  * Separately, it replaces expressions used as group keys inside the aggregates with references:
- * STATS max(a + b + 1) BY a + b
+ * {@code STATS max(a + b + 1) BY a + b}
  * becomes
- * STATS max($x + 1) BY $x = a + b
+ * {@code STATS max($x + 1) BY $x = a + b}
  */
-public final class RemoveStatsOverride extends AnalyzerRules.AnalyzerRule<Aggregate> {
+public final class RemoveStatsOverride extends AnalyzerRules.AnalyzerRule<LogicalPlan> {
 
     @Override
     protected boolean skipResolved() {
@@ -38,19 +42,18 @@ protected boolean skipResolved() {
     }
 
     @Override
-    protected LogicalPlan rule(Aggregate agg) {
-        return agg.resolved() ? removeAggDuplicates(agg) : agg;
-    }
-
-    private static Aggregate removeAggDuplicates(Aggregate agg) {
-        var groupings = agg.groupings();
-        var aggregates = agg.aggregates();
-
-        groupings = removeDuplicateNames(groupings);
-        aggregates = removeDuplicateNames(aggregates);
-
-        // replace EsqlAggregate with Aggregate
-        return new Aggregate(agg.source(), agg.child(), agg.aggregateType(), groupings, aggregates);
+    protected LogicalPlan rule(LogicalPlan p) {
+        if (p.resolved() == false) {
+            return p;
+        }
+        if (p instanceof Stats stats) {
+            return (LogicalPlan) stats.with(
+                stats.child(),
+                removeDuplicateNames(stats.groupings()),
+                removeDuplicateNames(stats.aggregates())
+            );
+        }
+        return p;
     }
 
     private static <T extends Expression> List<T> removeDuplicateNames(List<T> list) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceOrderByExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceOrderByExpressionWithEval.java
index 02fc98428f14a..3886982a16aba 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceOrderByExpressionWithEval.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceOrderByExpressionWithEval.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
-import org.elasticsearch.xpack.esql.core.expression.Order;
+import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.OrderBy;
@@ -18,7 +18,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.elasticsearch.xpack.esql.optimizer.rules.SubstituteSurrogates.rawTemporaryName;
+import static org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer.rawTemporaryName;
 
 public final class ReplaceOrderByExpressionWithEval extends OptimizerRules.OptimizerRule<OrderBy> {
     private static int counter = 0;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java
index 31b543cd115df..ea0a302f7131d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
+import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
@@ -33,7 +34,7 @@
  * becomes
  * stats a1 = sum(a), a2 = min(b) by x | eval a = a1 + a2 | keep a, x
  * The rule also considers expressions applied over groups:
- * stats a = x + 1 by x becomes stats by x | eval a = x + 1 | keep a, x
+ * {@code STATS a = x + 1 BY x} becomes {@code STATS BY x | EVAL a = x + 1 | KEEP a, x}
  * And to combine the two:
  * stats a = x + count(*) by x
  * becomes
@@ -105,14 +106,7 @@ protected LogicalPlan rule(Aggregate aggregate) {
                         Alias alias = rootAggs.get(canonical);
                         if (alias == null) {
                             // create synthetic alias ove the found agg function
-                            alias = new Alias(
-                                af.source(),
-                                syntheticName(canonical, child, counter[0]++),
-                                as.qualifier(),
-                                canonical,
-                                null,
-                                true
-                            );
+                            alias = new Alias(af.source(), syntheticName(canonical, child, counter[0]++), canonical, null, true);
                             // and remember it to remove duplicates
                             rootAggs.put(canonical, alias);
                             // add it to the list of aggregates and continue
@@ -149,6 +143,6 @@ protected LogicalPlan rule(Aggregate aggregate) {
     }
 
     static String syntheticName(Expression expression, Expression af, int counter) {
-        return SubstituteSurrogates.temporaryName(expression, af, counter);
+        return LogicalPlanOptimizer.temporaryName(expression, af, counter);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java
index 0979b745a6607..02b39f6babef0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java
@@ -14,9 +14,10 @@
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction;
-import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
+import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.plan.logical.Stats;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -24,15 +25,26 @@
 import java.util.Map;
 
 /**
- * Replace nested expressions inside an aggregate with synthetic eval (which end up being projected away by the aggregate).
- * stats sum(a + 1) by x % 2
+ * Replace nested expressions inside a {@link Stats} with synthetic eval.
+ * {@code STATS SUM(a + 1) BY x % 2}
  * becomes
- * eval `a + 1` = a + 1, `x % 2` = x % 2 | stats sum(`a+1`_ref) by `x % 2`_ref
+ * {@code EVAL `a + 1` = a + 1, `x % 2` = x % 2 | STATS SUM(`a+1`_ref) BY `x % 2`_ref}
+ * and
+ * {@code INLINESTATS SUM(a + 1) BY x % 2}
+ * becomes
+ * {@code EVAL `a + 1` = a + 1, `x % 2` = x % 2 | INLINESTATS SUM(`a+1`_ref) BY `x % 2`_ref}
  */
-public final class ReplaceStatsNestedExpressionWithEval extends OptimizerRules.OptimizerRule<Aggregate> {
+public final class ReplaceStatsNestedExpressionWithEval extends OptimizerRules.OptimizerRule<LogicalPlan> {
 
     @Override
-    protected LogicalPlan rule(Aggregate aggregate) {
+    protected LogicalPlan rule(LogicalPlan p) {
+        if (p instanceof Stats stats) {
+            return rule(stats);
+        }
+        return p;
+    }
+
+    private LogicalPlan rule(Stats aggregate) {
         List<Alias> evals = new ArrayList<>();
         Map<String, Attribute> evalNames = new HashMap<>();
         Map<GroupingFunction, Attribute> groupingAttributes = new HashMap<>();
@@ -102,7 +114,7 @@ protected LogicalPlan rule(Aggregate aggregate) {
                     if (field instanceof Attribute == false && field.foldable() == false) {
                         // 3. create a new alias if one doesn't exist yet no reference
                         Attribute attr = expToAttribute.computeIfAbsent(field.canonical(), k -> {
-                            Alias newAlias = new Alias(k.source(), syntheticName(k, af, counter[0]++), null, k, null, true);
+                            Alias newAlias = new Alias(k.source(), syntheticName(k, af, counter[0]++), k, null, true);
                             evals.add(newAlias);
                             return newAlias.toAttribute();
                         });
@@ -133,13 +145,13 @@ protected LogicalPlan rule(Aggregate aggregate) {
             var aggregates = aggsChanged.get() ? newAggs : aggregate.aggregates();
 
             var newEval = new Eval(aggregate.source(), aggregate.child(), evals);
-            aggregate = new Aggregate(aggregate.source(), newEval, aggregate.aggregateType(), groupings, aggregates);
+            aggregate = aggregate.with(newEval, groupings, aggregates);
         }
 
-        return aggregate;
+        return (LogicalPlan) aggregate;
     }
 
     static String syntheticName(Expression expression, AggregateFunction af, int counter) {
-        return SubstituteSurrogates.temporaryName(expression, af, counter);
+        return LogicalPlanOptimizer.temporaryName(expression, af, counter);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java
index 4ef069ea16d04..fe83aeb647bf9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java
@@ -9,10 +9,10 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.ArithmeticOperation;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.ArithmeticOperation;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.BinaryComparisonInversible;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java
index 65fa0a5f51d52..4c5a04ff8353c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java
@@ -14,11 +14,11 @@
 import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
-import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate;
+import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
@@ -80,9 +80,9 @@ protected LogicalPlan rule(Aggregate aggregate) {
                         var attr = aggFuncToAttr.get(af);
                         // the agg doesn't exist in the Aggregate, create an alias for it and save its attribute
                         if (attr == null) {
-                            var temporaryName = temporaryName(af, agg, counter[0]++);
+                            var temporaryName = LogicalPlanOptimizer.temporaryName(af, agg, counter[0]++);
                             // create a synthetic alias (so it doesn't clash with a user defined name)
-                            var newAlias = new Alias(agg.source(), temporaryName, null, af, null, true);
+                            var newAlias = new Alias(agg.source(), temporaryName, af, null, true);
                             attr = newAlias.toAttribute();
                             aggFuncToAttr.put(af, attr);
                             newAggs.add(newAlias);
@@ -92,7 +92,7 @@ protected LogicalPlan rule(Aggregate aggregate) {
                     // 4. move the expression as an eval using the original alias
                     // copy the original alias id so that other nodes using it down stream (e.g. eval referring to the original agg)
                     // don't have to updated
-                    var aliased = new Alias(agg.source(), agg.name(), null, surrogateWithRefs, agg.toAttribute().id());
+                    var aliased = new Alias(agg.source(), agg.name(), surrogateWithRefs, agg.toAttribute().id());
                     transientEval.add(aliased);
                 }
                 // the replacement is another aggregate function, so replace it in place
@@ -133,28 +133,4 @@ protected LogicalPlan rule(Aggregate aggregate) {
 
         return plan;
     }
-
-    public static String temporaryName(Expression inner, Expression outer, int suffix) {
-        String in = toString(inner);
-        String out = toString(outer);
-        return rawTemporaryName(in, out, String.valueOf(suffix));
-    }
-
-    public static String rawTemporaryName(String inner, String outer, String suffix) {
-        return FieldAttribute.SYNTHETIC_ATTRIBUTE_NAME_PREFIX + inner + "$" + outer + "$" + suffix;
-    }
-
-    static int TO_STRING_LIMIT = 16;
-
-    static String toString(Expression ex) {
-        return ex instanceof AggregateFunction af ? af.functionName() : extractString(ex);
-    }
-
-    static String extractString(Expression ex) {
-        return ex instanceof NamedExpression ne ? ne.name() : limitToString(ex.sourceText()).replace(' ', '_');
-    }
-
-    static String limitToString(String string) {
-        return string.length() > 16 ? string.substring(0, TO_STRING_LIMIT - 1) + ">" : string;
-    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/TranslateMetricsAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/TranslateMetricsAggregate.java
index 10c7a7325debc..f2926fe92ca3f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/TranslateMetricsAggregate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/TranslateMetricsAggregate.java
@@ -147,12 +147,12 @@ LogicalPlan translate(Aggregate metrics) {
                     return rateAgg.toAttribute();
                 });
                 if (changed.get()) {
-                    secondPassAggs.add(new Alias(alias.source(), alias.name(), null, outerAgg, agg.id()));
+                    secondPassAggs.add(new Alias(alias.source(), alias.name(), outerAgg, agg.id()));
                 } else {
                     var toPartial = new Alias(agg.source(), alias.name(), new ToPartial(agg.source(), af.field(), af));
                     var fromPartial = new FromPartial(agg.source(), toPartial.toAttribute(), af);
                     firstPassAggs.add(toPartial);
-                    secondPassAggs.add(new Alias(alias.source(), alias.name(), null, fromPartial, alias.id()));
+                    secondPassAggs.add(new Alias(alias.source(), alias.name(), fromPartial, alias.id()));
                 }
             }
         }
@@ -200,10 +200,10 @@ LogicalPlan translate(Aggregate metrics) {
                 newFinalGroup = timeBucket.toAttribute();
                 firstPassGroupings.add(newFinalGroup);
             } else {
-                newFinalGroup = new Alias(g.source(), g.name(), null, new Values(g.source(), g), g.id());
+                newFinalGroup = new Alias(g.source(), g.name(), new Values(g.source(), g), g.id());
                 firstPassAggs.add(newFinalGroup);
             }
-            secondPassGroupings.add(new Alias(g.source(), g.name(), null, newFinalGroup.toAttribute(), g.id()));
+            secondPassGroupings.add(new Alias(g.source(), g.name(), newFinalGroup.toAttribute(), g.id()));
         }
         return newAggregate(
             newAggregate(metrics.child(), Aggregate.AggregateType.METRICS, firstPassAggs, firstPassGroupings),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java
index b40531ea9359c..17d317bedbb6f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java
@@ -121,6 +121,8 @@
  *         function implementations.</li>
  *     <li>{@link org.elasticsearch.xpack.esql.action.RestEsqlQueryAction Sync} and
  *         {@link org.elasticsearch.xpack.esql.action.RestEsqlAsyncQueryAction async} HTTP API entry points</li>
+ *     <li>{@link org.elasticsearch.xpack.esql.plan.logical.Phased} - Marks a {@link org.elasticsearch.xpack.esql.plan.logical.LogicalPlan}
+ *         node as requiring multiple ESQL executions to run. </li>
  * </ul>
  *
  * <h3>Query Planner</h3>
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/AbstractBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/AbstractBuilder.java
index ce8c743106411..f0723c810c6b9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/AbstractBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/AbstractBuilder.java
@@ -9,7 +9,6 @@
 
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.TerminalNode;
-import org.elasticsearch.xpack.esql.core.parser.ParserUtils;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
 abstract class AbstractBuilder extends EsqlBaseParserBaseVisitor<Object> {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/parser/CaseChangingCharStream.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/CaseChangingCharStream.java
similarity index 97%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/parser/CaseChangingCharStream.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/CaseChangingCharStream.java
index 6248004d73dac..cf9f727b240ce 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/parser/CaseChangingCharStream.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/CaseChangingCharStream.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core.parser;
+package org.elasticsearch.xpack.esql.parser;
 
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.misc.Interval;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/DelegatingTokenSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/DelegatingTokenSource.java
new file mode 100644
index 0000000000000..2ab4bc9d9fcc0
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/DelegatingTokenSource.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.parser;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenFactory;
+import org.antlr.v4.runtime.TokenSource;
+
+/**
+ * Utility class for filtering/processing TokenSource.
+ */
+abstract class DelegatingTokenSource implements TokenSource {
+    final TokenSource delegate;
+
+    DelegatingTokenSource(TokenSource delegate) {
+        this.delegate = delegate;
+    }
+
+    @Override
+    public Token nextToken() {
+        return delegate.nextToken();
+    }
+
+    @Override
+    public int getLine() {
+        return delegate.getLine();
+    }
+
+    @Override
+    public int getCharPositionInLine() {
+        return delegate.getCharPositionInLine();
+    }
+
+    @Override
+    public CharStream getInputStream() {
+        return delegate.getInputStream();
+    }
+
+    @Override
+    public String getSourceName() {
+        return delegate.getSourceName();
+    }
+
+    @Override
+    public void setTokenFactory(TokenFactory<?> tokenFactory) {
+        delegate.setTokenFactory(tokenFactory);
+    }
+
+    @Override
+    public TokenFactory<?> getTokenFactory() {
+        return delegate.getTokenFactory();
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp
index f5484f3e7070f..8122a56884280 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp
@@ -7,12 +7,9 @@ null
 'explain'
 'from'
 'grok'
-'inlinestats'
 'keep'
 'limit'
-'lookup'
 'meta'
-'metrics'
 'mv_expand'
 'rename'
 'row'
@@ -42,11 +39,11 @@ null
 '.'
 'false'
 'first'
-'last'
-'('
 'in'
 'is'
+'last'
 'like'
+'('
 'not'
 'null'
 'nulls'
@@ -75,6 +72,9 @@ null
 null
 null
 null
+null
+null
+null
 'metadata'
 null
 null
@@ -83,6 +83,7 @@ null
 null
 null
 null
+null
 'as'
 null
 null
@@ -99,21 +100,21 @@ null
 null
 null
 null
+'info'
 null
 null
 null
+'functions'
 null
 null
 null
-'info'
+':'
 null
 null
 null
-'functions'
 null
 null
 null
-':'
 null
 null
 null
@@ -134,12 +135,9 @@ EVAL
 EXPLAIN
 FROM
 GROK
-INLINESTATS
 KEEP
 LIMIT
-LOOKUP
 META
-METRICS
 MV_EXPAND
 RENAME
 ROW
@@ -147,14 +145,14 @@ SHOW
 SORT
 STATS
 WHERE
+DEV_INLINESTATS
+DEV_LOOKUP
+DEV_MATCH
+DEV_METRICS
 UNKNOWN_CMD
 LINE_COMMENT
 MULTILINE_COMMENT
 WS
-UNQUOTED_SOURCE
-EXPLAIN_WS
-EXPLAIN_LINE_COMMENT
-EXPLAIN_MULTILINE_COMMENT
 PIPE
 QUOTED_STRING
 INTEGER_LITERAL
@@ -169,11 +167,11 @@ DESC
 DOT
 FALSE
 FIRST
-LAST
-LP
 IN
 IS
+LAST
 LIKE
+LP
 NOT
 NULL
 NULLS
@@ -202,7 +200,11 @@ QUOTED_IDENTIFIER
 EXPR_LINE_COMMENT
 EXPR_MULTILINE_COMMENT
 EXPR_WS
+EXPLAIN_WS
+EXPLAIN_LINE_COMMENT
+EXPLAIN_MULTILINE_COMMENT
 METADATA
+UNQUOTED_SOURCE
 FROM_LINE_COMMENT
 FROM_MULTILINE_COMMENT
 FROM_WS
@@ -223,12 +225,6 @@ ENRICH_WS
 ENRICH_FIELD_LINE_COMMENT
 ENRICH_FIELD_MULTILINE_COMMENT
 ENRICH_FIELD_WS
-LOOKUP_LINE_COMMENT
-LOOKUP_MULTILINE_COMMENT
-LOOKUP_WS
-LOOKUP_FIELD_LINE_COMMENT
-LOOKUP_FIELD_MULTILINE_COMMENT
-LOOKUP_FIELD_WS
 MVEXPAND_LINE_COMMENT
 MVEXPAND_MULTILINE_COMMENT
 MVEXPAND_WS
@@ -245,6 +241,12 @@ SETTING
 SETTING_LINE_COMMENT
 SETTTING_MULTILINE_COMMENT
 SETTING_WS
+LOOKUP_LINE_COMMENT
+LOOKUP_MULTILINE_COMMENT
+LOOKUP_WS
+LOOKUP_FIELD_LINE_COMMENT
+LOOKUP_FIELD_MULTILINE_COMMENT
+LOOKUP_FIELD_WS
 METRICS_LINE_COMMENT
 METRICS_MULTILINE_COMMENT
 METRICS_WS
@@ -260,12 +262,9 @@ EVAL
 EXPLAIN
 FROM
 GROK
-INLINESTATS
 KEEP
 LIMIT
-LOOKUP
 META
-METRICS
 MV_EXPAND
 RENAME
 ROW
@@ -273,17 +272,14 @@ SHOW
 SORT
 STATS
 WHERE
+DEV_INLINESTATS
+DEV_LOOKUP
+DEV_MATCH
+DEV_METRICS
 UNKNOWN_CMD
 LINE_COMMENT
 MULTILINE_COMMENT
 WS
-UNQUOTED_SOURCE_PART
-UNQUOTED_SOURCE
-EXPLAIN_OPENING_BRACKET
-EXPLAIN_PIPE
-EXPLAIN_WS
-EXPLAIN_LINE_COMMENT
-EXPLAIN_MULTILINE_COMMENT
 PIPE
 DIGIT
 LETTER
@@ -308,11 +304,11 @@ DESC
 DOT
 FALSE
 FIRST
-LAST
-LP
 IN
 IS
+LAST
 LIKE
+LP
 NOT
 NULL
 NULLS
@@ -333,6 +329,7 @@ MINUS
 ASTERISK
 SLASH
 PERCENT
+DEV_MATCH_OP
 NAMED_OR_POSITIONAL_PARAM
 OPENING_BRACKET
 CLOSING_BRACKET
@@ -342,6 +339,11 @@ QUOTED_IDENTIFIER
 EXPR_LINE_COMMENT
 EXPR_MULTILINE_COMMENT
 EXPR_WS
+EXPLAIN_OPENING_BRACKET
+EXPLAIN_PIPE
+EXPLAIN_WS
+EXPLAIN_LINE_COMMENT
+EXPLAIN_MULTILINE_COMMENT
 FROM_PIPE
 FROM_OPENING_BRACKET
 FROM_CLOSING_BRACKET
@@ -349,6 +351,8 @@ FROM_COLON
 FROM_COMMA
 FROM_ASSIGN
 METADATA
+UNQUOTED_SOURCE_PART
+UNQUOTED_SOURCE
 FROM_UNQUOTED_SOURCE
 FROM_QUOTED_SOURCE
 FROM_LINE_COMMENT
@@ -392,23 +396,6 @@ ENRICH_FIELD_QUOTED_IDENTIFIER
 ENRICH_FIELD_LINE_COMMENT
 ENRICH_FIELD_MULTILINE_COMMENT
 ENRICH_FIELD_WS
-LOOKUP_PIPE
-LOOKUP_COLON
-LOOKUP_COMMA
-LOOKUP_DOT
-LOOKUP_ON
-LOOKUP_UNQUOTED_SOURCE
-LOOKUP_QUOTED_SOURCE
-LOOKUP_LINE_COMMENT
-LOOKUP_MULTILINE_COMMENT
-LOOKUP_WS
-LOOKUP_FIELD_PIPE
-LOOKUP_FIELD_COMMA
-LOOKUP_FIELD_DOT
-LOOKUP_FIELD_ID_PATTERN
-LOOKUP_FIELD_LINE_COMMENT
-LOOKUP_FIELD_MULTILINE_COMMENT
-LOOKUP_FIELD_WS
 MVEXPAND_PIPE
 MVEXPAND_DOT
 MVEXPAND_QUOTED_IDENTIFIER
@@ -432,6 +419,23 @@ SETTING
 SETTING_LINE_COMMENT
 SETTTING_MULTILINE_COMMENT
 SETTING_WS
+LOOKUP_PIPE
+LOOKUP_COLON
+LOOKUP_COMMA
+LOOKUP_DOT
+LOOKUP_ON
+LOOKUP_UNQUOTED_SOURCE
+LOOKUP_QUOTED_SOURCE
+LOOKUP_LINE_COMMENT
+LOOKUP_MULTILINE_COMMENT
+LOOKUP_WS
+LOOKUP_FIELD_PIPE
+LOOKUP_FIELD_COMMA
+LOOKUP_FIELD_DOT
+LOOKUP_FIELD_ID_PATTERN
+LOOKUP_FIELD_LINE_COMMENT
+LOOKUP_FIELD_MULTILINE_COMMENT
+LOOKUP_FIELD_WS
 METRICS_PIPE
 METRICS_UNQUOTED_SOURCE
 METRICS_QUOTED_SOURCE
@@ -454,21 +458,21 @@ HIDDEN
 
 mode names:
 DEFAULT_MODE
-EXPLAIN_MODE
 EXPRESSION_MODE
+EXPLAIN_MODE
 FROM_MODE
 PROJECT_MODE
 RENAME_MODE
 ENRICH_MODE
 ENRICH_FIELD_MODE
-LOOKUP_MODE
-LOOKUP_FIELD_MODE
 MVEXPAND_MODE
 SHOW_MODE
 META_MODE
 SETTING_MODE
+LOOKUP_MODE
+LOOKUP_FIELD_MODE
 METRICS_MODE
 CLOSING_METRICS_MODE
 
 atn:
-[4, 0, 124, 1450, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 4, 20, 575, 8, 20, 11, 20, 12, 20, 576, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 585, 8, 21, 10, 21, 12, 21, 588, 9, 21, 1, 21, 3, 21, 591, 8, 21, 1, 21, 3, 21, 594, 8, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 603, 8, 22, 10, 22, 12, 22, 606, 9, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 4, 23, 614, 8, 23, 11, 23, 12, 23, 615, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 3, 24, 623, 8, 24, 1, 25, 4, 25, 626, 8, 25, 11, 25, 12, 25, 627, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 28, 1, 28, 1, 29, 1, 29, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 1, 35, 1, 35, 1, 36, 1, 36, 3, 36, 667, 8, 36, 1, 36, 4, 36, 670, 8, 36, 11, 36, 12, 36, 671, 1, 37, 1, 37, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 3, 39, 681, 8, 39, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 3, 41, 688, 8, 41, 1, 42, 1, 42, 1, 42, 5, 42, 693, 8, 42, 10, 42, 12, 42, 696, 9, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 704, 8, 42, 10, 42, 12, 42, 707, 9, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 3, 42, 714, 8, 42, 1, 42, 3, 42, 717, 8, 42, 3, 42, 719, 8, 42, 1, 43, 4, 43, 722, 8, 43, 11, 43, 12, 43, 723, 1, 44, 4, 44, 727, 8, 44, 11, 44, 12, 44, 728, 1, 44, 1, 44, 5, 44, 733, 8, 44, 10, 44, 12, 44, 736, 9, 44, 1, 44, 1, 44, 4, 44, 740, 8, 44, 11, 44, 12, 44, 741, 1, 44, 4, 44, 745, 8, 44, 11, 44, 12, 44, 746, 1, 44, 1, 44, 5, 44, 751, 8, 44, 10, 44, 12, 44, 754, 9, 44, 3, 44, 756, 8, 44, 1, 44, 1, 44, 1, 44, 1, 44, 4, 44, 762, 8, 44, 11, 44, 12, 44, 763, 1, 44, 1, 44, 3, 44, 768, 8, 44, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 69, 1, 70, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 76, 1, 76, 1, 77, 1, 77, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 5, 80, 890, 8, 80, 10, 80, 12, 80, 893, 9, 80, 1, 80, 1, 80, 4, 80, 897, 8, 80, 11, 80, 12, 80, 898, 3, 80, 901, 8, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 5, 83, 915, 8, 83, 10, 83, 12, 83, 918, 9, 83, 1, 83, 1, 83, 3, 83, 922, 8, 83, 1, 83, 4, 83, 925, 8, 83, 11, 83, 12, 83, 926, 3, 83, 929, 8, 83, 1, 84, 1, 84, 4, 84, 933, 8, 84, 11, 84, 12, 84, 934, 1, 84, 1, 84, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 3, 104, 1024, 8, 104, 1, 105, 1, 105, 3, 105, 1028, 8, 105, 1, 105, 5, 105, 1031, 8, 105, 10, 105, 12, 105, 1034, 9, 105, 1, 105, 1, 105, 3, 105, 1038, 8, 105, 1, 105, 4, 105, 1041, 8, 105, 11, 105, 12, 105, 1042, 3, 105, 1045, 8, 105, 1, 106, 1, 106, 4, 106, 1049, 8, 106, 11, 106, 12, 106, 1050, 1, 107, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 124, 4, 124, 1126, 8, 124, 11, 124, 12, 124, 1127, 1, 124, 1, 124, 3, 124, 1132, 8, 124, 1, 124, 4, 124, 1135, 8, 124, 11, 124, 12, 124, 1136, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 175, 4, 175, 1359, 8, 175, 11, 175, 12, 175, 1360, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 2, 604, 705, 0, 194, 16, 1, 18, 2, 20, 3, 22, 4, 24, 5, 26, 6, 28, 7, 30, 8, 32, 9, 34, 10, 36, 11, 38, 12, 40, 13, 42, 14, 44, 15, 46, 16, 48, 17, 50, 18, 52, 19, 54, 20, 56, 21, 58, 22, 60, 23, 62, 24, 64, 0, 66, 25, 68, 0, 70, 0, 72, 26, 74, 27, 76, 28, 78, 29, 80, 0, 82, 0, 84, 0, 86, 0, 88, 0, 90, 0, 92, 0, 94, 0, 96, 0, 98, 0, 100, 30, 102, 31, 104, 32, 106, 33, 108, 34, 110, 35, 112, 36, 114, 37, 116, 38, 118, 39, 120, 40, 122, 41, 124, 42, 126, 43, 128, 44, 130, 45, 132, 46, 134, 47, 136, 48, 138, 49, 140, 50, 142, 51, 144, 52, 146, 53, 148, 54, 150, 55, 152, 56, 154, 57, 156, 58, 158, 59, 160, 60, 162, 61, 164, 62, 166, 63, 168, 64, 170, 65, 172, 66, 174, 67, 176, 68, 178, 69, 180, 70, 182, 71, 184, 0, 186, 72, 188, 73, 190, 74, 192, 75, 194, 0, 196, 0, 198, 0, 200, 0, 202, 0, 204, 0, 206, 76, 208, 0, 210, 0, 212, 77, 214, 78, 216, 79, 218, 0, 220, 0, 222, 0, 224, 0, 226, 0, 228, 80, 230, 81, 232, 82, 234, 83, 236, 0, 238, 0, 240, 0, 242, 0, 244, 84, 246, 0, 248, 85, 250, 86, 252, 87, 254, 0, 256, 0, 258, 88, 260, 89, 262, 0, 264, 90, 266, 0, 268, 91, 270, 92, 272, 93, 274, 0, 276, 0, 278, 0, 280, 0, 282, 0, 284, 0, 286, 0, 288, 94, 290, 95, 292, 96, 294, 0, 296, 0, 298, 0, 300, 0, 302, 0, 304, 0, 306, 0, 308, 97, 310, 98, 312, 99, 314, 0, 316, 0, 318, 0, 320, 0, 322, 100, 324, 101, 326, 102, 328, 0, 330, 0, 332, 0, 334, 0, 336, 103, 338, 104, 340, 105, 342, 0, 344, 106, 346, 107, 348, 108, 350, 109, 352, 0, 354, 110, 356, 111, 358, 112, 360, 113, 362, 0, 364, 114, 366, 115, 368, 116, 370, 117, 372, 118, 374, 0, 376, 0, 378, 0, 380, 119, 382, 120, 384, 121, 386, 0, 388, 0, 390, 122, 392, 123, 394, 124, 396, 0, 398, 0, 400, 0, 402, 0, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 13, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 5, 0, 34, 34, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 69, 69, 101, 101, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1476, 0, 16, 1, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 50, 1, 0, 0, 0, 0, 52, 1, 0, 0, 0, 0, 54, 1, 0, 0, 0, 0, 56, 1, 0, 0, 0, 0, 58, 1, 0, 0, 0, 0, 60, 1, 0, 0, 0, 0, 62, 1, 0, 0, 0, 0, 66, 1, 0, 0, 0, 1, 68, 1, 0, 0, 0, 1, 70, 1, 0, 0, 0, 1, 72, 1, 0, 0, 0, 1, 74, 1, 0, 0, 0, 1, 76, 1, 0, 0, 0, 2, 78, 1, 0, 0, 0, 2, 100, 1, 0, 0, 0, 2, 102, 1, 0, 0, 0, 2, 104, 1, 0, 0, 0, 2, 106, 1, 0, 0, 0, 2, 108, 1, 0, 0, 0, 2, 110, 1, 0, 0, 0, 2, 112, 1, 0, 0, 0, 2, 114, 1, 0, 0, 0, 2, 116, 1, 0, 0, 0, 2, 118, 1, 0, 0, 0, 2, 120, 1, 0, 0, 0, 2, 122, 1, 0, 0, 0, 2, 124, 1, 0, 0, 0, 2, 126, 1, 0, 0, 0, 2, 128, 1, 0, 0, 0, 2, 130, 1, 0, 0, 0, 2, 132, 1, 0, 0, 0, 2, 134, 1, 0, 0, 0, 2, 136, 1, 0, 0, 0, 2, 138, 1, 0, 0, 0, 2, 140, 1, 0, 0, 0, 2, 142, 1, 0, 0, 0, 2, 144, 1, 0, 0, 0, 2, 146, 1, 0, 0, 0, 2, 148, 1, 0, 0, 0, 2, 150, 1, 0, 0, 0, 2, 152, 1, 0, 0, 0, 2, 154, 1, 0, 0, 0, 2, 156, 1, 0, 0, 0, 2, 158, 1, 0, 0, 0, 2, 160, 1, 0, 0, 0, 2, 162, 1, 0, 0, 0, 2, 164, 1, 0, 0, 0, 2, 166, 1, 0, 0, 0, 2, 168, 1, 0, 0, 0, 2, 170, 1, 0, 0, 0, 2, 172, 1, 0, 0, 0, 2, 174, 1, 0, 0, 0, 2, 176, 1, 0, 0, 0, 2, 178, 1, 0, 0, 0, 2, 180, 1, 0, 0, 0, 2, 182, 1, 0, 0, 0, 2, 186, 1, 0, 0, 0, 2, 188, 1, 0, 0, 0, 2, 190, 1, 0, 0, 0, 2, 192, 1, 0, 0, 0, 3, 194, 1, 0, 0, 0, 3, 196, 1, 0, 0, 0, 3, 198, 1, 0, 0, 0, 3, 200, 1, 0, 0, 0, 3, 202, 1, 0, 0, 0, 3, 204, 1, 0, 0, 0, 3, 206, 1, 0, 0, 0, 3, 208, 1, 0, 0, 0, 3, 210, 1, 0, 0, 0, 3, 212, 1, 0, 0, 0, 3, 214, 1, 0, 0, 0, 3, 216, 1, 0, 0, 0, 4, 218, 1, 0, 0, 0, 4, 220, 1, 0, 0, 0, 4, 222, 1, 0, 0, 0, 4, 228, 1, 0, 0, 0, 4, 230, 1, 0, 0, 0, 4, 232, 1, 0, 0, 0, 4, 234, 1, 0, 0, 0, 5, 236, 1, 0, 0, 0, 5, 238, 1, 0, 0, 0, 5, 240, 1, 0, 0, 0, 5, 242, 1, 0, 0, 0, 5, 244, 1, 0, 0, 0, 5, 246, 1, 0, 0, 0, 5, 248, 1, 0, 0, 0, 5, 250, 1, 0, 0, 0, 5, 252, 1, 0, 0, 0, 6, 254, 1, 0, 0, 0, 6, 256, 1, 0, 0, 0, 6, 258, 1, 0, 0, 0, 6, 260, 1, 0, 0, 0, 6, 264, 1, 0, 0, 0, 6, 266, 1, 0, 0, 0, 6, 268, 1, 0, 0, 0, 6, 270, 1, 0, 0, 0, 6, 272, 1, 0, 0, 0, 7, 274, 1, 0, 0, 0, 7, 276, 1, 0, 0, 0, 7, 278, 1, 0, 0, 0, 7, 280, 1, 0, 0, 0, 7, 282, 1, 0, 0, 0, 7, 284, 1, 0, 0, 0, 7, 286, 1, 0, 0, 0, 7, 288, 1, 0, 0, 0, 7, 290, 1, 0, 0, 0, 7, 292, 1, 0, 0, 0, 8, 294, 1, 0, 0, 0, 8, 296, 1, 0, 0, 0, 8, 298, 1, 0, 0, 0, 8, 300, 1, 0, 0, 0, 8, 302, 1, 0, 0, 0, 8, 304, 1, 0, 0, 0, 8, 306, 1, 0, 0, 0, 8, 308, 1, 0, 0, 0, 8, 310, 1, 0, 0, 0, 8, 312, 1, 0, 0, 0, 9, 314, 1, 0, 0, 0, 9, 316, 1, 0, 0, 0, 9, 318, 1, 0, 0, 0, 9, 320, 1, 0, 0, 0, 9, 322, 1, 0, 0, 0, 9, 324, 1, 0, 0, 0, 9, 326, 1, 0, 0, 0, 10, 328, 1, 0, 0, 0, 10, 330, 1, 0, 0, 0, 10, 332, 1, 0, 0, 0, 10, 334, 1, 0, 0, 0, 10, 336, 1, 0, 0, 0, 10, 338, 1, 0, 0, 0, 10, 340, 1, 0, 0, 0, 11, 342, 1, 0, 0, 0, 11, 344, 1, 0, 0, 0, 11, 346, 1, 0, 0, 0, 11, 348, 1, 0, 0, 0, 11, 350, 1, 0, 0, 0, 12, 352, 1, 0, 0, 0, 12, 354, 1, 0, 0, 0, 12, 356, 1, 0, 0, 0, 12, 358, 1, 0, 0, 0, 12, 360, 1, 0, 0, 0, 13, 362, 1, 0, 0, 0, 13, 364, 1, 0, 0, 0, 13, 366, 1, 0, 0, 0, 13, 368, 1, 0, 0, 0, 13, 370, 1, 0, 0, 0, 13, 372, 1, 0, 0, 0, 14, 374, 1, 0, 0, 0, 14, 376, 1, 0, 0, 0, 14, 378, 1, 0, 0, 0, 14, 380, 1, 0, 0, 0, 14, 382, 1, 0, 0, 0, 14, 384, 1, 0, 0, 0, 15, 386, 1, 0, 0, 0, 15, 388, 1, 0, 0, 0, 15, 390, 1, 0, 0, 0, 15, 392, 1, 0, 0, 0, 15, 394, 1, 0, 0, 0, 15, 396, 1, 0, 0, 0, 15, 398, 1, 0, 0, 0, 15, 400, 1, 0, 0, 0, 15, 402, 1, 0, 0, 0, 16, 404, 1, 0, 0, 0, 18, 414, 1, 0, 0, 0, 20, 421, 1, 0, 0, 0, 22, 430, 1, 0, 0, 0, 24, 437, 1, 0, 0, 0, 26, 447, 1, 0, 0, 0, 28, 454, 1, 0, 0, 0, 30, 461, 1, 0, 0, 0, 32, 475, 1, 0, 0, 0, 34, 482, 1, 0, 0, 0, 36, 490, 1, 0, 0, 0, 38, 499, 1, 0, 0, 0, 40, 506, 1, 0, 0, 0, 42, 516, 1, 0, 0, 0, 44, 528, 1, 0, 0, 0, 46, 537, 1, 0, 0, 0, 48, 543, 1, 0, 0, 0, 50, 550, 1, 0, 0, 0, 52, 557, 1, 0, 0, 0, 54, 565, 1, 0, 0, 0, 56, 574, 1, 0, 0, 0, 58, 580, 1, 0, 0, 0, 60, 597, 1, 0, 0, 0, 62, 613, 1, 0, 0, 0, 64, 622, 1, 0, 0, 0, 66, 625, 1, 0, 0, 0, 68, 629, 1, 0, 0, 0, 70, 634, 1, 0, 0, 0, 72, 639, 1, 0, 0, 0, 74, 643, 1, 0, 0, 0, 76, 647, 1, 0, 0, 0, 78, 651, 1, 0, 0, 0, 80, 655, 1, 0, 0, 0, 82, 657, 1, 0, 0, 0, 84, 659, 1, 0, 0, 0, 86, 662, 1, 0, 0, 0, 88, 664, 1, 0, 0, 0, 90, 673, 1, 0, 0, 0, 92, 675, 1, 0, 0, 0, 94, 680, 1, 0, 0, 0, 96, 682, 1, 0, 0, 0, 98, 687, 1, 0, 0, 0, 100, 718, 1, 0, 0, 0, 102, 721, 1, 0, 0, 0, 104, 767, 1, 0, 0, 0, 106, 769, 1, 0, 0, 0, 108, 772, 1, 0, 0, 0, 110, 776, 1, 0, 0, 0, 112, 780, 1, 0, 0, 0, 114, 782, 1, 0, 0, 0, 116, 785, 1, 0, 0, 0, 118, 787, 1, 0, 0, 0, 120, 792, 1, 0, 0, 0, 122, 794, 1, 0, 0, 0, 124, 800, 1, 0, 0, 0, 126, 806, 1, 0, 0, 0, 128, 811, 1, 0, 0, 0, 130, 813, 1, 0, 0, 0, 132, 816, 1, 0, 0, 0, 134, 819, 1, 0, 0, 0, 136, 824, 1, 0, 0, 0, 138, 828, 1, 0, 0, 0, 140, 833, 1, 0, 0, 0, 142, 839, 1, 0, 0, 0, 144, 842, 1, 0, 0, 0, 146, 844, 1, 0, 0, 0, 148, 850, 1, 0, 0, 0, 150, 852, 1, 0, 0, 0, 152, 857, 1, 0, 0, 0, 154, 860, 1, 0, 0, 0, 156, 863, 1, 0, 0, 0, 158, 866, 1, 0, 0, 0, 160, 868, 1, 0, 0, 0, 162, 871, 1, 0, 0, 0, 164, 873, 1, 0, 0, 0, 166, 876, 1, 0, 0, 0, 168, 878, 1, 0, 0, 0, 170, 880, 1, 0, 0, 0, 172, 882, 1, 0, 0, 0, 174, 884, 1, 0, 0, 0, 176, 900, 1, 0, 0, 0, 178, 902, 1, 0, 0, 0, 180, 907, 1, 0, 0, 0, 182, 928, 1, 0, 0, 0, 184, 930, 1, 0, 0, 0, 186, 938, 1, 0, 0, 0, 188, 940, 1, 0, 0, 0, 190, 944, 1, 0, 0, 0, 192, 948, 1, 0, 0, 0, 194, 952, 1, 0, 0, 0, 196, 957, 1, 0, 0, 0, 198, 961, 1, 0, 0, 0, 200, 965, 1, 0, 0, 0, 202, 969, 1, 0, 0, 0, 204, 973, 1, 0, 0, 0, 206, 977, 1, 0, 0, 0, 208, 986, 1, 0, 0, 0, 210, 990, 1, 0, 0, 0, 212, 994, 1, 0, 0, 0, 214, 998, 1, 0, 0, 0, 216, 1002, 1, 0, 0, 0, 218, 1006, 1, 0, 0, 0, 220, 1011, 1, 0, 0, 0, 222, 1015, 1, 0, 0, 0, 224, 1023, 1, 0, 0, 0, 226, 1044, 1, 0, 0, 0, 228, 1048, 1, 0, 0, 0, 230, 1052, 1, 0, 0, 0, 232, 1056, 1, 0, 0, 0, 234, 1060, 1, 0, 0, 0, 236, 1064, 1, 0, 0, 0, 238, 1069, 1, 0, 0, 0, 240, 1073, 1, 0, 0, 0, 242, 1077, 1, 0, 0, 0, 244, 1081, 1, 0, 0, 0, 246, 1084, 1, 0, 0, 0, 248, 1088, 1, 0, 0, 0, 250, 1092, 1, 0, 0, 0, 252, 1096, 1, 0, 0, 0, 254, 1100, 1, 0, 0, 0, 256, 1105, 1, 0, 0, 0, 258, 1110, 1, 0, 0, 0, 260, 1115, 1, 0, 0, 0, 262, 1122, 1, 0, 0, 0, 264, 1131, 1, 0, 0, 0, 266, 1138, 1, 0, 0, 0, 268, 1142, 1, 0, 0, 0, 270, 1146, 1, 0, 0, 0, 272, 1150, 1, 0, 0, 0, 274, 1154, 1, 0, 0, 0, 276, 1160, 1, 0, 0, 0, 278, 1164, 1, 0, 0, 0, 280, 1168, 1, 0, 0, 0, 282, 1172, 1, 0, 0, 0, 284, 1176, 1, 0, 0, 0, 286, 1180, 1, 0, 0, 0, 288, 1184, 1, 0, 0, 0, 290, 1188, 1, 0, 0, 0, 292, 1192, 1, 0, 0, 0, 294, 1196, 1, 0, 0, 0, 296, 1201, 1, 0, 0, 0, 298, 1205, 1, 0, 0, 0, 300, 1209, 1, 0, 0, 0, 302, 1213, 1, 0, 0, 0, 304, 1218, 1, 0, 0, 0, 306, 1222, 1, 0, 0, 0, 308, 1226, 1, 0, 0, 0, 310, 1230, 1, 0, 0, 0, 312, 1234, 1, 0, 0, 0, 314, 1238, 1, 0, 0, 0, 316, 1244, 1, 0, 0, 0, 318, 1248, 1, 0, 0, 0, 320, 1252, 1, 0, 0, 0, 322, 1256, 1, 0, 0, 0, 324, 1260, 1, 0, 0, 0, 326, 1264, 1, 0, 0, 0, 328, 1268, 1, 0, 0, 0, 330, 1273, 1, 0, 0, 0, 332, 1277, 1, 0, 0, 0, 334, 1281, 1, 0, 0, 0, 336, 1285, 1, 0, 0, 0, 338, 1289, 1, 0, 0, 0, 340, 1293, 1, 0, 0, 0, 342, 1297, 1, 0, 0, 0, 344, 1302, 1, 0, 0, 0, 346, 1307, 1, 0, 0, 0, 348, 1311, 1, 0, 0, 0, 350, 1315, 1, 0, 0, 0, 352, 1319, 1, 0, 0, 0, 354, 1324, 1, 0, 0, 0, 356, 1334, 1, 0, 0, 0, 358, 1338, 1, 0, 0, 0, 360, 1342, 1, 0, 0, 0, 362, 1346, 1, 0, 0, 0, 364, 1351, 1, 0, 0, 0, 366, 1358, 1, 0, 0, 0, 368, 1362, 1, 0, 0, 0, 370, 1366, 1, 0, 0, 0, 372, 1370, 1, 0, 0, 0, 374, 1374, 1, 0, 0, 0, 376, 1379, 1, 0, 0, 0, 378, 1385, 1, 0, 0, 0, 380, 1391, 1, 0, 0, 0, 382, 1395, 1, 0, 0, 0, 384, 1399, 1, 0, 0, 0, 386, 1403, 1, 0, 0, 0, 388, 1409, 1, 0, 0, 0, 390, 1415, 1, 0, 0, 0, 392, 1419, 1, 0, 0, 0, 394, 1423, 1, 0, 0, 0, 396, 1427, 1, 0, 0, 0, 398, 1433, 1, 0, 0, 0, 400, 1439, 1, 0, 0, 0, 402, 1445, 1, 0, 0, 0, 404, 405, 5, 100, 0, 0, 405, 406, 5, 105, 0, 0, 406, 407, 5, 115, 0, 0, 407, 408, 5, 115, 0, 0, 408, 409, 5, 101, 0, 0, 409, 410, 5, 99, 0, 0, 410, 411, 5, 116, 0, 0, 411, 412, 1, 0, 0, 0, 412, 413, 6, 0, 0, 0, 413, 17, 1, 0, 0, 0, 414, 415, 5, 100, 0, 0, 415, 416, 5, 114, 0, 0, 416, 417, 5, 111, 0, 0, 417, 418, 5, 112, 0, 0, 418, 419, 1, 0, 0, 0, 419, 420, 6, 1, 1, 0, 420, 19, 1, 0, 0, 0, 421, 422, 5, 101, 0, 0, 422, 423, 5, 110, 0, 0, 423, 424, 5, 114, 0, 0, 424, 425, 5, 105, 0, 0, 425, 426, 5, 99, 0, 0, 426, 427, 5, 104, 0, 0, 427, 428, 1, 0, 0, 0, 428, 429, 6, 2, 2, 0, 429, 21, 1, 0, 0, 0, 430, 431, 5, 101, 0, 0, 431, 432, 5, 118, 0, 0, 432, 433, 5, 97, 0, 0, 433, 434, 5, 108, 0, 0, 434, 435, 1, 0, 0, 0, 435, 436, 6, 3, 0, 0, 436, 23, 1, 0, 0, 0, 437, 438, 5, 101, 0, 0, 438, 439, 5, 120, 0, 0, 439, 440, 5, 112, 0, 0, 440, 441, 5, 108, 0, 0, 441, 442, 5, 97, 0, 0, 442, 443, 5, 105, 0, 0, 443, 444, 5, 110, 0, 0, 444, 445, 1, 0, 0, 0, 445, 446, 6, 4, 3, 0, 446, 25, 1, 0, 0, 0, 447, 448, 5, 102, 0, 0, 448, 449, 5, 114, 0, 0, 449, 450, 5, 111, 0, 0, 450, 451, 5, 109, 0, 0, 451, 452, 1, 0, 0, 0, 452, 453, 6, 5, 4, 0, 453, 27, 1, 0, 0, 0, 454, 455, 5, 103, 0, 0, 455, 456, 5, 114, 0, 0, 456, 457, 5, 111, 0, 0, 457, 458, 5, 107, 0, 0, 458, 459, 1, 0, 0, 0, 459, 460, 6, 6, 0, 0, 460, 29, 1, 0, 0, 0, 461, 462, 5, 105, 0, 0, 462, 463, 5, 110, 0, 0, 463, 464, 5, 108, 0, 0, 464, 465, 5, 105, 0, 0, 465, 466, 5, 110, 0, 0, 466, 467, 5, 101, 0, 0, 467, 468, 5, 115, 0, 0, 468, 469, 5, 116, 0, 0, 469, 470, 5, 97, 0, 0, 470, 471, 5, 116, 0, 0, 471, 472, 5, 115, 0, 0, 472, 473, 1, 0, 0, 0, 473, 474, 6, 7, 0, 0, 474, 31, 1, 0, 0, 0, 475, 476, 5, 107, 0, 0, 476, 477, 5, 101, 0, 0, 477, 478, 5, 101, 0, 0, 478, 479, 5, 112, 0, 0, 479, 480, 1, 0, 0, 0, 480, 481, 6, 8, 1, 0, 481, 33, 1, 0, 0, 0, 482, 483, 5, 108, 0, 0, 483, 484, 5, 105, 0, 0, 484, 485, 5, 109, 0, 0, 485, 486, 5, 105, 0, 0, 486, 487, 5, 116, 0, 0, 487, 488, 1, 0, 0, 0, 488, 489, 6, 9, 0, 0, 489, 35, 1, 0, 0, 0, 490, 491, 5, 108, 0, 0, 491, 492, 5, 111, 0, 0, 492, 493, 5, 111, 0, 0, 493, 494, 5, 107, 0, 0, 494, 495, 5, 117, 0, 0, 495, 496, 5, 112, 0, 0, 496, 497, 1, 0, 0, 0, 497, 498, 6, 10, 5, 0, 498, 37, 1, 0, 0, 0, 499, 500, 5, 109, 0, 0, 500, 501, 5, 101, 0, 0, 501, 502, 5, 116, 0, 0, 502, 503, 5, 97, 0, 0, 503, 504, 1, 0, 0, 0, 504, 505, 6, 11, 6, 0, 505, 39, 1, 0, 0, 0, 506, 507, 5, 109, 0, 0, 507, 508, 5, 101, 0, 0, 508, 509, 5, 116, 0, 0, 509, 510, 5, 114, 0, 0, 510, 511, 5, 105, 0, 0, 511, 512, 5, 99, 0, 0, 512, 513, 5, 115, 0, 0, 513, 514, 1, 0, 0, 0, 514, 515, 6, 12, 7, 0, 515, 41, 1, 0, 0, 0, 516, 517, 5, 109, 0, 0, 517, 518, 5, 118, 0, 0, 518, 519, 5, 95, 0, 0, 519, 520, 5, 101, 0, 0, 520, 521, 5, 120, 0, 0, 521, 522, 5, 112, 0, 0, 522, 523, 5, 97, 0, 0, 523, 524, 5, 110, 0, 0, 524, 525, 5, 100, 0, 0, 525, 526, 1, 0, 0, 0, 526, 527, 6, 13, 8, 0, 527, 43, 1, 0, 0, 0, 528, 529, 5, 114, 0, 0, 529, 530, 5, 101, 0, 0, 530, 531, 5, 110, 0, 0, 531, 532, 5, 97, 0, 0, 532, 533, 5, 109, 0, 0, 533, 534, 5, 101, 0, 0, 534, 535, 1, 0, 0, 0, 535, 536, 6, 14, 9, 0, 536, 45, 1, 0, 0, 0, 537, 538, 5, 114, 0, 0, 538, 539, 5, 111, 0, 0, 539, 540, 5, 119, 0, 0, 540, 541, 1, 0, 0, 0, 541, 542, 6, 15, 0, 0, 542, 47, 1, 0, 0, 0, 543, 544, 5, 115, 0, 0, 544, 545, 5, 104, 0, 0, 545, 546, 5, 111, 0, 0, 546, 547, 5, 119, 0, 0, 547, 548, 1, 0, 0, 0, 548, 549, 6, 16, 10, 0, 549, 49, 1, 0, 0, 0, 550, 551, 5, 115, 0, 0, 551, 552, 5, 111, 0, 0, 552, 553, 5, 114, 0, 0, 553, 554, 5, 116, 0, 0, 554, 555, 1, 0, 0, 0, 555, 556, 6, 17, 0, 0, 556, 51, 1, 0, 0, 0, 557, 558, 5, 115, 0, 0, 558, 559, 5, 116, 0, 0, 559, 560, 5, 97, 0, 0, 560, 561, 5, 116, 0, 0, 561, 562, 5, 115, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 6, 18, 0, 0, 564, 53, 1, 0, 0, 0, 565, 566, 5, 119, 0, 0, 566, 567, 5, 104, 0, 0, 567, 568, 5, 101, 0, 0, 568, 569, 5, 114, 0, 0, 569, 570, 5, 101, 0, 0, 570, 571, 1, 0, 0, 0, 571, 572, 6, 19, 0, 0, 572, 55, 1, 0, 0, 0, 573, 575, 8, 0, 0, 0, 574, 573, 1, 0, 0, 0, 575, 576, 1, 0, 0, 0, 576, 574, 1, 0, 0, 0, 576, 577, 1, 0, 0, 0, 577, 578, 1, 0, 0, 0, 578, 579, 6, 20, 0, 0, 579, 57, 1, 0, 0, 0, 580, 581, 5, 47, 0, 0, 581, 582, 5, 47, 0, 0, 582, 586, 1, 0, 0, 0, 583, 585, 8, 1, 0, 0, 584, 583, 1, 0, 0, 0, 585, 588, 1, 0, 0, 0, 586, 584, 1, 0, 0, 0, 586, 587, 1, 0, 0, 0, 587, 590, 1, 0, 0, 0, 588, 586, 1, 0, 0, 0, 589, 591, 5, 13, 0, 0, 590, 589, 1, 0, 0, 0, 590, 591, 1, 0, 0, 0, 591, 593, 1, 0, 0, 0, 592, 594, 5, 10, 0, 0, 593, 592, 1, 0, 0, 0, 593, 594, 1, 0, 0, 0, 594, 595, 1, 0, 0, 0, 595, 596, 6, 21, 11, 0, 596, 59, 1, 0, 0, 0, 597, 598, 5, 47, 0, 0, 598, 599, 5, 42, 0, 0, 599, 604, 1, 0, 0, 0, 600, 603, 3, 60, 22, 0, 601, 603, 9, 0, 0, 0, 602, 600, 1, 0, 0, 0, 602, 601, 1, 0, 0, 0, 603, 606, 1, 0, 0, 0, 604, 605, 1, 0, 0, 0, 604, 602, 1, 0, 0, 0, 605, 607, 1, 0, 0, 0, 606, 604, 1, 0, 0, 0, 607, 608, 5, 42, 0, 0, 608, 609, 5, 47, 0, 0, 609, 610, 1, 0, 0, 0, 610, 611, 6, 22, 11, 0, 611, 61, 1, 0, 0, 0, 612, 614, 7, 2, 0, 0, 613, 612, 1, 0, 0, 0, 614, 615, 1, 0, 0, 0, 615, 613, 1, 0, 0, 0, 615, 616, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 617, 618, 6, 23, 11, 0, 618, 63, 1, 0, 0, 0, 619, 623, 8, 3, 0, 0, 620, 621, 5, 47, 0, 0, 621, 623, 8, 4, 0, 0, 622, 619, 1, 0, 0, 0, 622, 620, 1, 0, 0, 0, 623, 65, 1, 0, 0, 0, 624, 626, 3, 64, 24, 0, 625, 624, 1, 0, 0, 0, 626, 627, 1, 0, 0, 0, 627, 625, 1, 0, 0, 0, 627, 628, 1, 0, 0, 0, 628, 67, 1, 0, 0, 0, 629, 630, 3, 178, 81, 0, 630, 631, 1, 0, 0, 0, 631, 632, 6, 26, 12, 0, 632, 633, 6, 26, 13, 0, 633, 69, 1, 0, 0, 0, 634, 635, 3, 78, 31, 0, 635, 636, 1, 0, 0, 0, 636, 637, 6, 27, 14, 0, 637, 638, 6, 27, 15, 0, 638, 71, 1, 0, 0, 0, 639, 640, 3, 62, 23, 0, 640, 641, 1, 0, 0, 0, 641, 642, 6, 28, 11, 0, 642, 73, 1, 0, 0, 0, 643, 644, 3, 58, 21, 0, 644, 645, 1, 0, 0, 0, 645, 646, 6, 29, 11, 0, 646, 75, 1, 0, 0, 0, 647, 648, 3, 60, 22, 0, 648, 649, 1, 0, 0, 0, 649, 650, 6, 30, 11, 0, 650, 77, 1, 0, 0, 0, 651, 652, 5, 124, 0, 0, 652, 653, 1, 0, 0, 0, 653, 654, 6, 31, 15, 0, 654, 79, 1, 0, 0, 0, 655, 656, 7, 5, 0, 0, 656, 81, 1, 0, 0, 0, 657, 658, 7, 6, 0, 0, 658, 83, 1, 0, 0, 0, 659, 660, 5, 92, 0, 0, 660, 661, 7, 7, 0, 0, 661, 85, 1, 0, 0, 0, 662, 663, 8, 8, 0, 0, 663, 87, 1, 0, 0, 0, 664, 666, 7, 9, 0, 0, 665, 667, 7, 10, 0, 0, 666, 665, 1, 0, 0, 0, 666, 667, 1, 0, 0, 0, 667, 669, 1, 0, 0, 0, 668, 670, 3, 80, 32, 0, 669, 668, 1, 0, 0, 0, 670, 671, 1, 0, 0, 0, 671, 669, 1, 0, 0, 0, 671, 672, 1, 0, 0, 0, 672, 89, 1, 0, 0, 0, 673, 674, 5, 64, 0, 0, 674, 91, 1, 0, 0, 0, 675, 676, 5, 96, 0, 0, 676, 93, 1, 0, 0, 0, 677, 681, 8, 11, 0, 0, 678, 679, 5, 96, 0, 0, 679, 681, 5, 96, 0, 0, 680, 677, 1, 0, 0, 0, 680, 678, 1, 0, 0, 0, 681, 95, 1, 0, 0, 0, 682, 683, 5, 95, 0, 0, 683, 97, 1, 0, 0, 0, 684, 688, 3, 82, 33, 0, 685, 688, 3, 80, 32, 0, 686, 688, 3, 96, 40, 0, 687, 684, 1, 0, 0, 0, 687, 685, 1, 0, 0, 0, 687, 686, 1, 0, 0, 0, 688, 99, 1, 0, 0, 0, 689, 694, 5, 34, 0, 0, 690, 693, 3, 84, 34, 0, 691, 693, 3, 86, 35, 0, 692, 690, 1, 0, 0, 0, 692, 691, 1, 0, 0, 0, 693, 696, 1, 0, 0, 0, 694, 692, 1, 0, 0, 0, 694, 695, 1, 0, 0, 0, 695, 697, 1, 0, 0, 0, 696, 694, 1, 0, 0, 0, 697, 719, 5, 34, 0, 0, 698, 699, 5, 34, 0, 0, 699, 700, 5, 34, 0, 0, 700, 701, 5, 34, 0, 0, 701, 705, 1, 0, 0, 0, 702, 704, 8, 1, 0, 0, 703, 702, 1, 0, 0, 0, 704, 707, 1, 0, 0, 0, 705, 706, 1, 0, 0, 0, 705, 703, 1, 0, 0, 0, 706, 708, 1, 0, 0, 0, 707, 705, 1, 0, 0, 0, 708, 709, 5, 34, 0, 0, 709, 710, 5, 34, 0, 0, 710, 711, 5, 34, 0, 0, 711, 713, 1, 0, 0, 0, 712, 714, 5, 34, 0, 0, 713, 712, 1, 0, 0, 0, 713, 714, 1, 0, 0, 0, 714, 716, 1, 0, 0, 0, 715, 717, 5, 34, 0, 0, 716, 715, 1, 0, 0, 0, 716, 717, 1, 0, 0, 0, 717, 719, 1, 0, 0, 0, 718, 689, 1, 0, 0, 0, 718, 698, 1, 0, 0, 0, 719, 101, 1, 0, 0, 0, 720, 722, 3, 80, 32, 0, 721, 720, 1, 0, 0, 0, 722, 723, 1, 0, 0, 0, 723, 721, 1, 0, 0, 0, 723, 724, 1, 0, 0, 0, 724, 103, 1, 0, 0, 0, 725, 727, 3, 80, 32, 0, 726, 725, 1, 0, 0, 0, 727, 728, 1, 0, 0, 0, 728, 726, 1, 0, 0, 0, 728, 729, 1, 0, 0, 0, 729, 730, 1, 0, 0, 0, 730, 734, 3, 120, 52, 0, 731, 733, 3, 80, 32, 0, 732, 731, 1, 0, 0, 0, 733, 736, 1, 0, 0, 0, 734, 732, 1, 0, 0, 0, 734, 735, 1, 0, 0, 0, 735, 768, 1, 0, 0, 0, 736, 734, 1, 0, 0, 0, 737, 739, 3, 120, 52, 0, 738, 740, 3, 80, 32, 0, 739, 738, 1, 0, 0, 0, 740, 741, 1, 0, 0, 0, 741, 739, 1, 0, 0, 0, 741, 742, 1, 0, 0, 0, 742, 768, 1, 0, 0, 0, 743, 745, 3, 80, 32, 0, 744, 743, 1, 0, 0, 0, 745, 746, 1, 0, 0, 0, 746, 744, 1, 0, 0, 0, 746, 747, 1, 0, 0, 0, 747, 755, 1, 0, 0, 0, 748, 752, 3, 120, 52, 0, 749, 751, 3, 80, 32, 0, 750, 749, 1, 0, 0, 0, 751, 754, 1, 0, 0, 0, 752, 750, 1, 0, 0, 0, 752, 753, 1, 0, 0, 0, 753, 756, 1, 0, 0, 0, 754, 752, 1, 0, 0, 0, 755, 748, 1, 0, 0, 0, 755, 756, 1, 0, 0, 0, 756, 757, 1, 0, 0, 0, 757, 758, 3, 88, 36, 0, 758, 768, 1, 0, 0, 0, 759, 761, 3, 120, 52, 0, 760, 762, 3, 80, 32, 0, 761, 760, 1, 0, 0, 0, 762, 763, 1, 0, 0, 0, 763, 761, 1, 0, 0, 0, 763, 764, 1, 0, 0, 0, 764, 765, 1, 0, 0, 0, 765, 766, 3, 88, 36, 0, 766, 768, 1, 0, 0, 0, 767, 726, 1, 0, 0, 0, 767, 737, 1, 0, 0, 0, 767, 744, 1, 0, 0, 0, 767, 759, 1, 0, 0, 0, 768, 105, 1, 0, 0, 0, 769, 770, 5, 98, 0, 0, 770, 771, 5, 121, 0, 0, 771, 107, 1, 0, 0, 0, 772, 773, 5, 97, 0, 0, 773, 774, 5, 110, 0, 0, 774, 775, 5, 100, 0, 0, 775, 109, 1, 0, 0, 0, 776, 777, 5, 97, 0, 0, 777, 778, 5, 115, 0, 0, 778, 779, 5, 99, 0, 0, 779, 111, 1, 0, 0, 0, 780, 781, 5, 61, 0, 0, 781, 113, 1, 0, 0, 0, 782, 783, 5, 58, 0, 0, 783, 784, 5, 58, 0, 0, 784, 115, 1, 0, 0, 0, 785, 786, 5, 44, 0, 0, 786, 117, 1, 0, 0, 0, 787, 788, 5, 100, 0, 0, 788, 789, 5, 101, 0, 0, 789, 790, 5, 115, 0, 0, 790, 791, 5, 99, 0, 0, 791, 119, 1, 0, 0, 0, 792, 793, 5, 46, 0, 0, 793, 121, 1, 0, 0, 0, 794, 795, 5, 102, 0, 0, 795, 796, 5, 97, 0, 0, 796, 797, 5, 108, 0, 0, 797, 798, 5, 115, 0, 0, 798, 799, 5, 101, 0, 0, 799, 123, 1, 0, 0, 0, 800, 801, 5, 102, 0, 0, 801, 802, 5, 105, 0, 0, 802, 803, 5, 114, 0, 0, 803, 804, 5, 115, 0, 0, 804, 805, 5, 116, 0, 0, 805, 125, 1, 0, 0, 0, 806, 807, 5, 108, 0, 0, 807, 808, 5, 97, 0, 0, 808, 809, 5, 115, 0, 0, 809, 810, 5, 116, 0, 0, 810, 127, 1, 0, 0, 0, 811, 812, 5, 40, 0, 0, 812, 129, 1, 0, 0, 0, 813, 814, 5, 105, 0, 0, 814, 815, 5, 110, 0, 0, 815, 131, 1, 0, 0, 0, 816, 817, 5, 105, 0, 0, 817, 818, 5, 115, 0, 0, 818, 133, 1, 0, 0, 0, 819, 820, 5, 108, 0, 0, 820, 821, 5, 105, 0, 0, 821, 822, 5, 107, 0, 0, 822, 823, 5, 101, 0, 0, 823, 135, 1, 0, 0, 0, 824, 825, 5, 110, 0, 0, 825, 826, 5, 111, 0, 0, 826, 827, 5, 116, 0, 0, 827, 137, 1, 0, 0, 0, 828, 829, 5, 110, 0, 0, 829, 830, 5, 117, 0, 0, 830, 831, 5, 108, 0, 0, 831, 832, 5, 108, 0, 0, 832, 139, 1, 0, 0, 0, 833, 834, 5, 110, 0, 0, 834, 835, 5, 117, 0, 0, 835, 836, 5, 108, 0, 0, 836, 837, 5, 108, 0, 0, 837, 838, 5, 115, 0, 0, 838, 141, 1, 0, 0, 0, 839, 840, 5, 111, 0, 0, 840, 841, 5, 114, 0, 0, 841, 143, 1, 0, 0, 0, 842, 843, 5, 63, 0, 0, 843, 145, 1, 0, 0, 0, 844, 845, 5, 114, 0, 0, 845, 846, 5, 108, 0, 0, 846, 847, 5, 105, 0, 0, 847, 848, 5, 107, 0, 0, 848, 849, 5, 101, 0, 0, 849, 147, 1, 0, 0, 0, 850, 851, 5, 41, 0, 0, 851, 149, 1, 0, 0, 0, 852, 853, 5, 116, 0, 0, 853, 854, 5, 114, 0, 0, 854, 855, 5, 117, 0, 0, 855, 856, 5, 101, 0, 0, 856, 151, 1, 0, 0, 0, 857, 858, 5, 61, 0, 0, 858, 859, 5, 61, 0, 0, 859, 153, 1, 0, 0, 0, 860, 861, 5, 61, 0, 0, 861, 862, 5, 126, 0, 0, 862, 155, 1, 0, 0, 0, 863, 864, 5, 33, 0, 0, 864, 865, 5, 61, 0, 0, 865, 157, 1, 0, 0, 0, 866, 867, 5, 60, 0, 0, 867, 159, 1, 0, 0, 0, 868, 869, 5, 60, 0, 0, 869, 870, 5, 61, 0, 0, 870, 161, 1, 0, 0, 0, 871, 872, 5, 62, 0, 0, 872, 163, 1, 0, 0, 0, 873, 874, 5, 62, 0, 0, 874, 875, 5, 61, 0, 0, 875, 165, 1, 0, 0, 0, 876, 877, 5, 43, 0, 0, 877, 167, 1, 0, 0, 0, 878, 879, 5, 45, 0, 0, 879, 169, 1, 0, 0, 0, 880, 881, 5, 42, 0, 0, 881, 171, 1, 0, 0, 0, 882, 883, 5, 47, 0, 0, 883, 173, 1, 0, 0, 0, 884, 885, 5, 37, 0, 0, 885, 175, 1, 0, 0, 0, 886, 887, 3, 144, 64, 0, 887, 891, 3, 82, 33, 0, 888, 890, 3, 98, 41, 0, 889, 888, 1, 0, 0, 0, 890, 893, 1, 0, 0, 0, 891, 889, 1, 0, 0, 0, 891, 892, 1, 0, 0, 0, 892, 901, 1, 0, 0, 0, 893, 891, 1, 0, 0, 0, 894, 896, 3, 144, 64, 0, 895, 897, 3, 80, 32, 0, 896, 895, 1, 0, 0, 0, 897, 898, 1, 0, 0, 0, 898, 896, 1, 0, 0, 0, 898, 899, 1, 0, 0, 0, 899, 901, 1, 0, 0, 0, 900, 886, 1, 0, 0, 0, 900, 894, 1, 0, 0, 0, 901, 177, 1, 0, 0, 0, 902, 903, 5, 91, 0, 0, 903, 904, 1, 0, 0, 0, 904, 905, 6, 81, 0, 0, 905, 906, 6, 81, 0, 0, 906, 179, 1, 0, 0, 0, 907, 908, 5, 93, 0, 0, 908, 909, 1, 0, 0, 0, 909, 910, 6, 82, 15, 0, 910, 911, 6, 82, 15, 0, 911, 181, 1, 0, 0, 0, 912, 916, 3, 82, 33, 0, 913, 915, 3, 98, 41, 0, 914, 913, 1, 0, 0, 0, 915, 918, 1, 0, 0, 0, 916, 914, 1, 0, 0, 0, 916, 917, 1, 0, 0, 0, 917, 929, 1, 0, 0, 0, 918, 916, 1, 0, 0, 0, 919, 922, 3, 96, 40, 0, 920, 922, 3, 90, 37, 0, 921, 919, 1, 0, 0, 0, 921, 920, 1, 0, 0, 0, 922, 924, 1, 0, 0, 0, 923, 925, 3, 98, 41, 0, 924, 923, 1, 0, 0, 0, 925, 926, 1, 0, 0, 0, 926, 924, 1, 0, 0, 0, 926, 927, 1, 0, 0, 0, 927, 929, 1, 0, 0, 0, 928, 912, 1, 0, 0, 0, 928, 921, 1, 0, 0, 0, 929, 183, 1, 0, 0, 0, 930, 932, 3, 92, 38, 0, 931, 933, 3, 94, 39, 0, 932, 931, 1, 0, 0, 0, 933, 934, 1, 0, 0, 0, 934, 932, 1, 0, 0, 0, 934, 935, 1, 0, 0, 0, 935, 936, 1, 0, 0, 0, 936, 937, 3, 92, 38, 0, 937, 185, 1, 0, 0, 0, 938, 939, 3, 184, 84, 0, 939, 187, 1, 0, 0, 0, 940, 941, 3, 58, 21, 0, 941, 942, 1, 0, 0, 0, 942, 943, 6, 86, 11, 0, 943, 189, 1, 0, 0, 0, 944, 945, 3, 60, 22, 0, 945, 946, 1, 0, 0, 0, 946, 947, 6, 87, 11, 0, 947, 191, 1, 0, 0, 0, 948, 949, 3, 62, 23, 0, 949, 950, 1, 0, 0, 0, 950, 951, 6, 88, 11, 0, 951, 193, 1, 0, 0, 0, 952, 953, 3, 78, 31, 0, 953, 954, 1, 0, 0, 0, 954, 955, 6, 89, 14, 0, 955, 956, 6, 89, 15, 0, 956, 195, 1, 0, 0, 0, 957, 958, 3, 178, 81, 0, 958, 959, 1, 0, 0, 0, 959, 960, 6, 90, 12, 0, 960, 197, 1, 0, 0, 0, 961, 962, 3, 180, 82, 0, 962, 963, 1, 0, 0, 0, 963, 964, 6, 91, 16, 0, 964, 199, 1, 0, 0, 0, 965, 966, 3, 364, 174, 0, 966, 967, 1, 0, 0, 0, 967, 968, 6, 92, 17, 0, 968, 201, 1, 0, 0, 0, 969, 970, 3, 116, 50, 0, 970, 971, 1, 0, 0, 0, 971, 972, 6, 93, 18, 0, 972, 203, 1, 0, 0, 0, 973, 974, 3, 112, 48, 0, 974, 975, 1, 0, 0, 0, 975, 976, 6, 94, 19, 0, 976, 205, 1, 0, 0, 0, 977, 978, 5, 109, 0, 0, 978, 979, 5, 101, 0, 0, 979, 980, 5, 116, 0, 0, 980, 981, 5, 97, 0, 0, 981, 982, 5, 100, 0, 0, 982, 983, 5, 97, 0, 0, 983, 984, 5, 116, 0, 0, 984, 985, 5, 97, 0, 0, 985, 207, 1, 0, 0, 0, 986, 987, 3, 66, 25, 0, 987, 988, 1, 0, 0, 0, 988, 989, 6, 96, 20, 0, 989, 209, 1, 0, 0, 0, 990, 991, 3, 100, 42, 0, 991, 992, 1, 0, 0, 0, 992, 993, 6, 97, 21, 0, 993, 211, 1, 0, 0, 0, 994, 995, 3, 58, 21, 0, 995, 996, 1, 0, 0, 0, 996, 997, 6, 98, 11, 0, 997, 213, 1, 0, 0, 0, 998, 999, 3, 60, 22, 0, 999, 1000, 1, 0, 0, 0, 1000, 1001, 6, 99, 11, 0, 1001, 215, 1, 0, 0, 0, 1002, 1003, 3, 62, 23, 0, 1003, 1004, 1, 0, 0, 0, 1004, 1005, 6, 100, 11, 0, 1005, 217, 1, 0, 0, 0, 1006, 1007, 3, 78, 31, 0, 1007, 1008, 1, 0, 0, 0, 1008, 1009, 6, 101, 14, 0, 1009, 1010, 6, 101, 15, 0, 1010, 219, 1, 0, 0, 0, 1011, 1012, 3, 120, 52, 0, 1012, 1013, 1, 0, 0, 0, 1013, 1014, 6, 102, 22, 0, 1014, 221, 1, 0, 0, 0, 1015, 1016, 3, 116, 50, 0, 1016, 1017, 1, 0, 0, 0, 1017, 1018, 6, 103, 18, 0, 1018, 223, 1, 0, 0, 0, 1019, 1024, 3, 82, 33, 0, 1020, 1024, 3, 80, 32, 0, 1021, 1024, 3, 96, 40, 0, 1022, 1024, 3, 170, 77, 0, 1023, 1019, 1, 0, 0, 0, 1023, 1020, 1, 0, 0, 0, 1023, 1021, 1, 0, 0, 0, 1023, 1022, 1, 0, 0, 0, 1024, 225, 1, 0, 0, 0, 1025, 1028, 3, 82, 33, 0, 1026, 1028, 3, 170, 77, 0, 1027, 1025, 1, 0, 0, 0, 1027, 1026, 1, 0, 0, 0, 1028, 1032, 1, 0, 0, 0, 1029, 1031, 3, 224, 104, 0, 1030, 1029, 1, 0, 0, 0, 1031, 1034, 1, 0, 0, 0, 1032, 1030, 1, 0, 0, 0, 1032, 1033, 1, 0, 0, 0, 1033, 1045, 1, 0, 0, 0, 1034, 1032, 1, 0, 0, 0, 1035, 1038, 3, 96, 40, 0, 1036, 1038, 3, 90, 37, 0, 1037, 1035, 1, 0, 0, 0, 1037, 1036, 1, 0, 0, 0, 1038, 1040, 1, 0, 0, 0, 1039, 1041, 3, 224, 104, 0, 1040, 1039, 1, 0, 0, 0, 1041, 1042, 1, 0, 0, 0, 1042, 1040, 1, 0, 0, 0, 1042, 1043, 1, 0, 0, 0, 1043, 1045, 1, 0, 0, 0, 1044, 1027, 1, 0, 0, 0, 1044, 1037, 1, 0, 0, 0, 1045, 227, 1, 0, 0, 0, 1046, 1049, 3, 226, 105, 0, 1047, 1049, 3, 184, 84, 0, 1048, 1046, 1, 0, 0, 0, 1048, 1047, 1, 0, 0, 0, 1049, 1050, 1, 0, 0, 0, 1050, 1048, 1, 0, 0, 0, 1050, 1051, 1, 0, 0, 0, 1051, 229, 1, 0, 0, 0, 1052, 1053, 3, 58, 21, 0, 1053, 1054, 1, 0, 0, 0, 1054, 1055, 6, 107, 11, 0, 1055, 231, 1, 0, 0, 0, 1056, 1057, 3, 60, 22, 0, 1057, 1058, 1, 0, 0, 0, 1058, 1059, 6, 108, 11, 0, 1059, 233, 1, 0, 0, 0, 1060, 1061, 3, 62, 23, 0, 1061, 1062, 1, 0, 0, 0, 1062, 1063, 6, 109, 11, 0, 1063, 235, 1, 0, 0, 0, 1064, 1065, 3, 78, 31, 0, 1065, 1066, 1, 0, 0, 0, 1066, 1067, 6, 110, 14, 0, 1067, 1068, 6, 110, 15, 0, 1068, 237, 1, 0, 0, 0, 1069, 1070, 3, 112, 48, 0, 1070, 1071, 1, 0, 0, 0, 1071, 1072, 6, 111, 19, 0, 1072, 239, 1, 0, 0, 0, 1073, 1074, 3, 116, 50, 0, 1074, 1075, 1, 0, 0, 0, 1075, 1076, 6, 112, 18, 0, 1076, 241, 1, 0, 0, 0, 1077, 1078, 3, 120, 52, 0, 1078, 1079, 1, 0, 0, 0, 1079, 1080, 6, 113, 22, 0, 1080, 243, 1, 0, 0, 0, 1081, 1082, 5, 97, 0, 0, 1082, 1083, 5, 115, 0, 0, 1083, 245, 1, 0, 0, 0, 1084, 1085, 3, 228, 106, 0, 1085, 1086, 1, 0, 0, 0, 1086, 1087, 6, 115, 23, 0, 1087, 247, 1, 0, 0, 0, 1088, 1089, 3, 58, 21, 0, 1089, 1090, 1, 0, 0, 0, 1090, 1091, 6, 116, 11, 0, 1091, 249, 1, 0, 0, 0, 1092, 1093, 3, 60, 22, 0, 1093, 1094, 1, 0, 0, 0, 1094, 1095, 6, 117, 11, 0, 1095, 251, 1, 0, 0, 0, 1096, 1097, 3, 62, 23, 0, 1097, 1098, 1, 0, 0, 0, 1098, 1099, 6, 118, 11, 0, 1099, 253, 1, 0, 0, 0, 1100, 1101, 3, 78, 31, 0, 1101, 1102, 1, 0, 0, 0, 1102, 1103, 6, 119, 14, 0, 1103, 1104, 6, 119, 15, 0, 1104, 255, 1, 0, 0, 0, 1105, 1106, 3, 178, 81, 0, 1106, 1107, 1, 0, 0, 0, 1107, 1108, 6, 120, 12, 0, 1108, 1109, 6, 120, 24, 0, 1109, 257, 1, 0, 0, 0, 1110, 1111, 5, 111, 0, 0, 1111, 1112, 5, 110, 0, 0, 1112, 1113, 1, 0, 0, 0, 1113, 1114, 6, 121, 25, 0, 1114, 259, 1, 0, 0, 0, 1115, 1116, 5, 119, 0, 0, 1116, 1117, 5, 105, 0, 0, 1117, 1118, 5, 116, 0, 0, 1118, 1119, 5, 104, 0, 0, 1119, 1120, 1, 0, 0, 0, 1120, 1121, 6, 122, 25, 0, 1121, 261, 1, 0, 0, 0, 1122, 1123, 8, 12, 0, 0, 1123, 263, 1, 0, 0, 0, 1124, 1126, 3, 262, 123, 0, 1125, 1124, 1, 0, 0, 0, 1126, 1127, 1, 0, 0, 0, 1127, 1125, 1, 0, 0, 0, 1127, 1128, 1, 0, 0, 0, 1128, 1129, 1, 0, 0, 0, 1129, 1130, 3, 364, 174, 0, 1130, 1132, 1, 0, 0, 0, 1131, 1125, 1, 0, 0, 0, 1131, 1132, 1, 0, 0, 0, 1132, 1134, 1, 0, 0, 0, 1133, 1135, 3, 262, 123, 0, 1134, 1133, 1, 0, 0, 0, 1135, 1136, 1, 0, 0, 0, 1136, 1134, 1, 0, 0, 0, 1136, 1137, 1, 0, 0, 0, 1137, 265, 1, 0, 0, 0, 1138, 1139, 3, 264, 124, 0, 1139, 1140, 1, 0, 0, 0, 1140, 1141, 6, 125, 26, 0, 1141, 267, 1, 0, 0, 0, 1142, 1143, 3, 58, 21, 0, 1143, 1144, 1, 0, 0, 0, 1144, 1145, 6, 126, 11, 0, 1145, 269, 1, 0, 0, 0, 1146, 1147, 3, 60, 22, 0, 1147, 1148, 1, 0, 0, 0, 1148, 1149, 6, 127, 11, 0, 1149, 271, 1, 0, 0, 0, 1150, 1151, 3, 62, 23, 0, 1151, 1152, 1, 0, 0, 0, 1152, 1153, 6, 128, 11, 0, 1153, 273, 1, 0, 0, 0, 1154, 1155, 3, 78, 31, 0, 1155, 1156, 1, 0, 0, 0, 1156, 1157, 6, 129, 14, 0, 1157, 1158, 6, 129, 15, 0, 1158, 1159, 6, 129, 15, 0, 1159, 275, 1, 0, 0, 0, 1160, 1161, 3, 112, 48, 0, 1161, 1162, 1, 0, 0, 0, 1162, 1163, 6, 130, 19, 0, 1163, 277, 1, 0, 0, 0, 1164, 1165, 3, 116, 50, 0, 1165, 1166, 1, 0, 0, 0, 1166, 1167, 6, 131, 18, 0, 1167, 279, 1, 0, 0, 0, 1168, 1169, 3, 120, 52, 0, 1169, 1170, 1, 0, 0, 0, 1170, 1171, 6, 132, 22, 0, 1171, 281, 1, 0, 0, 0, 1172, 1173, 3, 260, 122, 0, 1173, 1174, 1, 0, 0, 0, 1174, 1175, 6, 133, 27, 0, 1175, 283, 1, 0, 0, 0, 1176, 1177, 3, 228, 106, 0, 1177, 1178, 1, 0, 0, 0, 1178, 1179, 6, 134, 23, 0, 1179, 285, 1, 0, 0, 0, 1180, 1181, 3, 186, 85, 0, 1181, 1182, 1, 0, 0, 0, 1182, 1183, 6, 135, 28, 0, 1183, 287, 1, 0, 0, 0, 1184, 1185, 3, 58, 21, 0, 1185, 1186, 1, 0, 0, 0, 1186, 1187, 6, 136, 11, 0, 1187, 289, 1, 0, 0, 0, 1188, 1189, 3, 60, 22, 0, 1189, 1190, 1, 0, 0, 0, 1190, 1191, 6, 137, 11, 0, 1191, 291, 1, 0, 0, 0, 1192, 1193, 3, 62, 23, 0, 1193, 1194, 1, 0, 0, 0, 1194, 1195, 6, 138, 11, 0, 1195, 293, 1, 0, 0, 0, 1196, 1197, 3, 78, 31, 0, 1197, 1198, 1, 0, 0, 0, 1198, 1199, 6, 139, 14, 0, 1199, 1200, 6, 139, 15, 0, 1200, 295, 1, 0, 0, 0, 1201, 1202, 3, 364, 174, 0, 1202, 1203, 1, 0, 0, 0, 1203, 1204, 6, 140, 17, 0, 1204, 297, 1, 0, 0, 0, 1205, 1206, 3, 116, 50, 0, 1206, 1207, 1, 0, 0, 0, 1207, 1208, 6, 141, 18, 0, 1208, 299, 1, 0, 0, 0, 1209, 1210, 3, 120, 52, 0, 1210, 1211, 1, 0, 0, 0, 1211, 1212, 6, 142, 22, 0, 1212, 301, 1, 0, 0, 0, 1213, 1214, 3, 258, 121, 0, 1214, 1215, 1, 0, 0, 0, 1215, 1216, 6, 143, 29, 0, 1216, 1217, 6, 143, 30, 0, 1217, 303, 1, 0, 0, 0, 1218, 1219, 3, 66, 25, 0, 1219, 1220, 1, 0, 0, 0, 1220, 1221, 6, 144, 20, 0, 1221, 305, 1, 0, 0, 0, 1222, 1223, 3, 100, 42, 0, 1223, 1224, 1, 0, 0, 0, 1224, 1225, 6, 145, 21, 0, 1225, 307, 1, 0, 0, 0, 1226, 1227, 3, 58, 21, 0, 1227, 1228, 1, 0, 0, 0, 1228, 1229, 6, 146, 11, 0, 1229, 309, 1, 0, 0, 0, 1230, 1231, 3, 60, 22, 0, 1231, 1232, 1, 0, 0, 0, 1232, 1233, 6, 147, 11, 0, 1233, 311, 1, 0, 0, 0, 1234, 1235, 3, 62, 23, 0, 1235, 1236, 1, 0, 0, 0, 1236, 1237, 6, 148, 11, 0, 1237, 313, 1, 0, 0, 0, 1238, 1239, 3, 78, 31, 0, 1239, 1240, 1, 0, 0, 0, 1240, 1241, 6, 149, 14, 0, 1241, 1242, 6, 149, 15, 0, 1242, 1243, 6, 149, 15, 0, 1243, 315, 1, 0, 0, 0, 1244, 1245, 3, 116, 50, 0, 1245, 1246, 1, 0, 0, 0, 1246, 1247, 6, 150, 18, 0, 1247, 317, 1, 0, 0, 0, 1248, 1249, 3, 120, 52, 0, 1249, 1250, 1, 0, 0, 0, 1250, 1251, 6, 151, 22, 0, 1251, 319, 1, 0, 0, 0, 1252, 1253, 3, 228, 106, 0, 1253, 1254, 1, 0, 0, 0, 1254, 1255, 6, 152, 23, 0, 1255, 321, 1, 0, 0, 0, 1256, 1257, 3, 58, 21, 0, 1257, 1258, 1, 0, 0, 0, 1258, 1259, 6, 153, 11, 0, 1259, 323, 1, 0, 0, 0, 1260, 1261, 3, 60, 22, 0, 1261, 1262, 1, 0, 0, 0, 1262, 1263, 6, 154, 11, 0, 1263, 325, 1, 0, 0, 0, 1264, 1265, 3, 62, 23, 0, 1265, 1266, 1, 0, 0, 0, 1266, 1267, 6, 155, 11, 0, 1267, 327, 1, 0, 0, 0, 1268, 1269, 3, 78, 31, 0, 1269, 1270, 1, 0, 0, 0, 1270, 1271, 6, 156, 14, 0, 1271, 1272, 6, 156, 15, 0, 1272, 329, 1, 0, 0, 0, 1273, 1274, 3, 120, 52, 0, 1274, 1275, 1, 0, 0, 0, 1275, 1276, 6, 157, 22, 0, 1276, 331, 1, 0, 0, 0, 1277, 1278, 3, 186, 85, 0, 1278, 1279, 1, 0, 0, 0, 1279, 1280, 6, 158, 28, 0, 1280, 333, 1, 0, 0, 0, 1281, 1282, 3, 182, 83, 0, 1282, 1283, 1, 0, 0, 0, 1283, 1284, 6, 159, 31, 0, 1284, 335, 1, 0, 0, 0, 1285, 1286, 3, 58, 21, 0, 1286, 1287, 1, 0, 0, 0, 1287, 1288, 6, 160, 11, 0, 1288, 337, 1, 0, 0, 0, 1289, 1290, 3, 60, 22, 0, 1290, 1291, 1, 0, 0, 0, 1291, 1292, 6, 161, 11, 0, 1292, 339, 1, 0, 0, 0, 1293, 1294, 3, 62, 23, 0, 1294, 1295, 1, 0, 0, 0, 1295, 1296, 6, 162, 11, 0, 1296, 341, 1, 0, 0, 0, 1297, 1298, 3, 78, 31, 0, 1298, 1299, 1, 0, 0, 0, 1299, 1300, 6, 163, 14, 0, 1300, 1301, 6, 163, 15, 0, 1301, 343, 1, 0, 0, 0, 1302, 1303, 5, 105, 0, 0, 1303, 1304, 5, 110, 0, 0, 1304, 1305, 5, 102, 0, 0, 1305, 1306, 5, 111, 0, 0, 1306, 345, 1, 0, 0, 0, 1307, 1308, 3, 58, 21, 0, 1308, 1309, 1, 0, 0, 0, 1309, 1310, 6, 165, 11, 0, 1310, 347, 1, 0, 0, 0, 1311, 1312, 3, 60, 22, 0, 1312, 1313, 1, 0, 0, 0, 1313, 1314, 6, 166, 11, 0, 1314, 349, 1, 0, 0, 0, 1315, 1316, 3, 62, 23, 0, 1316, 1317, 1, 0, 0, 0, 1317, 1318, 6, 167, 11, 0, 1318, 351, 1, 0, 0, 0, 1319, 1320, 3, 78, 31, 0, 1320, 1321, 1, 0, 0, 0, 1321, 1322, 6, 168, 14, 0, 1322, 1323, 6, 168, 15, 0, 1323, 353, 1, 0, 0, 0, 1324, 1325, 5, 102, 0, 0, 1325, 1326, 5, 117, 0, 0, 1326, 1327, 5, 110, 0, 0, 1327, 1328, 5, 99, 0, 0, 1328, 1329, 5, 116, 0, 0, 1329, 1330, 5, 105, 0, 0, 1330, 1331, 5, 111, 0, 0, 1331, 1332, 5, 110, 0, 0, 1332, 1333, 5, 115, 0, 0, 1333, 355, 1, 0, 0, 0, 1334, 1335, 3, 58, 21, 0, 1335, 1336, 1, 0, 0, 0, 1336, 1337, 6, 170, 11, 0, 1337, 357, 1, 0, 0, 0, 1338, 1339, 3, 60, 22, 0, 1339, 1340, 1, 0, 0, 0, 1340, 1341, 6, 171, 11, 0, 1341, 359, 1, 0, 0, 0, 1342, 1343, 3, 62, 23, 0, 1343, 1344, 1, 0, 0, 0, 1344, 1345, 6, 172, 11, 0, 1345, 361, 1, 0, 0, 0, 1346, 1347, 3, 180, 82, 0, 1347, 1348, 1, 0, 0, 0, 1348, 1349, 6, 173, 16, 0, 1349, 1350, 6, 173, 15, 0, 1350, 363, 1, 0, 0, 0, 1351, 1352, 5, 58, 0, 0, 1352, 365, 1, 0, 0, 0, 1353, 1359, 3, 90, 37, 0, 1354, 1359, 3, 80, 32, 0, 1355, 1359, 3, 120, 52, 0, 1356, 1359, 3, 82, 33, 0, 1357, 1359, 3, 96, 40, 0, 1358, 1353, 1, 0, 0, 0, 1358, 1354, 1, 0, 0, 0, 1358, 1355, 1, 0, 0, 0, 1358, 1356, 1, 0, 0, 0, 1358, 1357, 1, 0, 0, 0, 1359, 1360, 1, 0, 0, 0, 1360, 1358, 1, 0, 0, 0, 1360, 1361, 1, 0, 0, 0, 1361, 367, 1, 0, 0, 0, 1362, 1363, 3, 58, 21, 0, 1363, 1364, 1, 0, 0, 0, 1364, 1365, 6, 176, 11, 0, 1365, 369, 1, 0, 0, 0, 1366, 1367, 3, 60, 22, 0, 1367, 1368, 1, 0, 0, 0, 1368, 1369, 6, 177, 11, 0, 1369, 371, 1, 0, 0, 0, 1370, 1371, 3, 62, 23, 0, 1371, 1372, 1, 0, 0, 0, 1372, 1373, 6, 178, 11, 0, 1373, 373, 1, 0, 0, 0, 1374, 1375, 3, 78, 31, 0, 1375, 1376, 1, 0, 0, 0, 1376, 1377, 6, 179, 14, 0, 1377, 1378, 6, 179, 15, 0, 1378, 375, 1, 0, 0, 0, 1379, 1380, 3, 66, 25, 0, 1380, 1381, 1, 0, 0, 0, 1381, 1382, 6, 180, 20, 0, 1382, 1383, 6, 180, 15, 0, 1383, 1384, 6, 180, 32, 0, 1384, 377, 1, 0, 0, 0, 1385, 1386, 3, 100, 42, 0, 1386, 1387, 1, 0, 0, 0, 1387, 1388, 6, 181, 21, 0, 1388, 1389, 6, 181, 15, 0, 1389, 1390, 6, 181, 32, 0, 1390, 379, 1, 0, 0, 0, 1391, 1392, 3, 58, 21, 0, 1392, 1393, 1, 0, 0, 0, 1393, 1394, 6, 182, 11, 0, 1394, 381, 1, 0, 0, 0, 1395, 1396, 3, 60, 22, 0, 1396, 1397, 1, 0, 0, 0, 1397, 1398, 6, 183, 11, 0, 1398, 383, 1, 0, 0, 0, 1399, 1400, 3, 62, 23, 0, 1400, 1401, 1, 0, 0, 0, 1401, 1402, 6, 184, 11, 0, 1402, 385, 1, 0, 0, 0, 1403, 1404, 3, 364, 174, 0, 1404, 1405, 1, 0, 0, 0, 1405, 1406, 6, 185, 17, 0, 1406, 1407, 6, 185, 15, 0, 1407, 1408, 6, 185, 7, 0, 1408, 387, 1, 0, 0, 0, 1409, 1410, 3, 116, 50, 0, 1410, 1411, 1, 0, 0, 0, 1411, 1412, 6, 186, 18, 0, 1412, 1413, 6, 186, 15, 0, 1413, 1414, 6, 186, 7, 0, 1414, 389, 1, 0, 0, 0, 1415, 1416, 3, 58, 21, 0, 1416, 1417, 1, 0, 0, 0, 1417, 1418, 6, 187, 11, 0, 1418, 391, 1, 0, 0, 0, 1419, 1420, 3, 60, 22, 0, 1420, 1421, 1, 0, 0, 0, 1421, 1422, 6, 188, 11, 0, 1422, 393, 1, 0, 0, 0, 1423, 1424, 3, 62, 23, 0, 1424, 1425, 1, 0, 0, 0, 1425, 1426, 6, 189, 11, 0, 1426, 395, 1, 0, 0, 0, 1427, 1428, 3, 186, 85, 0, 1428, 1429, 1, 0, 0, 0, 1429, 1430, 6, 190, 15, 0, 1430, 1431, 6, 190, 0, 0, 1431, 1432, 6, 190, 28, 0, 1432, 397, 1, 0, 0, 0, 1433, 1434, 3, 182, 83, 0, 1434, 1435, 1, 0, 0, 0, 1435, 1436, 6, 191, 15, 0, 1436, 1437, 6, 191, 0, 0, 1437, 1438, 6, 191, 31, 0, 1438, 399, 1, 0, 0, 0, 1439, 1440, 3, 106, 45, 0, 1440, 1441, 1, 0, 0, 0, 1441, 1442, 6, 192, 15, 0, 1442, 1443, 6, 192, 0, 0, 1443, 1444, 6, 192, 33, 0, 1444, 401, 1, 0, 0, 0, 1445, 1446, 3, 78, 31, 0, 1446, 1447, 1, 0, 0, 0, 1447, 1448, 6, 193, 14, 0, 1448, 1449, 6, 193, 15, 0, 1449, 403, 1, 0, 0, 0, 65, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 576, 586, 590, 593, 602, 604, 615, 622, 627, 666, 671, 680, 687, 692, 694, 705, 713, 716, 718, 723, 728, 734, 741, 746, 752, 755, 763, 767, 891, 898, 900, 916, 921, 926, 928, 934, 1023, 1027, 1032, 1037, 1042, 1044, 1048, 1050, 1127, 1131, 1136, 1358, 1360, 34, 5, 2, 0, 5, 4, 0, 5, 6, 0, 5, 1, 0, 5, 3, 0, 5, 8, 0, 5, 12, 0, 5, 14, 0, 5, 10, 0, 5, 5, 0, 5, 11, 0, 0, 1, 0, 7, 69, 0, 5, 0, 0, 7, 29, 0, 4, 0, 0, 7, 70, 0, 7, 114, 0, 7, 38, 0, 7, 36, 0, 7, 25, 0, 7, 30, 0, 7, 40, 0, 7, 80, 0, 5, 13, 0, 5, 7, 0, 7, 90, 0, 7, 89, 0, 7, 72, 0, 7, 88, 0, 5, 9, 0, 7, 71, 0, 5, 15, 0, 7, 33, 0]
\ No newline at end of file
+[4, 0, 125, 1474, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 4, 21, 591, 8, 21, 11, 21, 12, 21, 592, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 601, 8, 22, 10, 22, 12, 22, 604, 9, 22, 1, 22, 3, 22, 607, 8, 22, 1, 22, 3, 22, 610, 8, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 619, 8, 23, 10, 23, 12, 23, 622, 9, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 4, 24, 630, 8, 24, 11, 24, 12, 24, 631, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 3, 30, 651, 8, 30, 1, 30, 4, 30, 654, 8, 30, 11, 30, 12, 30, 655, 1, 31, 1, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 665, 8, 33, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 3, 35, 672, 8, 35, 1, 36, 1, 36, 1, 36, 5, 36, 677, 8, 36, 10, 36, 12, 36, 680, 9, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 688, 8, 36, 10, 36, 12, 36, 691, 9, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 3, 36, 698, 8, 36, 1, 36, 3, 36, 701, 8, 36, 3, 36, 703, 8, 36, 1, 37, 4, 37, 706, 8, 37, 11, 37, 12, 37, 707, 1, 38, 4, 38, 711, 8, 38, 11, 38, 12, 38, 712, 1, 38, 1, 38, 5, 38, 717, 8, 38, 10, 38, 12, 38, 720, 9, 38, 1, 38, 1, 38, 4, 38, 724, 8, 38, 11, 38, 12, 38, 725, 1, 38, 4, 38, 729, 8, 38, 11, 38, 12, 38, 730, 1, 38, 1, 38, 5, 38, 735, 8, 38, 10, 38, 12, 38, 738, 9, 38, 3, 38, 740, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 4, 38, 746, 8, 38, 11, 38, 12, 38, 747, 1, 38, 1, 38, 3, 38, 752, 8, 38, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 75, 3, 75, 879, 8, 75, 1, 75, 5, 75, 882, 8, 75, 10, 75, 12, 75, 885, 9, 75, 1, 75, 1, 75, 4, 75, 889, 8, 75, 11, 75, 12, 75, 890, 3, 75, 893, 8, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 1, 77, 1, 77, 1, 77, 1, 78, 1, 78, 5, 78, 907, 8, 78, 10, 78, 12, 78, 910, 9, 78, 1, 78, 1, 78, 3, 78, 914, 8, 78, 1, 78, 4, 78, 917, 8, 78, 11, 78, 12, 78, 918, 3, 78, 921, 8, 78, 1, 79, 1, 79, 4, 79, 925, 8, 79, 11, 79, 12, 79, 926, 1, 79, 1, 79, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 3, 96, 1004, 8, 96, 1, 97, 4, 97, 1007, 8, 97, 11, 97, 12, 97, 1008, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 3, 106, 1048, 8, 106, 1, 107, 1, 107, 3, 107, 1052, 8, 107, 1, 107, 5, 107, 1055, 8, 107, 10, 107, 12, 107, 1058, 9, 107, 1, 107, 1, 107, 3, 107, 1062, 8, 107, 1, 107, 4, 107, 1065, 8, 107, 11, 107, 12, 107, 1066, 3, 107, 1069, 8, 107, 1, 108, 1, 108, 4, 108, 1073, 8, 108, 11, 108, 12, 108, 1074, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 126, 4, 126, 1150, 8, 126, 11, 126, 12, 126, 1151, 1, 126, 1, 126, 3, 126, 1156, 8, 126, 1, 126, 4, 126, 1159, 8, 126, 11, 126, 12, 126, 1160, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 160, 4, 160, 1311, 8, 160, 11, 160, 12, 160, 1312, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 2, 620, 689, 0, 196, 16, 1, 18, 2, 20, 3, 22, 4, 24, 5, 26, 6, 28, 7, 30, 8, 32, 9, 34, 10, 36, 11, 38, 12, 40, 13, 42, 14, 44, 15, 46, 16, 48, 17, 50, 18, 52, 19, 54, 20, 56, 21, 58, 22, 60, 23, 62, 24, 64, 25, 66, 26, 68, 0, 70, 0, 72, 0, 74, 0, 76, 0, 78, 0, 80, 0, 82, 0, 84, 0, 86, 0, 88, 27, 90, 28, 92, 29, 94, 30, 96, 31, 98, 32, 100, 33, 102, 34, 104, 35, 106, 36, 108, 37, 110, 38, 112, 39, 114, 40, 116, 41, 118, 42, 120, 43, 122, 44, 124, 45, 126, 46, 128, 47, 130, 48, 132, 49, 134, 50, 136, 51, 138, 52, 140, 53, 142, 54, 144, 55, 146, 56, 148, 57, 150, 58, 152, 59, 154, 60, 156, 61, 158, 62, 160, 63, 162, 64, 164, 0, 166, 65, 168, 66, 170, 67, 172, 68, 174, 0, 176, 69, 178, 70, 180, 71, 182, 72, 184, 0, 186, 0, 188, 73, 190, 74, 192, 75, 194, 0, 196, 0, 198, 0, 200, 0, 202, 0, 204, 0, 206, 76, 208, 0, 210, 77, 212, 0, 214, 0, 216, 78, 218, 79, 220, 80, 222, 0, 224, 0, 226, 0, 228, 0, 230, 0, 232, 81, 234, 82, 236, 83, 238, 84, 240, 0, 242, 0, 244, 0, 246, 0, 248, 85, 250, 0, 252, 86, 254, 87, 256, 88, 258, 0, 260, 0, 262, 89, 264, 90, 266, 0, 268, 91, 270, 0, 272, 92, 274, 93, 276, 94, 278, 0, 280, 0, 282, 0, 284, 0, 286, 0, 288, 0, 290, 0, 292, 95, 294, 96, 296, 97, 298, 0, 300, 0, 302, 0, 304, 0, 306, 98, 308, 99, 310, 100, 312, 0, 314, 101, 316, 102, 318, 103, 320, 104, 322, 0, 324, 105, 326, 106, 328, 107, 330, 108, 332, 0, 334, 109, 336, 110, 338, 111, 340, 112, 342, 113, 344, 0, 346, 0, 348, 0, 350, 0, 352, 0, 354, 0, 356, 0, 358, 114, 360, 115, 362, 116, 364, 0, 366, 0, 368, 0, 370, 0, 372, 117, 374, 118, 376, 119, 378, 0, 380, 0, 382, 0, 384, 120, 386, 121, 388, 122, 390, 0, 392, 0, 394, 123, 396, 124, 398, 125, 400, 0, 402, 0, 404, 0, 406, 0, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 35, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1501, 0, 16, 1, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 50, 1, 0, 0, 0, 0, 52, 1, 0, 0, 0, 0, 54, 1, 0, 0, 0, 0, 56, 1, 0, 0, 0, 0, 58, 1, 0, 0, 0, 0, 60, 1, 0, 0, 0, 0, 62, 1, 0, 0, 0, 0, 64, 1, 0, 0, 0, 1, 66, 1, 0, 0, 0, 1, 88, 1, 0, 0, 0, 1, 90, 1, 0, 0, 0, 1, 92, 1, 0, 0, 0, 1, 94, 1, 0, 0, 0, 1, 96, 1, 0, 0, 0, 1, 98, 1, 0, 0, 0, 1, 100, 1, 0, 0, 0, 1, 102, 1, 0, 0, 0, 1, 104, 1, 0, 0, 0, 1, 106, 1, 0, 0, 0, 1, 108, 1, 0, 0, 0, 1, 110, 1, 0, 0, 0, 1, 112, 1, 0, 0, 0, 1, 114, 1, 0, 0, 0, 1, 116, 1, 0, 0, 0, 1, 118, 1, 0, 0, 0, 1, 120, 1, 0, 0, 0, 1, 122, 1, 0, 0, 0, 1, 124, 1, 0, 0, 0, 1, 126, 1, 0, 0, 0, 1, 128, 1, 0, 0, 0, 1, 130, 1, 0, 0, 0, 1, 132, 1, 0, 0, 0, 1, 134, 1, 0, 0, 0, 1, 136, 1, 0, 0, 0, 1, 138, 1, 0, 0, 0, 1, 140, 1, 0, 0, 0, 1, 142, 1, 0, 0, 0, 1, 144, 1, 0, 0, 0, 1, 146, 1, 0, 0, 0, 1, 148, 1, 0, 0, 0, 1, 150, 1, 0, 0, 0, 1, 152, 1, 0, 0, 0, 1, 154, 1, 0, 0, 0, 1, 156, 1, 0, 0, 0, 1, 158, 1, 0, 0, 0, 1, 160, 1, 0, 0, 0, 1, 162, 1, 0, 0, 0, 1, 164, 1, 0, 0, 0, 1, 166, 1, 0, 0, 0, 1, 168, 1, 0, 0, 0, 1, 170, 1, 0, 0, 0, 1, 172, 1, 0, 0, 0, 1, 176, 1, 0, 0, 0, 1, 178, 1, 0, 0, 0, 1, 180, 1, 0, 0, 0, 1, 182, 1, 0, 0, 0, 2, 184, 1, 0, 0, 0, 2, 186, 1, 0, 0, 0, 2, 188, 1, 0, 0, 0, 2, 190, 1, 0, 0, 0, 2, 192, 1, 0, 0, 0, 3, 194, 1, 0, 0, 0, 3, 196, 1, 0, 0, 0, 3, 198, 1, 0, 0, 0, 3, 200, 1, 0, 0, 0, 3, 202, 1, 0, 0, 0, 3, 204, 1, 0, 0, 0, 3, 206, 1, 0, 0, 0, 3, 210, 1, 0, 0, 0, 3, 212, 1, 0, 0, 0, 3, 214, 1, 0, 0, 0, 3, 216, 1, 0, 0, 0, 3, 218, 1, 0, 0, 0, 3, 220, 1, 0, 0, 0, 4, 222, 1, 0, 0, 0, 4, 224, 1, 0, 0, 0, 4, 226, 1, 0, 0, 0, 4, 232, 1, 0, 0, 0, 4, 234, 1, 0, 0, 0, 4, 236, 1, 0, 0, 0, 4, 238, 1, 0, 0, 0, 5, 240, 1, 0, 0, 0, 5, 242, 1, 0, 0, 0, 5, 244, 1, 0, 0, 0, 5, 246, 1, 0, 0, 0, 5, 248, 1, 0, 0, 0, 5, 250, 1, 0, 0, 0, 5, 252, 1, 0, 0, 0, 5, 254, 1, 0, 0, 0, 5, 256, 1, 0, 0, 0, 6, 258, 1, 0, 0, 0, 6, 260, 1, 0, 0, 0, 6, 262, 1, 0, 0, 0, 6, 264, 1, 0, 0, 0, 6, 268, 1, 0, 0, 0, 6, 270, 1, 0, 0, 0, 6, 272, 1, 0, 0, 0, 6, 274, 1, 0, 0, 0, 6, 276, 1, 0, 0, 0, 7, 278, 1, 0, 0, 0, 7, 280, 1, 0, 0, 0, 7, 282, 1, 0, 0, 0, 7, 284, 1, 0, 0, 0, 7, 286, 1, 0, 0, 0, 7, 288, 1, 0, 0, 0, 7, 290, 1, 0, 0, 0, 7, 292, 1, 0, 0, 0, 7, 294, 1, 0, 0, 0, 7, 296, 1, 0, 0, 0, 8, 298, 1, 0, 0, 0, 8, 300, 1, 0, 0, 0, 8, 302, 1, 0, 0, 0, 8, 304, 1, 0, 0, 0, 8, 306, 1, 0, 0, 0, 8, 308, 1, 0, 0, 0, 8, 310, 1, 0, 0, 0, 9, 312, 1, 0, 0, 0, 9, 314, 1, 0, 0, 0, 9, 316, 1, 0, 0, 0, 9, 318, 1, 0, 0, 0, 9, 320, 1, 0, 0, 0, 10, 322, 1, 0, 0, 0, 10, 324, 1, 0, 0, 0, 10, 326, 1, 0, 0, 0, 10, 328, 1, 0, 0, 0, 10, 330, 1, 0, 0, 0, 11, 332, 1, 0, 0, 0, 11, 334, 1, 0, 0, 0, 11, 336, 1, 0, 0, 0, 11, 338, 1, 0, 0, 0, 11, 340, 1, 0, 0, 0, 11, 342, 1, 0, 0, 0, 12, 344, 1, 0, 0, 0, 12, 346, 1, 0, 0, 0, 12, 348, 1, 0, 0, 0, 12, 350, 1, 0, 0, 0, 12, 352, 1, 0, 0, 0, 12, 354, 1, 0, 0, 0, 12, 356, 1, 0, 0, 0, 12, 358, 1, 0, 0, 0, 12, 360, 1, 0, 0, 0, 12, 362, 1, 0, 0, 0, 13, 364, 1, 0, 0, 0, 13, 366, 1, 0, 0, 0, 13, 368, 1, 0, 0, 0, 13, 370, 1, 0, 0, 0, 13, 372, 1, 0, 0, 0, 13, 374, 1, 0, 0, 0, 13, 376, 1, 0, 0, 0, 14, 378, 1, 0, 0, 0, 14, 380, 1, 0, 0, 0, 14, 382, 1, 0, 0, 0, 14, 384, 1, 0, 0, 0, 14, 386, 1, 0, 0, 0, 14, 388, 1, 0, 0, 0, 15, 390, 1, 0, 0, 0, 15, 392, 1, 0, 0, 0, 15, 394, 1, 0, 0, 0, 15, 396, 1, 0, 0, 0, 15, 398, 1, 0, 0, 0, 15, 400, 1, 0, 0, 0, 15, 402, 1, 0, 0, 0, 15, 404, 1, 0, 0, 0, 15, 406, 1, 0, 0, 0, 16, 408, 1, 0, 0, 0, 18, 418, 1, 0, 0, 0, 20, 425, 1, 0, 0, 0, 22, 434, 1, 0, 0, 0, 24, 441, 1, 0, 0, 0, 26, 451, 1, 0, 0, 0, 28, 458, 1, 0, 0, 0, 30, 465, 1, 0, 0, 0, 32, 472, 1, 0, 0, 0, 34, 480, 1, 0, 0, 0, 36, 487, 1, 0, 0, 0, 38, 499, 1, 0, 0, 0, 40, 508, 1, 0, 0, 0, 42, 514, 1, 0, 0, 0, 44, 521, 1, 0, 0, 0, 46, 528, 1, 0, 0, 0, 48, 536, 1, 0, 0, 0, 50, 544, 1, 0, 0, 0, 52, 559, 1, 0, 0, 0, 54, 569, 1, 0, 0, 0, 56, 578, 1, 0, 0, 0, 58, 590, 1, 0, 0, 0, 60, 596, 1, 0, 0, 0, 62, 613, 1, 0, 0, 0, 64, 629, 1, 0, 0, 0, 66, 635, 1, 0, 0, 0, 68, 639, 1, 0, 0, 0, 70, 641, 1, 0, 0, 0, 72, 643, 1, 0, 0, 0, 74, 646, 1, 0, 0, 0, 76, 648, 1, 0, 0, 0, 78, 657, 1, 0, 0, 0, 80, 659, 1, 0, 0, 0, 82, 664, 1, 0, 0, 0, 84, 666, 1, 0, 0, 0, 86, 671, 1, 0, 0, 0, 88, 702, 1, 0, 0, 0, 90, 705, 1, 0, 0, 0, 92, 751, 1, 0, 0, 0, 94, 753, 1, 0, 0, 0, 96, 756, 1, 0, 0, 0, 98, 760, 1, 0, 0, 0, 100, 764, 1, 0, 0, 0, 102, 766, 1, 0, 0, 0, 104, 769, 1, 0, 0, 0, 106, 771, 1, 0, 0, 0, 108, 776, 1, 0, 0, 0, 110, 778, 1, 0, 0, 0, 112, 784, 1, 0, 0, 0, 114, 790, 1, 0, 0, 0, 116, 793, 1, 0, 0, 0, 118, 796, 1, 0, 0, 0, 120, 801, 1, 0, 0, 0, 122, 806, 1, 0, 0, 0, 124, 808, 1, 0, 0, 0, 126, 812, 1, 0, 0, 0, 128, 817, 1, 0, 0, 0, 130, 823, 1, 0, 0, 0, 132, 826, 1, 0, 0, 0, 134, 828, 1, 0, 0, 0, 136, 834, 1, 0, 0, 0, 138, 836, 1, 0, 0, 0, 140, 841, 1, 0, 0, 0, 142, 844, 1, 0, 0, 0, 144, 847, 1, 0, 0, 0, 146, 850, 1, 0, 0, 0, 148, 852, 1, 0, 0, 0, 150, 855, 1, 0, 0, 0, 152, 857, 1, 0, 0, 0, 154, 860, 1, 0, 0, 0, 156, 862, 1, 0, 0, 0, 158, 864, 1, 0, 0, 0, 160, 866, 1, 0, 0, 0, 162, 868, 1, 0, 0, 0, 164, 870, 1, 0, 0, 0, 166, 892, 1, 0, 0, 0, 168, 894, 1, 0, 0, 0, 170, 899, 1, 0, 0, 0, 172, 920, 1, 0, 0, 0, 174, 922, 1, 0, 0, 0, 176, 930, 1, 0, 0, 0, 178, 932, 1, 0, 0, 0, 180, 936, 1, 0, 0, 0, 182, 940, 1, 0, 0, 0, 184, 944, 1, 0, 0, 0, 186, 949, 1, 0, 0, 0, 188, 954, 1, 0, 0, 0, 190, 958, 1, 0, 0, 0, 192, 962, 1, 0, 0, 0, 194, 966, 1, 0, 0, 0, 196, 971, 1, 0, 0, 0, 198, 975, 1, 0, 0, 0, 200, 979, 1, 0, 0, 0, 202, 983, 1, 0, 0, 0, 204, 987, 1, 0, 0, 0, 206, 991, 1, 0, 0, 0, 208, 1003, 1, 0, 0, 0, 210, 1006, 1, 0, 0, 0, 212, 1010, 1, 0, 0, 0, 214, 1014, 1, 0, 0, 0, 216, 1018, 1, 0, 0, 0, 218, 1022, 1, 0, 0, 0, 220, 1026, 1, 0, 0, 0, 222, 1030, 1, 0, 0, 0, 224, 1035, 1, 0, 0, 0, 226, 1039, 1, 0, 0, 0, 228, 1047, 1, 0, 0, 0, 230, 1068, 1, 0, 0, 0, 232, 1072, 1, 0, 0, 0, 234, 1076, 1, 0, 0, 0, 236, 1080, 1, 0, 0, 0, 238, 1084, 1, 0, 0, 0, 240, 1088, 1, 0, 0, 0, 242, 1093, 1, 0, 0, 0, 244, 1097, 1, 0, 0, 0, 246, 1101, 1, 0, 0, 0, 248, 1105, 1, 0, 0, 0, 250, 1108, 1, 0, 0, 0, 252, 1112, 1, 0, 0, 0, 254, 1116, 1, 0, 0, 0, 256, 1120, 1, 0, 0, 0, 258, 1124, 1, 0, 0, 0, 260, 1129, 1, 0, 0, 0, 262, 1134, 1, 0, 0, 0, 264, 1139, 1, 0, 0, 0, 266, 1146, 1, 0, 0, 0, 268, 1155, 1, 0, 0, 0, 270, 1162, 1, 0, 0, 0, 272, 1166, 1, 0, 0, 0, 274, 1170, 1, 0, 0, 0, 276, 1174, 1, 0, 0, 0, 278, 1178, 1, 0, 0, 0, 280, 1184, 1, 0, 0, 0, 282, 1188, 1, 0, 0, 0, 284, 1192, 1, 0, 0, 0, 286, 1196, 1, 0, 0, 0, 288, 1200, 1, 0, 0, 0, 290, 1204, 1, 0, 0, 0, 292, 1208, 1, 0, 0, 0, 294, 1212, 1, 0, 0, 0, 296, 1216, 1, 0, 0, 0, 298, 1220, 1, 0, 0, 0, 300, 1225, 1, 0, 0, 0, 302, 1229, 1, 0, 0, 0, 304, 1233, 1, 0, 0, 0, 306, 1237, 1, 0, 0, 0, 308, 1241, 1, 0, 0, 0, 310, 1245, 1, 0, 0, 0, 312, 1249, 1, 0, 0, 0, 314, 1254, 1, 0, 0, 0, 316, 1259, 1, 0, 0, 0, 318, 1263, 1, 0, 0, 0, 320, 1267, 1, 0, 0, 0, 322, 1271, 1, 0, 0, 0, 324, 1276, 1, 0, 0, 0, 326, 1286, 1, 0, 0, 0, 328, 1290, 1, 0, 0, 0, 330, 1294, 1, 0, 0, 0, 332, 1298, 1, 0, 0, 0, 334, 1303, 1, 0, 0, 0, 336, 1310, 1, 0, 0, 0, 338, 1314, 1, 0, 0, 0, 340, 1318, 1, 0, 0, 0, 342, 1322, 1, 0, 0, 0, 344, 1326, 1, 0, 0, 0, 346, 1331, 1, 0, 0, 0, 348, 1335, 1, 0, 0, 0, 350, 1339, 1, 0, 0, 0, 352, 1343, 1, 0, 0, 0, 354, 1348, 1, 0, 0, 0, 356, 1352, 1, 0, 0, 0, 358, 1356, 1, 0, 0, 0, 360, 1360, 1, 0, 0, 0, 362, 1364, 1, 0, 0, 0, 364, 1368, 1, 0, 0, 0, 366, 1374, 1, 0, 0, 0, 368, 1378, 1, 0, 0, 0, 370, 1382, 1, 0, 0, 0, 372, 1386, 1, 0, 0, 0, 374, 1390, 1, 0, 0, 0, 376, 1394, 1, 0, 0, 0, 378, 1398, 1, 0, 0, 0, 380, 1403, 1, 0, 0, 0, 382, 1409, 1, 0, 0, 0, 384, 1415, 1, 0, 0, 0, 386, 1419, 1, 0, 0, 0, 388, 1423, 1, 0, 0, 0, 390, 1427, 1, 0, 0, 0, 392, 1433, 1, 0, 0, 0, 394, 1439, 1, 0, 0, 0, 396, 1443, 1, 0, 0, 0, 398, 1447, 1, 0, 0, 0, 400, 1451, 1, 0, 0, 0, 402, 1457, 1, 0, 0, 0, 404, 1463, 1, 0, 0, 0, 406, 1469, 1, 0, 0, 0, 408, 409, 7, 0, 0, 0, 409, 410, 7, 1, 0, 0, 410, 411, 7, 2, 0, 0, 411, 412, 7, 2, 0, 0, 412, 413, 7, 3, 0, 0, 413, 414, 7, 4, 0, 0, 414, 415, 7, 5, 0, 0, 415, 416, 1, 0, 0, 0, 416, 417, 6, 0, 0, 0, 417, 17, 1, 0, 0, 0, 418, 419, 7, 0, 0, 0, 419, 420, 7, 6, 0, 0, 420, 421, 7, 7, 0, 0, 421, 422, 7, 8, 0, 0, 422, 423, 1, 0, 0, 0, 423, 424, 6, 1, 1, 0, 424, 19, 1, 0, 0, 0, 425, 426, 7, 3, 0, 0, 426, 427, 7, 9, 0, 0, 427, 428, 7, 6, 0, 0, 428, 429, 7, 1, 0, 0, 429, 430, 7, 4, 0, 0, 430, 431, 7, 10, 0, 0, 431, 432, 1, 0, 0, 0, 432, 433, 6, 2, 2, 0, 433, 21, 1, 0, 0, 0, 434, 435, 7, 3, 0, 0, 435, 436, 7, 11, 0, 0, 436, 437, 7, 12, 0, 0, 437, 438, 7, 13, 0, 0, 438, 439, 1, 0, 0, 0, 439, 440, 6, 3, 0, 0, 440, 23, 1, 0, 0, 0, 441, 442, 7, 3, 0, 0, 442, 443, 7, 14, 0, 0, 443, 444, 7, 8, 0, 0, 444, 445, 7, 13, 0, 0, 445, 446, 7, 12, 0, 0, 446, 447, 7, 1, 0, 0, 447, 448, 7, 9, 0, 0, 448, 449, 1, 0, 0, 0, 449, 450, 6, 4, 3, 0, 450, 25, 1, 0, 0, 0, 451, 452, 7, 15, 0, 0, 452, 453, 7, 6, 0, 0, 453, 454, 7, 7, 0, 0, 454, 455, 7, 16, 0, 0, 455, 456, 1, 0, 0, 0, 456, 457, 6, 5, 4, 0, 457, 27, 1, 0, 0, 0, 458, 459, 7, 17, 0, 0, 459, 460, 7, 6, 0, 0, 460, 461, 7, 7, 0, 0, 461, 462, 7, 18, 0, 0, 462, 463, 1, 0, 0, 0, 463, 464, 6, 6, 0, 0, 464, 29, 1, 0, 0, 0, 465, 466, 7, 18, 0, 0, 466, 467, 7, 3, 0, 0, 467, 468, 7, 3, 0, 0, 468, 469, 7, 8, 0, 0, 469, 470, 1, 0, 0, 0, 470, 471, 6, 7, 1, 0, 471, 31, 1, 0, 0, 0, 472, 473, 7, 13, 0, 0, 473, 474, 7, 1, 0, 0, 474, 475, 7, 16, 0, 0, 475, 476, 7, 1, 0, 0, 476, 477, 7, 5, 0, 0, 477, 478, 1, 0, 0, 0, 478, 479, 6, 8, 0, 0, 479, 33, 1, 0, 0, 0, 480, 481, 7, 16, 0, 0, 481, 482, 7, 3, 0, 0, 482, 483, 7, 5, 0, 0, 483, 484, 7, 12, 0, 0, 484, 485, 1, 0, 0, 0, 485, 486, 6, 9, 5, 0, 486, 35, 1, 0, 0, 0, 487, 488, 7, 16, 0, 0, 488, 489, 7, 11, 0, 0, 489, 490, 5, 95, 0, 0, 490, 491, 7, 3, 0, 0, 491, 492, 7, 14, 0, 0, 492, 493, 7, 8, 0, 0, 493, 494, 7, 12, 0, 0, 494, 495, 7, 9, 0, 0, 495, 496, 7, 0, 0, 0, 496, 497, 1, 0, 0, 0, 497, 498, 6, 10, 6, 0, 498, 37, 1, 0, 0, 0, 499, 500, 7, 6, 0, 0, 500, 501, 7, 3, 0, 0, 501, 502, 7, 9, 0, 0, 502, 503, 7, 12, 0, 0, 503, 504, 7, 16, 0, 0, 504, 505, 7, 3, 0, 0, 505, 506, 1, 0, 0, 0, 506, 507, 6, 11, 7, 0, 507, 39, 1, 0, 0, 0, 508, 509, 7, 6, 0, 0, 509, 510, 7, 7, 0, 0, 510, 511, 7, 19, 0, 0, 511, 512, 1, 0, 0, 0, 512, 513, 6, 12, 0, 0, 513, 41, 1, 0, 0, 0, 514, 515, 7, 2, 0, 0, 515, 516, 7, 10, 0, 0, 516, 517, 7, 7, 0, 0, 517, 518, 7, 19, 0, 0, 518, 519, 1, 0, 0, 0, 519, 520, 6, 13, 8, 0, 520, 43, 1, 0, 0, 0, 521, 522, 7, 2, 0, 0, 522, 523, 7, 7, 0, 0, 523, 524, 7, 6, 0, 0, 524, 525, 7, 5, 0, 0, 525, 526, 1, 0, 0, 0, 526, 527, 6, 14, 0, 0, 527, 45, 1, 0, 0, 0, 528, 529, 7, 2, 0, 0, 529, 530, 7, 5, 0, 0, 530, 531, 7, 12, 0, 0, 531, 532, 7, 5, 0, 0, 532, 533, 7, 2, 0, 0, 533, 534, 1, 0, 0, 0, 534, 535, 6, 15, 0, 0, 535, 47, 1, 0, 0, 0, 536, 537, 7, 19, 0, 0, 537, 538, 7, 10, 0, 0, 538, 539, 7, 3, 0, 0, 539, 540, 7, 6, 0, 0, 540, 541, 7, 3, 0, 0, 541, 542, 1, 0, 0, 0, 542, 543, 6, 16, 0, 0, 543, 49, 1, 0, 0, 0, 544, 545, 4, 17, 0, 0, 545, 546, 7, 1, 0, 0, 546, 547, 7, 9, 0, 0, 547, 548, 7, 13, 0, 0, 548, 549, 7, 1, 0, 0, 549, 550, 7, 9, 0, 0, 550, 551, 7, 3, 0, 0, 551, 552, 7, 2, 0, 0, 552, 553, 7, 5, 0, 0, 553, 554, 7, 12, 0, 0, 554, 555, 7, 5, 0, 0, 555, 556, 7, 2, 0, 0, 556, 557, 1, 0, 0, 0, 557, 558, 6, 17, 0, 0, 558, 51, 1, 0, 0, 0, 559, 560, 4, 18, 1, 0, 560, 561, 7, 13, 0, 0, 561, 562, 7, 7, 0, 0, 562, 563, 7, 7, 0, 0, 563, 564, 7, 18, 0, 0, 564, 565, 7, 20, 0, 0, 565, 566, 7, 8, 0, 0, 566, 567, 1, 0, 0, 0, 567, 568, 6, 18, 9, 0, 568, 53, 1, 0, 0, 0, 569, 570, 4, 19, 2, 0, 570, 571, 7, 16, 0, 0, 571, 572, 7, 12, 0, 0, 572, 573, 7, 5, 0, 0, 573, 574, 7, 4, 0, 0, 574, 575, 7, 10, 0, 0, 575, 576, 1, 0, 0, 0, 576, 577, 6, 19, 0, 0, 577, 55, 1, 0, 0, 0, 578, 579, 4, 20, 3, 0, 579, 580, 7, 16, 0, 0, 580, 581, 7, 3, 0, 0, 581, 582, 7, 5, 0, 0, 582, 583, 7, 6, 0, 0, 583, 584, 7, 1, 0, 0, 584, 585, 7, 4, 0, 0, 585, 586, 7, 2, 0, 0, 586, 587, 1, 0, 0, 0, 587, 588, 6, 20, 10, 0, 588, 57, 1, 0, 0, 0, 589, 591, 8, 21, 0, 0, 590, 589, 1, 0, 0, 0, 591, 592, 1, 0, 0, 0, 592, 590, 1, 0, 0, 0, 592, 593, 1, 0, 0, 0, 593, 594, 1, 0, 0, 0, 594, 595, 6, 21, 0, 0, 595, 59, 1, 0, 0, 0, 596, 597, 5, 47, 0, 0, 597, 598, 5, 47, 0, 0, 598, 602, 1, 0, 0, 0, 599, 601, 8, 22, 0, 0, 600, 599, 1, 0, 0, 0, 601, 604, 1, 0, 0, 0, 602, 600, 1, 0, 0, 0, 602, 603, 1, 0, 0, 0, 603, 606, 1, 0, 0, 0, 604, 602, 1, 0, 0, 0, 605, 607, 5, 13, 0, 0, 606, 605, 1, 0, 0, 0, 606, 607, 1, 0, 0, 0, 607, 609, 1, 0, 0, 0, 608, 610, 5, 10, 0, 0, 609, 608, 1, 0, 0, 0, 609, 610, 1, 0, 0, 0, 610, 611, 1, 0, 0, 0, 611, 612, 6, 22, 11, 0, 612, 61, 1, 0, 0, 0, 613, 614, 5, 47, 0, 0, 614, 615, 5, 42, 0, 0, 615, 620, 1, 0, 0, 0, 616, 619, 3, 62, 23, 0, 617, 619, 9, 0, 0, 0, 618, 616, 1, 0, 0, 0, 618, 617, 1, 0, 0, 0, 619, 622, 1, 0, 0, 0, 620, 621, 1, 0, 0, 0, 620, 618, 1, 0, 0, 0, 621, 623, 1, 0, 0, 0, 622, 620, 1, 0, 0, 0, 623, 624, 5, 42, 0, 0, 624, 625, 5, 47, 0, 0, 625, 626, 1, 0, 0, 0, 626, 627, 6, 23, 11, 0, 627, 63, 1, 0, 0, 0, 628, 630, 7, 23, 0, 0, 629, 628, 1, 0, 0, 0, 630, 631, 1, 0, 0, 0, 631, 629, 1, 0, 0, 0, 631, 632, 1, 0, 0, 0, 632, 633, 1, 0, 0, 0, 633, 634, 6, 24, 11, 0, 634, 65, 1, 0, 0, 0, 635, 636, 5, 124, 0, 0, 636, 637, 1, 0, 0, 0, 637, 638, 6, 25, 12, 0, 638, 67, 1, 0, 0, 0, 639, 640, 7, 24, 0, 0, 640, 69, 1, 0, 0, 0, 641, 642, 7, 25, 0, 0, 642, 71, 1, 0, 0, 0, 643, 644, 5, 92, 0, 0, 644, 645, 7, 26, 0, 0, 645, 73, 1, 0, 0, 0, 646, 647, 8, 27, 0, 0, 647, 75, 1, 0, 0, 0, 648, 650, 7, 3, 0, 0, 649, 651, 7, 28, 0, 0, 650, 649, 1, 0, 0, 0, 650, 651, 1, 0, 0, 0, 651, 653, 1, 0, 0, 0, 652, 654, 3, 68, 26, 0, 653, 652, 1, 0, 0, 0, 654, 655, 1, 0, 0, 0, 655, 653, 1, 0, 0, 0, 655, 656, 1, 0, 0, 0, 656, 77, 1, 0, 0, 0, 657, 658, 5, 64, 0, 0, 658, 79, 1, 0, 0, 0, 659, 660, 5, 96, 0, 0, 660, 81, 1, 0, 0, 0, 661, 665, 8, 29, 0, 0, 662, 663, 5, 96, 0, 0, 663, 665, 5, 96, 0, 0, 664, 661, 1, 0, 0, 0, 664, 662, 1, 0, 0, 0, 665, 83, 1, 0, 0, 0, 666, 667, 5, 95, 0, 0, 667, 85, 1, 0, 0, 0, 668, 672, 3, 70, 27, 0, 669, 672, 3, 68, 26, 0, 670, 672, 3, 84, 34, 0, 671, 668, 1, 0, 0, 0, 671, 669, 1, 0, 0, 0, 671, 670, 1, 0, 0, 0, 672, 87, 1, 0, 0, 0, 673, 678, 5, 34, 0, 0, 674, 677, 3, 72, 28, 0, 675, 677, 3, 74, 29, 0, 676, 674, 1, 0, 0, 0, 676, 675, 1, 0, 0, 0, 677, 680, 1, 0, 0, 0, 678, 676, 1, 0, 0, 0, 678, 679, 1, 0, 0, 0, 679, 681, 1, 0, 0, 0, 680, 678, 1, 0, 0, 0, 681, 703, 5, 34, 0, 0, 682, 683, 5, 34, 0, 0, 683, 684, 5, 34, 0, 0, 684, 685, 5, 34, 0, 0, 685, 689, 1, 0, 0, 0, 686, 688, 8, 22, 0, 0, 687, 686, 1, 0, 0, 0, 688, 691, 1, 0, 0, 0, 689, 690, 1, 0, 0, 0, 689, 687, 1, 0, 0, 0, 690, 692, 1, 0, 0, 0, 691, 689, 1, 0, 0, 0, 692, 693, 5, 34, 0, 0, 693, 694, 5, 34, 0, 0, 694, 695, 5, 34, 0, 0, 695, 697, 1, 0, 0, 0, 696, 698, 5, 34, 0, 0, 697, 696, 1, 0, 0, 0, 697, 698, 1, 0, 0, 0, 698, 700, 1, 0, 0, 0, 699, 701, 5, 34, 0, 0, 700, 699, 1, 0, 0, 0, 700, 701, 1, 0, 0, 0, 701, 703, 1, 0, 0, 0, 702, 673, 1, 0, 0, 0, 702, 682, 1, 0, 0, 0, 703, 89, 1, 0, 0, 0, 704, 706, 3, 68, 26, 0, 705, 704, 1, 0, 0, 0, 706, 707, 1, 0, 0, 0, 707, 705, 1, 0, 0, 0, 707, 708, 1, 0, 0, 0, 708, 91, 1, 0, 0, 0, 709, 711, 3, 68, 26, 0, 710, 709, 1, 0, 0, 0, 711, 712, 1, 0, 0, 0, 712, 710, 1, 0, 0, 0, 712, 713, 1, 0, 0, 0, 713, 714, 1, 0, 0, 0, 714, 718, 3, 108, 46, 0, 715, 717, 3, 68, 26, 0, 716, 715, 1, 0, 0, 0, 717, 720, 1, 0, 0, 0, 718, 716, 1, 0, 0, 0, 718, 719, 1, 0, 0, 0, 719, 752, 1, 0, 0, 0, 720, 718, 1, 0, 0, 0, 721, 723, 3, 108, 46, 0, 722, 724, 3, 68, 26, 0, 723, 722, 1, 0, 0, 0, 724, 725, 1, 0, 0, 0, 725, 723, 1, 0, 0, 0, 725, 726, 1, 0, 0, 0, 726, 752, 1, 0, 0, 0, 727, 729, 3, 68, 26, 0, 728, 727, 1, 0, 0, 0, 729, 730, 1, 0, 0, 0, 730, 728, 1, 0, 0, 0, 730, 731, 1, 0, 0, 0, 731, 739, 1, 0, 0, 0, 732, 736, 3, 108, 46, 0, 733, 735, 3, 68, 26, 0, 734, 733, 1, 0, 0, 0, 735, 738, 1, 0, 0, 0, 736, 734, 1, 0, 0, 0, 736, 737, 1, 0, 0, 0, 737, 740, 1, 0, 0, 0, 738, 736, 1, 0, 0, 0, 739, 732, 1, 0, 0, 0, 739, 740, 1, 0, 0, 0, 740, 741, 1, 0, 0, 0, 741, 742, 3, 76, 30, 0, 742, 752, 1, 0, 0, 0, 743, 745, 3, 108, 46, 0, 744, 746, 3, 68, 26, 0, 745, 744, 1, 0, 0, 0, 746, 747, 1, 0, 0, 0, 747, 745, 1, 0, 0, 0, 747, 748, 1, 0, 0, 0, 748, 749, 1, 0, 0, 0, 749, 750, 3, 76, 30, 0, 750, 752, 1, 0, 0, 0, 751, 710, 1, 0, 0, 0, 751, 721, 1, 0, 0, 0, 751, 728, 1, 0, 0, 0, 751, 743, 1, 0, 0, 0, 752, 93, 1, 0, 0, 0, 753, 754, 7, 30, 0, 0, 754, 755, 7, 31, 0, 0, 755, 95, 1, 0, 0, 0, 756, 757, 7, 12, 0, 0, 757, 758, 7, 9, 0, 0, 758, 759, 7, 0, 0, 0, 759, 97, 1, 0, 0, 0, 760, 761, 7, 12, 0, 0, 761, 762, 7, 2, 0, 0, 762, 763, 7, 4, 0, 0, 763, 99, 1, 0, 0, 0, 764, 765, 5, 61, 0, 0, 765, 101, 1, 0, 0, 0, 766, 767, 5, 58, 0, 0, 767, 768, 5, 58, 0, 0, 768, 103, 1, 0, 0, 0, 769, 770, 5, 44, 0, 0, 770, 105, 1, 0, 0, 0, 771, 772, 7, 0, 0, 0, 772, 773, 7, 3, 0, 0, 773, 774, 7, 2, 0, 0, 774, 775, 7, 4, 0, 0, 775, 107, 1, 0, 0, 0, 776, 777, 5, 46, 0, 0, 777, 109, 1, 0, 0, 0, 778, 779, 7, 15, 0, 0, 779, 780, 7, 12, 0, 0, 780, 781, 7, 13, 0, 0, 781, 782, 7, 2, 0, 0, 782, 783, 7, 3, 0, 0, 783, 111, 1, 0, 0, 0, 784, 785, 7, 15, 0, 0, 785, 786, 7, 1, 0, 0, 786, 787, 7, 6, 0, 0, 787, 788, 7, 2, 0, 0, 788, 789, 7, 5, 0, 0, 789, 113, 1, 0, 0, 0, 790, 791, 7, 1, 0, 0, 791, 792, 7, 9, 0, 0, 792, 115, 1, 0, 0, 0, 793, 794, 7, 1, 0, 0, 794, 795, 7, 2, 0, 0, 795, 117, 1, 0, 0, 0, 796, 797, 7, 13, 0, 0, 797, 798, 7, 12, 0, 0, 798, 799, 7, 2, 0, 0, 799, 800, 7, 5, 0, 0, 800, 119, 1, 0, 0, 0, 801, 802, 7, 13, 0, 0, 802, 803, 7, 1, 0, 0, 803, 804, 7, 18, 0, 0, 804, 805, 7, 3, 0, 0, 805, 121, 1, 0, 0, 0, 806, 807, 5, 40, 0, 0, 807, 123, 1, 0, 0, 0, 808, 809, 7, 9, 0, 0, 809, 810, 7, 7, 0, 0, 810, 811, 7, 5, 0, 0, 811, 125, 1, 0, 0, 0, 812, 813, 7, 9, 0, 0, 813, 814, 7, 20, 0, 0, 814, 815, 7, 13, 0, 0, 815, 816, 7, 13, 0, 0, 816, 127, 1, 0, 0, 0, 817, 818, 7, 9, 0, 0, 818, 819, 7, 20, 0, 0, 819, 820, 7, 13, 0, 0, 820, 821, 7, 13, 0, 0, 821, 822, 7, 2, 0, 0, 822, 129, 1, 0, 0, 0, 823, 824, 7, 7, 0, 0, 824, 825, 7, 6, 0, 0, 825, 131, 1, 0, 0, 0, 826, 827, 5, 63, 0, 0, 827, 133, 1, 0, 0, 0, 828, 829, 7, 6, 0, 0, 829, 830, 7, 13, 0, 0, 830, 831, 7, 1, 0, 0, 831, 832, 7, 18, 0, 0, 832, 833, 7, 3, 0, 0, 833, 135, 1, 0, 0, 0, 834, 835, 5, 41, 0, 0, 835, 137, 1, 0, 0, 0, 836, 837, 7, 5, 0, 0, 837, 838, 7, 6, 0, 0, 838, 839, 7, 20, 0, 0, 839, 840, 7, 3, 0, 0, 840, 139, 1, 0, 0, 0, 841, 842, 5, 61, 0, 0, 842, 843, 5, 61, 0, 0, 843, 141, 1, 0, 0, 0, 844, 845, 5, 61, 0, 0, 845, 846, 5, 126, 0, 0, 846, 143, 1, 0, 0, 0, 847, 848, 5, 33, 0, 0, 848, 849, 5, 61, 0, 0, 849, 145, 1, 0, 0, 0, 850, 851, 5, 60, 0, 0, 851, 147, 1, 0, 0, 0, 852, 853, 5, 60, 0, 0, 853, 854, 5, 61, 0, 0, 854, 149, 1, 0, 0, 0, 855, 856, 5, 62, 0, 0, 856, 151, 1, 0, 0, 0, 857, 858, 5, 62, 0, 0, 858, 859, 5, 61, 0, 0, 859, 153, 1, 0, 0, 0, 860, 861, 5, 43, 0, 0, 861, 155, 1, 0, 0, 0, 862, 863, 5, 45, 0, 0, 863, 157, 1, 0, 0, 0, 864, 865, 5, 42, 0, 0, 865, 159, 1, 0, 0, 0, 866, 867, 5, 47, 0, 0, 867, 161, 1, 0, 0, 0, 868, 869, 5, 37, 0, 0, 869, 163, 1, 0, 0, 0, 870, 871, 4, 74, 4, 0, 871, 872, 3, 54, 19, 0, 872, 873, 1, 0, 0, 0, 873, 874, 6, 74, 13, 0, 874, 165, 1, 0, 0, 0, 875, 878, 3, 132, 58, 0, 876, 879, 3, 70, 27, 0, 877, 879, 3, 84, 34, 0, 878, 876, 1, 0, 0, 0, 878, 877, 1, 0, 0, 0, 879, 883, 1, 0, 0, 0, 880, 882, 3, 86, 35, 0, 881, 880, 1, 0, 0, 0, 882, 885, 1, 0, 0, 0, 883, 881, 1, 0, 0, 0, 883, 884, 1, 0, 0, 0, 884, 893, 1, 0, 0, 0, 885, 883, 1, 0, 0, 0, 886, 888, 3, 132, 58, 0, 887, 889, 3, 68, 26, 0, 888, 887, 1, 0, 0, 0, 889, 890, 1, 0, 0, 0, 890, 888, 1, 0, 0, 0, 890, 891, 1, 0, 0, 0, 891, 893, 1, 0, 0, 0, 892, 875, 1, 0, 0, 0, 892, 886, 1, 0, 0, 0, 893, 167, 1, 0, 0, 0, 894, 895, 5, 91, 0, 0, 895, 896, 1, 0, 0, 0, 896, 897, 6, 76, 0, 0, 897, 898, 6, 76, 0, 0, 898, 169, 1, 0, 0, 0, 899, 900, 5, 93, 0, 0, 900, 901, 1, 0, 0, 0, 901, 902, 6, 77, 12, 0, 902, 903, 6, 77, 12, 0, 903, 171, 1, 0, 0, 0, 904, 908, 3, 70, 27, 0, 905, 907, 3, 86, 35, 0, 906, 905, 1, 0, 0, 0, 907, 910, 1, 0, 0, 0, 908, 906, 1, 0, 0, 0, 908, 909, 1, 0, 0, 0, 909, 921, 1, 0, 0, 0, 910, 908, 1, 0, 0, 0, 911, 914, 3, 84, 34, 0, 912, 914, 3, 78, 31, 0, 913, 911, 1, 0, 0, 0, 913, 912, 1, 0, 0, 0, 914, 916, 1, 0, 0, 0, 915, 917, 3, 86, 35, 0, 916, 915, 1, 0, 0, 0, 917, 918, 1, 0, 0, 0, 918, 916, 1, 0, 0, 0, 918, 919, 1, 0, 0, 0, 919, 921, 1, 0, 0, 0, 920, 904, 1, 0, 0, 0, 920, 913, 1, 0, 0, 0, 921, 173, 1, 0, 0, 0, 922, 924, 3, 80, 32, 0, 923, 925, 3, 82, 33, 0, 924, 923, 1, 0, 0, 0, 925, 926, 1, 0, 0, 0, 926, 924, 1, 0, 0, 0, 926, 927, 1, 0, 0, 0, 927, 928, 1, 0, 0, 0, 928, 929, 3, 80, 32, 0, 929, 175, 1, 0, 0, 0, 930, 931, 3, 174, 79, 0, 931, 177, 1, 0, 0, 0, 932, 933, 3, 60, 22, 0, 933, 934, 1, 0, 0, 0, 934, 935, 6, 81, 11, 0, 935, 179, 1, 0, 0, 0, 936, 937, 3, 62, 23, 0, 937, 938, 1, 0, 0, 0, 938, 939, 6, 82, 11, 0, 939, 181, 1, 0, 0, 0, 940, 941, 3, 64, 24, 0, 941, 942, 1, 0, 0, 0, 942, 943, 6, 83, 11, 0, 943, 183, 1, 0, 0, 0, 944, 945, 3, 168, 76, 0, 945, 946, 1, 0, 0, 0, 946, 947, 6, 84, 14, 0, 947, 948, 6, 84, 15, 0, 948, 185, 1, 0, 0, 0, 949, 950, 3, 66, 25, 0, 950, 951, 1, 0, 0, 0, 951, 952, 6, 85, 16, 0, 952, 953, 6, 85, 12, 0, 953, 187, 1, 0, 0, 0, 954, 955, 3, 64, 24, 0, 955, 956, 1, 0, 0, 0, 956, 957, 6, 86, 11, 0, 957, 189, 1, 0, 0, 0, 958, 959, 3, 60, 22, 0, 959, 960, 1, 0, 0, 0, 960, 961, 6, 87, 11, 0, 961, 191, 1, 0, 0, 0, 962, 963, 3, 62, 23, 0, 963, 964, 1, 0, 0, 0, 964, 965, 6, 88, 11, 0, 965, 193, 1, 0, 0, 0, 966, 967, 3, 66, 25, 0, 967, 968, 1, 0, 0, 0, 968, 969, 6, 89, 16, 0, 969, 970, 6, 89, 12, 0, 970, 195, 1, 0, 0, 0, 971, 972, 3, 168, 76, 0, 972, 973, 1, 0, 0, 0, 973, 974, 6, 90, 14, 0, 974, 197, 1, 0, 0, 0, 975, 976, 3, 170, 77, 0, 976, 977, 1, 0, 0, 0, 977, 978, 6, 91, 17, 0, 978, 199, 1, 0, 0, 0, 979, 980, 3, 334, 159, 0, 980, 981, 1, 0, 0, 0, 981, 982, 6, 92, 18, 0, 982, 201, 1, 0, 0, 0, 983, 984, 3, 104, 44, 0, 984, 985, 1, 0, 0, 0, 985, 986, 6, 93, 19, 0, 986, 203, 1, 0, 0, 0, 987, 988, 3, 100, 42, 0, 988, 989, 1, 0, 0, 0, 989, 990, 6, 94, 20, 0, 990, 205, 1, 0, 0, 0, 991, 992, 7, 16, 0, 0, 992, 993, 7, 3, 0, 0, 993, 994, 7, 5, 0, 0, 994, 995, 7, 12, 0, 0, 995, 996, 7, 0, 0, 0, 996, 997, 7, 12, 0, 0, 997, 998, 7, 5, 0, 0, 998, 999, 7, 12, 0, 0, 999, 207, 1, 0, 0, 0, 1000, 1004, 8, 32, 0, 0, 1001, 1002, 5, 47, 0, 0, 1002, 1004, 8, 33, 0, 0, 1003, 1000, 1, 0, 0, 0, 1003, 1001, 1, 0, 0, 0, 1004, 209, 1, 0, 0, 0, 1005, 1007, 3, 208, 96, 0, 1006, 1005, 1, 0, 0, 0, 1007, 1008, 1, 0, 0, 0, 1008, 1006, 1, 0, 0, 0, 1008, 1009, 1, 0, 0, 0, 1009, 211, 1, 0, 0, 0, 1010, 1011, 3, 210, 97, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1013, 6, 98, 21, 0, 1013, 213, 1, 0, 0, 0, 1014, 1015, 3, 88, 36, 0, 1015, 1016, 1, 0, 0, 0, 1016, 1017, 6, 99, 22, 0, 1017, 215, 1, 0, 0, 0, 1018, 1019, 3, 60, 22, 0, 1019, 1020, 1, 0, 0, 0, 1020, 1021, 6, 100, 11, 0, 1021, 217, 1, 0, 0, 0, 1022, 1023, 3, 62, 23, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 101, 11, 0, 1025, 219, 1, 0, 0, 0, 1026, 1027, 3, 64, 24, 0, 1027, 1028, 1, 0, 0, 0, 1028, 1029, 6, 102, 11, 0, 1029, 221, 1, 0, 0, 0, 1030, 1031, 3, 66, 25, 0, 1031, 1032, 1, 0, 0, 0, 1032, 1033, 6, 103, 16, 0, 1033, 1034, 6, 103, 12, 0, 1034, 223, 1, 0, 0, 0, 1035, 1036, 3, 108, 46, 0, 1036, 1037, 1, 0, 0, 0, 1037, 1038, 6, 104, 23, 0, 1038, 225, 1, 0, 0, 0, 1039, 1040, 3, 104, 44, 0, 1040, 1041, 1, 0, 0, 0, 1041, 1042, 6, 105, 19, 0, 1042, 227, 1, 0, 0, 0, 1043, 1048, 3, 70, 27, 0, 1044, 1048, 3, 68, 26, 0, 1045, 1048, 3, 84, 34, 0, 1046, 1048, 3, 158, 71, 0, 1047, 1043, 1, 0, 0, 0, 1047, 1044, 1, 0, 0, 0, 1047, 1045, 1, 0, 0, 0, 1047, 1046, 1, 0, 0, 0, 1048, 229, 1, 0, 0, 0, 1049, 1052, 3, 70, 27, 0, 1050, 1052, 3, 158, 71, 0, 1051, 1049, 1, 0, 0, 0, 1051, 1050, 1, 0, 0, 0, 1052, 1056, 1, 0, 0, 0, 1053, 1055, 3, 228, 106, 0, 1054, 1053, 1, 0, 0, 0, 1055, 1058, 1, 0, 0, 0, 1056, 1054, 1, 0, 0, 0, 1056, 1057, 1, 0, 0, 0, 1057, 1069, 1, 0, 0, 0, 1058, 1056, 1, 0, 0, 0, 1059, 1062, 3, 84, 34, 0, 1060, 1062, 3, 78, 31, 0, 1061, 1059, 1, 0, 0, 0, 1061, 1060, 1, 0, 0, 0, 1062, 1064, 1, 0, 0, 0, 1063, 1065, 3, 228, 106, 0, 1064, 1063, 1, 0, 0, 0, 1065, 1066, 1, 0, 0, 0, 1066, 1064, 1, 0, 0, 0, 1066, 1067, 1, 0, 0, 0, 1067, 1069, 1, 0, 0, 0, 1068, 1051, 1, 0, 0, 0, 1068, 1061, 1, 0, 0, 0, 1069, 231, 1, 0, 0, 0, 1070, 1073, 3, 230, 107, 0, 1071, 1073, 3, 174, 79, 0, 1072, 1070, 1, 0, 0, 0, 1072, 1071, 1, 0, 0, 0, 1073, 1074, 1, 0, 0, 0, 1074, 1072, 1, 0, 0, 0, 1074, 1075, 1, 0, 0, 0, 1075, 233, 1, 0, 0, 0, 1076, 1077, 3, 60, 22, 0, 1077, 1078, 1, 0, 0, 0, 1078, 1079, 6, 109, 11, 0, 1079, 235, 1, 0, 0, 0, 1080, 1081, 3, 62, 23, 0, 1081, 1082, 1, 0, 0, 0, 1082, 1083, 6, 110, 11, 0, 1083, 237, 1, 0, 0, 0, 1084, 1085, 3, 64, 24, 0, 1085, 1086, 1, 0, 0, 0, 1086, 1087, 6, 111, 11, 0, 1087, 239, 1, 0, 0, 0, 1088, 1089, 3, 66, 25, 0, 1089, 1090, 1, 0, 0, 0, 1090, 1091, 6, 112, 16, 0, 1091, 1092, 6, 112, 12, 0, 1092, 241, 1, 0, 0, 0, 1093, 1094, 3, 100, 42, 0, 1094, 1095, 1, 0, 0, 0, 1095, 1096, 6, 113, 20, 0, 1096, 243, 1, 0, 0, 0, 1097, 1098, 3, 104, 44, 0, 1098, 1099, 1, 0, 0, 0, 1099, 1100, 6, 114, 19, 0, 1100, 245, 1, 0, 0, 0, 1101, 1102, 3, 108, 46, 0, 1102, 1103, 1, 0, 0, 0, 1103, 1104, 6, 115, 23, 0, 1104, 247, 1, 0, 0, 0, 1105, 1106, 7, 12, 0, 0, 1106, 1107, 7, 2, 0, 0, 1107, 249, 1, 0, 0, 0, 1108, 1109, 3, 232, 108, 0, 1109, 1110, 1, 0, 0, 0, 1110, 1111, 6, 117, 24, 0, 1111, 251, 1, 0, 0, 0, 1112, 1113, 3, 60, 22, 0, 1113, 1114, 1, 0, 0, 0, 1114, 1115, 6, 118, 11, 0, 1115, 253, 1, 0, 0, 0, 1116, 1117, 3, 62, 23, 0, 1117, 1118, 1, 0, 0, 0, 1118, 1119, 6, 119, 11, 0, 1119, 255, 1, 0, 0, 0, 1120, 1121, 3, 64, 24, 0, 1121, 1122, 1, 0, 0, 0, 1122, 1123, 6, 120, 11, 0, 1123, 257, 1, 0, 0, 0, 1124, 1125, 3, 66, 25, 0, 1125, 1126, 1, 0, 0, 0, 1126, 1127, 6, 121, 16, 0, 1127, 1128, 6, 121, 12, 0, 1128, 259, 1, 0, 0, 0, 1129, 1130, 3, 168, 76, 0, 1130, 1131, 1, 0, 0, 0, 1131, 1132, 6, 122, 14, 0, 1132, 1133, 6, 122, 25, 0, 1133, 261, 1, 0, 0, 0, 1134, 1135, 7, 7, 0, 0, 1135, 1136, 7, 9, 0, 0, 1136, 1137, 1, 0, 0, 0, 1137, 1138, 6, 123, 26, 0, 1138, 263, 1, 0, 0, 0, 1139, 1140, 7, 19, 0, 0, 1140, 1141, 7, 1, 0, 0, 1141, 1142, 7, 5, 0, 0, 1142, 1143, 7, 10, 0, 0, 1143, 1144, 1, 0, 0, 0, 1144, 1145, 6, 124, 26, 0, 1145, 265, 1, 0, 0, 0, 1146, 1147, 8, 34, 0, 0, 1147, 267, 1, 0, 0, 0, 1148, 1150, 3, 266, 125, 0, 1149, 1148, 1, 0, 0, 0, 1150, 1151, 1, 0, 0, 0, 1151, 1149, 1, 0, 0, 0, 1151, 1152, 1, 0, 0, 0, 1152, 1153, 1, 0, 0, 0, 1153, 1154, 3, 334, 159, 0, 1154, 1156, 1, 0, 0, 0, 1155, 1149, 1, 0, 0, 0, 1155, 1156, 1, 0, 0, 0, 1156, 1158, 1, 0, 0, 0, 1157, 1159, 3, 266, 125, 0, 1158, 1157, 1, 0, 0, 0, 1159, 1160, 1, 0, 0, 0, 1160, 1158, 1, 0, 0, 0, 1160, 1161, 1, 0, 0, 0, 1161, 269, 1, 0, 0, 0, 1162, 1163, 3, 268, 126, 0, 1163, 1164, 1, 0, 0, 0, 1164, 1165, 6, 127, 27, 0, 1165, 271, 1, 0, 0, 0, 1166, 1167, 3, 60, 22, 0, 1167, 1168, 1, 0, 0, 0, 1168, 1169, 6, 128, 11, 0, 1169, 273, 1, 0, 0, 0, 1170, 1171, 3, 62, 23, 0, 1171, 1172, 1, 0, 0, 0, 1172, 1173, 6, 129, 11, 0, 1173, 275, 1, 0, 0, 0, 1174, 1175, 3, 64, 24, 0, 1175, 1176, 1, 0, 0, 0, 1176, 1177, 6, 130, 11, 0, 1177, 277, 1, 0, 0, 0, 1178, 1179, 3, 66, 25, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1181, 6, 131, 16, 0, 1181, 1182, 6, 131, 12, 0, 1182, 1183, 6, 131, 12, 0, 1183, 279, 1, 0, 0, 0, 1184, 1185, 3, 100, 42, 0, 1185, 1186, 1, 0, 0, 0, 1186, 1187, 6, 132, 20, 0, 1187, 281, 1, 0, 0, 0, 1188, 1189, 3, 104, 44, 0, 1189, 1190, 1, 0, 0, 0, 1190, 1191, 6, 133, 19, 0, 1191, 283, 1, 0, 0, 0, 1192, 1193, 3, 108, 46, 0, 1193, 1194, 1, 0, 0, 0, 1194, 1195, 6, 134, 23, 0, 1195, 285, 1, 0, 0, 0, 1196, 1197, 3, 264, 124, 0, 1197, 1198, 1, 0, 0, 0, 1198, 1199, 6, 135, 28, 0, 1199, 287, 1, 0, 0, 0, 1200, 1201, 3, 232, 108, 0, 1201, 1202, 1, 0, 0, 0, 1202, 1203, 6, 136, 24, 0, 1203, 289, 1, 0, 0, 0, 1204, 1205, 3, 176, 80, 0, 1205, 1206, 1, 0, 0, 0, 1206, 1207, 6, 137, 29, 0, 1207, 291, 1, 0, 0, 0, 1208, 1209, 3, 60, 22, 0, 1209, 1210, 1, 0, 0, 0, 1210, 1211, 6, 138, 11, 0, 1211, 293, 1, 0, 0, 0, 1212, 1213, 3, 62, 23, 0, 1213, 1214, 1, 0, 0, 0, 1214, 1215, 6, 139, 11, 0, 1215, 295, 1, 0, 0, 0, 1216, 1217, 3, 64, 24, 0, 1217, 1218, 1, 0, 0, 0, 1218, 1219, 6, 140, 11, 0, 1219, 297, 1, 0, 0, 0, 1220, 1221, 3, 66, 25, 0, 1221, 1222, 1, 0, 0, 0, 1222, 1223, 6, 141, 16, 0, 1223, 1224, 6, 141, 12, 0, 1224, 299, 1, 0, 0, 0, 1225, 1226, 3, 108, 46, 0, 1226, 1227, 1, 0, 0, 0, 1227, 1228, 6, 142, 23, 0, 1228, 301, 1, 0, 0, 0, 1229, 1230, 3, 176, 80, 0, 1230, 1231, 1, 0, 0, 0, 1231, 1232, 6, 143, 29, 0, 1232, 303, 1, 0, 0, 0, 1233, 1234, 3, 172, 78, 0, 1234, 1235, 1, 0, 0, 0, 1235, 1236, 6, 144, 30, 0, 1236, 305, 1, 0, 0, 0, 1237, 1238, 3, 60, 22, 0, 1238, 1239, 1, 0, 0, 0, 1239, 1240, 6, 145, 11, 0, 1240, 307, 1, 0, 0, 0, 1241, 1242, 3, 62, 23, 0, 1242, 1243, 1, 0, 0, 0, 1243, 1244, 6, 146, 11, 0, 1244, 309, 1, 0, 0, 0, 1245, 1246, 3, 64, 24, 0, 1246, 1247, 1, 0, 0, 0, 1247, 1248, 6, 147, 11, 0, 1248, 311, 1, 0, 0, 0, 1249, 1250, 3, 66, 25, 0, 1250, 1251, 1, 0, 0, 0, 1251, 1252, 6, 148, 16, 0, 1252, 1253, 6, 148, 12, 0, 1253, 313, 1, 0, 0, 0, 1254, 1255, 7, 1, 0, 0, 1255, 1256, 7, 9, 0, 0, 1256, 1257, 7, 15, 0, 0, 1257, 1258, 7, 7, 0, 0, 1258, 315, 1, 0, 0, 0, 1259, 1260, 3, 60, 22, 0, 1260, 1261, 1, 0, 0, 0, 1261, 1262, 6, 150, 11, 0, 1262, 317, 1, 0, 0, 0, 1263, 1264, 3, 62, 23, 0, 1264, 1265, 1, 0, 0, 0, 1265, 1266, 6, 151, 11, 0, 1266, 319, 1, 0, 0, 0, 1267, 1268, 3, 64, 24, 0, 1268, 1269, 1, 0, 0, 0, 1269, 1270, 6, 152, 11, 0, 1270, 321, 1, 0, 0, 0, 1271, 1272, 3, 66, 25, 0, 1272, 1273, 1, 0, 0, 0, 1273, 1274, 6, 153, 16, 0, 1274, 1275, 6, 153, 12, 0, 1275, 323, 1, 0, 0, 0, 1276, 1277, 7, 15, 0, 0, 1277, 1278, 7, 20, 0, 0, 1278, 1279, 7, 9, 0, 0, 1279, 1280, 7, 4, 0, 0, 1280, 1281, 7, 5, 0, 0, 1281, 1282, 7, 1, 0, 0, 1282, 1283, 7, 7, 0, 0, 1283, 1284, 7, 9, 0, 0, 1284, 1285, 7, 2, 0, 0, 1285, 325, 1, 0, 0, 0, 1286, 1287, 3, 60, 22, 0, 1287, 1288, 1, 0, 0, 0, 1288, 1289, 6, 155, 11, 0, 1289, 327, 1, 0, 0, 0, 1290, 1291, 3, 62, 23, 0, 1291, 1292, 1, 0, 0, 0, 1292, 1293, 6, 156, 11, 0, 1293, 329, 1, 0, 0, 0, 1294, 1295, 3, 64, 24, 0, 1295, 1296, 1, 0, 0, 0, 1296, 1297, 6, 157, 11, 0, 1297, 331, 1, 0, 0, 0, 1298, 1299, 3, 170, 77, 0, 1299, 1300, 1, 0, 0, 0, 1300, 1301, 6, 158, 17, 0, 1301, 1302, 6, 158, 12, 0, 1302, 333, 1, 0, 0, 0, 1303, 1304, 5, 58, 0, 0, 1304, 335, 1, 0, 0, 0, 1305, 1311, 3, 78, 31, 0, 1306, 1311, 3, 68, 26, 0, 1307, 1311, 3, 108, 46, 0, 1308, 1311, 3, 70, 27, 0, 1309, 1311, 3, 84, 34, 0, 1310, 1305, 1, 0, 0, 0, 1310, 1306, 1, 0, 0, 0, 1310, 1307, 1, 0, 0, 0, 1310, 1308, 1, 0, 0, 0, 1310, 1309, 1, 0, 0, 0, 1311, 1312, 1, 0, 0, 0, 1312, 1310, 1, 0, 0, 0, 1312, 1313, 1, 0, 0, 0, 1313, 337, 1, 0, 0, 0, 1314, 1315, 3, 60, 22, 0, 1315, 1316, 1, 0, 0, 0, 1316, 1317, 6, 161, 11, 0, 1317, 339, 1, 0, 0, 0, 1318, 1319, 3, 62, 23, 0, 1319, 1320, 1, 0, 0, 0, 1320, 1321, 6, 162, 11, 0, 1321, 341, 1, 0, 0, 0, 1322, 1323, 3, 64, 24, 0, 1323, 1324, 1, 0, 0, 0, 1324, 1325, 6, 163, 11, 0, 1325, 343, 1, 0, 0, 0, 1326, 1327, 3, 66, 25, 0, 1327, 1328, 1, 0, 0, 0, 1328, 1329, 6, 164, 16, 0, 1329, 1330, 6, 164, 12, 0, 1330, 345, 1, 0, 0, 0, 1331, 1332, 3, 334, 159, 0, 1332, 1333, 1, 0, 0, 0, 1333, 1334, 6, 165, 18, 0, 1334, 347, 1, 0, 0, 0, 1335, 1336, 3, 104, 44, 0, 1336, 1337, 1, 0, 0, 0, 1337, 1338, 6, 166, 19, 0, 1338, 349, 1, 0, 0, 0, 1339, 1340, 3, 108, 46, 0, 1340, 1341, 1, 0, 0, 0, 1341, 1342, 6, 167, 23, 0, 1342, 351, 1, 0, 0, 0, 1343, 1344, 3, 262, 123, 0, 1344, 1345, 1, 0, 0, 0, 1345, 1346, 6, 168, 31, 0, 1346, 1347, 6, 168, 32, 0, 1347, 353, 1, 0, 0, 0, 1348, 1349, 3, 210, 97, 0, 1349, 1350, 1, 0, 0, 0, 1350, 1351, 6, 169, 21, 0, 1351, 355, 1, 0, 0, 0, 1352, 1353, 3, 88, 36, 0, 1353, 1354, 1, 0, 0, 0, 1354, 1355, 6, 170, 22, 0, 1355, 357, 1, 0, 0, 0, 1356, 1357, 3, 60, 22, 0, 1357, 1358, 1, 0, 0, 0, 1358, 1359, 6, 171, 11, 0, 1359, 359, 1, 0, 0, 0, 1360, 1361, 3, 62, 23, 0, 1361, 1362, 1, 0, 0, 0, 1362, 1363, 6, 172, 11, 0, 1363, 361, 1, 0, 0, 0, 1364, 1365, 3, 64, 24, 0, 1365, 1366, 1, 0, 0, 0, 1366, 1367, 6, 173, 11, 0, 1367, 363, 1, 0, 0, 0, 1368, 1369, 3, 66, 25, 0, 1369, 1370, 1, 0, 0, 0, 1370, 1371, 6, 174, 16, 0, 1371, 1372, 6, 174, 12, 0, 1372, 1373, 6, 174, 12, 0, 1373, 365, 1, 0, 0, 0, 1374, 1375, 3, 104, 44, 0, 1375, 1376, 1, 0, 0, 0, 1376, 1377, 6, 175, 19, 0, 1377, 367, 1, 0, 0, 0, 1378, 1379, 3, 108, 46, 0, 1379, 1380, 1, 0, 0, 0, 1380, 1381, 6, 176, 23, 0, 1381, 369, 1, 0, 0, 0, 1382, 1383, 3, 232, 108, 0, 1383, 1384, 1, 0, 0, 0, 1384, 1385, 6, 177, 24, 0, 1385, 371, 1, 0, 0, 0, 1386, 1387, 3, 60, 22, 0, 1387, 1388, 1, 0, 0, 0, 1388, 1389, 6, 178, 11, 0, 1389, 373, 1, 0, 0, 0, 1390, 1391, 3, 62, 23, 0, 1391, 1392, 1, 0, 0, 0, 1392, 1393, 6, 179, 11, 0, 1393, 375, 1, 0, 0, 0, 1394, 1395, 3, 64, 24, 0, 1395, 1396, 1, 0, 0, 0, 1396, 1397, 6, 180, 11, 0, 1397, 377, 1, 0, 0, 0, 1398, 1399, 3, 66, 25, 0, 1399, 1400, 1, 0, 0, 0, 1400, 1401, 6, 181, 16, 0, 1401, 1402, 6, 181, 12, 0, 1402, 379, 1, 0, 0, 0, 1403, 1404, 3, 210, 97, 0, 1404, 1405, 1, 0, 0, 0, 1405, 1406, 6, 182, 21, 0, 1406, 1407, 6, 182, 12, 0, 1407, 1408, 6, 182, 33, 0, 1408, 381, 1, 0, 0, 0, 1409, 1410, 3, 88, 36, 0, 1410, 1411, 1, 0, 0, 0, 1411, 1412, 6, 183, 22, 0, 1412, 1413, 6, 183, 12, 0, 1413, 1414, 6, 183, 33, 0, 1414, 383, 1, 0, 0, 0, 1415, 1416, 3, 60, 22, 0, 1416, 1417, 1, 0, 0, 0, 1417, 1418, 6, 184, 11, 0, 1418, 385, 1, 0, 0, 0, 1419, 1420, 3, 62, 23, 0, 1420, 1421, 1, 0, 0, 0, 1421, 1422, 6, 185, 11, 0, 1422, 387, 1, 0, 0, 0, 1423, 1424, 3, 64, 24, 0, 1424, 1425, 1, 0, 0, 0, 1425, 1426, 6, 186, 11, 0, 1426, 389, 1, 0, 0, 0, 1427, 1428, 3, 334, 159, 0, 1428, 1429, 1, 0, 0, 0, 1429, 1430, 6, 187, 18, 0, 1430, 1431, 6, 187, 12, 0, 1431, 1432, 6, 187, 10, 0, 1432, 391, 1, 0, 0, 0, 1433, 1434, 3, 104, 44, 0, 1434, 1435, 1, 0, 0, 0, 1435, 1436, 6, 188, 19, 0, 1436, 1437, 6, 188, 12, 0, 1437, 1438, 6, 188, 10, 0, 1438, 393, 1, 0, 0, 0, 1439, 1440, 3, 60, 22, 0, 1440, 1441, 1, 0, 0, 0, 1441, 1442, 6, 189, 11, 0, 1442, 395, 1, 0, 0, 0, 1443, 1444, 3, 62, 23, 0, 1444, 1445, 1, 0, 0, 0, 1445, 1446, 6, 190, 11, 0, 1446, 397, 1, 0, 0, 0, 1447, 1448, 3, 64, 24, 0, 1448, 1449, 1, 0, 0, 0, 1449, 1450, 6, 191, 11, 0, 1450, 399, 1, 0, 0, 0, 1451, 1452, 3, 176, 80, 0, 1452, 1453, 1, 0, 0, 0, 1453, 1454, 6, 192, 12, 0, 1454, 1455, 6, 192, 0, 0, 1455, 1456, 6, 192, 29, 0, 1456, 401, 1, 0, 0, 0, 1457, 1458, 3, 172, 78, 0, 1458, 1459, 1, 0, 0, 0, 1459, 1460, 6, 193, 12, 0, 1460, 1461, 6, 193, 0, 0, 1461, 1462, 6, 193, 30, 0, 1462, 403, 1, 0, 0, 0, 1463, 1464, 3, 94, 39, 0, 1464, 1465, 1, 0, 0, 0, 1465, 1466, 6, 194, 12, 0, 1466, 1467, 6, 194, 0, 0, 1467, 1468, 6, 194, 34, 0, 1468, 405, 1, 0, 0, 0, 1469, 1470, 3, 66, 25, 0, 1470, 1471, 1, 0, 0, 0, 1471, 1472, 6, 195, 16, 0, 1472, 1473, 6, 195, 12, 0, 1473, 407, 1, 0, 0, 0, 66, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 592, 602, 606, 609, 618, 620, 631, 650, 655, 664, 671, 676, 678, 689, 697, 700, 702, 707, 712, 718, 725, 730, 736, 739, 747, 751, 878, 883, 890, 892, 908, 913, 918, 920, 926, 1003, 1008, 1047, 1051, 1056, 1061, 1066, 1068, 1072, 1074, 1151, 1155, 1160, 1310, 1312, 35, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 10, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 12, 0, 5, 14, 0, 0, 1, 0, 4, 0, 0, 7, 20, 0, 7, 66, 0, 5, 0, 0, 7, 26, 0, 7, 67, 0, 7, 109, 0, 7, 35, 0, 7, 33, 0, 7, 77, 0, 7, 27, 0, 7, 37, 0, 7, 81, 0, 5, 11, 0, 5, 7, 0, 7, 91, 0, 7, 90, 0, 7, 69, 0, 7, 68, 0, 7, 89, 0, 5, 13, 0, 5, 15, 0, 7, 30, 0]
\ No newline at end of file
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java
index de837d1764791..a746a0d49004f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java
@@ -1,5 +1,13 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.xpack.esql.parser;
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
 import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Token;
@@ -10,72 +18,73 @@
 import org.antlr.v4.runtime.misc.*;
 
 @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue", "this-escape"})
-public class EsqlBaseLexer extends Lexer {
+public class EsqlBaseLexer extends LexerConfig {
   static { RuntimeMetaData.checkVersion("4.13.1", RuntimeMetaData.VERSION); }
 
   protected static final DFA[] _decisionToDFA;
   protected static final PredictionContextCache _sharedContextCache =
     new PredictionContextCache();
   public static final int
-    DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, INLINESTATS=8, 
-    KEEP=9, LIMIT=10, LOOKUP=11, META=12, METRICS=13, MV_EXPAND=14, RENAME=15, 
-    ROW=16, SHOW=17, SORT=18, STATS=19, WHERE=20, UNKNOWN_CMD=21, LINE_COMMENT=22, 
-    MULTILINE_COMMENT=23, WS=24, UNQUOTED_SOURCE=25, EXPLAIN_WS=26, EXPLAIN_LINE_COMMENT=27, 
-    EXPLAIN_MULTILINE_COMMENT=28, PIPE=29, QUOTED_STRING=30, INTEGER_LITERAL=31, 
-    DECIMAL_LITERAL=32, BY=33, AND=34, ASC=35, ASSIGN=36, CAST_OP=37, COMMA=38, 
-    DESC=39, DOT=40, FALSE=41, FIRST=42, LAST=43, LP=44, IN=45, IS=46, LIKE=47, 
-    NOT=48, NULL=49, NULLS=50, OR=51, PARAM=52, RLIKE=53, RP=54, TRUE=55, 
-    EQ=56, CIEQ=57, NEQ=58, LT=59, LTE=60, GT=61, GTE=62, PLUS=63, MINUS=64, 
-    ASTERISK=65, SLASH=66, PERCENT=67, NAMED_OR_POSITIONAL_PARAM=68, OPENING_BRACKET=69, 
-    CLOSING_BRACKET=70, UNQUOTED_IDENTIFIER=71, QUOTED_IDENTIFIER=72, EXPR_LINE_COMMENT=73, 
-    EXPR_MULTILINE_COMMENT=74, EXPR_WS=75, METADATA=76, FROM_LINE_COMMENT=77, 
-    FROM_MULTILINE_COMMENT=78, FROM_WS=79, ID_PATTERN=80, PROJECT_LINE_COMMENT=81, 
-    PROJECT_MULTILINE_COMMENT=82, PROJECT_WS=83, AS=84, RENAME_LINE_COMMENT=85, 
-    RENAME_MULTILINE_COMMENT=86, RENAME_WS=87, ON=88, WITH=89, ENRICH_POLICY_NAME=90, 
-    ENRICH_LINE_COMMENT=91, ENRICH_MULTILINE_COMMENT=92, ENRICH_WS=93, ENRICH_FIELD_LINE_COMMENT=94, 
-    ENRICH_FIELD_MULTILINE_COMMENT=95, ENRICH_FIELD_WS=96, LOOKUP_LINE_COMMENT=97, 
-    LOOKUP_MULTILINE_COMMENT=98, LOOKUP_WS=99, LOOKUP_FIELD_LINE_COMMENT=100, 
-    LOOKUP_FIELD_MULTILINE_COMMENT=101, LOOKUP_FIELD_WS=102, MVEXPAND_LINE_COMMENT=103, 
-    MVEXPAND_MULTILINE_COMMENT=104, MVEXPAND_WS=105, INFO=106, SHOW_LINE_COMMENT=107, 
-    SHOW_MULTILINE_COMMENT=108, SHOW_WS=109, FUNCTIONS=110, META_LINE_COMMENT=111, 
-    META_MULTILINE_COMMENT=112, META_WS=113, COLON=114, SETTING=115, SETTING_LINE_COMMENT=116, 
-    SETTTING_MULTILINE_COMMENT=117, SETTING_WS=118, METRICS_LINE_COMMENT=119, 
-    METRICS_MULTILINE_COMMENT=120, METRICS_WS=121, CLOSING_METRICS_LINE_COMMENT=122, 
-    CLOSING_METRICS_MULTILINE_COMMENT=123, CLOSING_METRICS_WS=124;
+    DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, 
+    LIMIT=9, META=10, MV_EXPAND=11, RENAME=12, ROW=13, SHOW=14, SORT=15, STATS=16, 
+    WHERE=17, DEV_INLINESTATS=18, DEV_LOOKUP=19, DEV_MATCH=20, DEV_METRICS=21, 
+    UNKNOWN_CMD=22, LINE_COMMENT=23, MULTILINE_COMMENT=24, WS=25, PIPE=26, 
+    QUOTED_STRING=27, INTEGER_LITERAL=28, DECIMAL_LITERAL=29, BY=30, AND=31, 
+    ASC=32, ASSIGN=33, CAST_OP=34, COMMA=35, DESC=36, DOT=37, FALSE=38, FIRST=39, 
+    IN=40, IS=41, LAST=42, LIKE=43, LP=44, NOT=45, NULL=46, NULLS=47, OR=48, 
+    PARAM=49, RLIKE=50, RP=51, TRUE=52, EQ=53, CIEQ=54, NEQ=55, LT=56, LTE=57, 
+    GT=58, GTE=59, PLUS=60, MINUS=61, ASTERISK=62, SLASH=63, PERCENT=64, NAMED_OR_POSITIONAL_PARAM=65, 
+    OPENING_BRACKET=66, CLOSING_BRACKET=67, UNQUOTED_IDENTIFIER=68, QUOTED_IDENTIFIER=69, 
+    EXPR_LINE_COMMENT=70, EXPR_MULTILINE_COMMENT=71, EXPR_WS=72, EXPLAIN_WS=73, 
+    EXPLAIN_LINE_COMMENT=74, EXPLAIN_MULTILINE_COMMENT=75, METADATA=76, UNQUOTED_SOURCE=77, 
+    FROM_LINE_COMMENT=78, FROM_MULTILINE_COMMENT=79, FROM_WS=80, ID_PATTERN=81, 
+    PROJECT_LINE_COMMENT=82, PROJECT_MULTILINE_COMMENT=83, PROJECT_WS=84, 
+    AS=85, RENAME_LINE_COMMENT=86, RENAME_MULTILINE_COMMENT=87, RENAME_WS=88, 
+    ON=89, WITH=90, ENRICH_POLICY_NAME=91, ENRICH_LINE_COMMENT=92, ENRICH_MULTILINE_COMMENT=93, 
+    ENRICH_WS=94, ENRICH_FIELD_LINE_COMMENT=95, ENRICH_FIELD_MULTILINE_COMMENT=96, 
+    ENRICH_FIELD_WS=97, MVEXPAND_LINE_COMMENT=98, MVEXPAND_MULTILINE_COMMENT=99, 
+    MVEXPAND_WS=100, INFO=101, SHOW_LINE_COMMENT=102, SHOW_MULTILINE_COMMENT=103, 
+    SHOW_WS=104, FUNCTIONS=105, META_LINE_COMMENT=106, META_MULTILINE_COMMENT=107, 
+    META_WS=108, COLON=109, SETTING=110, SETTING_LINE_COMMENT=111, SETTTING_MULTILINE_COMMENT=112, 
+    SETTING_WS=113, LOOKUP_LINE_COMMENT=114, LOOKUP_MULTILINE_COMMENT=115, 
+    LOOKUP_WS=116, LOOKUP_FIELD_LINE_COMMENT=117, LOOKUP_FIELD_MULTILINE_COMMENT=118, 
+    LOOKUP_FIELD_WS=119, METRICS_LINE_COMMENT=120, METRICS_MULTILINE_COMMENT=121, 
+    METRICS_WS=122, CLOSING_METRICS_LINE_COMMENT=123, CLOSING_METRICS_MULTILINE_COMMENT=124, 
+    CLOSING_METRICS_WS=125;
   public static final int
-    EXPLAIN_MODE=1, EXPRESSION_MODE=2, FROM_MODE=3, PROJECT_MODE=4, RENAME_MODE=5, 
-    ENRICH_MODE=6, ENRICH_FIELD_MODE=7, LOOKUP_MODE=8, LOOKUP_FIELD_MODE=9, 
-    MVEXPAND_MODE=10, SHOW_MODE=11, META_MODE=12, SETTING_MODE=13, METRICS_MODE=14, 
+    EXPRESSION_MODE=1, EXPLAIN_MODE=2, FROM_MODE=3, PROJECT_MODE=4, RENAME_MODE=5, 
+    ENRICH_MODE=6, ENRICH_FIELD_MODE=7, MVEXPAND_MODE=8, SHOW_MODE=9, META_MODE=10, 
+    SETTING_MODE=11, LOOKUP_MODE=12, LOOKUP_FIELD_MODE=13, METRICS_MODE=14, 
     CLOSING_METRICS_MODE=15;
   public static String[] channelNames = {
     "DEFAULT_TOKEN_CHANNEL", "HIDDEN"
   };
 
   public static String[] modeNames = {
-    "DEFAULT_MODE", "EXPLAIN_MODE", "EXPRESSION_MODE", "FROM_MODE", "PROJECT_MODE", 
-    "RENAME_MODE", "ENRICH_MODE", "ENRICH_FIELD_MODE", "LOOKUP_MODE", "LOOKUP_FIELD_MODE", 
-    "MVEXPAND_MODE", "SHOW_MODE", "META_MODE", "SETTING_MODE", "METRICS_MODE", 
+    "DEFAULT_MODE", "EXPRESSION_MODE", "EXPLAIN_MODE", "FROM_MODE", "PROJECT_MODE", 
+    "RENAME_MODE", "ENRICH_MODE", "ENRICH_FIELD_MODE", "MVEXPAND_MODE", "SHOW_MODE", 
+    "META_MODE", "SETTING_MODE", "LOOKUP_MODE", "LOOKUP_FIELD_MODE", "METRICS_MODE", 
     "CLOSING_METRICS_MODE"
   };
 
   private static String[] makeRuleNames() {
     return new String[] {
-      "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "INLINESTATS", 
-      "KEEP", "LIMIT", "LOOKUP", "META", "METRICS", "MV_EXPAND", "RENAME", 
-      "ROW", "SHOW", "SORT", "STATS", "WHERE", "UNKNOWN_CMD", "LINE_COMMENT", 
-      "MULTILINE_COMMENT", "WS", "UNQUOTED_SOURCE_PART", "UNQUOTED_SOURCE", 
-      "EXPLAIN_OPENING_BRACKET", "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", 
-      "EXPLAIN_MULTILINE_COMMENT", "PIPE", "DIGIT", "LETTER", "ESCAPE_SEQUENCE", 
-      "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", "BACKQUOTE_BLOCK", 
-      "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", "INTEGER_LITERAL", 
-      "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", 
-      "DESC", "DOT", "FALSE", "FIRST", "LAST", "LP", "IN", "IS", "LIKE", "NOT", 
-      "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", 
-      "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", 
-      "PERCENT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", 
-      "UNQUOTED_IDENTIFIER", "QUOTED_ID", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", 
-      "EXPR_MULTILINE_COMMENT", "EXPR_WS", "FROM_PIPE", "FROM_OPENING_BRACKET", 
-      "FROM_CLOSING_BRACKET", "FROM_COLON", "FROM_COMMA", "FROM_ASSIGN", "METADATA", 
+      "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", 
+      "LIMIT", "META", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", 
+      "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", 
+      "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", 
+      "LETTER", "ESCAPE_SEQUENCE", "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", 
+      "BACKQUOTE", "BACKQUOTE_BLOCK", "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", 
+      "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", 
+      "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", 
+      "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", 
+      "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", 
+      "SLASH", "PERCENT", "DEV_MATCH_OP", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", 
+      "CLOSING_BRACKET", "UNQUOTED_IDENTIFIER", "QUOTED_ID", "QUOTED_IDENTIFIER", 
+      "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_OPENING_BRACKET", 
+      "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", 
+      "FROM_PIPE", "FROM_OPENING_BRACKET", "FROM_CLOSING_BRACKET", "FROM_COLON", 
+      "FROM_COMMA", "FROM_ASSIGN", "METADATA", "UNQUOTED_SOURCE_PART", "UNQUOTED_SOURCE", 
       "FROM_UNQUOTED_SOURCE", "FROM_QUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", 
       "FROM_WS", "PROJECT_PIPE", "PROJECT_DOT", "PROJECT_COMMA", "UNQUOTED_ID_BODY_WITH_PATTERN", 
       "UNQUOTED_ID_PATTERN", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", 
@@ -86,21 +95,21 @@ private static String[] makeRuleNames() {
       "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_PIPE", "ENRICH_FIELD_ASSIGN", 
       "ENRICH_FIELD_COMMA", "ENRICH_FIELD_DOT", "ENRICH_FIELD_WITH", "ENRICH_FIELD_ID_PATTERN", 
       "ENRICH_FIELD_QUOTED_IDENTIFIER", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", 
-      "ENRICH_FIELD_WS", "LOOKUP_PIPE", "LOOKUP_COLON", "LOOKUP_COMMA", "LOOKUP_DOT", 
-      "LOOKUP_ON", "LOOKUP_UNQUOTED_SOURCE", "LOOKUP_QUOTED_SOURCE", "LOOKUP_LINE_COMMENT", 
-      "LOOKUP_MULTILINE_COMMENT", "LOOKUP_WS", "LOOKUP_FIELD_PIPE", "LOOKUP_FIELD_COMMA", 
-      "LOOKUP_FIELD_DOT", "LOOKUP_FIELD_ID_PATTERN", "LOOKUP_FIELD_LINE_COMMENT", 
-      "LOOKUP_FIELD_MULTILINE_COMMENT", "LOOKUP_FIELD_WS", "MVEXPAND_PIPE", 
-      "MVEXPAND_DOT", "MVEXPAND_QUOTED_IDENTIFIER", "MVEXPAND_UNQUOTED_IDENTIFIER", 
-      "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", 
-      "SHOW_PIPE", "INFO", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", "SHOW_WS", 
-      "META_PIPE", "FUNCTIONS", "META_LINE_COMMENT", "META_MULTILINE_COMMENT", 
+      "ENRICH_FIELD_WS", "MVEXPAND_PIPE", "MVEXPAND_DOT", "MVEXPAND_QUOTED_IDENTIFIER", 
+      "MVEXPAND_UNQUOTED_IDENTIFIER", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", 
+      "MVEXPAND_WS", "SHOW_PIPE", "INFO", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", 
+      "SHOW_WS", "META_PIPE", "FUNCTIONS", "META_LINE_COMMENT", "META_MULTILINE_COMMENT", 
       "META_WS", "SETTING_CLOSING_BRACKET", "COLON", "SETTING", "SETTING_LINE_COMMENT", 
-      "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "METRICS_PIPE", "METRICS_UNQUOTED_SOURCE", 
-      "METRICS_QUOTED_SOURCE", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", 
-      "METRICS_WS", "CLOSING_METRICS_COLON", "CLOSING_METRICS_COMMA", "CLOSING_METRICS_LINE_COMMENT", 
-      "CLOSING_METRICS_MULTILINE_COMMENT", "CLOSING_METRICS_WS", "CLOSING_METRICS_QUOTED_IDENTIFIER", 
-      "CLOSING_METRICS_UNQUOTED_IDENTIFIER", "CLOSING_METRICS_BY", "CLOSING_METRICS_PIPE"
+      "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_PIPE", "LOOKUP_COLON", 
+      "LOOKUP_COMMA", "LOOKUP_DOT", "LOOKUP_ON", "LOOKUP_UNQUOTED_SOURCE", 
+      "LOOKUP_QUOTED_SOURCE", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", 
+      "LOOKUP_WS", "LOOKUP_FIELD_PIPE", "LOOKUP_FIELD_COMMA", "LOOKUP_FIELD_DOT", 
+      "LOOKUP_FIELD_ID_PATTERN", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", 
+      "LOOKUP_FIELD_WS", "METRICS_PIPE", "METRICS_UNQUOTED_SOURCE", "METRICS_QUOTED_SOURCE", 
+      "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", "METRICS_WS", "CLOSING_METRICS_COLON", 
+      "CLOSING_METRICS_COMMA", "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", 
+      "CLOSING_METRICS_WS", "CLOSING_METRICS_QUOTED_IDENTIFIER", "CLOSING_METRICS_UNQUOTED_IDENTIFIER", 
+      "CLOSING_METRICS_BY", "CLOSING_METRICS_PIPE"
     };
   }
   public static final String[] ruleNames = makeRuleNames();
@@ -108,46 +117,45 @@ private static String[] makeRuleNames() {
   private static String[] makeLiteralNames() {
     return new String[] {
       null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", 
-      "'grok'", "'inlinestats'", "'keep'", "'limit'", "'lookup'", "'meta'", 
-      "'metrics'", "'mv_expand'", "'rename'", "'row'", "'show'", "'sort'", 
-      "'stats'", "'where'", null, null, null, null, null, null, null, null, 
-      "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "','", 
-      "'desc'", "'.'", "'false'", "'first'", "'last'", "'('", "'in'", "'is'", 
-      "'like'", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", 
-      "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", 
-      "'-'", "'*'", "'/'", "'%'", null, null, "']'", null, null, null, null, 
-      null, "'metadata'", null, null, null, null, null, null, null, "'as'", 
-      null, null, null, "'on'", "'with'", null, null, null, null, null, null, 
-      null, null, null, null, null, null, null, null, null, null, "'info'", 
-      null, null, null, "'functions'", null, null, null, "':'"
+      "'grok'", "'keep'", "'limit'", "'meta'", "'mv_expand'", "'rename'", "'row'", 
+      "'show'", "'sort'", "'stats'", "'where'", null, null, null, null, null, 
+      null, null, null, "'|'", null, null, null, "'by'", "'and'", "'asc'", 
+      "'='", "'::'", "','", "'desc'", "'.'", "'false'", "'first'", "'in'", 
+      "'is'", "'last'", "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", 
+      "'?'", "'rlike'", "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", 
+      "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", 
+      null, null, null, null, null, null, null, null, "'metadata'", null, null, 
+      null, null, null, null, null, null, "'as'", null, null, null, "'on'", 
+      "'with'", null, null, null, null, null, null, null, null, null, null, 
+      "'info'", null, null, null, "'functions'", null, null, null, "':'"
     };
   }
   private static final String[] _LITERAL_NAMES = makeLiteralNames();
   private static String[] makeSymbolicNames() {
     return new String[] {
       null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", 
-      "INLINESTATS", "KEEP", "LIMIT", "LOOKUP", "META", "METRICS", "MV_EXPAND", 
-      "RENAME", "ROW", "SHOW", "SORT", "STATS", "WHERE", "UNKNOWN_CMD", "LINE_COMMENT", 
-      "MULTILINE_COMMENT", "WS", "UNQUOTED_SOURCE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", 
-      "EXPLAIN_MULTILINE_COMMENT", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", 
-      "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", 
-      "DESC", "DOT", "FALSE", "FIRST", "LAST", "LP", "IN", "IS", "LIKE", "NOT", 
-      "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", 
-      "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", 
-      "PERCENT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", 
+      "KEEP", "LIMIT", "META", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", 
+      "STATS", "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", 
+      "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", 
+      "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", 
+      "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", 
+      "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", 
+      "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", 
+      "SLASH", "PERCENT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", 
       "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", 
-      "EXPR_WS", "METADATA", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", 
+      "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", 
+      "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", 
       "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", 
       "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", 
       "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", "ENRICH_LINE_COMMENT", 
       "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", 
-      "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "LOOKUP_LINE_COMMENT", 
-      "LOOKUP_MULTILINE_COMMENT", "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", 
-      "LOOKUP_FIELD_MULTILINE_COMMENT", "LOOKUP_FIELD_WS", "MVEXPAND_LINE_COMMENT", 
+      "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", 
       "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", 
       "SHOW_MULTILINE_COMMENT", "SHOW_WS", "FUNCTIONS", "META_LINE_COMMENT", 
       "META_MULTILINE_COMMENT", "META_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", 
-      "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", 
+      "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", 
+      "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", 
+      "LOOKUP_FIELD_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", 
       "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", 
       "CLOSING_METRICS_WS"
     };
@@ -211,8 +219,60 @@ public EsqlBaseLexer(CharStream input) {
   @Override
   public ATN getATN() { return _ATN; }
 
+  @Override
+  public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
+    switch (ruleIndex) {
+    case 17:
+      return DEV_INLINESTATS_sempred((RuleContext)_localctx, predIndex);
+    case 18:
+      return DEV_LOOKUP_sempred((RuleContext)_localctx, predIndex);
+    case 19:
+      return DEV_MATCH_sempred((RuleContext)_localctx, predIndex);
+    case 20:
+      return DEV_METRICS_sempred((RuleContext)_localctx, predIndex);
+    case 74:
+      return DEV_MATCH_OP_sempred((RuleContext)_localctx, predIndex);
+    }
+    return true;
+  }
+  private boolean DEV_INLINESTATS_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 0:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean DEV_LOOKUP_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 1:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean DEV_MATCH_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 2:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean DEV_METRICS_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 3:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean DEV_MATCH_OP_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 4:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+
   public static final String _serializedATN =
-    "\u0004\u0000|\u05aa\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
+    "\u0004\u0000}\u05c2\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
     "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
     "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
     "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
@@ -267,875 +327,908 @@ public EsqlBaseLexer(CharStream input) {
     "\u0002\u00b7\u0007\u00b7\u0002\u00b8\u0007\u00b8\u0002\u00b9\u0007\u00b9"+
     "\u0002\u00ba\u0007\u00ba\u0002\u00bb\u0007\u00bb\u0002\u00bc\u0007\u00bc"+
     "\u0002\u00bd\u0007\u00bd\u0002\u00be\u0007\u00be\u0002\u00bf\u0007\u00bf"+
-    "\u0002\u00c0\u0007\u00c0\u0002\u00c1\u0007\u00c1\u0001\u0000\u0001\u0000"+
+    "\u0002\u00c0\u0007\u00c0\u0002\u00c1\u0007\u00c1\u0002\u00c2\u0007\u00c2"+
+    "\u0002\u00c3\u0007\u00c3\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000"+
     "\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000"+
-    "\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+
-    "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002"+
-    "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"+
-    "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+
-    "\u0001\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004"+
-    "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005"+
-    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+
-    "\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+
-    "\u0001\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007"+
+    "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+
+    "\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"+
+    "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003"+
+    "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0004"+
+    "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004"+
+    "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005"+
+    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006"+
+    "\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0007"+
     "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007"+
-    "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001"+
-    "\b\u0001\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+
-    "\t\u0001\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+
-    "\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+
-    "\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+
-    "\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001"+
-    "\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\u000e"+
-    "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e"+
-    "\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f"+
-    "\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"+
-    "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011"+
-    "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0012\u0001\u0012"+
-    "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+
-    "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"+
-    "\u0001\u0013\u0001\u0013\u0001\u0014\u0004\u0014\u023f\b\u0014\u000b\u0014"+
-    "\f\u0014\u0240\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015\u0001\u0015"+
-    "\u0001\u0015\u0005\u0015\u0249\b\u0015\n\u0015\f\u0015\u024c\t\u0015\u0001"+
-    "\u0015\u0003\u0015\u024f\b\u0015\u0001\u0015\u0003\u0015\u0252\b\u0015"+
-    "\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016"+
-    "\u0001\u0016\u0005\u0016\u025b\b\u0016\n\u0016\f\u0016\u025e\t\u0016\u0001"+
-    "\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0004"+
-    "\u0017\u0266\b\u0017\u000b\u0017\f\u0017\u0267\u0001\u0017\u0001\u0017"+
-    "\u0001\u0018\u0001\u0018\u0001\u0018\u0003\u0018\u026f\b\u0018\u0001\u0019"+
-    "\u0004\u0019\u0272\b\u0019\u000b\u0019\f\u0019\u0273\u0001\u001a\u0001"+
-    "\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001"+
-    "\u001b\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001c\u0001"+
-    "\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001e\u0001"+
-    "\u001e\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0001"+
-    "\u001f\u0001 \u0001 \u0001!\u0001!\u0001\"\u0001\"\u0001\"\u0001#\u0001"+
-    "#\u0001$\u0001$\u0003$\u029b\b$\u0001$\u0004$\u029e\b$\u000b$\f$\u029f"+
-    "\u0001%\u0001%\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0003\'\u02a9\b\'"+
-    "\u0001(\u0001(\u0001)\u0001)\u0001)\u0003)\u02b0\b)\u0001*\u0001*\u0001"+
-    "*\u0005*\u02b5\b*\n*\f*\u02b8\t*\u0001*\u0001*\u0001*\u0001*\u0001*\u0001"+
-    "*\u0005*\u02c0\b*\n*\f*\u02c3\t*\u0001*\u0001*\u0001*\u0001*\u0001*\u0003"+
-    "*\u02ca\b*\u0001*\u0003*\u02cd\b*\u0003*\u02cf\b*\u0001+\u0004+\u02d2"+
-    "\b+\u000b+\f+\u02d3\u0001,\u0004,\u02d7\b,\u000b,\f,\u02d8\u0001,\u0001"+
-    ",\u0005,\u02dd\b,\n,\f,\u02e0\t,\u0001,\u0001,\u0004,\u02e4\b,\u000b,"+
-    "\f,\u02e5\u0001,\u0004,\u02e9\b,\u000b,\f,\u02ea\u0001,\u0001,\u0005,"+
-    "\u02ef\b,\n,\f,\u02f2\t,\u0003,\u02f4\b,\u0001,\u0001,\u0001,\u0001,\u0004"+
-    ",\u02fa\b,\u000b,\f,\u02fb\u0001,\u0001,\u0003,\u0300\b,\u0001-\u0001"+
-    "-\u0001-\u0001.\u0001.\u0001.\u0001.\u0001/\u0001/\u0001/\u0001/\u0001"+
-    "0\u00010\u00011\u00011\u00011\u00012\u00012\u00013\u00013\u00013\u0001"+
-    "3\u00013\u00014\u00014\u00015\u00015\u00015\u00015\u00015\u00015\u0001"+
-    "6\u00016\u00016\u00016\u00016\u00016\u00017\u00017\u00017\u00017\u0001"+
-    "7\u00018\u00018\u00019\u00019\u00019\u0001:\u0001:\u0001:\u0001;\u0001"+
-    ";\u0001;\u0001;\u0001;\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001"+
-    "=\u0001=\u0001=\u0001>\u0001>\u0001>\u0001>\u0001>\u0001>\u0001?\u0001"+
-    "?\u0001?\u0001@\u0001@\u0001A\u0001A\u0001A\u0001A\u0001A\u0001A\u0001"+
-    "B\u0001B\u0001C\u0001C\u0001C\u0001C\u0001C\u0001D\u0001D\u0001D\u0001"+
-    "E\u0001E\u0001E\u0001F\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001"+
-    "H\u0001I\u0001I\u0001J\u0001J\u0001J\u0001K\u0001K\u0001L\u0001L\u0001"+
-    "M\u0001M\u0001N\u0001N\u0001O\u0001O\u0001P\u0001P\u0001P\u0005P\u037a"+
-    "\bP\nP\fP\u037d\tP\u0001P\u0001P\u0004P\u0381\bP\u000bP\fP\u0382\u0003"+
-    "P\u0385\bP\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001"+
-    "R\u0001R\u0001S\u0001S\u0005S\u0393\bS\nS\fS\u0396\tS\u0001S\u0001S\u0003"+
-    "S\u039a\bS\u0001S\u0004S\u039d\bS\u000bS\fS\u039e\u0003S\u03a1\bS\u0001"+
-    "T\u0001T\u0004T\u03a5\bT\u000bT\fT\u03a6\u0001T\u0001T\u0001U\u0001U\u0001"+
-    "V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001W\u0001W\u0001X\u0001X\u0001"+
-    "X\u0001X\u0001Y\u0001Y\u0001Y\u0001Y\u0001Y\u0001Z\u0001Z\u0001Z\u0001"+
-    "Z\u0001[\u0001[\u0001[\u0001[\u0001\\\u0001\\\u0001\\\u0001\\\u0001]\u0001"+
-    "]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001"+
-    "_\u0001_\u0001_\u0001_\u0001_\u0001_\u0001`\u0001`\u0001`\u0001`\u0001"+
-    "a\u0001a\u0001a\u0001a\u0001b\u0001b\u0001b\u0001b\u0001c\u0001c\u0001"+
-    "c\u0001c\u0001d\u0001d\u0001d\u0001d\u0001e\u0001e\u0001e\u0001e\u0001"+
-    "e\u0001f\u0001f\u0001f\u0001f\u0001g\u0001g\u0001g\u0001g\u0001h\u0001"+
-    "h\u0001h\u0001h\u0003h\u0400\bh\u0001i\u0001i\u0003i\u0404\bi\u0001i\u0005"+
-    "i\u0407\bi\ni\fi\u040a\ti\u0001i\u0001i\u0003i\u040e\bi\u0001i\u0004i"+
-    "\u0411\bi\u000bi\fi\u0412\u0003i\u0415\bi\u0001j\u0001j\u0004j\u0419\b"+
-    "j\u000bj\fj\u041a\u0001k\u0001k\u0001k\u0001k\u0001l\u0001l\u0001l\u0001"+
-    "l\u0001m\u0001m\u0001m\u0001m\u0001n\u0001n\u0001n\u0001n\u0001n\u0001"+
-    "o\u0001o\u0001o\u0001o\u0001p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001"+
-    "q\u0001q\u0001r\u0001r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001"+
-    "t\u0001t\u0001t\u0001u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001"+
-    "v\u0001w\u0001w\u0001w\u0001w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001"+
-    "x\u0001y\u0001y\u0001y\u0001y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001"+
-    "z\u0001z\u0001z\u0001{\u0001{\u0001|\u0004|\u0466\b|\u000b|\f|\u0467\u0001"+
-    "|\u0001|\u0003|\u046c\b|\u0001|\u0004|\u046f\b|\u000b|\f|\u0470\u0001"+
-    "}\u0001}\u0001}\u0001}\u0001~\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f"+
-    "\u0001\u007f\u0001\u007f\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0080"+
-    "\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0081"+
-    "\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0083\u0001\u0083"+
-    "\u0001\u0083\u0001\u0083\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0084"+
-    "\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0086\u0001\u0086"+
-    "\u0001\u0086\u0001\u0086\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0087"+
-    "\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0089\u0001\u0089"+
-    "\u0001\u0089\u0001\u0089\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a"+
-    "\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008b\u0001\u008c"+
-    "\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008d\u0001\u008d\u0001\u008d"+
-    "\u0001\u008d\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008f"+
-    "\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u0090\u0001\u0090"+
-    "\u0001\u0090\u0001\u0090\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0091"+
-    "\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0093\u0001\u0093"+
-    "\u0001\u0093\u0001\u0093\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0094"+
-    "\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095"+
-    "\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0097\u0001\u0097"+
-    "\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098\u0001\u0098\u0001\u0098"+
-    "\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u009a\u0001\u009a"+
-    "\u0001\u009a\u0001\u009a\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009b"+
-    "\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009d"+
-    "\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009e\u0001\u009e\u0001\u009e"+
-    "\u0001\u009e\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u00a0"+
-    "\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a1\u0001\u00a1\u0001\u00a1"+
-    "\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a3"+
-    "\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4"+
-    "\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a5\u0001\u00a5\u0001\u00a5"+
-    "\u0001\u00a5\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7"+
-    "\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8"+
-    "\u0001\u00a8\u0001\u00a8\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9"+
-    "\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9"+
-    "\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab\u0001\u00ab"+
-    "\u0001\u00ab\u0001\u00ab\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac"+
-    "\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ae"+
-    "\u0001\u00ae\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00af"+
-    "\u0004\u00af\u054f\b\u00af\u000b\u00af\f\u00af\u0550\u0001\u00b0\u0001"+
-    "\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001"+
-    "\u00b1\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b3\u0001"+
-    "\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b4\u0001\u00b4\u0001"+
+    "\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001"+
+    "\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001"+
+    "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+
+    "\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+
+    "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+
+    "\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001"+
+    "\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001"+
+    "\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001"+
+    "\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001"+
+    "\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001"+
+    "\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001"+
+    "\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001"+
+    "\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001"+
+    "\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001"+
+    "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001"+
+    "\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001"+
+    "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001"+
+    "\u0014\u0001\u0014\u0001\u0015\u0004\u0015\u024f\b\u0015\u000b\u0015\f"+
+    "\u0015\u0250\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016"+
+    "\u0001\u0016\u0005\u0016\u0259\b\u0016\n\u0016\f\u0016\u025c\t\u0016\u0001"+
+    "\u0016\u0003\u0016\u025f\b\u0016\u0001\u0016\u0003\u0016\u0262\b\u0016"+
+    "\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017"+
+    "\u0001\u0017\u0005\u0017\u026b\b\u0017\n\u0017\f\u0017\u026e\t\u0017\u0001"+
+    "\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0004"+
+    "\u0018\u0276\b\u0018\u000b\u0018\f\u0018\u0277\u0001\u0018\u0001\u0018"+
+    "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a"+
+    "\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001d"+
+    "\u0001\u001d\u0001\u001e\u0001\u001e\u0003\u001e\u028b\b\u001e\u0001\u001e"+
+    "\u0004\u001e\u028e\b\u001e\u000b\u001e\f\u001e\u028f\u0001\u001f\u0001"+
+    "\u001f\u0001 \u0001 \u0001!\u0001!\u0001!\u0003!\u0299\b!\u0001\"\u0001"+
+    "\"\u0001#\u0001#\u0001#\u0003#\u02a0\b#\u0001$\u0001$\u0001$\u0005$\u02a5"+
+    "\b$\n$\f$\u02a8\t$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0005$\u02b0"+
+    "\b$\n$\f$\u02b3\t$\u0001$\u0001$\u0001$\u0001$\u0001$\u0003$\u02ba\b$"+
+    "\u0001$\u0003$\u02bd\b$\u0003$\u02bf\b$\u0001%\u0004%\u02c2\b%\u000b%"+
+    "\f%\u02c3\u0001&\u0004&\u02c7\b&\u000b&\f&\u02c8\u0001&\u0001&\u0005&"+
+    "\u02cd\b&\n&\f&\u02d0\t&\u0001&\u0001&\u0004&\u02d4\b&\u000b&\f&\u02d5"+
+    "\u0001&\u0004&\u02d9\b&\u000b&\f&\u02da\u0001&\u0001&\u0005&\u02df\b&"+
+    "\n&\f&\u02e2\t&\u0003&\u02e4\b&\u0001&\u0001&\u0001&\u0001&\u0004&\u02ea"+
+    "\b&\u000b&\f&\u02eb\u0001&\u0001&\u0003&\u02f0\b&\u0001\'\u0001\'\u0001"+
+    "\'\u0001(\u0001(\u0001(\u0001(\u0001)\u0001)\u0001)\u0001)\u0001*\u0001"+
+    "*\u0001+\u0001+\u0001+\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001"+
+    "-\u0001.\u0001.\u0001/\u0001/\u0001/\u0001/\u0001/\u0001/\u00010\u0001"+
+    "0\u00010\u00010\u00010\u00010\u00011\u00011\u00011\u00012\u00012\u0001"+
+    "2\u00013\u00013\u00013\u00013\u00013\u00014\u00014\u00014\u00014\u0001"+
+    "4\u00015\u00015\u00016\u00016\u00016\u00016\u00017\u00017\u00017\u0001"+
+    "7\u00017\u00018\u00018\u00018\u00018\u00018\u00018\u00019\u00019\u0001"+
+    "9\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001;\u0001;\u0001<\u0001"+
+    "<\u0001=\u0001=\u0001=\u0001=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001"+
+    "?\u0001?\u0001@\u0001@\u0001@\u0001A\u0001A\u0001B\u0001B\u0001B\u0001"+
+    "C\u0001C\u0001D\u0001D\u0001D\u0001E\u0001E\u0001F\u0001F\u0001G\u0001"+
+    "G\u0001H\u0001H\u0001I\u0001I\u0001J\u0001J\u0001J\u0001J\u0001J\u0001"+
+    "K\u0001K\u0001K\u0003K\u036f\bK\u0001K\u0005K\u0372\bK\nK\fK\u0375\tK"+
+    "\u0001K\u0001K\u0004K\u0379\bK\u000bK\fK\u037a\u0003K\u037d\bK\u0001L"+
+    "\u0001L\u0001L\u0001L\u0001L\u0001M\u0001M\u0001M\u0001M\u0001M\u0001"+
+    "N\u0001N\u0005N\u038b\bN\nN\fN\u038e\tN\u0001N\u0001N\u0003N\u0392\bN"+
+    "\u0001N\u0004N\u0395\bN\u000bN\fN\u0396\u0003N\u0399\bN\u0001O\u0001O"+
+    "\u0004O\u039d\bO\u000bO\fO\u039e\u0001O\u0001O\u0001P\u0001P\u0001Q\u0001"+
+    "Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001S\u0001S\u0001S\u0001"+
+    "S\u0001T\u0001T\u0001T\u0001T\u0001T\u0001U\u0001U\u0001U\u0001U\u0001"+
+    "U\u0001V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001W\u0001W\u0001X\u0001"+
+    "X\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001Y\u0001Y\u0001Z\u0001Z\u0001"+
+    "Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001\\\u0001\\\u0001\\\u0001\\\u0001"+
+    "]\u0001]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001"+
+    "_\u0001_\u0001_\u0001_\u0001_\u0001_\u0001_\u0001`\u0001`\u0001`\u0003"+
+    "`\u03ec\b`\u0001a\u0004a\u03ef\ba\u000ba\fa\u03f0\u0001b\u0001b\u0001"+
+    "b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001d\u0001d\u0001"+
+    "e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001f\u0001g\u0001g\u0001"+
+    "g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001i\u0001i\u0001i\u0001"+
+    "i\u0001j\u0001j\u0001j\u0001j\u0003j\u0418\bj\u0001k\u0001k\u0003k\u041c"+
+    "\bk\u0001k\u0005k\u041f\bk\nk\fk\u0422\tk\u0001k\u0001k\u0003k\u0426\b"+
+    "k\u0001k\u0004k\u0429\bk\u000bk\fk\u042a\u0003k\u042d\bk\u0001l\u0001"+
+    "l\u0004l\u0431\bl\u000bl\fl\u0432\u0001m\u0001m\u0001m\u0001m\u0001n\u0001"+
+    "n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001p\u0001p\u0001p\u0001"+
+    "p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001r\u0001r\u0001r\u0001r\u0001"+
+    "s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001u\u0001u\u0001u\u0001"+
+    "u\u0001v\u0001v\u0001v\u0001v\u0001w\u0001w\u0001w\u0001w\u0001x\u0001"+
+    "x\u0001x\u0001x\u0001y\u0001y\u0001y\u0001y\u0001y\u0001z\u0001z\u0001"+
+    "z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001{\u0001|\u0001|\u0001"+
+    "|\u0001|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001~\u0004~\u047e\b~\u000b"+
+    "~\f~\u047f\u0001~\u0001~\u0003~\u0484\b~\u0001~\u0004~\u0487\b~\u000b"+
+    "~\f~\u0488\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u0080"+
+    "\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0081\u0001\u0081\u0001\u0081"+
+    "\u0001\u0081\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0083"+
+    "\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084"+
+    "\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085"+
+    "\u0001\u0085\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0087"+
+    "\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0088\u0001\u0088\u0001\u0088"+
+    "\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u008a"+
+    "\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001\u008b\u0001\u008b"+
+    "\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008d"+
+    "\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001\u008e"+
+    "\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u008f"+
+    "\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0091\u0001\u0091"+
+    "\u0001\u0091\u0001\u0091\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0092"+
+    "\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0094\u0001\u0094"+
+    "\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0095\u0001\u0095\u0001\u0095"+
+    "\u0001\u0095\u0001\u0095\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096"+
+    "\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098"+
+    "\u0001\u0098\u0001\u0098\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099"+
+    "\u0001\u0099\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a"+
+    "\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009b"+
+    "\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c"+
+    "\u0001\u009c\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009e"+
+    "\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f"+
+    "\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0004\u00a0"+
+    "\u051f\b\u00a0\u000b\u00a0\f\u00a0\u0520\u0001\u00a1\u0001\u00a1\u0001"+
+    "\u00a1\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001"+
+    "\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4\u0001"+
+    "\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001"+
+    "\u00a5\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7\u0001"+
+    "\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001"+
+    "\u00a8\u0001\u00a8\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001"+
+    "\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab\u0001\u00ab\u0001"+
+    "\u00ab\u0001\u00ab\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001"+
+    "\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ae\u0001\u00ae\u0001"+
+    "\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00af\u0001\u00af\u0001"+
+    "\u00af\u0001\u00af\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001"+
+    "\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001\u00b2\u0001"+
+    "\u00b2\u0001\u00b2\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001"+
     "\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b5\u0001\u00b5\u0001"+
-    "\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001"+
-    "\u00b6\u0001\u00b6\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001"+
-    "\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9\u0001\u00b9\u0001"+
-    "\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001\u00ba\u0001"+
-    "\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001"+
-    "\u00bb\u0001\u00bb\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001"+
-    "\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00be\u0001\u00be\u0001"+
-    "\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00bf\u0001\u00bf\u0001"+
-    "\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00c0\u0001\u00c0\u0001"+
-    "\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c1\u0001\u00c1\u0001"+
-    "\u00c1\u0001\u00c1\u0001\u00c1\u0002\u025c\u02c1\u0000\u00c2\u0010\u0001"+
-    "\u0012\u0002\u0014\u0003\u0016\u0004\u0018\u0005\u001a\u0006\u001c\u0007"+
-    "\u001e\b \t\"\n$\u000b&\f(\r*\u000e,\u000f.\u00100\u00112\u00124\u0013"+
-    "6\u00148\u0015:\u0016<\u0017>\u0018@\u0000B\u0019D\u0000F\u0000H\u001a"+
-    "J\u001bL\u001cN\u001dP\u0000R\u0000T\u0000V\u0000X\u0000Z\u0000\\\u0000"+
-    "^\u0000`\u0000b\u0000d\u001ef\u001fh j!l\"n#p$r%t&v\'x(z)|*~+\u0080,\u0082"+
-    "-\u0084.\u0086/\u00880\u008a1\u008c2\u008e3\u00904\u00925\u00946\u0096"+
-    "7\u00988\u009a9\u009c:\u009e;\u00a0<\u00a2=\u00a4>\u00a6?\u00a8@\u00aa"+
-    "A\u00acB\u00aeC\u00b0D\u00b2E\u00b4F\u00b6G\u00b8\u0000\u00baH\u00bcI"+
-    "\u00beJ\u00c0K\u00c2\u0000\u00c4\u0000\u00c6\u0000\u00c8\u0000\u00ca\u0000"+
-    "\u00cc\u0000\u00ceL\u00d0\u0000\u00d2\u0000\u00d4M\u00d6N\u00d8O\u00da"+
-    "\u0000\u00dc\u0000\u00de\u0000\u00e0\u0000\u00e2\u0000\u00e4P\u00e6Q\u00e8"+
-    "R\u00eaS\u00ec\u0000\u00ee\u0000\u00f0\u0000\u00f2\u0000\u00f4T\u00f6"+
-    "\u0000\u00f8U\u00faV\u00fcW\u00fe\u0000\u0100\u0000\u0102X\u0104Y\u0106"+
-    "\u0000\u0108Z\u010a\u0000\u010c[\u010e\\\u0110]\u0112\u0000\u0114\u0000"+
-    "\u0116\u0000\u0118\u0000\u011a\u0000\u011c\u0000\u011e\u0000\u0120^\u0122"+
-    "_\u0124`\u0126\u0000\u0128\u0000\u012a\u0000\u012c\u0000\u012e\u0000\u0130"+
-    "\u0000\u0132\u0000\u0134a\u0136b\u0138c\u013a\u0000\u013c\u0000\u013e"+
-    "\u0000\u0140\u0000\u0142d\u0144e\u0146f\u0148\u0000\u014a\u0000\u014c"+
-    "\u0000\u014e\u0000\u0150g\u0152h\u0154i\u0156\u0000\u0158j\u015ak\u015c"+
-    "l\u015em\u0160\u0000\u0162n\u0164o\u0166p\u0168q\u016a\u0000\u016cr\u016e"+
-    "s\u0170t\u0172u\u0174v\u0176\u0000\u0178\u0000\u017a\u0000\u017cw\u017e"+
-    "x\u0180y\u0182\u0000\u0184\u0000\u0186z\u0188{\u018a|\u018c\u0000\u018e"+
-    "\u0000\u0190\u0000\u0192\u0000\u0010\u0000\u0001\u0002\u0003\u0004\u0005"+
-    "\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\r\u0006\u0000\t\n\r\r  //[[]"+
-    "]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r\r  \u000b\u0000\t\n\r\r  \"\""+
-    ",,//::==[[]]||\u0002\u0000**//\u0001\u000009\u0002\u0000AZaz\u0005\u0000"+
-    "\"\"\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002\u0000EEee\u0002\u0000"+
-    "++--\u0001\u0000``\u000b\u0000\t\n\r\r  \"#,,//::<<>?\\\\||\u05c4\u0000"+
-    "\u0010\u0001\u0000\u0000\u0000\u0000\u0012\u0001\u0000\u0000\u0000\u0000"+
-    "\u0014\u0001\u0000\u0000\u0000\u0000\u0016\u0001\u0000\u0000\u0000\u0000"+
-    "\u0018\u0001\u0000\u0000\u0000\u0000\u001a\u0001\u0000\u0000\u0000\u0000"+
-    "\u001c\u0001\u0000\u0000\u0000\u0000\u001e\u0001\u0000\u0000\u0000\u0000"+
-    " \u0001\u0000\u0000\u0000\u0000\"\u0001\u0000\u0000\u0000\u0000$\u0001"+
-    "\u0000\u0000\u0000\u0000&\u0001\u0000\u0000\u0000\u0000(\u0001\u0000\u0000"+
-    "\u0000\u0000*\u0001\u0000\u0000\u0000\u0000,\u0001\u0000\u0000\u0000\u0000"+
-    ".\u0001\u0000\u0000\u0000\u00000\u0001\u0000\u0000\u0000\u00002\u0001"+
-    "\u0000\u0000\u0000\u00004\u0001\u0000\u0000\u0000\u00006\u0001\u0000\u0000"+
-    "\u0000\u00008\u0001\u0000\u0000\u0000\u0000:\u0001\u0000\u0000\u0000\u0000"+
-    "<\u0001\u0000\u0000\u0000\u0000>\u0001\u0000\u0000\u0000\u0000B\u0001"+
-    "\u0000\u0000\u0000\u0001D\u0001\u0000\u0000\u0000\u0001F\u0001\u0000\u0000"+
-    "\u0000\u0001H\u0001\u0000\u0000\u0000\u0001J\u0001\u0000\u0000\u0000\u0001"+
-    "L\u0001\u0000\u0000\u0000\u0002N\u0001\u0000\u0000\u0000\u0002d\u0001"+
-    "\u0000\u0000\u0000\u0002f\u0001\u0000\u0000\u0000\u0002h\u0001\u0000\u0000"+
-    "\u0000\u0002j\u0001\u0000\u0000\u0000\u0002l\u0001\u0000\u0000\u0000\u0002"+
-    "n\u0001\u0000\u0000\u0000\u0002p\u0001\u0000\u0000\u0000\u0002r\u0001"+
-    "\u0000\u0000\u0000\u0002t\u0001\u0000\u0000\u0000\u0002v\u0001\u0000\u0000"+
-    "\u0000\u0002x\u0001\u0000\u0000\u0000\u0002z\u0001\u0000\u0000\u0000\u0002"+
-    "|\u0001\u0000\u0000\u0000\u0002~\u0001\u0000\u0000\u0000\u0002\u0080\u0001"+
-    "\u0000\u0000\u0000\u0002\u0082\u0001\u0000\u0000\u0000\u0002\u0084\u0001"+
-    "\u0000\u0000\u0000\u0002\u0086\u0001\u0000\u0000\u0000\u0002\u0088\u0001"+
-    "\u0000\u0000\u0000\u0002\u008a\u0001\u0000\u0000\u0000\u0002\u008c\u0001"+
-    "\u0000\u0000\u0000\u0002\u008e\u0001\u0000\u0000\u0000\u0002\u0090\u0001"+
-    "\u0000\u0000\u0000\u0002\u0092\u0001\u0000\u0000\u0000\u0002\u0094\u0001"+
-    "\u0000\u0000\u0000\u0002\u0096\u0001\u0000\u0000\u0000\u0002\u0098\u0001"+
-    "\u0000\u0000\u0000\u0002\u009a\u0001\u0000\u0000\u0000\u0002\u009c\u0001"+
-    "\u0000\u0000\u0000\u0002\u009e\u0001\u0000\u0000\u0000\u0002\u00a0\u0001"+
-    "\u0000\u0000\u0000\u0002\u00a2\u0001\u0000\u0000\u0000\u0002\u00a4\u0001"+
-    "\u0000\u0000\u0000\u0002\u00a6\u0001\u0000\u0000\u0000\u0002\u00a8\u0001"+
-    "\u0000\u0000\u0000\u0002\u00aa\u0001\u0000\u0000\u0000\u0002\u00ac\u0001"+
-    "\u0000\u0000\u0000\u0002\u00ae\u0001\u0000\u0000\u0000\u0002\u00b0\u0001"+
-    "\u0000\u0000\u0000\u0002\u00b2\u0001\u0000\u0000\u0000\u0002\u00b4\u0001"+
-    "\u0000\u0000\u0000\u0002\u00b6\u0001\u0000\u0000\u0000\u0002\u00ba\u0001"+
-    "\u0000\u0000\u0000\u0002\u00bc\u0001\u0000\u0000\u0000\u0002\u00be\u0001"+
-    "\u0000\u0000\u0000\u0002\u00c0\u0001\u0000\u0000\u0000\u0003\u00c2\u0001"+
-    "\u0000\u0000\u0000\u0003\u00c4\u0001\u0000\u0000\u0000\u0003\u00c6\u0001"+
-    "\u0000\u0000\u0000\u0003\u00c8\u0001\u0000\u0000\u0000\u0003\u00ca\u0001"+
-    "\u0000\u0000\u0000\u0003\u00cc\u0001\u0000\u0000\u0000\u0003\u00ce\u0001"+
-    "\u0000\u0000\u0000\u0003\u00d0\u0001\u0000\u0000\u0000\u0003\u00d2\u0001"+
-    "\u0000\u0000\u0000\u0003\u00d4\u0001\u0000\u0000\u0000\u0003\u00d6\u0001"+
-    "\u0000\u0000\u0000\u0003\u00d8\u0001\u0000\u0000\u0000\u0004\u00da\u0001"+
-    "\u0000\u0000\u0000\u0004\u00dc\u0001\u0000\u0000\u0000\u0004\u00de\u0001"+
-    "\u0000\u0000\u0000\u0004\u00e4\u0001\u0000\u0000\u0000\u0004\u00e6\u0001"+
-    "\u0000\u0000\u0000\u0004\u00e8\u0001\u0000\u0000\u0000\u0004\u00ea\u0001"+
-    "\u0000\u0000\u0000\u0005\u00ec\u0001\u0000\u0000\u0000\u0005\u00ee\u0001"+
-    "\u0000\u0000\u0000\u0005\u00f0\u0001\u0000\u0000\u0000\u0005\u00f2\u0001"+
-    "\u0000\u0000\u0000\u0005\u00f4\u0001\u0000\u0000\u0000\u0005\u00f6\u0001"+
-    "\u0000\u0000\u0000\u0005\u00f8\u0001\u0000\u0000\u0000\u0005\u00fa\u0001"+
-    "\u0000\u0000\u0000\u0005\u00fc\u0001\u0000\u0000\u0000\u0006\u00fe\u0001"+
-    "\u0000\u0000\u0000\u0006\u0100\u0001\u0000\u0000\u0000\u0006\u0102\u0001"+
-    "\u0000\u0000\u0000\u0006\u0104\u0001\u0000\u0000\u0000\u0006\u0108\u0001"+
-    "\u0000\u0000\u0000\u0006\u010a\u0001\u0000\u0000\u0000\u0006\u010c\u0001"+
-    "\u0000\u0000\u0000\u0006\u010e\u0001\u0000\u0000\u0000\u0006\u0110\u0001"+
-    "\u0000\u0000\u0000\u0007\u0112\u0001\u0000\u0000\u0000\u0007\u0114\u0001"+
-    "\u0000\u0000\u0000\u0007\u0116\u0001\u0000\u0000\u0000\u0007\u0118\u0001"+
-    "\u0000\u0000\u0000\u0007\u011a\u0001\u0000\u0000\u0000\u0007\u011c\u0001"+
-    "\u0000\u0000\u0000\u0007\u011e\u0001\u0000\u0000\u0000\u0007\u0120\u0001"+
-    "\u0000\u0000\u0000\u0007\u0122\u0001\u0000\u0000\u0000\u0007\u0124\u0001"+
-    "\u0000\u0000\u0000\b\u0126\u0001\u0000\u0000\u0000\b\u0128\u0001\u0000"+
-    "\u0000\u0000\b\u012a\u0001\u0000\u0000\u0000\b\u012c\u0001\u0000\u0000"+
-    "\u0000\b\u012e\u0001\u0000\u0000\u0000\b\u0130\u0001\u0000\u0000\u0000"+
-    "\b\u0132\u0001\u0000\u0000\u0000\b\u0134\u0001\u0000\u0000\u0000\b\u0136"+
-    "\u0001\u0000\u0000\u0000\b\u0138\u0001\u0000\u0000\u0000\t\u013a\u0001"+
-    "\u0000\u0000\u0000\t\u013c\u0001\u0000\u0000\u0000\t\u013e\u0001\u0000"+
-    "\u0000\u0000\t\u0140\u0001\u0000\u0000\u0000\t\u0142\u0001\u0000\u0000"+
-    "\u0000\t\u0144\u0001\u0000\u0000\u0000\t\u0146\u0001\u0000\u0000\u0000"+
-    "\n\u0148\u0001\u0000\u0000\u0000\n\u014a\u0001\u0000\u0000\u0000\n\u014c"+
-    "\u0001\u0000\u0000\u0000\n\u014e\u0001\u0000\u0000\u0000\n\u0150\u0001"+
-    "\u0000\u0000\u0000\n\u0152\u0001\u0000\u0000\u0000\n\u0154\u0001\u0000"+
-    "\u0000\u0000\u000b\u0156\u0001\u0000\u0000\u0000\u000b\u0158\u0001\u0000"+
-    "\u0000\u0000\u000b\u015a\u0001\u0000\u0000\u0000\u000b\u015c\u0001\u0000"+
-    "\u0000\u0000\u000b\u015e\u0001\u0000\u0000\u0000\f\u0160\u0001\u0000\u0000"+
-    "\u0000\f\u0162\u0001\u0000\u0000\u0000\f\u0164\u0001\u0000\u0000\u0000"+
-    "\f\u0166\u0001\u0000\u0000\u0000\f\u0168\u0001\u0000\u0000\u0000\r\u016a"+
-    "\u0001\u0000\u0000\u0000\r\u016c\u0001\u0000\u0000\u0000\r\u016e\u0001"+
-    "\u0000\u0000\u0000\r\u0170\u0001\u0000\u0000\u0000\r\u0172\u0001\u0000"+
-    "\u0000\u0000\r\u0174\u0001\u0000\u0000\u0000\u000e\u0176\u0001\u0000\u0000"+
-    "\u0000\u000e\u0178\u0001\u0000\u0000\u0000\u000e\u017a\u0001\u0000\u0000"+
-    "\u0000\u000e\u017c\u0001\u0000\u0000\u0000\u000e\u017e\u0001\u0000\u0000"+
-    "\u0000\u000e\u0180\u0001\u0000\u0000\u0000\u000f\u0182\u0001\u0000\u0000"+
-    "\u0000\u000f\u0184\u0001\u0000\u0000\u0000\u000f\u0186\u0001\u0000\u0000"+
-    "\u0000\u000f\u0188\u0001\u0000\u0000\u0000\u000f\u018a\u0001\u0000\u0000"+
-    "\u0000\u000f\u018c\u0001\u0000\u0000\u0000\u000f\u018e\u0001\u0000\u0000"+
-    "\u0000\u000f\u0190\u0001\u0000\u0000\u0000\u000f\u0192\u0001\u0000\u0000"+
-    "\u0000\u0010\u0194\u0001\u0000\u0000\u0000\u0012\u019e\u0001\u0000\u0000"+
-    "\u0000\u0014\u01a5\u0001\u0000\u0000\u0000\u0016\u01ae\u0001\u0000\u0000"+
-    "\u0000\u0018\u01b5\u0001\u0000\u0000\u0000\u001a\u01bf\u0001\u0000\u0000"+
-    "\u0000\u001c\u01c6\u0001\u0000\u0000\u0000\u001e\u01cd\u0001\u0000\u0000"+
-    "\u0000 \u01db\u0001\u0000\u0000\u0000\"\u01e2\u0001\u0000\u0000\u0000"+
-    "$\u01ea\u0001\u0000\u0000\u0000&\u01f3\u0001\u0000\u0000\u0000(\u01fa"+
-    "\u0001\u0000\u0000\u0000*\u0204\u0001\u0000\u0000\u0000,\u0210\u0001\u0000"+
-    "\u0000\u0000.\u0219\u0001\u0000\u0000\u00000\u021f\u0001\u0000\u0000\u0000"+
-    "2\u0226\u0001\u0000\u0000\u00004\u022d\u0001\u0000\u0000\u00006\u0235"+
-    "\u0001\u0000\u0000\u00008\u023e\u0001\u0000\u0000\u0000:\u0244\u0001\u0000"+
-    "\u0000\u0000<\u0255\u0001\u0000\u0000\u0000>\u0265\u0001\u0000\u0000\u0000"+
-    "@\u026e\u0001\u0000\u0000\u0000B\u0271\u0001\u0000\u0000\u0000D\u0275"+
-    "\u0001\u0000\u0000\u0000F\u027a\u0001\u0000\u0000\u0000H\u027f\u0001\u0000"+
-    "\u0000\u0000J\u0283\u0001\u0000\u0000\u0000L\u0287\u0001\u0000\u0000\u0000"+
-    "N\u028b\u0001\u0000\u0000\u0000P\u028f\u0001\u0000\u0000\u0000R\u0291"+
-    "\u0001\u0000\u0000\u0000T\u0293\u0001\u0000\u0000\u0000V\u0296\u0001\u0000"+
-    "\u0000\u0000X\u0298\u0001\u0000\u0000\u0000Z\u02a1\u0001\u0000\u0000\u0000"+
-    "\\\u02a3\u0001\u0000\u0000\u0000^\u02a8\u0001\u0000\u0000\u0000`\u02aa"+
-    "\u0001\u0000\u0000\u0000b\u02af\u0001\u0000\u0000\u0000d\u02ce\u0001\u0000"+
-    "\u0000\u0000f\u02d1\u0001\u0000\u0000\u0000h\u02ff\u0001\u0000\u0000\u0000"+
-    "j\u0301\u0001\u0000\u0000\u0000l\u0304\u0001\u0000\u0000\u0000n\u0308"+
-    "\u0001\u0000\u0000\u0000p\u030c\u0001\u0000\u0000\u0000r\u030e\u0001\u0000"+
-    "\u0000\u0000t\u0311\u0001\u0000\u0000\u0000v\u0313\u0001\u0000\u0000\u0000"+
-    "x\u0318\u0001\u0000\u0000\u0000z\u031a\u0001\u0000\u0000\u0000|\u0320"+
-    "\u0001\u0000\u0000\u0000~\u0326\u0001\u0000\u0000\u0000\u0080\u032b\u0001"+
-    "\u0000\u0000\u0000\u0082\u032d\u0001\u0000\u0000\u0000\u0084\u0330\u0001"+
-    "\u0000\u0000\u0000\u0086\u0333\u0001\u0000\u0000\u0000\u0088\u0338\u0001"+
-    "\u0000\u0000\u0000\u008a\u033c\u0001\u0000\u0000\u0000\u008c\u0341\u0001"+
-    "\u0000\u0000\u0000\u008e\u0347\u0001\u0000\u0000\u0000\u0090\u034a\u0001"+
-    "\u0000\u0000\u0000\u0092\u034c\u0001\u0000\u0000\u0000\u0094\u0352\u0001"+
-    "\u0000\u0000\u0000\u0096\u0354\u0001\u0000\u0000\u0000\u0098\u0359\u0001"+
-    "\u0000\u0000\u0000\u009a\u035c\u0001\u0000\u0000\u0000\u009c\u035f\u0001"+
-    "\u0000\u0000\u0000\u009e\u0362\u0001\u0000\u0000\u0000\u00a0\u0364\u0001"+
-    "\u0000\u0000\u0000\u00a2\u0367\u0001\u0000\u0000\u0000\u00a4\u0369\u0001"+
-    "\u0000\u0000\u0000\u00a6\u036c\u0001\u0000\u0000\u0000\u00a8\u036e\u0001"+
-    "\u0000\u0000\u0000\u00aa\u0370\u0001\u0000\u0000\u0000\u00ac\u0372\u0001"+
-    "\u0000\u0000\u0000\u00ae\u0374\u0001\u0000\u0000\u0000\u00b0\u0384\u0001"+
-    "\u0000\u0000\u0000\u00b2\u0386\u0001\u0000\u0000\u0000\u00b4\u038b\u0001"+
-    "\u0000\u0000\u0000\u00b6\u03a0\u0001\u0000\u0000\u0000\u00b8\u03a2\u0001"+
-    "\u0000\u0000\u0000\u00ba\u03aa\u0001\u0000\u0000\u0000\u00bc\u03ac\u0001"+
-    "\u0000\u0000\u0000\u00be\u03b0\u0001\u0000\u0000\u0000\u00c0\u03b4\u0001"+
-    "\u0000\u0000\u0000\u00c2\u03b8\u0001\u0000\u0000\u0000\u00c4\u03bd\u0001"+
-    "\u0000\u0000\u0000\u00c6\u03c1\u0001\u0000\u0000\u0000\u00c8\u03c5\u0001"+
-    "\u0000\u0000\u0000\u00ca\u03c9\u0001\u0000\u0000\u0000\u00cc\u03cd\u0001"+
-    "\u0000\u0000\u0000\u00ce\u03d1\u0001\u0000\u0000\u0000\u00d0\u03da\u0001"+
-    "\u0000\u0000\u0000\u00d2\u03de\u0001\u0000\u0000\u0000\u00d4\u03e2\u0001"+
-    "\u0000\u0000\u0000\u00d6\u03e6\u0001\u0000\u0000\u0000\u00d8\u03ea\u0001"+
-    "\u0000\u0000\u0000\u00da\u03ee\u0001\u0000\u0000\u0000\u00dc\u03f3\u0001"+
-    "\u0000\u0000\u0000\u00de\u03f7\u0001\u0000\u0000\u0000\u00e0\u03ff\u0001"+
-    "\u0000\u0000\u0000\u00e2\u0414\u0001\u0000\u0000\u0000\u00e4\u0418\u0001"+
-    "\u0000\u0000\u0000\u00e6\u041c\u0001\u0000\u0000\u0000\u00e8\u0420\u0001"+
-    "\u0000\u0000\u0000\u00ea\u0424\u0001\u0000\u0000\u0000\u00ec\u0428\u0001"+
-    "\u0000\u0000\u0000\u00ee\u042d\u0001\u0000\u0000\u0000\u00f0\u0431\u0001"+
-    "\u0000\u0000\u0000\u00f2\u0435\u0001\u0000\u0000\u0000\u00f4\u0439\u0001"+
-    "\u0000\u0000\u0000\u00f6\u043c\u0001\u0000\u0000\u0000\u00f8\u0440\u0001"+
-    "\u0000\u0000\u0000\u00fa\u0444\u0001\u0000\u0000\u0000\u00fc\u0448\u0001"+
-    "\u0000\u0000\u0000\u00fe\u044c\u0001\u0000\u0000\u0000\u0100\u0451\u0001"+
-    "\u0000\u0000\u0000\u0102\u0456\u0001\u0000\u0000\u0000\u0104\u045b\u0001"+
-    "\u0000\u0000\u0000\u0106\u0462\u0001\u0000\u0000\u0000\u0108\u046b\u0001"+
-    "\u0000\u0000\u0000\u010a\u0472\u0001\u0000\u0000\u0000\u010c\u0476\u0001"+
-    "\u0000\u0000\u0000\u010e\u047a\u0001\u0000\u0000\u0000\u0110\u047e\u0001"+
-    "\u0000\u0000\u0000\u0112\u0482\u0001\u0000\u0000\u0000\u0114\u0488\u0001"+
-    "\u0000\u0000\u0000\u0116\u048c\u0001\u0000\u0000\u0000\u0118\u0490\u0001"+
-    "\u0000\u0000\u0000\u011a\u0494\u0001\u0000\u0000\u0000\u011c\u0498\u0001"+
-    "\u0000\u0000\u0000\u011e\u049c\u0001\u0000\u0000\u0000\u0120\u04a0\u0001"+
-    "\u0000\u0000\u0000\u0122\u04a4\u0001\u0000\u0000\u0000\u0124\u04a8\u0001"+
-    "\u0000\u0000\u0000\u0126\u04ac\u0001\u0000\u0000\u0000\u0128\u04b1\u0001"+
-    "\u0000\u0000\u0000\u012a\u04b5\u0001\u0000\u0000\u0000\u012c\u04b9\u0001"+
-    "\u0000\u0000\u0000\u012e\u04bd\u0001\u0000\u0000\u0000\u0130\u04c2\u0001"+
-    "\u0000\u0000\u0000\u0132\u04c6\u0001\u0000\u0000\u0000\u0134\u04ca\u0001"+
-    "\u0000\u0000\u0000\u0136\u04ce\u0001\u0000\u0000\u0000\u0138\u04d2\u0001"+
-    "\u0000\u0000\u0000\u013a\u04d6\u0001\u0000\u0000\u0000\u013c\u04dc\u0001"+
-    "\u0000\u0000\u0000\u013e\u04e0\u0001\u0000\u0000\u0000\u0140\u04e4\u0001"+
-    "\u0000\u0000\u0000\u0142\u04e8\u0001\u0000\u0000\u0000\u0144\u04ec\u0001"+
-    "\u0000\u0000\u0000\u0146\u04f0\u0001\u0000\u0000\u0000\u0148\u04f4\u0001"+
-    "\u0000\u0000\u0000\u014a\u04f9\u0001\u0000\u0000\u0000\u014c\u04fd\u0001"+
-    "\u0000\u0000\u0000\u014e\u0501\u0001\u0000\u0000\u0000\u0150\u0505\u0001"+
-    "\u0000\u0000\u0000\u0152\u0509\u0001\u0000\u0000\u0000\u0154\u050d\u0001"+
-    "\u0000\u0000\u0000\u0156\u0511\u0001\u0000\u0000\u0000\u0158\u0516\u0001"+
-    "\u0000\u0000\u0000\u015a\u051b\u0001\u0000\u0000\u0000\u015c\u051f\u0001"+
-    "\u0000\u0000\u0000\u015e\u0523\u0001\u0000\u0000\u0000\u0160\u0527\u0001"+
-    "\u0000\u0000\u0000\u0162\u052c\u0001\u0000\u0000\u0000\u0164\u0536\u0001"+
-    "\u0000\u0000\u0000\u0166\u053a\u0001\u0000\u0000\u0000\u0168\u053e\u0001"+
-    "\u0000\u0000\u0000\u016a\u0542\u0001\u0000\u0000\u0000\u016c\u0547\u0001"+
-    "\u0000\u0000\u0000\u016e\u054e\u0001\u0000\u0000\u0000\u0170\u0552\u0001"+
-    "\u0000\u0000\u0000\u0172\u0556\u0001\u0000\u0000\u0000\u0174\u055a\u0001"+
-    "\u0000\u0000\u0000\u0176\u055e\u0001\u0000\u0000\u0000\u0178\u0563\u0001"+
-    "\u0000\u0000\u0000\u017a\u0569\u0001\u0000\u0000\u0000\u017c\u056f\u0001"+
-    "\u0000\u0000\u0000\u017e\u0573\u0001\u0000\u0000\u0000\u0180\u0577\u0001"+
-    "\u0000\u0000\u0000\u0182\u057b\u0001\u0000\u0000\u0000\u0184\u0581\u0001"+
-    "\u0000\u0000\u0000\u0186\u0587\u0001\u0000\u0000\u0000\u0188\u058b\u0001"+
-    "\u0000\u0000\u0000\u018a\u058f\u0001\u0000\u0000\u0000\u018c\u0593\u0001"+
-    "\u0000\u0000\u0000\u018e\u0599\u0001\u0000\u0000\u0000\u0190\u059f\u0001"+
-    "\u0000\u0000\u0000\u0192\u05a5\u0001\u0000\u0000\u0000\u0194\u0195\u0005"+
-    "d\u0000\u0000\u0195\u0196\u0005i\u0000\u0000\u0196\u0197\u0005s\u0000"+
-    "\u0000\u0197\u0198\u0005s\u0000\u0000\u0198\u0199\u0005e\u0000\u0000\u0199"+
-    "\u019a\u0005c\u0000\u0000\u019a\u019b\u0005t\u0000\u0000\u019b\u019c\u0001"+
-    "\u0000\u0000\u0000\u019c\u019d\u0006\u0000\u0000\u0000\u019d\u0011\u0001"+
-    "\u0000\u0000\u0000\u019e\u019f\u0005d\u0000\u0000\u019f\u01a0\u0005r\u0000"+
-    "\u0000\u01a0\u01a1\u0005o\u0000\u0000\u01a1\u01a2\u0005p\u0000\u0000\u01a2"+
-    "\u01a3\u0001\u0000\u0000\u0000\u01a3\u01a4\u0006\u0001\u0001\u0000\u01a4"+
-    "\u0013\u0001\u0000\u0000\u0000\u01a5\u01a6\u0005e\u0000\u0000\u01a6\u01a7"+
-    "\u0005n\u0000\u0000\u01a7\u01a8\u0005r\u0000\u0000\u01a8\u01a9\u0005i"+
-    "\u0000\u0000\u01a9\u01aa\u0005c\u0000\u0000\u01aa\u01ab\u0005h\u0000\u0000"+
-    "\u01ab\u01ac\u0001\u0000\u0000\u0000\u01ac\u01ad\u0006\u0002\u0002\u0000"+
-    "\u01ad\u0015\u0001\u0000\u0000\u0000\u01ae\u01af\u0005e\u0000\u0000\u01af"+
-    "\u01b0\u0005v\u0000\u0000\u01b0\u01b1\u0005a\u0000\u0000\u01b1\u01b2\u0005"+
-    "l\u0000\u0000\u01b2\u01b3\u0001\u0000\u0000\u0000\u01b3\u01b4\u0006\u0003"+
-    "\u0000\u0000\u01b4\u0017\u0001\u0000\u0000\u0000\u01b5\u01b6\u0005e\u0000"+
-    "\u0000\u01b6\u01b7\u0005x\u0000\u0000\u01b7\u01b8\u0005p\u0000\u0000\u01b8"+
-    "\u01b9\u0005l\u0000\u0000\u01b9\u01ba\u0005a\u0000\u0000\u01ba\u01bb\u0005"+
-    "i\u0000\u0000\u01bb\u01bc\u0005n\u0000\u0000\u01bc\u01bd\u0001\u0000\u0000"+
-    "\u0000\u01bd\u01be\u0006\u0004\u0003\u0000\u01be\u0019\u0001\u0000\u0000"+
-    "\u0000\u01bf\u01c0\u0005f\u0000\u0000\u01c0\u01c1\u0005r\u0000\u0000\u01c1"+
-    "\u01c2\u0005o\u0000\u0000\u01c2\u01c3\u0005m\u0000\u0000\u01c3\u01c4\u0001"+
-    "\u0000\u0000\u0000\u01c4\u01c5\u0006\u0005\u0004\u0000\u01c5\u001b\u0001"+
-    "\u0000\u0000\u0000\u01c6\u01c7\u0005g\u0000\u0000\u01c7\u01c8\u0005r\u0000"+
-    "\u0000\u01c8\u01c9\u0005o\u0000\u0000\u01c9\u01ca\u0005k\u0000\u0000\u01ca"+
-    "\u01cb\u0001\u0000\u0000\u0000\u01cb\u01cc\u0006\u0006\u0000\u0000\u01cc"+
-    "\u001d\u0001\u0000\u0000\u0000\u01cd\u01ce\u0005i\u0000\u0000\u01ce\u01cf"+
-    "\u0005n\u0000\u0000\u01cf\u01d0\u0005l\u0000\u0000\u01d0\u01d1\u0005i"+
-    "\u0000\u0000\u01d1\u01d2\u0005n\u0000\u0000\u01d2\u01d3\u0005e\u0000\u0000"+
-    "\u01d3\u01d4\u0005s\u0000\u0000\u01d4\u01d5\u0005t\u0000\u0000\u01d5\u01d6"+
-    "\u0005a\u0000\u0000\u01d6\u01d7\u0005t\u0000\u0000\u01d7\u01d8\u0005s"+
-    "\u0000\u0000\u01d8\u01d9\u0001\u0000\u0000\u0000\u01d9\u01da\u0006\u0007"+
-    "\u0000\u0000\u01da\u001f\u0001\u0000\u0000\u0000\u01db\u01dc\u0005k\u0000"+
-    "\u0000\u01dc\u01dd\u0005e\u0000\u0000\u01dd\u01de\u0005e\u0000\u0000\u01de"+
-    "\u01df\u0005p\u0000\u0000\u01df\u01e0\u0001\u0000\u0000\u0000\u01e0\u01e1"+
-    "\u0006\b\u0001\u0000\u01e1!\u0001\u0000\u0000\u0000\u01e2\u01e3\u0005"+
-    "l\u0000\u0000\u01e3\u01e4\u0005i\u0000\u0000\u01e4\u01e5\u0005m\u0000"+
-    "\u0000\u01e5\u01e6\u0005i\u0000\u0000\u01e6\u01e7\u0005t\u0000\u0000\u01e7"+
-    "\u01e8\u0001\u0000\u0000\u0000\u01e8\u01e9\u0006\t\u0000\u0000\u01e9#"+
-    "\u0001\u0000\u0000\u0000\u01ea\u01eb\u0005l\u0000\u0000\u01eb\u01ec\u0005"+
-    "o\u0000\u0000\u01ec\u01ed\u0005o\u0000\u0000\u01ed\u01ee\u0005k\u0000"+
-    "\u0000\u01ee\u01ef\u0005u\u0000\u0000\u01ef\u01f0\u0005p\u0000\u0000\u01f0"+
-    "\u01f1\u0001\u0000\u0000\u0000\u01f1\u01f2\u0006\n\u0005\u0000\u01f2%"+
-    "\u0001\u0000\u0000\u0000\u01f3\u01f4\u0005m\u0000\u0000\u01f4\u01f5\u0005"+
-    "e\u0000\u0000\u01f5\u01f6\u0005t\u0000\u0000\u01f6\u01f7\u0005a\u0000"+
-    "\u0000\u01f7\u01f8\u0001\u0000\u0000\u0000\u01f8\u01f9\u0006\u000b\u0006"+
-    "\u0000\u01f9\'\u0001\u0000\u0000\u0000\u01fa\u01fb\u0005m\u0000\u0000"+
-    "\u01fb\u01fc\u0005e\u0000\u0000\u01fc\u01fd\u0005t\u0000\u0000\u01fd\u01fe"+
-    "\u0005r\u0000\u0000\u01fe\u01ff\u0005i\u0000\u0000\u01ff\u0200\u0005c"+
-    "\u0000\u0000\u0200\u0201\u0005s\u0000\u0000\u0201\u0202\u0001\u0000\u0000"+
-    "\u0000\u0202\u0203\u0006\f\u0007\u0000\u0203)\u0001\u0000\u0000\u0000"+
-    "\u0204\u0205\u0005m\u0000\u0000\u0205\u0206\u0005v\u0000\u0000\u0206\u0207"+
-    "\u0005_\u0000\u0000\u0207\u0208\u0005e\u0000\u0000\u0208\u0209\u0005x"+
-    "\u0000\u0000\u0209\u020a\u0005p\u0000\u0000\u020a\u020b\u0005a\u0000\u0000"+
-    "\u020b\u020c\u0005n\u0000\u0000\u020c\u020d\u0005d\u0000\u0000\u020d\u020e"+
-    "\u0001\u0000\u0000\u0000\u020e\u020f\u0006\r\b\u0000\u020f+\u0001\u0000"+
-    "\u0000\u0000\u0210\u0211\u0005r\u0000\u0000\u0211\u0212\u0005e\u0000\u0000"+
-    "\u0212\u0213\u0005n\u0000\u0000\u0213\u0214\u0005a\u0000\u0000\u0214\u0215"+
-    "\u0005m\u0000\u0000\u0215\u0216\u0005e\u0000\u0000\u0216\u0217\u0001\u0000"+
-    "\u0000\u0000\u0217\u0218\u0006\u000e\t\u0000\u0218-\u0001\u0000\u0000"+
-    "\u0000\u0219\u021a\u0005r\u0000\u0000\u021a\u021b\u0005o\u0000\u0000\u021b"+
-    "\u021c\u0005w\u0000\u0000\u021c\u021d\u0001\u0000\u0000\u0000\u021d\u021e"+
-    "\u0006\u000f\u0000\u0000\u021e/\u0001\u0000\u0000\u0000\u021f\u0220\u0005"+
-    "s\u0000\u0000\u0220\u0221\u0005h\u0000\u0000\u0221\u0222\u0005o\u0000"+
-    "\u0000\u0222\u0223\u0005w\u0000\u0000\u0223\u0224\u0001\u0000\u0000\u0000"+
-    "\u0224\u0225\u0006\u0010\n\u0000\u02251\u0001\u0000\u0000\u0000\u0226"+
-    "\u0227\u0005s\u0000\u0000\u0227\u0228\u0005o\u0000\u0000\u0228\u0229\u0005"+
-    "r\u0000\u0000\u0229\u022a\u0005t\u0000\u0000\u022a\u022b\u0001\u0000\u0000"+
-    "\u0000\u022b\u022c\u0006\u0011\u0000\u0000\u022c3\u0001\u0000\u0000\u0000"+
-    "\u022d\u022e\u0005s\u0000\u0000\u022e\u022f\u0005t\u0000\u0000\u022f\u0230"+
-    "\u0005a\u0000\u0000\u0230\u0231\u0005t\u0000\u0000\u0231\u0232\u0005s"+
-    "\u0000\u0000\u0232\u0233\u0001\u0000\u0000\u0000\u0233\u0234\u0006\u0012"+
-    "\u0000\u0000\u02345\u0001\u0000\u0000\u0000\u0235\u0236\u0005w\u0000\u0000"+
-    "\u0236\u0237\u0005h\u0000\u0000\u0237\u0238\u0005e\u0000\u0000\u0238\u0239"+
-    "\u0005r\u0000\u0000\u0239\u023a\u0005e\u0000\u0000\u023a\u023b\u0001\u0000"+
-    "\u0000\u0000\u023b\u023c\u0006\u0013\u0000\u0000\u023c7\u0001\u0000\u0000"+
-    "\u0000\u023d\u023f\b\u0000\u0000\u0000\u023e\u023d\u0001\u0000\u0000\u0000"+
-    "\u023f\u0240\u0001\u0000\u0000\u0000\u0240\u023e\u0001\u0000\u0000\u0000"+
-    "\u0240\u0241\u0001\u0000\u0000\u0000\u0241\u0242\u0001\u0000\u0000\u0000"+
-    "\u0242\u0243\u0006\u0014\u0000\u0000\u02439\u0001\u0000\u0000\u0000\u0244"+
-    "\u0245\u0005/\u0000\u0000\u0245\u0246\u0005/\u0000\u0000\u0246\u024a\u0001"+
-    "\u0000\u0000\u0000\u0247\u0249\b\u0001\u0000\u0000\u0248\u0247\u0001\u0000"+
-    "\u0000\u0000\u0249\u024c\u0001\u0000\u0000\u0000\u024a\u0248\u0001\u0000"+
-    "\u0000\u0000\u024a\u024b\u0001\u0000\u0000\u0000\u024b\u024e\u0001\u0000"+
-    "\u0000\u0000\u024c\u024a\u0001\u0000\u0000\u0000\u024d\u024f\u0005\r\u0000"+
-    "\u0000\u024e\u024d\u0001\u0000\u0000\u0000\u024e\u024f\u0001\u0000\u0000"+
-    "\u0000\u024f\u0251\u0001\u0000\u0000\u0000\u0250\u0252\u0005\n\u0000\u0000"+
-    "\u0251\u0250\u0001\u0000\u0000\u0000\u0251\u0252\u0001\u0000\u0000\u0000"+
-    "\u0252\u0253\u0001\u0000\u0000\u0000\u0253\u0254\u0006\u0015\u000b\u0000"+
-    "\u0254;\u0001\u0000\u0000\u0000\u0255\u0256\u0005/\u0000\u0000\u0256\u0257"+
-    "\u0005*\u0000\u0000\u0257\u025c\u0001\u0000\u0000\u0000\u0258\u025b\u0003"+
-    "<\u0016\u0000\u0259\u025b\t\u0000\u0000\u0000\u025a\u0258\u0001\u0000"+
-    "\u0000\u0000\u025a\u0259\u0001\u0000\u0000\u0000\u025b\u025e\u0001\u0000"+
-    "\u0000\u0000\u025c\u025d\u0001\u0000\u0000\u0000\u025c\u025a\u0001\u0000"+
-    "\u0000\u0000\u025d\u025f\u0001\u0000\u0000\u0000\u025e\u025c\u0001\u0000"+
-    "\u0000\u0000\u025f\u0260\u0005*\u0000\u0000\u0260\u0261\u0005/\u0000\u0000"+
-    "\u0261\u0262\u0001\u0000\u0000\u0000\u0262\u0263\u0006\u0016\u000b\u0000"+
-    "\u0263=\u0001\u0000\u0000\u0000\u0264\u0266\u0007\u0002\u0000\u0000\u0265"+
-    "\u0264\u0001\u0000\u0000\u0000\u0266\u0267\u0001\u0000\u0000\u0000\u0267"+
-    "\u0265\u0001\u0000\u0000\u0000\u0267\u0268\u0001\u0000\u0000\u0000\u0268"+
-    "\u0269\u0001\u0000\u0000\u0000\u0269\u026a\u0006\u0017\u000b\u0000\u026a"+
-    "?\u0001\u0000\u0000\u0000\u026b\u026f\b\u0003\u0000\u0000\u026c\u026d"+
-    "\u0005/\u0000\u0000\u026d\u026f\b\u0004\u0000\u0000\u026e\u026b\u0001"+
-    "\u0000\u0000\u0000\u026e\u026c\u0001\u0000\u0000\u0000\u026fA\u0001\u0000"+
-    "\u0000\u0000\u0270\u0272\u0003@\u0018\u0000\u0271\u0270\u0001\u0000\u0000"+
-    "\u0000\u0272\u0273\u0001\u0000\u0000\u0000\u0273\u0271\u0001\u0000\u0000"+
-    "\u0000\u0273\u0274\u0001\u0000\u0000\u0000\u0274C\u0001\u0000\u0000\u0000"+
-    "\u0275\u0276\u0003\u00b2Q\u0000\u0276\u0277\u0001\u0000\u0000\u0000\u0277"+
-    "\u0278\u0006\u001a\f\u0000\u0278\u0279\u0006\u001a\r\u0000\u0279E\u0001"+
-    "\u0000\u0000\u0000\u027a\u027b\u0003N\u001f\u0000\u027b\u027c\u0001\u0000"+
-    "\u0000\u0000\u027c\u027d\u0006\u001b\u000e\u0000\u027d\u027e\u0006\u001b"+
-    "\u000f\u0000\u027eG\u0001\u0000\u0000\u0000\u027f\u0280\u0003>\u0017\u0000"+
-    "\u0280\u0281\u0001\u0000\u0000\u0000\u0281\u0282\u0006\u001c\u000b\u0000"+
-    "\u0282I\u0001\u0000\u0000\u0000\u0283\u0284\u0003:\u0015\u0000\u0284\u0285"+
-    "\u0001\u0000\u0000\u0000\u0285\u0286\u0006\u001d\u000b\u0000\u0286K\u0001"+
-    "\u0000\u0000\u0000\u0287\u0288\u0003<\u0016\u0000\u0288\u0289\u0001\u0000"+
-    "\u0000\u0000\u0289\u028a\u0006\u001e\u000b\u0000\u028aM\u0001\u0000\u0000"+
-    "\u0000\u028b\u028c\u0005|\u0000\u0000\u028c\u028d\u0001\u0000\u0000\u0000"+
-    "\u028d\u028e\u0006\u001f\u000f\u0000\u028eO\u0001\u0000\u0000\u0000\u028f"+
-    "\u0290\u0007\u0005\u0000\u0000\u0290Q\u0001\u0000\u0000\u0000\u0291\u0292"+
-    "\u0007\u0006\u0000\u0000\u0292S\u0001\u0000\u0000\u0000\u0293\u0294\u0005"+
-    "\\\u0000\u0000\u0294\u0295\u0007\u0007\u0000\u0000\u0295U\u0001\u0000"+
-    "\u0000\u0000\u0296\u0297\b\b\u0000\u0000\u0297W\u0001\u0000\u0000\u0000"+
-    "\u0298\u029a\u0007\t\u0000\u0000\u0299\u029b\u0007\n\u0000\u0000\u029a"+
-    "\u0299\u0001\u0000\u0000\u0000\u029a\u029b\u0001\u0000\u0000\u0000\u029b"+
-    "\u029d\u0001\u0000\u0000\u0000\u029c\u029e\u0003P \u0000\u029d\u029c\u0001"+
-    "\u0000\u0000\u0000\u029e\u029f\u0001\u0000\u0000\u0000\u029f\u029d\u0001"+
-    "\u0000\u0000\u0000\u029f\u02a0\u0001\u0000\u0000\u0000\u02a0Y\u0001\u0000"+
-    "\u0000\u0000\u02a1\u02a2\u0005@\u0000\u0000\u02a2[\u0001\u0000\u0000\u0000"+
-    "\u02a3\u02a4\u0005`\u0000\u0000\u02a4]\u0001\u0000\u0000\u0000\u02a5\u02a9"+
-    "\b\u000b\u0000\u0000\u02a6\u02a7\u0005`\u0000\u0000\u02a7\u02a9\u0005"+
-    "`\u0000\u0000\u02a8\u02a5\u0001\u0000\u0000\u0000\u02a8\u02a6\u0001\u0000"+
-    "\u0000\u0000\u02a9_\u0001\u0000\u0000\u0000\u02aa\u02ab\u0005_\u0000\u0000"+
-    "\u02aba\u0001\u0000\u0000\u0000\u02ac\u02b0\u0003R!\u0000\u02ad\u02b0"+
-    "\u0003P \u0000\u02ae\u02b0\u0003`(\u0000\u02af\u02ac\u0001\u0000\u0000"+
-    "\u0000\u02af\u02ad\u0001\u0000\u0000\u0000\u02af\u02ae\u0001\u0000\u0000"+
-    "\u0000\u02b0c\u0001\u0000\u0000\u0000\u02b1\u02b6\u0005\"\u0000\u0000"+
-    "\u02b2\u02b5\u0003T\"\u0000\u02b3\u02b5\u0003V#\u0000\u02b4\u02b2\u0001"+
-    "\u0000\u0000\u0000\u02b4\u02b3\u0001\u0000\u0000\u0000\u02b5\u02b8\u0001"+
-    "\u0000\u0000\u0000\u02b6\u02b4\u0001\u0000\u0000\u0000\u02b6\u02b7\u0001"+
-    "\u0000\u0000\u0000\u02b7\u02b9\u0001\u0000\u0000\u0000\u02b8\u02b6\u0001"+
-    "\u0000\u0000\u0000\u02b9\u02cf\u0005\"\u0000\u0000\u02ba\u02bb\u0005\""+
-    "\u0000\u0000\u02bb\u02bc\u0005\"\u0000\u0000\u02bc\u02bd\u0005\"\u0000"+
-    "\u0000\u02bd\u02c1\u0001\u0000\u0000\u0000\u02be\u02c0\b\u0001\u0000\u0000"+
-    "\u02bf\u02be\u0001\u0000\u0000\u0000\u02c0\u02c3\u0001\u0000\u0000\u0000"+
-    "\u02c1\u02c2\u0001\u0000\u0000\u0000\u02c1\u02bf\u0001\u0000\u0000\u0000"+
-    "\u02c2\u02c4\u0001\u0000\u0000\u0000\u02c3\u02c1\u0001\u0000\u0000\u0000"+
-    "\u02c4\u02c5\u0005\"\u0000\u0000\u02c5\u02c6\u0005\"\u0000\u0000\u02c6"+
-    "\u02c7\u0005\"\u0000\u0000\u02c7\u02c9\u0001\u0000\u0000\u0000\u02c8\u02ca"+
-    "\u0005\"\u0000\u0000\u02c9\u02c8\u0001\u0000\u0000\u0000\u02c9\u02ca\u0001"+
-    "\u0000\u0000\u0000\u02ca\u02cc\u0001\u0000\u0000\u0000\u02cb\u02cd\u0005"+
-    "\"\u0000\u0000\u02cc\u02cb\u0001\u0000\u0000\u0000\u02cc\u02cd\u0001\u0000"+
-    "\u0000\u0000\u02cd\u02cf\u0001\u0000\u0000\u0000\u02ce\u02b1\u0001\u0000"+
-    "\u0000\u0000\u02ce\u02ba\u0001\u0000\u0000\u0000\u02cfe\u0001\u0000\u0000"+
-    "\u0000\u02d0\u02d2\u0003P \u0000\u02d1\u02d0\u0001\u0000\u0000\u0000\u02d2"+
-    "\u02d3\u0001\u0000\u0000\u0000\u02d3\u02d1\u0001\u0000\u0000\u0000\u02d3"+
-    "\u02d4\u0001\u0000\u0000\u0000\u02d4g\u0001\u0000\u0000\u0000\u02d5\u02d7"+
-    "\u0003P \u0000\u02d6\u02d5\u0001\u0000\u0000\u0000\u02d7\u02d8\u0001\u0000"+
-    "\u0000\u0000\u02d8\u02d6\u0001\u0000\u0000\u0000\u02d8\u02d9\u0001\u0000"+
-    "\u0000\u0000\u02d9\u02da\u0001\u0000\u0000\u0000\u02da\u02de\u0003x4\u0000"+
-    "\u02db\u02dd\u0003P \u0000\u02dc\u02db\u0001\u0000\u0000\u0000\u02dd\u02e0"+
-    "\u0001\u0000\u0000\u0000\u02de\u02dc\u0001\u0000\u0000\u0000\u02de\u02df"+
-    "\u0001\u0000\u0000\u0000\u02df\u0300\u0001\u0000\u0000\u0000\u02e0\u02de"+
-    "\u0001\u0000\u0000\u0000\u02e1\u02e3\u0003x4\u0000\u02e2\u02e4\u0003P"+
-    " \u0000\u02e3\u02e2\u0001\u0000\u0000\u0000\u02e4\u02e5\u0001\u0000\u0000"+
-    "\u0000\u02e5\u02e3\u0001\u0000\u0000\u0000\u02e5\u02e6\u0001\u0000\u0000"+
-    "\u0000\u02e6\u0300\u0001\u0000\u0000\u0000\u02e7\u02e9\u0003P \u0000\u02e8"+
-    "\u02e7\u0001\u0000\u0000\u0000\u02e9\u02ea\u0001\u0000\u0000\u0000\u02ea"+
-    "\u02e8\u0001\u0000\u0000\u0000\u02ea\u02eb\u0001\u0000\u0000\u0000\u02eb"+
-    "\u02f3\u0001\u0000\u0000\u0000\u02ec\u02f0\u0003x4\u0000\u02ed\u02ef\u0003"+
-    "P \u0000\u02ee\u02ed\u0001\u0000\u0000\u0000\u02ef\u02f2\u0001\u0000\u0000"+
-    "\u0000\u02f0\u02ee\u0001\u0000\u0000\u0000\u02f0\u02f1\u0001\u0000\u0000"+
-    "\u0000\u02f1\u02f4\u0001\u0000\u0000\u0000\u02f2\u02f0\u0001\u0000\u0000"+
-    "\u0000\u02f3\u02ec\u0001\u0000\u0000\u0000\u02f3\u02f4\u0001\u0000\u0000"+
-    "\u0000\u02f4\u02f5\u0001\u0000\u0000\u0000\u02f5\u02f6\u0003X$\u0000\u02f6"+
-    "\u0300\u0001\u0000\u0000\u0000\u02f7\u02f9\u0003x4\u0000\u02f8\u02fa\u0003"+
-    "P \u0000\u02f9\u02f8\u0001\u0000\u0000\u0000\u02fa\u02fb\u0001\u0000\u0000"+
-    "\u0000\u02fb\u02f9\u0001\u0000\u0000\u0000\u02fb\u02fc\u0001\u0000\u0000"+
-    "\u0000\u02fc\u02fd\u0001\u0000\u0000\u0000\u02fd\u02fe\u0003X$\u0000\u02fe"+
-    "\u0300\u0001\u0000\u0000\u0000\u02ff\u02d6\u0001\u0000\u0000\u0000\u02ff"+
-    "\u02e1\u0001\u0000\u0000\u0000\u02ff\u02e8\u0001\u0000\u0000\u0000\u02ff"+
-    "\u02f7\u0001\u0000\u0000\u0000\u0300i\u0001\u0000\u0000\u0000\u0301\u0302"+
-    "\u0005b\u0000\u0000\u0302\u0303\u0005y\u0000\u0000\u0303k\u0001\u0000"+
-    "\u0000\u0000\u0304\u0305\u0005a\u0000\u0000\u0305\u0306\u0005n\u0000\u0000"+
-    "\u0306\u0307\u0005d\u0000\u0000\u0307m\u0001\u0000\u0000\u0000\u0308\u0309"+
-    "\u0005a\u0000\u0000\u0309\u030a\u0005s\u0000\u0000\u030a\u030b\u0005c"+
-    "\u0000\u0000\u030bo\u0001\u0000\u0000\u0000\u030c\u030d\u0005=\u0000\u0000"+
-    "\u030dq\u0001\u0000\u0000\u0000\u030e\u030f\u0005:\u0000\u0000\u030f\u0310"+
-    "\u0005:\u0000\u0000\u0310s\u0001\u0000\u0000\u0000\u0311\u0312\u0005,"+
-    "\u0000\u0000\u0312u\u0001\u0000\u0000\u0000\u0313\u0314\u0005d\u0000\u0000"+
-    "\u0314\u0315\u0005e\u0000\u0000\u0315\u0316\u0005s\u0000\u0000\u0316\u0317"+
-    "\u0005c\u0000\u0000\u0317w\u0001\u0000\u0000\u0000\u0318\u0319\u0005."+
-    "\u0000\u0000\u0319y\u0001\u0000\u0000\u0000\u031a\u031b\u0005f\u0000\u0000"+
-    "\u031b\u031c\u0005a\u0000\u0000\u031c\u031d\u0005l\u0000\u0000\u031d\u031e"+
-    "\u0005s\u0000\u0000\u031e\u031f\u0005e\u0000\u0000\u031f{\u0001\u0000"+
-    "\u0000\u0000\u0320\u0321\u0005f\u0000\u0000\u0321\u0322\u0005i\u0000\u0000"+
-    "\u0322\u0323\u0005r\u0000\u0000\u0323\u0324\u0005s\u0000\u0000\u0324\u0325"+
-    "\u0005t\u0000\u0000\u0325}\u0001\u0000\u0000\u0000\u0326\u0327\u0005l"+
-    "\u0000\u0000\u0327\u0328\u0005a\u0000\u0000\u0328\u0329\u0005s\u0000\u0000"+
-    "\u0329\u032a\u0005t\u0000\u0000\u032a\u007f\u0001\u0000\u0000\u0000\u032b"+
-    "\u032c\u0005(\u0000\u0000\u032c\u0081\u0001\u0000\u0000\u0000\u032d\u032e"+
-    "\u0005i\u0000\u0000\u032e\u032f\u0005n\u0000\u0000\u032f\u0083\u0001\u0000"+
-    "\u0000\u0000\u0330\u0331\u0005i\u0000\u0000\u0331\u0332\u0005s\u0000\u0000"+
-    "\u0332\u0085\u0001\u0000\u0000\u0000\u0333\u0334\u0005l\u0000\u0000\u0334"+
-    "\u0335\u0005i\u0000\u0000\u0335\u0336\u0005k\u0000\u0000\u0336\u0337\u0005"+
-    "e\u0000\u0000\u0337\u0087\u0001\u0000\u0000\u0000\u0338\u0339\u0005n\u0000"+
-    "\u0000\u0339\u033a\u0005o\u0000\u0000\u033a\u033b\u0005t\u0000\u0000\u033b"+
-    "\u0089\u0001\u0000\u0000\u0000\u033c\u033d\u0005n\u0000\u0000\u033d\u033e"+
-    "\u0005u\u0000\u0000\u033e\u033f\u0005l\u0000\u0000\u033f\u0340\u0005l"+
-    "\u0000\u0000\u0340\u008b\u0001\u0000\u0000\u0000\u0341\u0342\u0005n\u0000"+
-    "\u0000\u0342\u0343\u0005u\u0000\u0000\u0343\u0344\u0005l\u0000\u0000\u0344"+
-    "\u0345\u0005l\u0000\u0000\u0345\u0346\u0005s\u0000\u0000\u0346\u008d\u0001"+
-    "\u0000\u0000\u0000\u0347\u0348\u0005o\u0000\u0000\u0348\u0349\u0005r\u0000"+
-    "\u0000\u0349\u008f\u0001\u0000\u0000\u0000\u034a\u034b\u0005?\u0000\u0000"+
-    "\u034b\u0091\u0001\u0000\u0000\u0000\u034c\u034d\u0005r\u0000\u0000\u034d"+
-    "\u034e\u0005l\u0000\u0000\u034e\u034f\u0005i\u0000\u0000\u034f\u0350\u0005"+
-    "k\u0000\u0000\u0350\u0351\u0005e\u0000\u0000\u0351\u0093\u0001\u0000\u0000"+
-    "\u0000\u0352\u0353\u0005)\u0000\u0000\u0353\u0095\u0001\u0000\u0000\u0000"+
-    "\u0354\u0355\u0005t\u0000\u0000\u0355\u0356\u0005r\u0000\u0000\u0356\u0357"+
-    "\u0005u\u0000\u0000\u0357\u0358\u0005e\u0000\u0000\u0358\u0097\u0001\u0000"+
-    "\u0000\u0000\u0359\u035a\u0005=\u0000\u0000\u035a\u035b\u0005=\u0000\u0000"+
-    "\u035b\u0099\u0001\u0000\u0000\u0000\u035c\u035d\u0005=\u0000\u0000\u035d"+
-    "\u035e\u0005~\u0000\u0000\u035e\u009b\u0001\u0000\u0000\u0000\u035f\u0360"+
-    "\u0005!\u0000\u0000\u0360\u0361\u0005=\u0000\u0000\u0361\u009d\u0001\u0000"+
-    "\u0000\u0000\u0362\u0363\u0005<\u0000\u0000\u0363\u009f\u0001\u0000\u0000"+
-    "\u0000\u0364\u0365\u0005<\u0000\u0000\u0365\u0366\u0005=\u0000\u0000\u0366"+
-    "\u00a1\u0001\u0000\u0000\u0000\u0367\u0368\u0005>\u0000\u0000\u0368\u00a3"+
-    "\u0001\u0000\u0000\u0000\u0369\u036a\u0005>\u0000\u0000\u036a\u036b\u0005"+
-    "=\u0000\u0000\u036b\u00a5\u0001\u0000\u0000\u0000\u036c\u036d\u0005+\u0000"+
-    "\u0000\u036d\u00a7\u0001\u0000\u0000\u0000\u036e\u036f\u0005-\u0000\u0000"+
-    "\u036f\u00a9\u0001\u0000\u0000\u0000\u0370\u0371\u0005*\u0000\u0000\u0371"+
-    "\u00ab\u0001\u0000\u0000\u0000\u0372\u0373\u0005/\u0000\u0000\u0373\u00ad"+
-    "\u0001\u0000\u0000\u0000\u0374\u0375\u0005%\u0000\u0000\u0375\u00af\u0001"+
-    "\u0000\u0000\u0000\u0376\u0377\u0003\u0090@\u0000\u0377\u037b\u0003R!"+
-    "\u0000\u0378\u037a\u0003b)\u0000\u0379\u0378\u0001\u0000\u0000\u0000\u037a"+
-    "\u037d\u0001\u0000\u0000\u0000\u037b\u0379\u0001\u0000\u0000\u0000\u037b"+
-    "\u037c\u0001\u0000\u0000\u0000\u037c\u0385\u0001\u0000\u0000\u0000\u037d"+
-    "\u037b\u0001\u0000\u0000\u0000\u037e\u0380\u0003\u0090@\u0000\u037f\u0381"+
-    "\u0003P \u0000\u0380\u037f\u0001\u0000\u0000\u0000\u0381\u0382\u0001\u0000"+
-    "\u0000\u0000\u0382\u0380\u0001\u0000\u0000\u0000\u0382\u0383\u0001\u0000"+
-    "\u0000\u0000\u0383\u0385\u0001\u0000\u0000\u0000\u0384\u0376\u0001\u0000"+
-    "\u0000\u0000\u0384\u037e\u0001\u0000\u0000\u0000\u0385\u00b1\u0001\u0000"+
-    "\u0000\u0000\u0386\u0387\u0005[\u0000\u0000\u0387\u0388\u0001\u0000\u0000"+
-    "\u0000\u0388\u0389\u0006Q\u0000\u0000\u0389\u038a\u0006Q\u0000\u0000\u038a"+
-    "\u00b3\u0001\u0000\u0000\u0000\u038b\u038c\u0005]\u0000\u0000\u038c\u038d"+
-    "\u0001\u0000\u0000\u0000\u038d\u038e\u0006R\u000f\u0000\u038e\u038f\u0006"+
-    "R\u000f\u0000\u038f\u00b5\u0001\u0000\u0000\u0000\u0390\u0394\u0003R!"+
-    "\u0000\u0391\u0393\u0003b)\u0000\u0392\u0391\u0001\u0000\u0000\u0000\u0393"+
-    "\u0396\u0001\u0000\u0000\u0000\u0394\u0392\u0001\u0000\u0000\u0000\u0394"+
-    "\u0395\u0001\u0000\u0000\u0000\u0395\u03a1\u0001\u0000\u0000\u0000\u0396"+
-    "\u0394\u0001\u0000\u0000\u0000\u0397\u039a\u0003`(\u0000\u0398\u039a\u0003"+
-    "Z%\u0000\u0399\u0397\u0001\u0000\u0000\u0000\u0399\u0398\u0001\u0000\u0000"+
-    "\u0000\u039a\u039c\u0001\u0000\u0000\u0000\u039b\u039d\u0003b)\u0000\u039c"+
-    "\u039b\u0001\u0000\u0000\u0000\u039d\u039e\u0001\u0000\u0000\u0000\u039e"+
-    "\u039c\u0001\u0000\u0000\u0000\u039e\u039f\u0001\u0000\u0000\u0000\u039f"+
-    "\u03a1\u0001\u0000\u0000\u0000\u03a0\u0390\u0001\u0000\u0000\u0000\u03a0"+
-    "\u0399\u0001\u0000\u0000\u0000\u03a1\u00b7\u0001\u0000\u0000\u0000\u03a2"+
-    "\u03a4\u0003\\&\u0000\u03a3\u03a5\u0003^\'\u0000\u03a4\u03a3\u0001\u0000"+
-    "\u0000\u0000\u03a5\u03a6\u0001\u0000\u0000\u0000\u03a6\u03a4\u0001\u0000"+
-    "\u0000\u0000\u03a6\u03a7\u0001\u0000\u0000\u0000\u03a7\u03a8\u0001\u0000"+
-    "\u0000\u0000\u03a8\u03a9\u0003\\&\u0000\u03a9\u00b9\u0001\u0000\u0000"+
-    "\u0000\u03aa\u03ab\u0003\u00b8T\u0000\u03ab\u00bb\u0001\u0000\u0000\u0000"+
-    "\u03ac\u03ad\u0003:\u0015\u0000\u03ad\u03ae\u0001\u0000\u0000\u0000\u03ae"+
-    "\u03af\u0006V\u000b\u0000\u03af\u00bd\u0001\u0000\u0000\u0000\u03b0\u03b1"+
-    "\u0003<\u0016\u0000\u03b1\u03b2\u0001\u0000\u0000\u0000\u03b2\u03b3\u0006"+
-    "W\u000b\u0000\u03b3\u00bf\u0001\u0000\u0000\u0000\u03b4\u03b5\u0003>\u0017"+
-    "\u0000\u03b5\u03b6\u0001\u0000\u0000\u0000\u03b6\u03b7\u0006X\u000b\u0000"+
-    "\u03b7\u00c1\u0001\u0000\u0000\u0000\u03b8\u03b9\u0003N\u001f\u0000\u03b9"+
-    "\u03ba\u0001\u0000\u0000\u0000\u03ba\u03bb\u0006Y\u000e\u0000\u03bb\u03bc"+
-    "\u0006Y\u000f\u0000\u03bc\u00c3\u0001\u0000\u0000\u0000\u03bd\u03be\u0003"+
-    "\u00b2Q\u0000\u03be\u03bf\u0001\u0000\u0000\u0000\u03bf\u03c0\u0006Z\f"+
-    "\u0000\u03c0\u00c5\u0001\u0000\u0000\u0000\u03c1\u03c2\u0003\u00b4R\u0000"+
-    "\u03c2\u03c3\u0001\u0000\u0000\u0000\u03c3\u03c4\u0006[\u0010\u0000\u03c4"+
-    "\u00c7\u0001\u0000\u0000\u0000\u03c5\u03c6\u0003\u016c\u00ae\u0000\u03c6"+
-    "\u03c7\u0001\u0000\u0000\u0000\u03c7\u03c8\u0006\\\u0011\u0000\u03c8\u00c9"+
-    "\u0001\u0000\u0000\u0000\u03c9\u03ca\u0003t2\u0000\u03ca\u03cb\u0001\u0000"+
-    "\u0000\u0000\u03cb\u03cc\u0006]\u0012\u0000\u03cc\u00cb\u0001\u0000\u0000"+
-    "\u0000\u03cd\u03ce\u0003p0\u0000\u03ce\u03cf\u0001\u0000\u0000\u0000\u03cf"+
-    "\u03d0\u0006^\u0013\u0000\u03d0\u00cd\u0001\u0000\u0000\u0000\u03d1\u03d2"+
-    "\u0005m\u0000\u0000\u03d2\u03d3\u0005e\u0000\u0000\u03d3\u03d4\u0005t"+
-    "\u0000\u0000\u03d4\u03d5\u0005a\u0000\u0000\u03d5\u03d6\u0005d\u0000\u0000"+
-    "\u03d6\u03d7\u0005a\u0000\u0000\u03d7\u03d8\u0005t\u0000\u0000\u03d8\u03d9"+
-    "\u0005a\u0000\u0000\u03d9\u00cf\u0001\u0000\u0000\u0000\u03da\u03db\u0003"+
-    "B\u0019\u0000\u03db\u03dc\u0001\u0000\u0000\u0000\u03dc\u03dd\u0006`\u0014"+
-    "\u0000\u03dd\u00d1\u0001\u0000\u0000\u0000\u03de\u03df\u0003d*\u0000\u03df"+
-    "\u03e0\u0001\u0000\u0000\u0000\u03e0\u03e1\u0006a\u0015\u0000\u03e1\u00d3"+
-    "\u0001\u0000\u0000\u0000\u03e2\u03e3\u0003:\u0015\u0000\u03e3\u03e4\u0001"+
-    "\u0000\u0000\u0000\u03e4\u03e5\u0006b\u000b\u0000\u03e5\u00d5\u0001\u0000"+
-    "\u0000\u0000\u03e6\u03e7\u0003<\u0016\u0000\u03e7\u03e8\u0001\u0000\u0000"+
-    "\u0000\u03e8\u03e9\u0006c\u000b\u0000\u03e9\u00d7\u0001\u0000\u0000\u0000"+
-    "\u03ea\u03eb\u0003>\u0017\u0000\u03eb\u03ec\u0001\u0000\u0000\u0000\u03ec"+
-    "\u03ed\u0006d\u000b\u0000\u03ed\u00d9\u0001\u0000\u0000\u0000\u03ee\u03ef"+
-    "\u0003N\u001f\u0000\u03ef\u03f0\u0001\u0000\u0000\u0000\u03f0\u03f1\u0006"+
-    "e\u000e\u0000\u03f1\u03f2\u0006e\u000f\u0000\u03f2\u00db\u0001\u0000\u0000"+
-    "\u0000\u03f3\u03f4\u0003x4\u0000\u03f4\u03f5\u0001\u0000\u0000\u0000\u03f5"+
-    "\u03f6\u0006f\u0016\u0000\u03f6\u00dd\u0001\u0000\u0000\u0000\u03f7\u03f8"+
-    "\u0003t2\u0000\u03f8\u03f9\u0001\u0000\u0000\u0000\u03f9\u03fa\u0006g"+
-    "\u0012\u0000\u03fa\u00df\u0001\u0000\u0000\u0000\u03fb\u0400\u0003R!\u0000"+
-    "\u03fc\u0400\u0003P \u0000\u03fd\u0400\u0003`(\u0000\u03fe\u0400\u0003"+
-    "\u00aaM\u0000\u03ff\u03fb\u0001\u0000\u0000\u0000\u03ff\u03fc\u0001\u0000"+
-    "\u0000\u0000\u03ff\u03fd\u0001\u0000\u0000\u0000\u03ff\u03fe\u0001\u0000"+
-    "\u0000\u0000\u0400\u00e1\u0001\u0000\u0000\u0000\u0401\u0404\u0003R!\u0000"+
-    "\u0402\u0404\u0003\u00aaM\u0000\u0403\u0401\u0001\u0000\u0000\u0000\u0403"+
-    "\u0402\u0001\u0000\u0000\u0000\u0404\u0408\u0001\u0000\u0000\u0000\u0405"+
-    "\u0407\u0003\u00e0h\u0000\u0406\u0405\u0001\u0000\u0000\u0000\u0407\u040a"+
-    "\u0001\u0000\u0000\u0000\u0408\u0406\u0001\u0000\u0000\u0000\u0408\u0409"+
-    "\u0001\u0000\u0000\u0000\u0409\u0415\u0001\u0000\u0000\u0000\u040a\u0408"+
-    "\u0001\u0000\u0000\u0000\u040b\u040e\u0003`(\u0000\u040c\u040e\u0003Z"+
-    "%\u0000\u040d\u040b\u0001\u0000\u0000\u0000\u040d\u040c\u0001\u0000\u0000"+
-    "\u0000\u040e\u0410\u0001\u0000\u0000\u0000\u040f\u0411\u0003\u00e0h\u0000"+
-    "\u0410\u040f\u0001\u0000\u0000\u0000\u0411\u0412\u0001\u0000\u0000\u0000"+
-    "\u0412\u0410\u0001\u0000\u0000\u0000\u0412\u0413\u0001\u0000\u0000\u0000"+
-    "\u0413\u0415\u0001\u0000\u0000\u0000\u0414\u0403\u0001\u0000\u0000\u0000"+
-    "\u0414\u040d\u0001\u0000\u0000\u0000\u0415\u00e3\u0001\u0000\u0000\u0000"+
-    "\u0416\u0419\u0003\u00e2i\u0000\u0417\u0419\u0003\u00b8T\u0000\u0418\u0416"+
-    "\u0001\u0000\u0000\u0000\u0418\u0417\u0001\u0000\u0000\u0000\u0419\u041a"+
-    "\u0001\u0000\u0000\u0000\u041a\u0418\u0001\u0000\u0000\u0000\u041a\u041b"+
-    "\u0001\u0000\u0000\u0000\u041b\u00e5\u0001\u0000\u0000\u0000\u041c\u041d"+
-    "\u0003:\u0015\u0000\u041d\u041e\u0001\u0000\u0000\u0000\u041e\u041f\u0006"+
-    "k\u000b\u0000\u041f\u00e7\u0001\u0000\u0000\u0000\u0420\u0421\u0003<\u0016"+
-    "\u0000\u0421\u0422\u0001\u0000\u0000\u0000\u0422\u0423\u0006l\u000b\u0000"+
-    "\u0423\u00e9\u0001\u0000\u0000\u0000\u0424\u0425\u0003>\u0017\u0000\u0425"+
-    "\u0426\u0001\u0000\u0000\u0000\u0426\u0427\u0006m\u000b\u0000\u0427\u00eb"+
-    "\u0001\u0000\u0000\u0000\u0428\u0429\u0003N\u001f\u0000\u0429\u042a\u0001"+
-    "\u0000\u0000\u0000\u042a\u042b\u0006n\u000e\u0000\u042b\u042c\u0006n\u000f"+
-    "\u0000\u042c\u00ed\u0001\u0000\u0000\u0000\u042d\u042e\u0003p0\u0000\u042e"+
-    "\u042f\u0001\u0000\u0000\u0000\u042f\u0430\u0006o\u0013\u0000\u0430\u00ef"+
-    "\u0001\u0000\u0000\u0000\u0431\u0432\u0003t2\u0000\u0432\u0433\u0001\u0000"+
-    "\u0000\u0000\u0433\u0434\u0006p\u0012\u0000\u0434\u00f1\u0001\u0000\u0000"+
-    "\u0000\u0435\u0436\u0003x4\u0000\u0436\u0437\u0001\u0000\u0000\u0000\u0437"+
-    "\u0438\u0006q\u0016\u0000\u0438\u00f3\u0001\u0000\u0000\u0000\u0439\u043a"+
-    "\u0005a\u0000\u0000\u043a\u043b\u0005s\u0000\u0000\u043b\u00f5\u0001\u0000"+
-    "\u0000\u0000\u043c\u043d\u0003\u00e4j\u0000\u043d\u043e\u0001\u0000\u0000"+
-    "\u0000\u043e\u043f\u0006s\u0017\u0000\u043f\u00f7\u0001\u0000\u0000\u0000"+
-    "\u0440\u0441\u0003:\u0015\u0000\u0441\u0442\u0001\u0000\u0000\u0000\u0442"+
-    "\u0443\u0006t\u000b\u0000\u0443\u00f9\u0001\u0000\u0000\u0000\u0444\u0445"+
-    "\u0003<\u0016\u0000\u0445\u0446\u0001\u0000\u0000\u0000\u0446\u0447\u0006"+
-    "u\u000b\u0000\u0447\u00fb\u0001\u0000\u0000\u0000\u0448\u0449\u0003>\u0017"+
-    "\u0000\u0449\u044a\u0001\u0000\u0000\u0000\u044a\u044b\u0006v\u000b\u0000"+
-    "\u044b\u00fd\u0001\u0000\u0000\u0000\u044c\u044d\u0003N\u001f\u0000\u044d"+
-    "\u044e\u0001\u0000\u0000\u0000\u044e\u044f\u0006w\u000e\u0000\u044f\u0450"+
-    "\u0006w\u000f\u0000\u0450\u00ff\u0001\u0000\u0000\u0000\u0451\u0452\u0003"+
-    "\u00b2Q\u0000\u0452\u0453\u0001\u0000\u0000\u0000\u0453\u0454\u0006x\f"+
-    "\u0000\u0454\u0455\u0006x\u0018\u0000\u0455\u0101\u0001\u0000\u0000\u0000"+
-    "\u0456\u0457\u0005o\u0000\u0000\u0457\u0458\u0005n\u0000\u0000\u0458\u0459"+
-    "\u0001\u0000\u0000\u0000\u0459\u045a\u0006y\u0019\u0000\u045a\u0103\u0001"+
-    "\u0000\u0000\u0000\u045b\u045c\u0005w\u0000\u0000\u045c\u045d\u0005i\u0000"+
-    "\u0000\u045d\u045e\u0005t\u0000\u0000\u045e\u045f\u0005h\u0000\u0000\u045f"+
-    "\u0460\u0001\u0000\u0000\u0000\u0460\u0461\u0006z\u0019\u0000\u0461\u0105"+
-    "\u0001\u0000\u0000\u0000\u0462\u0463\b\f\u0000\u0000\u0463\u0107\u0001"+
-    "\u0000\u0000\u0000\u0464\u0466\u0003\u0106{\u0000\u0465\u0464\u0001\u0000"+
-    "\u0000\u0000\u0466\u0467\u0001\u0000\u0000\u0000\u0467\u0465\u0001\u0000"+
-    "\u0000\u0000\u0467\u0468\u0001\u0000\u0000\u0000\u0468\u0469\u0001\u0000"+
-    "\u0000\u0000\u0469\u046a\u0003\u016c\u00ae\u0000\u046a\u046c\u0001\u0000"+
-    "\u0000\u0000\u046b\u0465\u0001\u0000\u0000\u0000\u046b\u046c\u0001\u0000"+
-    "\u0000\u0000\u046c\u046e\u0001\u0000\u0000\u0000\u046d\u046f\u0003\u0106"+
-    "{\u0000\u046e\u046d\u0001\u0000\u0000\u0000\u046f\u0470\u0001\u0000\u0000"+
-    "\u0000\u0470\u046e\u0001\u0000\u0000\u0000\u0470\u0471\u0001\u0000\u0000"+
-    "\u0000\u0471\u0109\u0001\u0000\u0000\u0000\u0472\u0473\u0003\u0108|\u0000"+
-    "\u0473\u0474\u0001\u0000\u0000\u0000\u0474\u0475\u0006}\u001a\u0000\u0475"+
-    "\u010b\u0001\u0000\u0000\u0000\u0476\u0477\u0003:\u0015\u0000\u0477\u0478"+
-    "\u0001\u0000\u0000\u0000\u0478\u0479\u0006~\u000b\u0000\u0479\u010d\u0001"+
-    "\u0000\u0000\u0000\u047a\u047b\u0003<\u0016\u0000\u047b\u047c\u0001\u0000"+
-    "\u0000\u0000\u047c\u047d\u0006\u007f\u000b\u0000\u047d\u010f\u0001\u0000"+
-    "\u0000\u0000\u047e\u047f\u0003>\u0017\u0000\u047f\u0480\u0001\u0000\u0000"+
-    "\u0000\u0480\u0481\u0006\u0080\u000b\u0000\u0481\u0111\u0001\u0000\u0000"+
-    "\u0000\u0482\u0483\u0003N\u001f\u0000\u0483\u0484\u0001\u0000\u0000\u0000"+
-    "\u0484\u0485\u0006\u0081\u000e\u0000\u0485\u0486\u0006\u0081\u000f\u0000"+
-    "\u0486\u0487\u0006\u0081\u000f\u0000\u0487\u0113\u0001\u0000\u0000\u0000"+
-    "\u0488\u0489\u0003p0\u0000\u0489\u048a\u0001\u0000\u0000\u0000\u048a\u048b"+
-    "\u0006\u0082\u0013\u0000\u048b\u0115\u0001\u0000\u0000\u0000\u048c\u048d"+
-    "\u0003t2\u0000\u048d\u048e\u0001\u0000\u0000\u0000\u048e\u048f\u0006\u0083"+
-    "\u0012\u0000\u048f\u0117\u0001\u0000\u0000\u0000\u0490\u0491\u0003x4\u0000"+
-    "\u0491\u0492\u0001\u0000\u0000\u0000\u0492\u0493\u0006\u0084\u0016\u0000"+
-    "\u0493\u0119\u0001\u0000\u0000\u0000\u0494\u0495\u0003\u0104z\u0000\u0495"+
-    "\u0496\u0001\u0000\u0000\u0000\u0496\u0497\u0006\u0085\u001b\u0000\u0497"+
-    "\u011b\u0001\u0000\u0000\u0000\u0498\u0499\u0003\u00e4j\u0000\u0499\u049a"+
-    "\u0001\u0000\u0000\u0000\u049a\u049b\u0006\u0086\u0017\u0000\u049b\u011d"+
-    "\u0001\u0000\u0000\u0000\u049c\u049d\u0003\u00baU\u0000\u049d\u049e\u0001"+
-    "\u0000\u0000\u0000\u049e\u049f\u0006\u0087\u001c\u0000\u049f\u011f\u0001"+
-    "\u0000\u0000\u0000\u04a0\u04a1\u0003:\u0015\u0000\u04a1\u04a2\u0001\u0000"+
-    "\u0000\u0000\u04a2\u04a3\u0006\u0088\u000b\u0000\u04a3\u0121\u0001\u0000"+
-    "\u0000\u0000\u04a4\u04a5\u0003<\u0016\u0000\u04a5\u04a6\u0001\u0000\u0000"+
-    "\u0000\u04a6\u04a7\u0006\u0089\u000b\u0000\u04a7\u0123\u0001\u0000\u0000"+
-    "\u0000\u04a8\u04a9\u0003>\u0017\u0000\u04a9\u04aa\u0001\u0000\u0000\u0000"+
-    "\u04aa\u04ab\u0006\u008a\u000b\u0000\u04ab\u0125\u0001\u0000\u0000\u0000"+
-    "\u04ac\u04ad\u0003N\u001f\u0000\u04ad\u04ae\u0001\u0000\u0000\u0000\u04ae"+
-    "\u04af\u0006\u008b\u000e\u0000\u04af\u04b0\u0006\u008b\u000f\u0000\u04b0"+
-    "\u0127\u0001\u0000\u0000\u0000\u04b1\u04b2\u0003\u016c\u00ae\u0000\u04b2"+
-    "\u04b3\u0001\u0000\u0000\u0000\u04b3\u04b4\u0006\u008c\u0011\u0000\u04b4"+
-    "\u0129\u0001\u0000\u0000\u0000\u04b5\u04b6\u0003t2\u0000\u04b6\u04b7\u0001"+
-    "\u0000\u0000\u0000\u04b7\u04b8\u0006\u008d\u0012\u0000\u04b8\u012b\u0001"+
-    "\u0000\u0000\u0000\u04b9\u04ba\u0003x4\u0000\u04ba\u04bb\u0001\u0000\u0000"+
-    "\u0000\u04bb\u04bc\u0006\u008e\u0016\u0000\u04bc\u012d\u0001\u0000\u0000"+
-    "\u0000\u04bd\u04be\u0003\u0102y\u0000\u04be\u04bf\u0001\u0000\u0000\u0000"+
-    "\u04bf\u04c0\u0006\u008f\u001d\u0000\u04c0\u04c1\u0006\u008f\u001e\u0000"+
-    "\u04c1\u012f\u0001\u0000\u0000\u0000\u04c2\u04c3\u0003B\u0019\u0000\u04c3"+
-    "\u04c4\u0001\u0000\u0000\u0000\u04c4\u04c5\u0006\u0090\u0014\u0000\u04c5"+
-    "\u0131\u0001\u0000\u0000\u0000\u04c6\u04c7\u0003d*\u0000\u04c7\u04c8\u0001"+
-    "\u0000\u0000\u0000\u04c8\u04c9\u0006\u0091\u0015\u0000\u04c9\u0133\u0001"+
-    "\u0000\u0000\u0000\u04ca\u04cb\u0003:\u0015\u0000\u04cb\u04cc\u0001\u0000"+
-    "\u0000\u0000\u04cc\u04cd\u0006\u0092\u000b\u0000\u04cd\u0135\u0001\u0000"+
-    "\u0000\u0000\u04ce\u04cf\u0003<\u0016\u0000\u04cf\u04d0\u0001\u0000\u0000"+
-    "\u0000\u04d0\u04d1\u0006\u0093\u000b\u0000\u04d1\u0137\u0001\u0000\u0000"+
-    "\u0000\u04d2\u04d3\u0003>\u0017\u0000\u04d3\u04d4\u0001\u0000\u0000\u0000"+
-    "\u04d4\u04d5\u0006\u0094\u000b\u0000\u04d5\u0139\u0001\u0000\u0000\u0000"+
-    "\u04d6\u04d7\u0003N\u001f\u0000\u04d7\u04d8\u0001\u0000\u0000\u0000\u04d8"+
-    "\u04d9\u0006\u0095\u000e\u0000\u04d9\u04da\u0006\u0095\u000f\u0000\u04da"+
-    "\u04db\u0006\u0095\u000f\u0000\u04db\u013b\u0001\u0000\u0000\u0000\u04dc"+
-    "\u04dd\u0003t2\u0000\u04dd\u04de\u0001\u0000\u0000\u0000\u04de\u04df\u0006"+
-    "\u0096\u0012\u0000\u04df\u013d\u0001\u0000\u0000\u0000\u04e0\u04e1\u0003"+
-    "x4\u0000\u04e1\u04e2\u0001\u0000\u0000\u0000\u04e2\u04e3\u0006\u0097\u0016"+
-    "\u0000\u04e3\u013f\u0001\u0000\u0000\u0000\u04e4\u04e5\u0003\u00e4j\u0000"+
-    "\u04e5\u04e6\u0001\u0000\u0000\u0000\u04e6\u04e7\u0006\u0098\u0017\u0000"+
-    "\u04e7\u0141\u0001\u0000\u0000\u0000\u04e8\u04e9\u0003:\u0015\u0000\u04e9"+
-    "\u04ea\u0001\u0000\u0000\u0000\u04ea\u04eb\u0006\u0099\u000b\u0000\u04eb"+
-    "\u0143\u0001\u0000\u0000\u0000\u04ec\u04ed\u0003<\u0016\u0000\u04ed\u04ee"+
-    "\u0001\u0000\u0000\u0000\u04ee\u04ef\u0006\u009a\u000b\u0000\u04ef\u0145"+
-    "\u0001\u0000\u0000\u0000\u04f0\u04f1\u0003>\u0017\u0000\u04f1\u04f2\u0001"+
-    "\u0000\u0000\u0000\u04f2\u04f3\u0006\u009b\u000b\u0000\u04f3\u0147\u0001"+
-    "\u0000\u0000\u0000\u04f4\u04f5\u0003N\u001f\u0000\u04f5\u04f6\u0001\u0000"+
-    "\u0000\u0000\u04f6\u04f7\u0006\u009c\u000e\u0000\u04f7\u04f8\u0006\u009c"+
-    "\u000f\u0000\u04f8\u0149\u0001\u0000\u0000\u0000\u04f9\u04fa\u0003x4\u0000"+
-    "\u04fa\u04fb\u0001\u0000\u0000\u0000\u04fb\u04fc\u0006\u009d\u0016\u0000"+
-    "\u04fc\u014b\u0001\u0000\u0000\u0000\u04fd\u04fe\u0003\u00baU\u0000\u04fe"+
-    "\u04ff\u0001\u0000\u0000\u0000\u04ff\u0500\u0006\u009e\u001c\u0000\u0500"+
-    "\u014d\u0001\u0000\u0000\u0000\u0501\u0502\u0003\u00b6S\u0000\u0502\u0503"+
-    "\u0001\u0000\u0000\u0000\u0503\u0504\u0006\u009f\u001f\u0000\u0504\u014f"+
-    "\u0001\u0000\u0000\u0000\u0505\u0506\u0003:\u0015\u0000\u0506\u0507\u0001"+
-    "\u0000\u0000\u0000\u0507\u0508\u0006\u00a0\u000b\u0000\u0508\u0151\u0001"+
-    "\u0000\u0000\u0000\u0509\u050a\u0003<\u0016\u0000\u050a\u050b\u0001\u0000"+
-    "\u0000\u0000\u050b\u050c\u0006\u00a1\u000b\u0000\u050c\u0153\u0001\u0000"+
-    "\u0000\u0000\u050d\u050e\u0003>\u0017\u0000\u050e\u050f\u0001\u0000\u0000"+
-    "\u0000\u050f\u0510\u0006\u00a2\u000b\u0000\u0510\u0155\u0001\u0000\u0000"+
-    "\u0000\u0511\u0512\u0003N\u001f\u0000\u0512\u0513\u0001\u0000\u0000\u0000"+
-    "\u0513\u0514\u0006\u00a3\u000e\u0000\u0514\u0515\u0006\u00a3\u000f\u0000"+
-    "\u0515\u0157\u0001\u0000\u0000\u0000\u0516\u0517\u0005i\u0000\u0000\u0517"+
-    "\u0518\u0005n\u0000\u0000\u0518\u0519\u0005f\u0000\u0000\u0519\u051a\u0005"+
-    "o\u0000\u0000\u051a\u0159\u0001\u0000\u0000\u0000\u051b\u051c\u0003:\u0015"+
-    "\u0000\u051c\u051d\u0001\u0000\u0000\u0000\u051d\u051e\u0006\u00a5\u000b"+
-    "\u0000\u051e\u015b\u0001\u0000\u0000\u0000\u051f\u0520\u0003<\u0016\u0000"+
-    "\u0520\u0521\u0001\u0000\u0000\u0000\u0521\u0522\u0006\u00a6\u000b\u0000"+
-    "\u0522\u015d\u0001\u0000\u0000\u0000\u0523\u0524\u0003>\u0017\u0000\u0524"+
-    "\u0525\u0001\u0000\u0000\u0000\u0525\u0526\u0006\u00a7\u000b\u0000\u0526"+
-    "\u015f\u0001\u0000\u0000\u0000\u0527\u0528\u0003N\u001f\u0000\u0528\u0529"+
-    "\u0001\u0000\u0000\u0000\u0529\u052a\u0006\u00a8\u000e\u0000\u052a\u052b"+
-    "\u0006\u00a8\u000f\u0000\u052b\u0161\u0001\u0000\u0000\u0000\u052c\u052d"+
-    "\u0005f\u0000\u0000\u052d\u052e\u0005u\u0000\u0000\u052e\u052f\u0005n"+
-    "\u0000\u0000\u052f\u0530\u0005c\u0000\u0000\u0530\u0531\u0005t\u0000\u0000"+
-    "\u0531\u0532\u0005i\u0000\u0000\u0532\u0533\u0005o\u0000\u0000\u0533\u0534"+
-    "\u0005n\u0000\u0000\u0534\u0535\u0005s\u0000\u0000\u0535\u0163\u0001\u0000"+
-    "\u0000\u0000\u0536\u0537\u0003:\u0015\u0000\u0537\u0538\u0001\u0000\u0000"+
-    "\u0000\u0538\u0539\u0006\u00aa\u000b\u0000\u0539\u0165\u0001\u0000\u0000"+
-    "\u0000\u053a\u053b\u0003<\u0016\u0000\u053b\u053c\u0001\u0000\u0000\u0000"+
-    "\u053c\u053d\u0006\u00ab\u000b\u0000\u053d\u0167\u0001\u0000\u0000\u0000"+
-    "\u053e\u053f\u0003>\u0017\u0000\u053f\u0540\u0001\u0000\u0000\u0000\u0540"+
-    "\u0541\u0006\u00ac\u000b\u0000\u0541\u0169\u0001\u0000\u0000\u0000\u0542"+
-    "\u0543\u0003\u00b4R\u0000\u0543\u0544\u0001\u0000\u0000\u0000\u0544\u0545"+
-    "\u0006\u00ad\u0010\u0000\u0545\u0546\u0006\u00ad\u000f\u0000\u0546\u016b"+
-    "\u0001\u0000\u0000\u0000\u0547\u0548\u0005:\u0000\u0000\u0548\u016d\u0001"+
-    "\u0000\u0000\u0000\u0549\u054f\u0003Z%\u0000\u054a\u054f\u0003P \u0000"+
-    "\u054b\u054f\u0003x4\u0000\u054c\u054f\u0003R!\u0000\u054d\u054f\u0003"+
-    "`(\u0000\u054e\u0549\u0001\u0000\u0000\u0000\u054e\u054a\u0001\u0000\u0000"+
-    "\u0000\u054e\u054b\u0001\u0000\u0000\u0000\u054e\u054c\u0001\u0000\u0000"+
-    "\u0000\u054e\u054d\u0001\u0000\u0000\u0000\u054f\u0550\u0001\u0000\u0000"+
-    "\u0000\u0550\u054e\u0001\u0000\u0000\u0000\u0550\u0551\u0001\u0000\u0000"+
-    "\u0000\u0551\u016f\u0001\u0000\u0000\u0000\u0552\u0553\u0003:\u0015\u0000"+
-    "\u0553\u0554\u0001\u0000\u0000\u0000\u0554\u0555\u0006\u00b0\u000b\u0000"+
-    "\u0555\u0171\u0001\u0000\u0000\u0000\u0556\u0557\u0003<\u0016\u0000\u0557"+
-    "\u0558\u0001\u0000\u0000\u0000\u0558\u0559\u0006\u00b1\u000b\u0000\u0559"+
-    "\u0173\u0001\u0000\u0000\u0000\u055a\u055b\u0003>\u0017\u0000\u055b\u055c"+
-    "\u0001\u0000\u0000\u0000\u055c\u055d\u0006\u00b2\u000b\u0000\u055d\u0175"+
-    "\u0001\u0000\u0000\u0000\u055e\u055f\u0003N\u001f\u0000\u055f\u0560\u0001"+
-    "\u0000\u0000\u0000\u0560\u0561\u0006\u00b3\u000e\u0000\u0561\u0562\u0006"+
-    "\u00b3\u000f\u0000\u0562\u0177\u0001\u0000\u0000\u0000\u0563\u0564\u0003"+
-    "B\u0019\u0000\u0564\u0565\u0001\u0000\u0000\u0000\u0565\u0566\u0006\u00b4"+
-    "\u0014\u0000\u0566\u0567\u0006\u00b4\u000f\u0000\u0567\u0568\u0006\u00b4"+
-    " \u0000\u0568\u0179\u0001\u0000\u0000\u0000\u0569\u056a\u0003d*\u0000"+
-    "\u056a\u056b\u0001\u0000\u0000\u0000\u056b\u056c\u0006\u00b5\u0015\u0000"+
-    "\u056c\u056d\u0006\u00b5\u000f\u0000\u056d\u056e\u0006\u00b5 \u0000\u056e"+
-    "\u017b\u0001\u0000\u0000\u0000\u056f\u0570\u0003:\u0015\u0000\u0570\u0571"+
-    "\u0001\u0000\u0000\u0000\u0571\u0572\u0006\u00b6\u000b\u0000\u0572\u017d"+
-    "\u0001\u0000\u0000\u0000\u0573\u0574\u0003<\u0016\u0000\u0574\u0575\u0001"+
-    "\u0000\u0000\u0000\u0575\u0576\u0006\u00b7\u000b\u0000\u0576\u017f\u0001"+
-    "\u0000\u0000\u0000\u0577\u0578\u0003>\u0017\u0000\u0578\u0579\u0001\u0000"+
-    "\u0000\u0000\u0579\u057a\u0006\u00b8\u000b\u0000\u057a\u0181\u0001\u0000"+
-    "\u0000\u0000\u057b\u057c\u0003\u016c\u00ae\u0000\u057c\u057d\u0001\u0000"+
-    "\u0000\u0000\u057d\u057e\u0006\u00b9\u0011\u0000\u057e\u057f\u0006\u00b9"+
-    "\u000f\u0000\u057f\u0580\u0006\u00b9\u0007\u0000\u0580\u0183\u0001\u0000"+
-    "\u0000\u0000\u0581\u0582\u0003t2\u0000\u0582\u0583\u0001\u0000\u0000\u0000"+
-    "\u0583\u0584\u0006\u00ba\u0012\u0000\u0584\u0585\u0006\u00ba\u000f\u0000"+
-    "\u0585\u0586\u0006\u00ba\u0007\u0000\u0586\u0185\u0001\u0000\u0000\u0000"+
-    "\u0587\u0588\u0003:\u0015\u0000\u0588\u0589\u0001\u0000\u0000\u0000\u0589"+
-    "\u058a\u0006\u00bb\u000b\u0000\u058a\u0187\u0001\u0000\u0000\u0000\u058b"+
-    "\u058c\u0003<\u0016\u0000\u058c\u058d\u0001\u0000\u0000\u0000\u058d\u058e"+
-    "\u0006\u00bc\u000b\u0000\u058e\u0189\u0001\u0000\u0000\u0000\u058f\u0590"+
-    "\u0003>\u0017\u0000\u0590\u0591\u0001\u0000\u0000\u0000\u0591\u0592\u0006"+
-    "\u00bd\u000b\u0000\u0592\u018b\u0001\u0000\u0000\u0000\u0593\u0594\u0003"+
-    "\u00baU\u0000\u0594\u0595\u0001\u0000\u0000\u0000\u0595\u0596\u0006\u00be"+
-    "\u000f\u0000\u0596\u0597\u0006\u00be\u0000\u0000\u0597\u0598\u0006\u00be"+
-    "\u001c\u0000\u0598\u018d\u0001\u0000\u0000\u0000\u0599\u059a\u0003\u00b6"+
-    "S\u0000\u059a\u059b\u0001\u0000\u0000\u0000\u059b\u059c\u0006\u00bf\u000f"+
-    "\u0000\u059c\u059d\u0006\u00bf\u0000\u0000\u059d\u059e\u0006\u00bf\u001f"+
-    "\u0000\u059e\u018f\u0001\u0000\u0000\u0000\u059f\u05a0\u0003j-\u0000\u05a0"+
-    "\u05a1\u0001\u0000\u0000\u0000\u05a1\u05a2\u0006\u00c0\u000f\u0000\u05a2"+
-    "\u05a3\u0006\u00c0\u0000\u0000\u05a3\u05a4\u0006\u00c0!\u0000\u05a4\u0191"+
-    "\u0001\u0000\u0000\u0000\u05a5\u05a6\u0003N\u001f\u0000\u05a6\u05a7\u0001"+
-    "\u0000\u0000\u0000\u05a7\u05a8\u0006\u00c1\u000e\u0000\u05a8\u05a9\u0006"+
-    "\u00c1\u000f\u0000\u05a9\u0193\u0001\u0000\u0000\u0000A\u0000\u0001\u0002"+
-    "\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0240\u024a"+
-    "\u024e\u0251\u025a\u025c\u0267\u026e\u0273\u029a\u029f\u02a8\u02af\u02b4"+
-    "\u02b6\u02c1\u02c9\u02cc\u02ce\u02d3\u02d8\u02de\u02e5\u02ea\u02f0\u02f3"+
-    "\u02fb\u02ff\u037b\u0382\u0384\u0394\u0399\u039e\u03a0\u03a6\u03ff\u0403"+
-    "\u0408\u040d\u0412\u0414\u0418\u041a\u0467\u046b\u0470\u054e\u0550\"\u0005"+
-    "\u0002\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005\u0001\u0000\u0005"+
-    "\u0003\u0000\u0005\b\u0000\u0005\f\u0000\u0005\u000e\u0000\u0005\n\u0000"+
-    "\u0005\u0005\u0000\u0005\u000b\u0000\u0000\u0001\u0000\u0007E\u0000\u0005"+
-    "\u0000\u0000\u0007\u001d\u0000\u0004\u0000\u0000\u0007F\u0000\u0007r\u0000"+
-    "\u0007&\u0000\u0007$\u0000\u0007\u0019\u0000\u0007\u001e\u0000\u0007("+
-    "\u0000\u0007P\u0000\u0005\r\u0000\u0005\u0007\u0000\u0007Z\u0000\u0007"+
-    "Y\u0000\u0007H\u0000\u0007X\u0000\u0005\t\u0000\u0007G\u0000\u0005\u000f"+
-    "\u0000\u0007!\u0000";
+    "\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001"+
+    "\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001"+
+    "\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001"+
+    "\u00b8\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001"+
+    "\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001"+
+    "\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001"+
+    "\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001"+
+    "\u00bd\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00bf\u0001"+
+    "\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001"+
+    "\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001"+
+    "\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001"+
+    "\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001"+
+    "\u00c3\u0001\u00c3\u0002\u026c\u02b1\u0000\u00c4\u0010\u0001\u0012\u0002"+
+    "\u0014\u0003\u0016\u0004\u0018\u0005\u001a\u0006\u001c\u0007\u001e\b "+
+    "\t\"\n$\u000b&\f(\r*\u000e,\u000f.\u00100\u00112\u00124\u00136\u00148"+
+    "\u0015:\u0016<\u0017>\u0018@\u0019B\u001aD\u0000F\u0000H\u0000J\u0000"+
+    "L\u0000N\u0000P\u0000R\u0000T\u0000V\u0000X\u001bZ\u001c\\\u001d^\u001e"+
+    "`\u001fb d!f\"h#j$l%n&p\'r(t)v*x+z,|-~.\u0080/\u00820\u00841\u00862\u0088"+
+    "3\u008a4\u008c5\u008e6\u00907\u00928\u00949\u0096:\u0098;\u009a<\u009c"+
+    "=\u009e>\u00a0?\u00a2@\u00a4\u0000\u00a6A\u00a8B\u00aaC\u00acD\u00ae\u0000"+
+    "\u00b0E\u00b2F\u00b4G\u00b6H\u00b8\u0000\u00ba\u0000\u00bcI\u00beJ\u00c0"+
+    "K\u00c2\u0000\u00c4\u0000\u00c6\u0000\u00c8\u0000\u00ca\u0000\u00cc\u0000"+
+    "\u00ceL\u00d0\u0000\u00d2M\u00d4\u0000\u00d6\u0000\u00d8N\u00daO\u00dc"+
+    "P\u00de\u0000\u00e0\u0000\u00e2\u0000\u00e4\u0000\u00e6\u0000\u00e8Q\u00ea"+
+    "R\u00ecS\u00eeT\u00f0\u0000\u00f2\u0000\u00f4\u0000\u00f6\u0000\u00f8"+
+    "U\u00fa\u0000\u00fcV\u00feW\u0100X\u0102\u0000\u0104\u0000\u0106Y\u0108"+
+    "Z\u010a\u0000\u010c[\u010e\u0000\u0110\\\u0112]\u0114^\u0116\u0000\u0118"+
+    "\u0000\u011a\u0000\u011c\u0000\u011e\u0000\u0120\u0000\u0122\u0000\u0124"+
+    "_\u0126`\u0128a\u012a\u0000\u012c\u0000\u012e\u0000\u0130\u0000\u0132"+
+    "b\u0134c\u0136d\u0138\u0000\u013ae\u013cf\u013eg\u0140h\u0142\u0000\u0144"+
+    "i\u0146j\u0148k\u014al\u014c\u0000\u014em\u0150n\u0152o\u0154p\u0156q"+
+    "\u0158\u0000\u015a\u0000\u015c\u0000\u015e\u0000\u0160\u0000\u0162\u0000"+
+    "\u0164\u0000\u0166r\u0168s\u016at\u016c\u0000\u016e\u0000\u0170\u0000"+
+    "\u0172\u0000\u0174u\u0176v\u0178w\u017a\u0000\u017c\u0000\u017e\u0000"+
+    "\u0180x\u0182y\u0184z\u0186\u0000\u0188\u0000\u018a{\u018c|\u018e}\u0190"+
+    "\u0000\u0192\u0000\u0194\u0000\u0196\u0000\u0010\u0000\u0001\u0002\u0003"+
+    "\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f#\u0002\u0000DDdd"+
+    "\u0002\u0000IIii\u0002\u0000SSss\u0002\u0000EEee\u0002\u0000CCcc\u0002"+
+    "\u0000TTtt\u0002\u0000RRrr\u0002\u0000OOoo\u0002\u0000PPpp\u0002\u0000"+
+    "NNnn\u0002\u0000HHhh\u0002\u0000VVvv\u0002\u0000AAaa\u0002\u0000LLll\u0002"+
+    "\u0000XXxx\u0002\u0000FFff\u0002\u0000MMmm\u0002\u0000GGgg\u0002\u0000"+
+    "KKkk\u0002\u0000WWww\u0002\u0000UUuu\u0006\u0000\t\n\r\r  //[[]]\u0002"+
+    "\u0000\n\n\r\r\u0003\u0000\t\n\r\r  \u0001\u000009\u0002\u0000AZaz\b\u0000"+
+    "\"\"NNRRTT\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002\u0000++--\u0001"+
+    "\u0000``\u0002\u0000BBbb\u0002\u0000YYyy\u000b\u0000\t\n\r\r  \"\",,/"+
+    "/::==[[]]||\u0002\u0000**//\u000b\u0000\t\n\r\r  \"#,,//::<<>?\\\\||\u05dd"+
+    "\u0000\u0010\u0001\u0000\u0000\u0000\u0000\u0012\u0001\u0000\u0000\u0000"+
+    "\u0000\u0014\u0001\u0000\u0000\u0000\u0000\u0016\u0001\u0000\u0000\u0000"+
+    "\u0000\u0018\u0001\u0000\u0000\u0000\u0000\u001a\u0001\u0000\u0000\u0000"+
+    "\u0000\u001c\u0001\u0000\u0000\u0000\u0000\u001e\u0001\u0000\u0000\u0000"+
+    "\u0000 \u0001\u0000\u0000\u0000\u0000\"\u0001\u0000\u0000\u0000\u0000"+
+    "$\u0001\u0000\u0000\u0000\u0000&\u0001\u0000\u0000\u0000\u0000(\u0001"+
+    "\u0000\u0000\u0000\u0000*\u0001\u0000\u0000\u0000\u0000,\u0001\u0000\u0000"+
+    "\u0000\u0000.\u0001\u0000\u0000\u0000\u00000\u0001\u0000\u0000\u0000\u0000"+
+    "2\u0001\u0000\u0000\u0000\u00004\u0001\u0000\u0000\u0000\u00006\u0001"+
+    "\u0000\u0000\u0000\u00008\u0001\u0000\u0000\u0000\u0000:\u0001\u0000\u0000"+
+    "\u0000\u0000<\u0001\u0000\u0000\u0000\u0000>\u0001\u0000\u0000\u0000\u0000"+
+    "@\u0001\u0000\u0000\u0000\u0001B\u0001\u0000\u0000\u0000\u0001X\u0001"+
+    "\u0000\u0000\u0000\u0001Z\u0001\u0000\u0000\u0000\u0001\\\u0001\u0000"+
+    "\u0000\u0000\u0001^\u0001\u0000\u0000\u0000\u0001`\u0001\u0000\u0000\u0000"+
+    "\u0001b\u0001\u0000\u0000\u0000\u0001d\u0001\u0000\u0000\u0000\u0001f"+
+    "\u0001\u0000\u0000\u0000\u0001h\u0001\u0000\u0000\u0000\u0001j\u0001\u0000"+
+    "\u0000\u0000\u0001l\u0001\u0000\u0000\u0000\u0001n\u0001\u0000\u0000\u0000"+
+    "\u0001p\u0001\u0000\u0000\u0000\u0001r\u0001\u0000\u0000\u0000\u0001t"+
+    "\u0001\u0000\u0000\u0000\u0001v\u0001\u0000\u0000\u0000\u0001x\u0001\u0000"+
+    "\u0000\u0000\u0001z\u0001\u0000\u0000\u0000\u0001|\u0001\u0000\u0000\u0000"+
+    "\u0001~\u0001\u0000\u0000\u0000\u0001\u0080\u0001\u0000\u0000\u0000\u0001"+
+    "\u0082\u0001\u0000\u0000\u0000\u0001\u0084\u0001\u0000\u0000\u0000\u0001"+
+    "\u0086\u0001\u0000\u0000\u0000\u0001\u0088\u0001\u0000\u0000\u0000\u0001"+
+    "\u008a\u0001\u0000\u0000\u0000\u0001\u008c\u0001\u0000\u0000\u0000\u0001"+
+    "\u008e\u0001\u0000\u0000\u0000\u0001\u0090\u0001\u0000\u0000\u0000\u0001"+
+    "\u0092\u0001\u0000\u0000\u0000\u0001\u0094\u0001\u0000\u0000\u0000\u0001"+
+    "\u0096\u0001\u0000\u0000\u0000\u0001\u0098\u0001\u0000\u0000\u0000\u0001"+
+    "\u009a\u0001\u0000\u0000\u0000\u0001\u009c\u0001\u0000\u0000\u0000\u0001"+
+    "\u009e\u0001\u0000\u0000\u0000\u0001\u00a0\u0001\u0000\u0000\u0000\u0001"+
+    "\u00a2\u0001\u0000\u0000\u0000\u0001\u00a4\u0001\u0000\u0000\u0000\u0001"+
+    "\u00a6\u0001\u0000\u0000\u0000\u0001\u00a8\u0001\u0000\u0000\u0000\u0001"+
+    "\u00aa\u0001\u0000\u0000\u0000\u0001\u00ac\u0001\u0000\u0000\u0000\u0001"+
+    "\u00b0\u0001\u0000\u0000\u0000\u0001\u00b2\u0001\u0000\u0000\u0000\u0001"+
+    "\u00b4\u0001\u0000\u0000\u0000\u0001\u00b6\u0001\u0000\u0000\u0000\u0002"+
+    "\u00b8\u0001\u0000\u0000\u0000\u0002\u00ba\u0001\u0000\u0000\u0000\u0002"+
+    "\u00bc\u0001\u0000\u0000\u0000\u0002\u00be\u0001\u0000\u0000\u0000\u0002"+
+    "\u00c0\u0001\u0000\u0000\u0000\u0003\u00c2\u0001\u0000\u0000\u0000\u0003"+
+    "\u00c4\u0001\u0000\u0000\u0000\u0003\u00c6\u0001\u0000\u0000\u0000\u0003"+
+    "\u00c8\u0001\u0000\u0000\u0000\u0003\u00ca\u0001\u0000\u0000\u0000\u0003"+
+    "\u00cc\u0001\u0000\u0000\u0000\u0003\u00ce\u0001\u0000\u0000\u0000\u0003"+
+    "\u00d2\u0001\u0000\u0000\u0000\u0003\u00d4\u0001\u0000\u0000\u0000\u0003"+
+    "\u00d6\u0001\u0000\u0000\u0000\u0003\u00d8\u0001\u0000\u0000\u0000\u0003"+
+    "\u00da\u0001\u0000\u0000\u0000\u0003\u00dc\u0001\u0000\u0000\u0000\u0004"+
+    "\u00de\u0001\u0000\u0000\u0000\u0004\u00e0\u0001\u0000\u0000\u0000\u0004"+
+    "\u00e2\u0001\u0000\u0000\u0000\u0004\u00e8\u0001\u0000\u0000\u0000\u0004"+
+    "\u00ea\u0001\u0000\u0000\u0000\u0004\u00ec\u0001\u0000\u0000\u0000\u0004"+
+    "\u00ee\u0001\u0000\u0000\u0000\u0005\u00f0\u0001\u0000\u0000\u0000\u0005"+
+    "\u00f2\u0001\u0000\u0000\u0000\u0005\u00f4\u0001\u0000\u0000\u0000\u0005"+
+    "\u00f6\u0001\u0000\u0000\u0000\u0005\u00f8\u0001\u0000\u0000\u0000\u0005"+
+    "\u00fa\u0001\u0000\u0000\u0000\u0005\u00fc\u0001\u0000\u0000\u0000\u0005"+
+    "\u00fe\u0001\u0000\u0000\u0000\u0005\u0100\u0001\u0000\u0000\u0000\u0006"+
+    "\u0102\u0001\u0000\u0000\u0000\u0006\u0104\u0001\u0000\u0000\u0000\u0006"+
+    "\u0106\u0001\u0000\u0000\u0000\u0006\u0108\u0001\u0000\u0000\u0000\u0006"+
+    "\u010c\u0001\u0000\u0000\u0000\u0006\u010e\u0001\u0000\u0000\u0000\u0006"+
+    "\u0110\u0001\u0000\u0000\u0000\u0006\u0112\u0001\u0000\u0000\u0000\u0006"+
+    "\u0114\u0001\u0000\u0000\u0000\u0007\u0116\u0001\u0000\u0000\u0000\u0007"+
+    "\u0118\u0001\u0000\u0000\u0000\u0007\u011a\u0001\u0000\u0000\u0000\u0007"+
+    "\u011c\u0001\u0000\u0000\u0000\u0007\u011e\u0001\u0000\u0000\u0000\u0007"+
+    "\u0120\u0001\u0000\u0000\u0000\u0007\u0122\u0001\u0000\u0000\u0000\u0007"+
+    "\u0124\u0001\u0000\u0000\u0000\u0007\u0126\u0001\u0000\u0000\u0000\u0007"+
+    "\u0128\u0001\u0000\u0000\u0000\b\u012a\u0001\u0000\u0000\u0000\b\u012c"+
+    "\u0001\u0000\u0000\u0000\b\u012e\u0001\u0000\u0000\u0000\b\u0130\u0001"+
+    "\u0000\u0000\u0000\b\u0132\u0001\u0000\u0000\u0000\b\u0134\u0001\u0000"+
+    "\u0000\u0000\b\u0136\u0001\u0000\u0000\u0000\t\u0138\u0001\u0000\u0000"+
+    "\u0000\t\u013a\u0001\u0000\u0000\u0000\t\u013c\u0001\u0000\u0000\u0000"+
+    "\t\u013e\u0001\u0000\u0000\u0000\t\u0140\u0001\u0000\u0000\u0000\n\u0142"+
+    "\u0001\u0000\u0000\u0000\n\u0144\u0001\u0000\u0000\u0000\n\u0146\u0001"+
+    "\u0000\u0000\u0000\n\u0148\u0001\u0000\u0000\u0000\n\u014a\u0001\u0000"+
+    "\u0000\u0000\u000b\u014c\u0001\u0000\u0000\u0000\u000b\u014e\u0001\u0000"+
+    "\u0000\u0000\u000b\u0150\u0001\u0000\u0000\u0000\u000b\u0152\u0001\u0000"+
+    "\u0000\u0000\u000b\u0154\u0001\u0000\u0000\u0000\u000b\u0156\u0001\u0000"+
+    "\u0000\u0000\f\u0158\u0001\u0000\u0000\u0000\f\u015a\u0001\u0000\u0000"+
+    "\u0000\f\u015c\u0001\u0000\u0000\u0000\f\u015e\u0001\u0000\u0000\u0000"+
+    "\f\u0160\u0001\u0000\u0000\u0000\f\u0162\u0001\u0000\u0000\u0000\f\u0164"+
+    "\u0001\u0000\u0000\u0000\f\u0166\u0001\u0000\u0000\u0000\f\u0168\u0001"+
+    "\u0000\u0000\u0000\f\u016a\u0001\u0000\u0000\u0000\r\u016c\u0001\u0000"+
+    "\u0000\u0000\r\u016e\u0001\u0000\u0000\u0000\r\u0170\u0001\u0000\u0000"+
+    "\u0000\r\u0172\u0001\u0000\u0000\u0000\r\u0174\u0001\u0000\u0000\u0000"+
+    "\r\u0176\u0001\u0000\u0000\u0000\r\u0178\u0001\u0000\u0000\u0000\u000e"+
+    "\u017a\u0001\u0000\u0000\u0000\u000e\u017c\u0001\u0000\u0000\u0000\u000e"+
+    "\u017e\u0001\u0000\u0000\u0000\u000e\u0180\u0001\u0000\u0000\u0000\u000e"+
+    "\u0182\u0001\u0000\u0000\u0000\u000e\u0184\u0001\u0000\u0000\u0000\u000f"+
+    "\u0186\u0001\u0000\u0000\u0000\u000f\u0188\u0001\u0000\u0000\u0000\u000f"+
+    "\u018a\u0001\u0000\u0000\u0000\u000f\u018c\u0001\u0000\u0000\u0000\u000f"+
+    "\u018e\u0001\u0000\u0000\u0000\u000f\u0190\u0001\u0000\u0000\u0000\u000f"+
+    "\u0192\u0001\u0000\u0000\u0000\u000f\u0194\u0001\u0000\u0000\u0000\u000f"+
+    "\u0196\u0001\u0000\u0000\u0000\u0010\u0198\u0001\u0000\u0000\u0000\u0012"+
+    "\u01a2\u0001\u0000\u0000\u0000\u0014\u01a9\u0001\u0000\u0000\u0000\u0016"+
+    "\u01b2\u0001\u0000\u0000\u0000\u0018\u01b9\u0001\u0000\u0000\u0000\u001a"+
+    "\u01c3\u0001\u0000\u0000\u0000\u001c\u01ca\u0001\u0000\u0000\u0000\u001e"+
+    "\u01d1\u0001\u0000\u0000\u0000 \u01d8\u0001\u0000\u0000\u0000\"\u01e0"+
+    "\u0001\u0000\u0000\u0000$\u01e7\u0001\u0000\u0000\u0000&\u01f3\u0001\u0000"+
+    "\u0000\u0000(\u01fc\u0001\u0000\u0000\u0000*\u0202\u0001\u0000\u0000\u0000"+
+    ",\u0209\u0001\u0000\u0000\u0000.\u0210\u0001\u0000\u0000\u00000\u0218"+
+    "\u0001\u0000\u0000\u00002\u0220\u0001\u0000\u0000\u00004\u022f\u0001\u0000"+
+    "\u0000\u00006\u0239\u0001\u0000\u0000\u00008\u0242\u0001\u0000\u0000\u0000"+
+    ":\u024e\u0001\u0000\u0000\u0000<\u0254\u0001\u0000\u0000\u0000>\u0265"+
+    "\u0001\u0000\u0000\u0000@\u0275\u0001\u0000\u0000\u0000B\u027b\u0001\u0000"+
+    "\u0000\u0000D\u027f\u0001\u0000\u0000\u0000F\u0281\u0001\u0000\u0000\u0000"+
+    "H\u0283\u0001\u0000\u0000\u0000J\u0286\u0001\u0000\u0000\u0000L\u0288"+
+    "\u0001\u0000\u0000\u0000N\u0291\u0001\u0000\u0000\u0000P\u0293\u0001\u0000"+
+    "\u0000\u0000R\u0298\u0001\u0000\u0000\u0000T\u029a\u0001\u0000\u0000\u0000"+
+    "V\u029f\u0001\u0000\u0000\u0000X\u02be\u0001\u0000\u0000\u0000Z\u02c1"+
+    "\u0001\u0000\u0000\u0000\\\u02ef\u0001\u0000\u0000\u0000^\u02f1\u0001"+
+    "\u0000\u0000\u0000`\u02f4\u0001\u0000\u0000\u0000b\u02f8\u0001\u0000\u0000"+
+    "\u0000d\u02fc\u0001\u0000\u0000\u0000f\u02fe\u0001\u0000\u0000\u0000h"+
+    "\u0301\u0001\u0000\u0000\u0000j\u0303\u0001\u0000\u0000\u0000l\u0308\u0001"+
+    "\u0000\u0000\u0000n\u030a\u0001\u0000\u0000\u0000p\u0310\u0001\u0000\u0000"+
+    "\u0000r\u0316\u0001\u0000\u0000\u0000t\u0319\u0001\u0000\u0000\u0000v"+
+    "\u031c\u0001\u0000\u0000\u0000x\u0321\u0001\u0000\u0000\u0000z\u0326\u0001"+
+    "\u0000\u0000\u0000|\u0328\u0001\u0000\u0000\u0000~\u032c\u0001\u0000\u0000"+
+    "\u0000\u0080\u0331\u0001\u0000\u0000\u0000\u0082\u0337\u0001\u0000\u0000"+
+    "\u0000\u0084\u033a\u0001\u0000\u0000\u0000\u0086\u033c\u0001\u0000\u0000"+
+    "\u0000\u0088\u0342\u0001\u0000\u0000\u0000\u008a\u0344\u0001\u0000\u0000"+
+    "\u0000\u008c\u0349\u0001\u0000\u0000\u0000\u008e\u034c\u0001\u0000\u0000"+
+    "\u0000\u0090\u034f\u0001\u0000\u0000\u0000\u0092\u0352\u0001\u0000\u0000"+
+    "\u0000\u0094\u0354\u0001\u0000\u0000\u0000\u0096\u0357\u0001\u0000\u0000"+
+    "\u0000\u0098\u0359\u0001\u0000\u0000\u0000\u009a\u035c\u0001\u0000\u0000"+
+    "\u0000\u009c\u035e\u0001\u0000\u0000\u0000\u009e\u0360\u0001\u0000\u0000"+
+    "\u0000\u00a0\u0362\u0001\u0000\u0000\u0000\u00a2\u0364\u0001\u0000\u0000"+
+    "\u0000\u00a4\u0366\u0001\u0000\u0000\u0000\u00a6\u037c\u0001\u0000\u0000"+
+    "\u0000\u00a8\u037e\u0001\u0000\u0000\u0000\u00aa\u0383\u0001\u0000\u0000"+
+    "\u0000\u00ac\u0398\u0001\u0000\u0000\u0000\u00ae\u039a\u0001\u0000\u0000"+
+    "\u0000\u00b0\u03a2\u0001\u0000\u0000\u0000\u00b2\u03a4\u0001\u0000\u0000"+
+    "\u0000\u00b4\u03a8\u0001\u0000\u0000\u0000\u00b6\u03ac\u0001\u0000\u0000"+
+    "\u0000\u00b8\u03b0\u0001\u0000\u0000\u0000\u00ba\u03b5\u0001\u0000\u0000"+
+    "\u0000\u00bc\u03ba\u0001\u0000\u0000\u0000\u00be\u03be\u0001\u0000\u0000"+
+    "\u0000\u00c0\u03c2\u0001\u0000\u0000\u0000\u00c2\u03c6\u0001\u0000\u0000"+
+    "\u0000\u00c4\u03cb\u0001\u0000\u0000\u0000\u00c6\u03cf\u0001\u0000\u0000"+
+    "\u0000\u00c8\u03d3\u0001\u0000\u0000\u0000\u00ca\u03d7\u0001\u0000\u0000"+
+    "\u0000\u00cc\u03db\u0001\u0000\u0000\u0000\u00ce\u03df\u0001\u0000\u0000"+
+    "\u0000\u00d0\u03eb\u0001\u0000\u0000\u0000\u00d2\u03ee\u0001\u0000\u0000"+
+    "\u0000\u00d4\u03f2\u0001\u0000\u0000\u0000\u00d6\u03f6\u0001\u0000\u0000"+
+    "\u0000\u00d8\u03fa\u0001\u0000\u0000\u0000\u00da\u03fe\u0001\u0000\u0000"+
+    "\u0000\u00dc\u0402\u0001\u0000\u0000\u0000\u00de\u0406\u0001\u0000\u0000"+
+    "\u0000\u00e0\u040b\u0001\u0000\u0000\u0000\u00e2\u040f\u0001\u0000\u0000"+
+    "\u0000\u00e4\u0417\u0001\u0000\u0000\u0000\u00e6\u042c\u0001\u0000\u0000"+
+    "\u0000\u00e8\u0430\u0001\u0000\u0000\u0000\u00ea\u0434\u0001\u0000\u0000"+
+    "\u0000\u00ec\u0438\u0001\u0000\u0000\u0000\u00ee\u043c\u0001\u0000\u0000"+
+    "\u0000\u00f0\u0440\u0001\u0000\u0000\u0000\u00f2\u0445\u0001\u0000\u0000"+
+    "\u0000\u00f4\u0449\u0001\u0000\u0000\u0000\u00f6\u044d\u0001\u0000\u0000"+
+    "\u0000\u00f8\u0451\u0001\u0000\u0000\u0000\u00fa\u0454\u0001\u0000\u0000"+
+    "\u0000\u00fc\u0458\u0001\u0000\u0000\u0000\u00fe\u045c\u0001\u0000\u0000"+
+    "\u0000\u0100\u0460\u0001\u0000\u0000\u0000\u0102\u0464\u0001\u0000\u0000"+
+    "\u0000\u0104\u0469\u0001\u0000\u0000\u0000\u0106\u046e\u0001\u0000\u0000"+
+    "\u0000\u0108\u0473\u0001\u0000\u0000\u0000\u010a\u047a\u0001\u0000\u0000"+
+    "\u0000\u010c\u0483\u0001\u0000\u0000\u0000\u010e\u048a\u0001\u0000\u0000"+
+    "\u0000\u0110\u048e\u0001\u0000\u0000\u0000\u0112\u0492\u0001\u0000\u0000"+
+    "\u0000\u0114\u0496\u0001\u0000\u0000\u0000\u0116\u049a\u0001\u0000\u0000"+
+    "\u0000\u0118\u04a0\u0001\u0000\u0000\u0000\u011a\u04a4\u0001\u0000\u0000"+
+    "\u0000\u011c\u04a8\u0001\u0000\u0000\u0000\u011e\u04ac\u0001\u0000\u0000"+
+    "\u0000\u0120\u04b0\u0001\u0000\u0000\u0000\u0122\u04b4\u0001\u0000\u0000"+
+    "\u0000\u0124\u04b8\u0001\u0000\u0000\u0000\u0126\u04bc\u0001\u0000\u0000"+
+    "\u0000\u0128\u04c0\u0001\u0000\u0000\u0000\u012a\u04c4\u0001\u0000\u0000"+
+    "\u0000\u012c\u04c9\u0001\u0000\u0000\u0000\u012e\u04cd\u0001\u0000\u0000"+
+    "\u0000\u0130\u04d1\u0001\u0000\u0000\u0000\u0132\u04d5\u0001\u0000\u0000"+
+    "\u0000\u0134\u04d9\u0001\u0000\u0000\u0000\u0136\u04dd\u0001\u0000\u0000"+
+    "\u0000\u0138\u04e1\u0001\u0000\u0000\u0000\u013a\u04e6\u0001\u0000\u0000"+
+    "\u0000\u013c\u04eb\u0001\u0000\u0000\u0000\u013e\u04ef\u0001\u0000\u0000"+
+    "\u0000\u0140\u04f3\u0001\u0000\u0000\u0000\u0142\u04f7\u0001\u0000\u0000"+
+    "\u0000\u0144\u04fc\u0001\u0000\u0000\u0000\u0146\u0506\u0001\u0000\u0000"+
+    "\u0000\u0148\u050a\u0001\u0000\u0000\u0000\u014a\u050e\u0001\u0000\u0000"+
+    "\u0000\u014c\u0512\u0001\u0000\u0000\u0000\u014e\u0517\u0001\u0000\u0000"+
+    "\u0000\u0150\u051e\u0001\u0000\u0000\u0000\u0152\u0522\u0001\u0000\u0000"+
+    "\u0000\u0154\u0526\u0001\u0000\u0000\u0000\u0156\u052a\u0001\u0000\u0000"+
+    "\u0000\u0158\u052e\u0001\u0000\u0000\u0000\u015a\u0533\u0001\u0000\u0000"+
+    "\u0000\u015c\u0537\u0001\u0000\u0000\u0000\u015e\u053b\u0001\u0000\u0000"+
+    "\u0000\u0160\u053f\u0001\u0000\u0000\u0000\u0162\u0544\u0001\u0000\u0000"+
+    "\u0000\u0164\u0548\u0001\u0000\u0000\u0000\u0166\u054c\u0001\u0000\u0000"+
+    "\u0000\u0168\u0550\u0001\u0000\u0000\u0000\u016a\u0554\u0001\u0000\u0000"+
+    "\u0000\u016c\u0558\u0001\u0000\u0000\u0000\u016e\u055e\u0001\u0000\u0000"+
+    "\u0000\u0170\u0562\u0001\u0000\u0000\u0000\u0172\u0566\u0001\u0000\u0000"+
+    "\u0000\u0174\u056a\u0001\u0000\u0000\u0000\u0176\u056e\u0001\u0000\u0000"+
+    "\u0000\u0178\u0572\u0001\u0000\u0000\u0000\u017a\u0576\u0001\u0000\u0000"+
+    "\u0000\u017c\u057b\u0001\u0000\u0000\u0000\u017e\u0581\u0001\u0000\u0000"+
+    "\u0000\u0180\u0587\u0001\u0000\u0000\u0000\u0182\u058b\u0001\u0000\u0000"+
+    "\u0000\u0184\u058f\u0001\u0000\u0000\u0000\u0186\u0593\u0001\u0000\u0000"+
+    "\u0000\u0188\u0599\u0001\u0000\u0000\u0000\u018a\u059f\u0001\u0000\u0000"+
+    "\u0000\u018c\u05a3\u0001\u0000\u0000\u0000\u018e\u05a7\u0001\u0000\u0000"+
+    "\u0000\u0190\u05ab\u0001\u0000\u0000\u0000\u0192\u05b1\u0001\u0000\u0000"+
+    "\u0000\u0194\u05b7\u0001\u0000\u0000\u0000\u0196\u05bd\u0001\u0000\u0000"+
+    "\u0000\u0198\u0199\u0007\u0000\u0000\u0000\u0199\u019a\u0007\u0001\u0000"+
+    "\u0000\u019a\u019b\u0007\u0002\u0000\u0000\u019b\u019c\u0007\u0002\u0000"+
+    "\u0000\u019c\u019d\u0007\u0003\u0000\u0000\u019d\u019e\u0007\u0004\u0000"+
+    "\u0000\u019e\u019f\u0007\u0005\u0000\u0000\u019f\u01a0\u0001\u0000\u0000"+
+    "\u0000\u01a0\u01a1\u0006\u0000\u0000\u0000\u01a1\u0011\u0001\u0000\u0000"+
+    "\u0000\u01a2\u01a3\u0007\u0000\u0000\u0000\u01a3\u01a4\u0007\u0006\u0000"+
+    "\u0000\u01a4\u01a5\u0007\u0007\u0000\u0000\u01a5\u01a6\u0007\b\u0000\u0000"+
+    "\u01a6\u01a7\u0001\u0000\u0000\u0000\u01a7\u01a8\u0006\u0001\u0001\u0000"+
+    "\u01a8\u0013\u0001\u0000\u0000\u0000\u01a9\u01aa\u0007\u0003\u0000\u0000"+
+    "\u01aa\u01ab\u0007\t\u0000\u0000\u01ab\u01ac\u0007\u0006\u0000\u0000\u01ac"+
+    "\u01ad\u0007\u0001\u0000\u0000\u01ad\u01ae\u0007\u0004\u0000\u0000\u01ae"+
+    "\u01af\u0007\n\u0000\u0000\u01af\u01b0\u0001\u0000\u0000\u0000\u01b0\u01b1"+
+    "\u0006\u0002\u0002\u0000\u01b1\u0015\u0001\u0000\u0000\u0000\u01b2\u01b3"+
+    "\u0007\u0003\u0000\u0000\u01b3\u01b4\u0007\u000b\u0000\u0000\u01b4\u01b5"+
+    "\u0007\f\u0000\u0000\u01b5\u01b6\u0007\r\u0000\u0000\u01b6\u01b7\u0001"+
+    "\u0000\u0000\u0000\u01b7\u01b8\u0006\u0003\u0000\u0000\u01b8\u0017\u0001"+
+    "\u0000\u0000\u0000\u01b9\u01ba\u0007\u0003\u0000\u0000\u01ba\u01bb\u0007"+
+    "\u000e\u0000\u0000\u01bb\u01bc\u0007\b\u0000\u0000\u01bc\u01bd\u0007\r"+
+    "\u0000\u0000\u01bd\u01be\u0007\f\u0000\u0000\u01be\u01bf\u0007\u0001\u0000"+
+    "\u0000\u01bf\u01c0\u0007\t\u0000\u0000\u01c0\u01c1\u0001\u0000\u0000\u0000"+
+    "\u01c1\u01c2\u0006\u0004\u0003\u0000\u01c2\u0019\u0001\u0000\u0000\u0000"+
+    "\u01c3\u01c4\u0007\u000f\u0000\u0000\u01c4\u01c5\u0007\u0006\u0000\u0000"+
+    "\u01c5\u01c6\u0007\u0007\u0000\u0000\u01c6\u01c7\u0007\u0010\u0000\u0000"+
+    "\u01c7\u01c8\u0001\u0000\u0000\u0000\u01c8\u01c9\u0006\u0005\u0004\u0000"+
+    "\u01c9\u001b\u0001\u0000\u0000\u0000\u01ca\u01cb\u0007\u0011\u0000\u0000"+
+    "\u01cb\u01cc\u0007\u0006\u0000\u0000\u01cc\u01cd\u0007\u0007\u0000\u0000"+
+    "\u01cd\u01ce\u0007\u0012\u0000\u0000\u01ce\u01cf\u0001\u0000\u0000\u0000"+
+    "\u01cf\u01d0\u0006\u0006\u0000\u0000\u01d0\u001d\u0001\u0000\u0000\u0000"+
+    "\u01d1\u01d2\u0007\u0012\u0000\u0000\u01d2\u01d3\u0007\u0003\u0000\u0000"+
+    "\u01d3\u01d4\u0007\u0003\u0000\u0000\u01d4\u01d5\u0007\b\u0000\u0000\u01d5"+
+    "\u01d6\u0001\u0000\u0000\u0000\u01d6\u01d7\u0006\u0007\u0001\u0000\u01d7"+
+    "\u001f\u0001\u0000\u0000\u0000\u01d8\u01d9\u0007\r\u0000\u0000\u01d9\u01da"+
+    "\u0007\u0001\u0000\u0000\u01da\u01db\u0007\u0010\u0000\u0000\u01db\u01dc"+
+    "\u0007\u0001\u0000\u0000\u01dc\u01dd\u0007\u0005\u0000\u0000\u01dd\u01de"+
+    "\u0001\u0000\u0000\u0000\u01de\u01df\u0006\b\u0000\u0000\u01df!\u0001"+
+    "\u0000\u0000\u0000\u01e0\u01e1\u0007\u0010\u0000\u0000\u01e1\u01e2\u0007"+
+    "\u0003\u0000\u0000\u01e2\u01e3\u0007\u0005\u0000\u0000\u01e3\u01e4\u0007"+
+    "\f\u0000\u0000\u01e4\u01e5\u0001\u0000\u0000\u0000\u01e5\u01e6\u0006\t"+
+    "\u0005\u0000\u01e6#\u0001\u0000\u0000\u0000\u01e7\u01e8\u0007\u0010\u0000"+
+    "\u0000\u01e8\u01e9\u0007\u000b\u0000\u0000\u01e9\u01ea\u0005_\u0000\u0000"+
+    "\u01ea\u01eb\u0007\u0003\u0000\u0000\u01eb\u01ec\u0007\u000e\u0000\u0000"+
+    "\u01ec\u01ed\u0007\b\u0000\u0000\u01ed\u01ee\u0007\f\u0000\u0000\u01ee"+
+    "\u01ef\u0007\t\u0000\u0000\u01ef\u01f0\u0007\u0000\u0000\u0000\u01f0\u01f1"+
+    "\u0001\u0000\u0000\u0000\u01f1\u01f2\u0006\n\u0006\u0000\u01f2%\u0001"+
+    "\u0000\u0000\u0000\u01f3\u01f4\u0007\u0006\u0000\u0000\u01f4\u01f5\u0007"+
+    "\u0003\u0000\u0000\u01f5\u01f6\u0007\t\u0000\u0000\u01f6\u01f7\u0007\f"+
+    "\u0000\u0000\u01f7\u01f8\u0007\u0010\u0000\u0000\u01f8\u01f9\u0007\u0003"+
+    "\u0000\u0000\u01f9\u01fa\u0001\u0000\u0000\u0000\u01fa\u01fb\u0006\u000b"+
+    "\u0007\u0000\u01fb\'\u0001\u0000\u0000\u0000\u01fc\u01fd\u0007\u0006\u0000"+
+    "\u0000\u01fd\u01fe\u0007\u0007\u0000\u0000\u01fe\u01ff\u0007\u0013\u0000"+
+    "\u0000\u01ff\u0200\u0001\u0000\u0000\u0000\u0200\u0201\u0006\f\u0000\u0000"+
+    "\u0201)\u0001\u0000\u0000\u0000\u0202\u0203\u0007\u0002\u0000\u0000\u0203"+
+    "\u0204\u0007\n\u0000\u0000\u0204\u0205\u0007\u0007\u0000\u0000\u0205\u0206"+
+    "\u0007\u0013\u0000\u0000\u0206\u0207\u0001\u0000\u0000\u0000\u0207\u0208"+
+    "\u0006\r\b\u0000\u0208+\u0001\u0000\u0000\u0000\u0209\u020a\u0007\u0002"+
+    "\u0000\u0000\u020a\u020b\u0007\u0007\u0000\u0000\u020b\u020c\u0007\u0006"+
+    "\u0000\u0000\u020c\u020d\u0007\u0005\u0000\u0000\u020d\u020e\u0001\u0000"+
+    "\u0000\u0000\u020e\u020f\u0006\u000e\u0000\u0000\u020f-\u0001\u0000\u0000"+
+    "\u0000\u0210\u0211\u0007\u0002\u0000\u0000\u0211\u0212\u0007\u0005\u0000"+
+    "\u0000\u0212\u0213\u0007\f\u0000\u0000\u0213\u0214\u0007\u0005\u0000\u0000"+
+    "\u0214\u0215\u0007\u0002\u0000\u0000\u0215\u0216\u0001\u0000\u0000\u0000"+
+    "\u0216\u0217\u0006\u000f\u0000\u0000\u0217/\u0001\u0000\u0000\u0000\u0218"+
+    "\u0219\u0007\u0013\u0000\u0000\u0219\u021a\u0007\n\u0000\u0000\u021a\u021b"+
+    "\u0007\u0003\u0000\u0000\u021b\u021c\u0007\u0006\u0000\u0000\u021c\u021d"+
+    "\u0007\u0003\u0000\u0000\u021d\u021e\u0001\u0000\u0000\u0000\u021e\u021f"+
+    "\u0006\u0010\u0000\u0000\u021f1\u0001\u0000\u0000\u0000\u0220\u0221\u0004"+
+    "\u0011\u0000\u0000\u0221\u0222\u0007\u0001\u0000\u0000\u0222\u0223\u0007"+
+    "\t\u0000\u0000\u0223\u0224\u0007\r\u0000\u0000\u0224\u0225\u0007\u0001"+
+    "\u0000\u0000\u0225\u0226\u0007\t\u0000\u0000\u0226\u0227\u0007\u0003\u0000"+
+    "\u0000\u0227\u0228\u0007\u0002\u0000\u0000\u0228\u0229\u0007\u0005\u0000"+
+    "\u0000\u0229\u022a\u0007\f\u0000\u0000\u022a\u022b\u0007\u0005\u0000\u0000"+
+    "\u022b\u022c\u0007\u0002\u0000\u0000\u022c\u022d\u0001\u0000\u0000\u0000"+
+    "\u022d\u022e\u0006\u0011\u0000\u0000\u022e3\u0001\u0000\u0000\u0000\u022f"+
+    "\u0230\u0004\u0012\u0001\u0000\u0230\u0231\u0007\r\u0000\u0000\u0231\u0232"+
+    "\u0007\u0007\u0000\u0000\u0232\u0233\u0007\u0007\u0000\u0000\u0233\u0234"+
+    "\u0007\u0012\u0000\u0000\u0234\u0235\u0007\u0014\u0000\u0000\u0235\u0236"+
+    "\u0007\b\u0000\u0000\u0236\u0237\u0001\u0000\u0000\u0000\u0237\u0238\u0006"+
+    "\u0012\t\u0000\u02385\u0001\u0000\u0000\u0000\u0239\u023a\u0004\u0013"+
+    "\u0002\u0000\u023a\u023b\u0007\u0010\u0000\u0000\u023b\u023c\u0007\f\u0000"+
+    "\u0000\u023c\u023d\u0007\u0005\u0000\u0000\u023d\u023e\u0007\u0004\u0000"+
+    "\u0000\u023e\u023f\u0007\n\u0000\u0000\u023f\u0240\u0001\u0000\u0000\u0000"+
+    "\u0240\u0241\u0006\u0013\u0000\u0000\u02417\u0001\u0000\u0000\u0000\u0242"+
+    "\u0243\u0004\u0014\u0003\u0000\u0243\u0244\u0007\u0010\u0000\u0000\u0244"+
+    "\u0245\u0007\u0003\u0000\u0000\u0245\u0246\u0007\u0005\u0000\u0000\u0246"+
+    "\u0247\u0007\u0006\u0000\u0000\u0247\u0248\u0007\u0001\u0000\u0000\u0248"+
+    "\u0249\u0007\u0004\u0000\u0000\u0249\u024a\u0007\u0002\u0000\u0000\u024a"+
+    "\u024b\u0001\u0000\u0000\u0000\u024b\u024c\u0006\u0014\n\u0000\u024c9"+
+    "\u0001\u0000\u0000\u0000\u024d\u024f\b\u0015\u0000\u0000\u024e\u024d\u0001"+
+    "\u0000\u0000\u0000\u024f\u0250\u0001\u0000\u0000\u0000\u0250\u024e\u0001"+
+    "\u0000\u0000\u0000\u0250\u0251\u0001\u0000\u0000\u0000\u0251\u0252\u0001"+
+    "\u0000\u0000\u0000\u0252\u0253\u0006\u0015\u0000\u0000\u0253;\u0001\u0000"+
+    "\u0000\u0000\u0254\u0255\u0005/\u0000\u0000\u0255\u0256\u0005/\u0000\u0000"+
+    "\u0256\u025a\u0001\u0000\u0000\u0000\u0257\u0259\b\u0016\u0000\u0000\u0258"+
+    "\u0257\u0001\u0000\u0000\u0000\u0259\u025c\u0001\u0000\u0000\u0000\u025a"+
+    "\u0258\u0001\u0000\u0000\u0000\u025a\u025b\u0001\u0000\u0000\u0000\u025b"+
+    "\u025e\u0001\u0000\u0000\u0000\u025c\u025a\u0001\u0000\u0000\u0000\u025d"+
+    "\u025f\u0005\r\u0000\u0000\u025e\u025d\u0001\u0000\u0000\u0000\u025e\u025f"+
+    "\u0001\u0000\u0000\u0000\u025f\u0261\u0001\u0000\u0000\u0000\u0260\u0262"+
+    "\u0005\n\u0000\u0000\u0261\u0260\u0001\u0000\u0000\u0000\u0261\u0262\u0001"+
+    "\u0000\u0000\u0000\u0262\u0263\u0001\u0000\u0000\u0000\u0263\u0264\u0006"+
+    "\u0016\u000b\u0000\u0264=\u0001\u0000\u0000\u0000\u0265\u0266\u0005/\u0000"+
+    "\u0000\u0266\u0267\u0005*\u0000\u0000\u0267\u026c\u0001\u0000\u0000\u0000"+
+    "\u0268\u026b\u0003>\u0017\u0000\u0269\u026b\t\u0000\u0000\u0000\u026a"+
+    "\u0268\u0001\u0000\u0000\u0000\u026a\u0269\u0001\u0000\u0000\u0000\u026b"+
+    "\u026e\u0001\u0000\u0000\u0000\u026c\u026d\u0001\u0000\u0000\u0000\u026c"+
+    "\u026a\u0001\u0000\u0000\u0000\u026d\u026f\u0001\u0000\u0000\u0000\u026e"+
+    "\u026c\u0001\u0000\u0000\u0000\u026f\u0270\u0005*\u0000\u0000\u0270\u0271"+
+    "\u0005/\u0000\u0000\u0271\u0272\u0001\u0000\u0000\u0000\u0272\u0273\u0006"+
+    "\u0017\u000b\u0000\u0273?\u0001\u0000\u0000\u0000\u0274\u0276\u0007\u0017"+
+    "\u0000\u0000\u0275\u0274\u0001\u0000\u0000\u0000\u0276\u0277\u0001\u0000"+
+    "\u0000\u0000\u0277\u0275\u0001\u0000\u0000\u0000\u0277\u0278\u0001\u0000"+
+    "\u0000\u0000\u0278\u0279\u0001\u0000\u0000\u0000\u0279\u027a\u0006\u0018"+
+    "\u000b\u0000\u027aA\u0001\u0000\u0000\u0000\u027b\u027c\u0005|\u0000\u0000"+
+    "\u027c\u027d\u0001\u0000\u0000\u0000\u027d\u027e\u0006\u0019\f\u0000\u027e"+
+    "C\u0001\u0000\u0000\u0000\u027f\u0280\u0007\u0018\u0000\u0000\u0280E\u0001"+
+    "\u0000\u0000\u0000\u0281\u0282\u0007\u0019\u0000\u0000\u0282G\u0001\u0000"+
+    "\u0000\u0000\u0283\u0284\u0005\\\u0000\u0000\u0284\u0285\u0007\u001a\u0000"+
+    "\u0000\u0285I\u0001\u0000\u0000\u0000\u0286\u0287\b\u001b\u0000\u0000"+
+    "\u0287K\u0001\u0000\u0000\u0000\u0288\u028a\u0007\u0003\u0000\u0000\u0289"+
+    "\u028b\u0007\u001c\u0000\u0000\u028a\u0289\u0001\u0000\u0000\u0000\u028a"+
+    "\u028b\u0001\u0000\u0000\u0000\u028b\u028d\u0001\u0000\u0000\u0000\u028c"+
+    "\u028e\u0003D\u001a\u0000\u028d\u028c\u0001\u0000\u0000\u0000\u028e\u028f"+
+    "\u0001\u0000\u0000\u0000\u028f\u028d\u0001\u0000\u0000\u0000\u028f\u0290"+
+    "\u0001\u0000\u0000\u0000\u0290M\u0001\u0000\u0000\u0000\u0291\u0292\u0005"+
+    "@\u0000\u0000\u0292O\u0001\u0000\u0000\u0000\u0293\u0294\u0005`\u0000"+
+    "\u0000\u0294Q\u0001\u0000\u0000\u0000\u0295\u0299\b\u001d\u0000\u0000"+
+    "\u0296\u0297\u0005`\u0000\u0000\u0297\u0299\u0005`\u0000\u0000\u0298\u0295"+
+    "\u0001\u0000\u0000\u0000\u0298\u0296\u0001\u0000\u0000\u0000\u0299S\u0001"+
+    "\u0000\u0000\u0000\u029a\u029b\u0005_\u0000\u0000\u029bU\u0001\u0000\u0000"+
+    "\u0000\u029c\u02a0\u0003F\u001b\u0000\u029d\u02a0\u0003D\u001a\u0000\u029e"+
+    "\u02a0\u0003T\"\u0000\u029f\u029c\u0001\u0000\u0000\u0000\u029f\u029d"+
+    "\u0001\u0000\u0000\u0000\u029f\u029e\u0001\u0000\u0000\u0000\u02a0W\u0001"+
+    "\u0000\u0000\u0000\u02a1\u02a6\u0005\"\u0000\u0000\u02a2\u02a5\u0003H"+
+    "\u001c\u0000\u02a3\u02a5\u0003J\u001d\u0000\u02a4\u02a2\u0001\u0000\u0000"+
+    "\u0000\u02a4\u02a3\u0001\u0000\u0000\u0000\u02a5\u02a8\u0001\u0000\u0000"+
+    "\u0000\u02a6\u02a4\u0001\u0000\u0000\u0000\u02a6\u02a7\u0001\u0000\u0000"+
+    "\u0000\u02a7\u02a9\u0001\u0000\u0000\u0000\u02a8\u02a6\u0001\u0000\u0000"+
+    "\u0000\u02a9\u02bf\u0005\"\u0000\u0000\u02aa\u02ab\u0005\"\u0000\u0000"+
+    "\u02ab\u02ac\u0005\"\u0000\u0000\u02ac\u02ad\u0005\"\u0000\u0000\u02ad"+
+    "\u02b1\u0001\u0000\u0000\u0000\u02ae\u02b0\b\u0016\u0000\u0000\u02af\u02ae"+
+    "\u0001\u0000\u0000\u0000\u02b0\u02b3\u0001\u0000\u0000\u0000\u02b1\u02b2"+
+    "\u0001\u0000\u0000\u0000\u02b1\u02af\u0001\u0000\u0000\u0000\u02b2\u02b4"+
+    "\u0001\u0000\u0000\u0000\u02b3\u02b1\u0001\u0000\u0000\u0000\u02b4\u02b5"+
+    "\u0005\"\u0000\u0000\u02b5\u02b6\u0005\"\u0000\u0000\u02b6\u02b7\u0005"+
+    "\"\u0000\u0000\u02b7\u02b9\u0001\u0000\u0000\u0000\u02b8\u02ba\u0005\""+
+    "\u0000\u0000\u02b9\u02b8\u0001\u0000\u0000\u0000\u02b9\u02ba\u0001\u0000"+
+    "\u0000\u0000\u02ba\u02bc\u0001\u0000\u0000\u0000\u02bb\u02bd\u0005\"\u0000"+
+    "\u0000\u02bc\u02bb\u0001\u0000\u0000\u0000\u02bc\u02bd\u0001\u0000\u0000"+
+    "\u0000\u02bd\u02bf\u0001\u0000\u0000\u0000\u02be\u02a1\u0001\u0000\u0000"+
+    "\u0000\u02be\u02aa\u0001\u0000\u0000\u0000\u02bfY\u0001\u0000\u0000\u0000"+
+    "\u02c0\u02c2\u0003D\u001a\u0000\u02c1\u02c0\u0001\u0000\u0000\u0000\u02c2"+
+    "\u02c3\u0001\u0000\u0000\u0000\u02c3\u02c1\u0001\u0000\u0000\u0000\u02c3"+
+    "\u02c4\u0001\u0000\u0000\u0000\u02c4[\u0001\u0000\u0000\u0000\u02c5\u02c7"+
+    "\u0003D\u001a\u0000\u02c6\u02c5\u0001\u0000\u0000\u0000\u02c7\u02c8\u0001"+
+    "\u0000\u0000\u0000\u02c8\u02c6\u0001\u0000\u0000\u0000\u02c8\u02c9\u0001"+
+    "\u0000\u0000\u0000\u02c9\u02ca\u0001\u0000\u0000\u0000\u02ca\u02ce\u0003"+
+    "l.\u0000\u02cb\u02cd\u0003D\u001a\u0000\u02cc\u02cb\u0001\u0000\u0000"+
+    "\u0000\u02cd\u02d0\u0001\u0000\u0000\u0000\u02ce\u02cc\u0001\u0000\u0000"+
+    "\u0000\u02ce\u02cf\u0001\u0000\u0000\u0000\u02cf\u02f0\u0001\u0000\u0000"+
+    "\u0000\u02d0\u02ce\u0001\u0000\u0000\u0000\u02d1\u02d3\u0003l.\u0000\u02d2"+
+    "\u02d4\u0003D\u001a\u0000\u02d3\u02d2\u0001\u0000\u0000\u0000\u02d4\u02d5"+
+    "\u0001\u0000\u0000\u0000\u02d5\u02d3\u0001\u0000\u0000\u0000\u02d5\u02d6"+
+    "\u0001\u0000\u0000\u0000\u02d6\u02f0\u0001\u0000\u0000\u0000\u02d7\u02d9"+
+    "\u0003D\u001a\u0000\u02d8\u02d7\u0001\u0000\u0000\u0000\u02d9\u02da\u0001"+
+    "\u0000\u0000\u0000\u02da\u02d8\u0001\u0000\u0000\u0000\u02da\u02db\u0001"+
+    "\u0000\u0000\u0000\u02db\u02e3\u0001\u0000\u0000\u0000\u02dc\u02e0\u0003"+
+    "l.\u0000\u02dd\u02df\u0003D\u001a\u0000\u02de\u02dd\u0001\u0000\u0000"+
+    "\u0000\u02df\u02e2\u0001\u0000\u0000\u0000\u02e0\u02de\u0001\u0000\u0000"+
+    "\u0000\u02e0\u02e1\u0001\u0000\u0000\u0000\u02e1\u02e4\u0001\u0000\u0000"+
+    "\u0000\u02e2\u02e0\u0001\u0000\u0000\u0000\u02e3\u02dc\u0001\u0000\u0000"+
+    "\u0000\u02e3\u02e4\u0001\u0000\u0000\u0000\u02e4\u02e5\u0001\u0000\u0000"+
+    "\u0000\u02e5\u02e6\u0003L\u001e\u0000\u02e6\u02f0\u0001\u0000\u0000\u0000"+
+    "\u02e7\u02e9\u0003l.\u0000\u02e8\u02ea\u0003D\u001a\u0000\u02e9\u02e8"+
+    "\u0001\u0000\u0000\u0000\u02ea\u02eb\u0001\u0000\u0000\u0000\u02eb\u02e9"+
+    "\u0001\u0000\u0000\u0000\u02eb\u02ec\u0001\u0000\u0000\u0000\u02ec\u02ed"+
+    "\u0001\u0000\u0000\u0000\u02ed\u02ee\u0003L\u001e\u0000\u02ee\u02f0\u0001"+
+    "\u0000\u0000\u0000\u02ef\u02c6\u0001\u0000\u0000\u0000\u02ef\u02d1\u0001"+
+    "\u0000\u0000\u0000\u02ef\u02d8\u0001\u0000\u0000\u0000\u02ef\u02e7\u0001"+
+    "\u0000\u0000\u0000\u02f0]\u0001\u0000\u0000\u0000\u02f1\u02f2\u0007\u001e"+
+    "\u0000\u0000\u02f2\u02f3\u0007\u001f\u0000\u0000\u02f3_\u0001\u0000\u0000"+
+    "\u0000\u02f4\u02f5\u0007\f\u0000\u0000\u02f5\u02f6\u0007\t\u0000\u0000"+
+    "\u02f6\u02f7\u0007\u0000\u0000\u0000\u02f7a\u0001\u0000\u0000\u0000\u02f8"+
+    "\u02f9\u0007\f\u0000\u0000\u02f9\u02fa\u0007\u0002\u0000\u0000\u02fa\u02fb"+
+    "\u0007\u0004\u0000\u0000\u02fbc\u0001\u0000\u0000\u0000\u02fc\u02fd\u0005"+
+    "=\u0000\u0000\u02fde\u0001\u0000\u0000\u0000\u02fe\u02ff\u0005:\u0000"+
+    "\u0000\u02ff\u0300\u0005:\u0000\u0000\u0300g\u0001\u0000\u0000\u0000\u0301"+
+    "\u0302\u0005,\u0000\u0000\u0302i\u0001\u0000\u0000\u0000\u0303\u0304\u0007"+
+    "\u0000\u0000\u0000\u0304\u0305\u0007\u0003\u0000\u0000\u0305\u0306\u0007"+
+    "\u0002\u0000\u0000\u0306\u0307\u0007\u0004\u0000\u0000\u0307k\u0001\u0000"+
+    "\u0000\u0000\u0308\u0309\u0005.\u0000\u0000\u0309m\u0001\u0000\u0000\u0000"+
+    "\u030a\u030b\u0007\u000f\u0000\u0000\u030b\u030c\u0007\f\u0000\u0000\u030c"+
+    "\u030d\u0007\r\u0000\u0000\u030d\u030e\u0007\u0002\u0000\u0000\u030e\u030f"+
+    "\u0007\u0003\u0000\u0000\u030fo\u0001\u0000\u0000\u0000\u0310\u0311\u0007"+
+    "\u000f\u0000\u0000\u0311\u0312\u0007\u0001\u0000\u0000\u0312\u0313\u0007"+
+    "\u0006\u0000\u0000\u0313\u0314\u0007\u0002\u0000\u0000\u0314\u0315\u0007"+
+    "\u0005\u0000\u0000\u0315q\u0001\u0000\u0000\u0000\u0316\u0317\u0007\u0001"+
+    "\u0000\u0000\u0317\u0318\u0007\t\u0000\u0000\u0318s\u0001\u0000\u0000"+
+    "\u0000\u0319\u031a\u0007\u0001\u0000\u0000\u031a\u031b\u0007\u0002\u0000"+
+    "\u0000\u031bu\u0001\u0000\u0000\u0000\u031c\u031d\u0007\r\u0000\u0000"+
+    "\u031d\u031e\u0007\f\u0000\u0000\u031e\u031f\u0007\u0002\u0000\u0000\u031f"+
+    "\u0320\u0007\u0005\u0000\u0000\u0320w\u0001\u0000\u0000\u0000\u0321\u0322"+
+    "\u0007\r\u0000\u0000\u0322\u0323\u0007\u0001\u0000\u0000\u0323\u0324\u0007"+
+    "\u0012\u0000\u0000\u0324\u0325\u0007\u0003\u0000\u0000\u0325y\u0001\u0000"+
+    "\u0000\u0000\u0326\u0327\u0005(\u0000\u0000\u0327{\u0001\u0000\u0000\u0000"+
+    "\u0328\u0329\u0007\t\u0000\u0000\u0329\u032a\u0007\u0007\u0000\u0000\u032a"+
+    "\u032b\u0007\u0005\u0000\u0000\u032b}\u0001\u0000\u0000\u0000\u032c\u032d"+
+    "\u0007\t\u0000\u0000\u032d\u032e\u0007\u0014\u0000\u0000\u032e\u032f\u0007"+
+    "\r\u0000\u0000\u032f\u0330\u0007\r\u0000\u0000\u0330\u007f\u0001\u0000"+
+    "\u0000\u0000\u0331\u0332\u0007\t\u0000\u0000\u0332\u0333\u0007\u0014\u0000"+
+    "\u0000\u0333\u0334\u0007\r\u0000\u0000\u0334\u0335\u0007\r\u0000\u0000"+
+    "\u0335\u0336\u0007\u0002\u0000\u0000\u0336\u0081\u0001\u0000\u0000\u0000"+
+    "\u0337\u0338\u0007\u0007\u0000\u0000\u0338\u0339\u0007\u0006\u0000\u0000"+
+    "\u0339\u0083\u0001\u0000\u0000\u0000\u033a\u033b\u0005?\u0000\u0000\u033b"+
+    "\u0085\u0001\u0000\u0000\u0000\u033c\u033d\u0007\u0006\u0000\u0000\u033d"+
+    "\u033e\u0007\r\u0000\u0000\u033e\u033f\u0007\u0001\u0000\u0000\u033f\u0340"+
+    "\u0007\u0012\u0000\u0000\u0340\u0341\u0007\u0003\u0000\u0000\u0341\u0087"+
+    "\u0001\u0000\u0000\u0000\u0342\u0343\u0005)\u0000\u0000\u0343\u0089\u0001"+
+    "\u0000\u0000\u0000\u0344\u0345\u0007\u0005\u0000\u0000\u0345\u0346\u0007"+
+    "\u0006\u0000\u0000\u0346\u0347\u0007\u0014\u0000\u0000\u0347\u0348\u0007"+
+    "\u0003\u0000\u0000\u0348\u008b\u0001\u0000\u0000\u0000\u0349\u034a\u0005"+
+    "=\u0000\u0000\u034a\u034b\u0005=\u0000\u0000\u034b\u008d\u0001\u0000\u0000"+
+    "\u0000\u034c\u034d\u0005=\u0000\u0000\u034d\u034e\u0005~\u0000\u0000\u034e"+
+    "\u008f\u0001\u0000\u0000\u0000\u034f\u0350\u0005!\u0000\u0000\u0350\u0351"+
+    "\u0005=\u0000\u0000\u0351\u0091\u0001\u0000\u0000\u0000\u0352\u0353\u0005"+
+    "<\u0000\u0000\u0353\u0093\u0001\u0000\u0000\u0000\u0354\u0355\u0005<\u0000"+
+    "\u0000\u0355\u0356\u0005=\u0000\u0000\u0356\u0095\u0001\u0000\u0000\u0000"+
+    "\u0357\u0358\u0005>\u0000\u0000\u0358\u0097\u0001\u0000\u0000\u0000\u0359"+
+    "\u035a\u0005>\u0000\u0000\u035a\u035b\u0005=\u0000\u0000\u035b\u0099\u0001"+
+    "\u0000\u0000\u0000\u035c\u035d\u0005+\u0000\u0000\u035d\u009b\u0001\u0000"+
+    "\u0000\u0000\u035e\u035f\u0005-\u0000\u0000\u035f\u009d\u0001\u0000\u0000"+
+    "\u0000\u0360\u0361\u0005*\u0000\u0000\u0361\u009f\u0001\u0000\u0000\u0000"+
+    "\u0362\u0363\u0005/\u0000\u0000\u0363\u00a1\u0001\u0000\u0000\u0000\u0364"+
+    "\u0365\u0005%\u0000\u0000\u0365\u00a3\u0001\u0000\u0000\u0000\u0366\u0367"+
+    "\u0004J\u0004\u0000\u0367\u0368\u00036\u0013\u0000\u0368\u0369\u0001\u0000"+
+    "\u0000\u0000\u0369\u036a\u0006J\r\u0000\u036a\u00a5\u0001\u0000\u0000"+
+    "\u0000\u036b\u036e\u0003\u0084:\u0000\u036c\u036f\u0003F\u001b\u0000\u036d"+
+    "\u036f\u0003T\"\u0000\u036e\u036c\u0001\u0000\u0000\u0000\u036e\u036d"+
+    "\u0001\u0000\u0000\u0000\u036f\u0373\u0001\u0000\u0000\u0000\u0370\u0372"+
+    "\u0003V#\u0000\u0371\u0370\u0001\u0000\u0000\u0000\u0372\u0375\u0001\u0000"+
+    "\u0000\u0000\u0373\u0371\u0001\u0000\u0000\u0000\u0373\u0374\u0001\u0000"+
+    "\u0000\u0000\u0374\u037d\u0001\u0000\u0000\u0000\u0375\u0373\u0001\u0000"+
+    "\u0000\u0000\u0376\u0378\u0003\u0084:\u0000\u0377\u0379\u0003D\u001a\u0000"+
+    "\u0378\u0377\u0001\u0000\u0000\u0000\u0379\u037a\u0001\u0000\u0000\u0000"+
+    "\u037a\u0378\u0001\u0000\u0000\u0000\u037a\u037b\u0001\u0000\u0000\u0000"+
+    "\u037b\u037d\u0001\u0000\u0000\u0000\u037c\u036b\u0001\u0000\u0000\u0000"+
+    "\u037c\u0376\u0001\u0000\u0000\u0000\u037d\u00a7\u0001\u0000\u0000\u0000"+
+    "\u037e\u037f\u0005[\u0000\u0000\u037f\u0380\u0001\u0000\u0000\u0000\u0380"+
+    "\u0381\u0006L\u0000\u0000\u0381\u0382\u0006L\u0000\u0000\u0382\u00a9\u0001"+
+    "\u0000\u0000\u0000\u0383\u0384\u0005]\u0000\u0000\u0384\u0385\u0001\u0000"+
+    "\u0000\u0000\u0385\u0386\u0006M\f\u0000\u0386\u0387\u0006M\f\u0000\u0387"+
+    "\u00ab\u0001\u0000\u0000\u0000\u0388\u038c\u0003F\u001b\u0000\u0389\u038b"+
+    "\u0003V#\u0000\u038a\u0389\u0001\u0000\u0000\u0000\u038b\u038e\u0001\u0000"+
+    "\u0000\u0000\u038c\u038a\u0001\u0000\u0000\u0000\u038c\u038d\u0001\u0000"+
+    "\u0000\u0000\u038d\u0399\u0001\u0000\u0000\u0000\u038e\u038c\u0001\u0000"+
+    "\u0000\u0000\u038f\u0392\u0003T\"\u0000\u0390\u0392\u0003N\u001f\u0000"+
+    "\u0391\u038f\u0001\u0000\u0000\u0000\u0391\u0390\u0001\u0000\u0000\u0000"+
+    "\u0392\u0394\u0001\u0000\u0000\u0000\u0393\u0395\u0003V#\u0000\u0394\u0393"+
+    "\u0001\u0000\u0000\u0000\u0395\u0396\u0001\u0000\u0000\u0000\u0396\u0394"+
+    "\u0001\u0000\u0000\u0000\u0396\u0397\u0001\u0000\u0000\u0000\u0397\u0399"+
+    "\u0001\u0000\u0000\u0000\u0398\u0388\u0001\u0000\u0000\u0000\u0398\u0391"+
+    "\u0001\u0000\u0000\u0000\u0399\u00ad\u0001\u0000\u0000\u0000\u039a\u039c"+
+    "\u0003P \u0000\u039b\u039d\u0003R!\u0000\u039c\u039b\u0001\u0000\u0000"+
+    "\u0000\u039d\u039e\u0001\u0000\u0000\u0000\u039e\u039c\u0001\u0000\u0000"+
+    "\u0000\u039e\u039f\u0001\u0000\u0000\u0000\u039f\u03a0\u0001\u0000\u0000"+
+    "\u0000\u03a0\u03a1\u0003P \u0000\u03a1\u00af\u0001\u0000\u0000\u0000\u03a2"+
+    "\u03a3\u0003\u00aeO\u0000\u03a3\u00b1\u0001\u0000\u0000\u0000\u03a4\u03a5"+
+    "\u0003<\u0016\u0000\u03a5\u03a6\u0001\u0000\u0000\u0000\u03a6\u03a7\u0006"+
+    "Q\u000b\u0000\u03a7\u00b3\u0001\u0000\u0000\u0000\u03a8\u03a9\u0003>\u0017"+
+    "\u0000\u03a9\u03aa\u0001\u0000\u0000\u0000\u03aa\u03ab\u0006R\u000b\u0000"+
+    "\u03ab\u00b5\u0001\u0000\u0000\u0000\u03ac\u03ad\u0003@\u0018\u0000\u03ad"+
+    "\u03ae\u0001\u0000\u0000\u0000\u03ae\u03af\u0006S\u000b\u0000\u03af\u00b7"+
+    "\u0001\u0000\u0000\u0000\u03b0\u03b1\u0003\u00a8L\u0000\u03b1\u03b2\u0001"+
+    "\u0000\u0000\u0000\u03b2\u03b3\u0006T\u000e\u0000\u03b3\u03b4\u0006T\u000f"+
+    "\u0000\u03b4\u00b9\u0001\u0000\u0000\u0000\u03b5\u03b6\u0003B\u0019\u0000"+
+    "\u03b6\u03b7\u0001\u0000\u0000\u0000\u03b7\u03b8\u0006U\u0010\u0000\u03b8"+
+    "\u03b9\u0006U\f\u0000\u03b9\u00bb\u0001\u0000\u0000\u0000\u03ba\u03bb"+
+    "\u0003@\u0018\u0000\u03bb\u03bc\u0001\u0000\u0000\u0000\u03bc\u03bd\u0006"+
+    "V\u000b\u0000\u03bd\u00bd\u0001\u0000\u0000\u0000\u03be\u03bf\u0003<\u0016"+
+    "\u0000\u03bf\u03c0\u0001\u0000\u0000\u0000\u03c0\u03c1\u0006W\u000b\u0000"+
+    "\u03c1\u00bf\u0001\u0000\u0000\u0000\u03c2\u03c3\u0003>\u0017\u0000\u03c3"+
+    "\u03c4\u0001\u0000\u0000\u0000\u03c4\u03c5\u0006X\u000b\u0000\u03c5\u00c1"+
+    "\u0001\u0000\u0000\u0000\u03c6\u03c7\u0003B\u0019\u0000\u03c7\u03c8\u0001"+
+    "\u0000\u0000\u0000\u03c8\u03c9\u0006Y\u0010\u0000\u03c9\u03ca\u0006Y\f"+
+    "\u0000\u03ca\u00c3\u0001\u0000\u0000\u0000\u03cb\u03cc\u0003\u00a8L\u0000"+
+    "\u03cc\u03cd\u0001\u0000\u0000\u0000\u03cd\u03ce\u0006Z\u000e\u0000\u03ce"+
+    "\u00c5\u0001\u0000\u0000\u0000\u03cf\u03d0\u0003\u00aaM\u0000\u03d0\u03d1"+
+    "\u0001\u0000\u0000\u0000\u03d1\u03d2\u0006[\u0011\u0000\u03d2\u00c7\u0001"+
+    "\u0000\u0000\u0000\u03d3\u03d4\u0003\u014e\u009f\u0000\u03d4\u03d5\u0001"+
+    "\u0000\u0000\u0000\u03d5\u03d6\u0006\\\u0012\u0000\u03d6\u00c9\u0001\u0000"+
+    "\u0000\u0000\u03d7\u03d8\u0003h,\u0000\u03d8\u03d9\u0001\u0000\u0000\u0000"+
+    "\u03d9\u03da\u0006]\u0013\u0000\u03da\u00cb\u0001\u0000\u0000\u0000\u03db"+
+    "\u03dc\u0003d*\u0000\u03dc\u03dd\u0001\u0000\u0000\u0000\u03dd\u03de\u0006"+
+    "^\u0014\u0000\u03de\u00cd\u0001\u0000\u0000\u0000\u03df\u03e0\u0007\u0010"+
+    "\u0000\u0000\u03e0\u03e1\u0007\u0003\u0000\u0000\u03e1\u03e2\u0007\u0005"+
+    "\u0000\u0000\u03e2\u03e3\u0007\f\u0000\u0000\u03e3\u03e4\u0007\u0000\u0000"+
+    "\u0000\u03e4\u03e5\u0007\f\u0000\u0000\u03e5\u03e6\u0007\u0005\u0000\u0000"+
+    "\u03e6\u03e7\u0007\f\u0000\u0000\u03e7\u00cf\u0001\u0000\u0000\u0000\u03e8"+
+    "\u03ec\b \u0000\u0000\u03e9\u03ea\u0005/\u0000\u0000\u03ea\u03ec\b!\u0000"+
+    "\u0000\u03eb\u03e8\u0001\u0000\u0000\u0000\u03eb\u03e9\u0001\u0000\u0000"+
+    "\u0000\u03ec\u00d1\u0001\u0000\u0000\u0000\u03ed\u03ef\u0003\u00d0`\u0000"+
+    "\u03ee\u03ed\u0001\u0000\u0000\u0000\u03ef\u03f0\u0001\u0000\u0000\u0000"+
+    "\u03f0\u03ee\u0001\u0000\u0000\u0000\u03f0\u03f1\u0001\u0000\u0000\u0000"+
+    "\u03f1\u00d3\u0001\u0000\u0000\u0000\u03f2\u03f3\u0003\u00d2a\u0000\u03f3"+
+    "\u03f4\u0001\u0000\u0000\u0000\u03f4\u03f5\u0006b\u0015\u0000\u03f5\u00d5"+
+    "\u0001\u0000\u0000\u0000\u03f6\u03f7\u0003X$\u0000\u03f7\u03f8\u0001\u0000"+
+    "\u0000\u0000\u03f8\u03f9\u0006c\u0016\u0000\u03f9\u00d7\u0001\u0000\u0000"+
+    "\u0000\u03fa\u03fb\u0003<\u0016\u0000\u03fb\u03fc\u0001\u0000\u0000\u0000"+
+    "\u03fc\u03fd\u0006d\u000b\u0000\u03fd\u00d9\u0001\u0000\u0000\u0000\u03fe"+
+    "\u03ff\u0003>\u0017\u0000\u03ff\u0400\u0001\u0000\u0000\u0000\u0400\u0401"+
+    "\u0006e\u000b\u0000\u0401\u00db\u0001\u0000\u0000\u0000\u0402\u0403\u0003"+
+    "@\u0018\u0000\u0403\u0404\u0001\u0000\u0000\u0000\u0404\u0405\u0006f\u000b"+
+    "\u0000\u0405\u00dd\u0001\u0000\u0000\u0000\u0406\u0407\u0003B\u0019\u0000"+
+    "\u0407\u0408\u0001\u0000\u0000\u0000\u0408\u0409\u0006g\u0010\u0000\u0409"+
+    "\u040a\u0006g\f\u0000\u040a\u00df\u0001\u0000\u0000\u0000\u040b\u040c"+
+    "\u0003l.\u0000\u040c\u040d\u0001\u0000\u0000\u0000\u040d\u040e\u0006h"+
+    "\u0017\u0000\u040e\u00e1\u0001\u0000\u0000\u0000\u040f\u0410\u0003h,\u0000"+
+    "\u0410\u0411\u0001\u0000\u0000\u0000\u0411\u0412\u0006i\u0013\u0000\u0412"+
+    "\u00e3\u0001\u0000\u0000\u0000\u0413\u0418\u0003F\u001b\u0000\u0414\u0418"+
+    "\u0003D\u001a\u0000\u0415\u0418\u0003T\"\u0000\u0416\u0418\u0003\u009e"+
+    "G\u0000\u0417\u0413\u0001\u0000\u0000\u0000\u0417\u0414\u0001\u0000\u0000"+
+    "\u0000\u0417\u0415\u0001\u0000\u0000\u0000\u0417\u0416\u0001\u0000\u0000"+
+    "\u0000\u0418\u00e5\u0001\u0000\u0000\u0000\u0419\u041c\u0003F\u001b\u0000"+
+    "\u041a\u041c\u0003\u009eG\u0000\u041b\u0419\u0001\u0000\u0000\u0000\u041b"+
+    "\u041a\u0001\u0000\u0000\u0000\u041c\u0420\u0001\u0000\u0000\u0000\u041d"+
+    "\u041f\u0003\u00e4j\u0000\u041e\u041d\u0001\u0000\u0000\u0000\u041f\u0422"+
+    "\u0001\u0000\u0000\u0000\u0420\u041e\u0001\u0000\u0000\u0000\u0420\u0421"+
+    "\u0001\u0000\u0000\u0000\u0421\u042d\u0001\u0000\u0000\u0000\u0422\u0420"+
+    "\u0001\u0000\u0000\u0000\u0423\u0426\u0003T\"\u0000\u0424\u0426\u0003"+
+    "N\u001f\u0000\u0425\u0423\u0001\u0000\u0000\u0000\u0425\u0424\u0001\u0000"+
+    "\u0000\u0000\u0426\u0428\u0001\u0000\u0000\u0000\u0427\u0429\u0003\u00e4"+
+    "j\u0000\u0428\u0427\u0001\u0000\u0000\u0000\u0429\u042a\u0001\u0000\u0000"+
+    "\u0000\u042a\u0428\u0001\u0000\u0000\u0000\u042a\u042b\u0001\u0000\u0000"+
+    "\u0000\u042b\u042d\u0001\u0000\u0000\u0000\u042c\u041b\u0001\u0000\u0000"+
+    "\u0000\u042c\u0425\u0001\u0000\u0000\u0000\u042d\u00e7\u0001\u0000\u0000"+
+    "\u0000\u042e\u0431\u0003\u00e6k\u0000\u042f\u0431\u0003\u00aeO\u0000\u0430"+
+    "\u042e\u0001\u0000\u0000\u0000\u0430\u042f\u0001\u0000\u0000\u0000\u0431"+
+    "\u0432\u0001\u0000\u0000\u0000\u0432\u0430\u0001\u0000\u0000\u0000\u0432"+
+    "\u0433\u0001\u0000\u0000\u0000\u0433\u00e9\u0001\u0000\u0000\u0000\u0434"+
+    "\u0435\u0003<\u0016\u0000\u0435\u0436\u0001\u0000\u0000\u0000\u0436\u0437"+
+    "\u0006m\u000b\u0000\u0437\u00eb\u0001\u0000\u0000\u0000\u0438\u0439\u0003"+
+    ">\u0017\u0000\u0439\u043a\u0001\u0000\u0000\u0000\u043a\u043b\u0006n\u000b"+
+    "\u0000\u043b\u00ed\u0001\u0000\u0000\u0000\u043c\u043d\u0003@\u0018\u0000"+
+    "\u043d\u043e\u0001\u0000\u0000\u0000\u043e\u043f\u0006o\u000b\u0000\u043f"+
+    "\u00ef\u0001\u0000\u0000\u0000\u0440\u0441\u0003B\u0019\u0000\u0441\u0442"+
+    "\u0001\u0000\u0000\u0000\u0442\u0443\u0006p\u0010\u0000\u0443\u0444\u0006"+
+    "p\f\u0000\u0444\u00f1\u0001\u0000\u0000\u0000\u0445\u0446\u0003d*\u0000"+
+    "\u0446\u0447\u0001\u0000\u0000\u0000\u0447\u0448\u0006q\u0014\u0000\u0448"+
+    "\u00f3\u0001\u0000\u0000\u0000\u0449\u044a\u0003h,\u0000\u044a\u044b\u0001"+
+    "\u0000\u0000\u0000\u044b\u044c\u0006r\u0013\u0000\u044c\u00f5\u0001\u0000"+
+    "\u0000\u0000\u044d\u044e\u0003l.\u0000\u044e\u044f\u0001\u0000\u0000\u0000"+
+    "\u044f\u0450\u0006s\u0017\u0000\u0450\u00f7\u0001\u0000\u0000\u0000\u0451"+
+    "\u0452\u0007\f\u0000\u0000\u0452\u0453\u0007\u0002\u0000\u0000\u0453\u00f9"+
+    "\u0001\u0000\u0000\u0000\u0454\u0455\u0003\u00e8l\u0000\u0455\u0456\u0001"+
+    "\u0000\u0000\u0000\u0456\u0457\u0006u\u0018\u0000\u0457\u00fb\u0001\u0000"+
+    "\u0000\u0000\u0458\u0459\u0003<\u0016\u0000\u0459\u045a\u0001\u0000\u0000"+
+    "\u0000\u045a\u045b\u0006v\u000b\u0000\u045b\u00fd\u0001\u0000\u0000\u0000"+
+    "\u045c\u045d\u0003>\u0017\u0000\u045d\u045e\u0001\u0000\u0000\u0000\u045e"+
+    "\u045f\u0006w\u000b\u0000\u045f\u00ff\u0001\u0000\u0000\u0000\u0460\u0461"+
+    "\u0003@\u0018\u0000\u0461\u0462\u0001\u0000\u0000\u0000\u0462\u0463\u0006"+
+    "x\u000b\u0000\u0463\u0101\u0001\u0000\u0000\u0000\u0464\u0465\u0003B\u0019"+
+    "\u0000\u0465\u0466\u0001\u0000\u0000\u0000\u0466\u0467\u0006y\u0010\u0000"+
+    "\u0467\u0468\u0006y\f\u0000\u0468\u0103\u0001\u0000\u0000\u0000\u0469"+
+    "\u046a\u0003\u00a8L\u0000\u046a\u046b\u0001\u0000\u0000\u0000\u046b\u046c"+
+    "\u0006z\u000e\u0000\u046c\u046d\u0006z\u0019\u0000\u046d\u0105\u0001\u0000"+
+    "\u0000\u0000\u046e\u046f\u0007\u0007\u0000\u0000\u046f\u0470\u0007\t\u0000"+
+    "\u0000\u0470\u0471\u0001\u0000\u0000\u0000\u0471\u0472\u0006{\u001a\u0000"+
+    "\u0472\u0107\u0001\u0000\u0000\u0000\u0473\u0474\u0007\u0013\u0000\u0000"+
+    "\u0474\u0475\u0007\u0001\u0000\u0000\u0475\u0476\u0007\u0005\u0000\u0000"+
+    "\u0476\u0477\u0007\n\u0000\u0000\u0477\u0478\u0001\u0000\u0000\u0000\u0478"+
+    "\u0479\u0006|\u001a\u0000\u0479\u0109\u0001\u0000\u0000\u0000\u047a\u047b"+
+    "\b\"\u0000\u0000\u047b\u010b\u0001\u0000\u0000\u0000\u047c\u047e\u0003"+
+    "\u010a}\u0000\u047d\u047c\u0001\u0000\u0000\u0000\u047e\u047f\u0001\u0000"+
+    "\u0000\u0000\u047f\u047d\u0001\u0000\u0000\u0000\u047f\u0480\u0001\u0000"+
+    "\u0000\u0000\u0480\u0481\u0001\u0000\u0000\u0000\u0481\u0482\u0003\u014e"+
+    "\u009f\u0000\u0482\u0484\u0001\u0000\u0000\u0000\u0483\u047d\u0001\u0000"+
+    "\u0000\u0000\u0483\u0484\u0001\u0000\u0000\u0000\u0484\u0486\u0001\u0000"+
+    "\u0000\u0000\u0485\u0487\u0003\u010a}\u0000\u0486\u0485\u0001\u0000\u0000"+
+    "\u0000\u0487\u0488\u0001\u0000\u0000\u0000\u0488\u0486\u0001\u0000\u0000"+
+    "\u0000\u0488\u0489\u0001\u0000\u0000\u0000\u0489\u010d\u0001\u0000\u0000"+
+    "\u0000\u048a\u048b\u0003\u010c~\u0000\u048b\u048c\u0001\u0000\u0000\u0000"+
+    "\u048c\u048d\u0006\u007f\u001b\u0000\u048d\u010f\u0001\u0000\u0000\u0000"+
+    "\u048e\u048f\u0003<\u0016\u0000\u048f\u0490\u0001\u0000\u0000\u0000\u0490"+
+    "\u0491\u0006\u0080\u000b\u0000\u0491\u0111\u0001\u0000\u0000\u0000\u0492"+
+    "\u0493\u0003>\u0017\u0000\u0493\u0494\u0001\u0000\u0000\u0000\u0494\u0495"+
+    "\u0006\u0081\u000b\u0000\u0495\u0113\u0001\u0000\u0000\u0000\u0496\u0497"+
+    "\u0003@\u0018\u0000\u0497\u0498\u0001\u0000\u0000\u0000\u0498\u0499\u0006"+
+    "\u0082\u000b\u0000\u0499\u0115\u0001\u0000\u0000\u0000\u049a\u049b\u0003"+
+    "B\u0019\u0000\u049b\u049c\u0001\u0000\u0000\u0000\u049c\u049d\u0006\u0083"+
+    "\u0010\u0000\u049d\u049e\u0006\u0083\f\u0000\u049e\u049f\u0006\u0083\f"+
+    "\u0000\u049f\u0117\u0001\u0000\u0000\u0000\u04a0\u04a1\u0003d*\u0000\u04a1"+
+    "\u04a2\u0001\u0000\u0000\u0000\u04a2\u04a3\u0006\u0084\u0014\u0000\u04a3"+
+    "\u0119\u0001\u0000\u0000\u0000\u04a4\u04a5\u0003h,\u0000\u04a5\u04a6\u0001"+
+    "\u0000\u0000\u0000\u04a6\u04a7\u0006\u0085\u0013\u0000\u04a7\u011b\u0001"+
+    "\u0000\u0000\u0000\u04a8\u04a9\u0003l.\u0000\u04a9\u04aa\u0001\u0000\u0000"+
+    "\u0000\u04aa\u04ab\u0006\u0086\u0017\u0000\u04ab\u011d\u0001\u0000\u0000"+
+    "\u0000\u04ac\u04ad\u0003\u0108|\u0000\u04ad\u04ae\u0001\u0000\u0000\u0000"+
+    "\u04ae\u04af\u0006\u0087\u001c\u0000\u04af\u011f\u0001\u0000\u0000\u0000"+
+    "\u04b0\u04b1\u0003\u00e8l\u0000\u04b1\u04b2\u0001\u0000\u0000\u0000\u04b2"+
+    "\u04b3\u0006\u0088\u0018\u0000\u04b3\u0121\u0001\u0000\u0000\u0000\u04b4"+
+    "\u04b5\u0003\u00b0P\u0000\u04b5\u04b6\u0001\u0000\u0000\u0000\u04b6\u04b7"+
+    "\u0006\u0089\u001d\u0000\u04b7\u0123\u0001\u0000\u0000\u0000\u04b8\u04b9"+
+    "\u0003<\u0016\u0000\u04b9\u04ba\u0001\u0000\u0000\u0000\u04ba\u04bb\u0006"+
+    "\u008a\u000b\u0000\u04bb\u0125\u0001\u0000\u0000\u0000\u04bc\u04bd\u0003"+
+    ">\u0017\u0000\u04bd\u04be\u0001\u0000\u0000\u0000\u04be\u04bf\u0006\u008b"+
+    "\u000b\u0000\u04bf\u0127\u0001\u0000\u0000\u0000\u04c0\u04c1\u0003@\u0018"+
+    "\u0000\u04c1\u04c2\u0001\u0000\u0000\u0000\u04c2\u04c3\u0006\u008c\u000b"+
+    "\u0000\u04c3\u0129\u0001\u0000\u0000\u0000\u04c4\u04c5\u0003B\u0019\u0000"+
+    "\u04c5\u04c6\u0001\u0000\u0000\u0000\u04c6\u04c7\u0006\u008d\u0010\u0000"+
+    "\u04c7\u04c8\u0006\u008d\f\u0000\u04c8\u012b\u0001\u0000\u0000\u0000\u04c9"+
+    "\u04ca\u0003l.\u0000\u04ca\u04cb\u0001\u0000\u0000\u0000\u04cb\u04cc\u0006"+
+    "\u008e\u0017\u0000\u04cc\u012d\u0001\u0000\u0000\u0000\u04cd\u04ce\u0003"+
+    "\u00b0P\u0000\u04ce\u04cf\u0001\u0000\u0000\u0000\u04cf\u04d0\u0006\u008f"+
+    "\u001d\u0000\u04d0\u012f\u0001\u0000\u0000\u0000\u04d1\u04d2\u0003\u00ac"+
+    "N\u0000\u04d2\u04d3\u0001\u0000\u0000\u0000\u04d3\u04d4\u0006\u0090\u001e"+
+    "\u0000\u04d4\u0131\u0001\u0000\u0000\u0000\u04d5\u04d6\u0003<\u0016\u0000"+
+    "\u04d6\u04d7\u0001\u0000\u0000\u0000\u04d7\u04d8\u0006\u0091\u000b\u0000"+
+    "\u04d8\u0133\u0001\u0000\u0000\u0000\u04d9\u04da\u0003>\u0017\u0000\u04da"+
+    "\u04db\u0001\u0000\u0000\u0000\u04db\u04dc\u0006\u0092\u000b\u0000\u04dc"+
+    "\u0135\u0001\u0000\u0000\u0000\u04dd\u04de\u0003@\u0018\u0000\u04de\u04df"+
+    "\u0001\u0000\u0000\u0000\u04df\u04e0\u0006\u0093\u000b\u0000\u04e0\u0137"+
+    "\u0001\u0000\u0000\u0000\u04e1\u04e2\u0003B\u0019\u0000\u04e2\u04e3\u0001"+
+    "\u0000\u0000\u0000\u04e3\u04e4\u0006\u0094\u0010\u0000\u04e4\u04e5\u0006"+
+    "\u0094\f\u0000\u04e5\u0139\u0001\u0000\u0000\u0000\u04e6\u04e7\u0007\u0001"+
+    "\u0000\u0000\u04e7\u04e8\u0007\t\u0000\u0000\u04e8\u04e9\u0007\u000f\u0000"+
+    "\u0000\u04e9\u04ea\u0007\u0007\u0000\u0000\u04ea\u013b\u0001\u0000\u0000"+
+    "\u0000\u04eb\u04ec\u0003<\u0016\u0000\u04ec\u04ed\u0001\u0000\u0000\u0000"+
+    "\u04ed\u04ee\u0006\u0096\u000b\u0000\u04ee\u013d\u0001\u0000\u0000\u0000"+
+    "\u04ef\u04f0\u0003>\u0017\u0000\u04f0\u04f1\u0001\u0000\u0000\u0000\u04f1"+
+    "\u04f2\u0006\u0097\u000b\u0000\u04f2\u013f\u0001\u0000\u0000\u0000\u04f3"+
+    "\u04f4\u0003@\u0018\u0000\u04f4\u04f5\u0001\u0000\u0000\u0000\u04f5\u04f6"+
+    "\u0006\u0098\u000b\u0000\u04f6\u0141\u0001\u0000\u0000\u0000\u04f7\u04f8"+
+    "\u0003B\u0019\u0000\u04f8\u04f9\u0001\u0000\u0000\u0000\u04f9\u04fa\u0006"+
+    "\u0099\u0010\u0000\u04fa\u04fb\u0006\u0099\f\u0000\u04fb\u0143\u0001\u0000"+
+    "\u0000\u0000\u04fc\u04fd\u0007\u000f\u0000\u0000\u04fd\u04fe\u0007\u0014"+
+    "\u0000\u0000\u04fe\u04ff\u0007\t\u0000\u0000\u04ff\u0500\u0007\u0004\u0000"+
+    "\u0000\u0500\u0501\u0007\u0005\u0000\u0000\u0501\u0502\u0007\u0001\u0000"+
+    "\u0000\u0502\u0503\u0007\u0007\u0000\u0000\u0503\u0504\u0007\t\u0000\u0000"+
+    "\u0504\u0505\u0007\u0002\u0000\u0000\u0505\u0145\u0001\u0000\u0000\u0000"+
+    "\u0506\u0507\u0003<\u0016\u0000\u0507\u0508\u0001\u0000\u0000\u0000\u0508"+
+    "\u0509\u0006\u009b\u000b\u0000\u0509\u0147\u0001\u0000\u0000\u0000\u050a"+
+    "\u050b\u0003>\u0017\u0000\u050b\u050c\u0001\u0000\u0000\u0000\u050c\u050d"+
+    "\u0006\u009c\u000b\u0000\u050d\u0149\u0001\u0000\u0000\u0000\u050e\u050f"+
+    "\u0003@\u0018\u0000\u050f\u0510\u0001\u0000\u0000\u0000\u0510\u0511\u0006"+
+    "\u009d\u000b\u0000\u0511\u014b\u0001\u0000\u0000\u0000\u0512\u0513\u0003"+
+    "\u00aaM\u0000\u0513\u0514\u0001\u0000\u0000\u0000\u0514\u0515\u0006\u009e"+
+    "\u0011\u0000\u0515\u0516\u0006\u009e\f\u0000\u0516\u014d\u0001\u0000\u0000"+
+    "\u0000\u0517\u0518\u0005:\u0000\u0000\u0518\u014f\u0001\u0000\u0000\u0000"+
+    "\u0519\u051f\u0003N\u001f\u0000\u051a\u051f\u0003D\u001a\u0000\u051b\u051f"+
+    "\u0003l.\u0000\u051c\u051f\u0003F\u001b\u0000\u051d\u051f\u0003T\"\u0000"+
+    "\u051e\u0519\u0001\u0000\u0000\u0000\u051e\u051a\u0001\u0000\u0000\u0000"+
+    "\u051e\u051b\u0001\u0000\u0000\u0000\u051e\u051c\u0001\u0000\u0000\u0000"+
+    "\u051e\u051d\u0001\u0000\u0000\u0000\u051f\u0520\u0001\u0000\u0000\u0000"+
+    "\u0520\u051e\u0001\u0000\u0000\u0000\u0520\u0521\u0001\u0000\u0000\u0000"+
+    "\u0521\u0151\u0001\u0000\u0000\u0000\u0522\u0523\u0003<\u0016\u0000\u0523"+
+    "\u0524\u0001\u0000\u0000\u0000\u0524\u0525\u0006\u00a1\u000b\u0000\u0525"+
+    "\u0153\u0001\u0000\u0000\u0000\u0526\u0527\u0003>\u0017\u0000\u0527\u0528"+
+    "\u0001\u0000\u0000\u0000\u0528\u0529\u0006\u00a2\u000b\u0000\u0529\u0155"+
+    "\u0001\u0000\u0000\u0000\u052a\u052b\u0003@\u0018\u0000\u052b\u052c\u0001"+
+    "\u0000\u0000\u0000\u052c\u052d\u0006\u00a3\u000b\u0000\u052d\u0157\u0001"+
+    "\u0000\u0000\u0000\u052e\u052f\u0003B\u0019\u0000\u052f\u0530\u0001\u0000"+
+    "\u0000\u0000\u0530\u0531\u0006\u00a4\u0010\u0000\u0531\u0532\u0006\u00a4"+
+    "\f\u0000\u0532\u0159\u0001\u0000\u0000\u0000\u0533\u0534\u0003\u014e\u009f"+
+    "\u0000\u0534\u0535\u0001\u0000\u0000\u0000\u0535\u0536\u0006\u00a5\u0012"+
+    "\u0000\u0536\u015b\u0001\u0000\u0000\u0000\u0537\u0538\u0003h,\u0000\u0538"+
+    "\u0539\u0001\u0000\u0000\u0000\u0539\u053a\u0006\u00a6\u0013\u0000\u053a"+
+    "\u015d\u0001\u0000\u0000\u0000\u053b\u053c\u0003l.\u0000\u053c\u053d\u0001"+
+    "\u0000\u0000\u0000\u053d\u053e\u0006\u00a7\u0017\u0000\u053e\u015f\u0001"+
+    "\u0000\u0000\u0000\u053f\u0540\u0003\u0106{\u0000\u0540\u0541\u0001\u0000"+
+    "\u0000\u0000\u0541\u0542\u0006\u00a8\u001f\u0000\u0542\u0543\u0006\u00a8"+
+    " \u0000\u0543\u0161\u0001\u0000\u0000\u0000\u0544\u0545\u0003\u00d2a\u0000"+
+    "\u0545\u0546\u0001\u0000\u0000\u0000\u0546\u0547\u0006\u00a9\u0015\u0000"+
+    "\u0547\u0163\u0001\u0000\u0000\u0000\u0548\u0549\u0003X$\u0000\u0549\u054a"+
+    "\u0001\u0000\u0000\u0000\u054a\u054b\u0006\u00aa\u0016\u0000\u054b\u0165"+
+    "\u0001\u0000\u0000\u0000\u054c\u054d\u0003<\u0016\u0000\u054d\u054e\u0001"+
+    "\u0000\u0000\u0000\u054e\u054f\u0006\u00ab\u000b\u0000\u054f\u0167\u0001"+
+    "\u0000\u0000\u0000\u0550\u0551\u0003>\u0017\u0000\u0551\u0552\u0001\u0000"+
+    "\u0000\u0000\u0552\u0553\u0006\u00ac\u000b\u0000\u0553\u0169\u0001\u0000"+
+    "\u0000\u0000\u0554\u0555\u0003@\u0018\u0000\u0555\u0556\u0001\u0000\u0000"+
+    "\u0000\u0556\u0557\u0006\u00ad\u000b\u0000\u0557\u016b\u0001\u0000\u0000"+
+    "\u0000\u0558\u0559\u0003B\u0019\u0000\u0559\u055a\u0001\u0000\u0000\u0000"+
+    "\u055a\u055b\u0006\u00ae\u0010\u0000\u055b\u055c\u0006\u00ae\f\u0000\u055c"+
+    "\u055d\u0006\u00ae\f\u0000\u055d\u016d\u0001\u0000\u0000\u0000\u055e\u055f"+
+    "\u0003h,\u0000\u055f\u0560\u0001\u0000\u0000\u0000\u0560\u0561\u0006\u00af"+
+    "\u0013\u0000\u0561\u016f\u0001\u0000\u0000\u0000\u0562\u0563\u0003l.\u0000"+
+    "\u0563\u0564\u0001\u0000\u0000\u0000\u0564\u0565\u0006\u00b0\u0017\u0000"+
+    "\u0565\u0171\u0001\u0000\u0000\u0000\u0566\u0567\u0003\u00e8l\u0000\u0567"+
+    "\u0568\u0001\u0000\u0000\u0000\u0568\u0569\u0006\u00b1\u0018\u0000\u0569"+
+    "\u0173\u0001\u0000\u0000\u0000\u056a\u056b\u0003<\u0016\u0000\u056b\u056c"+
+    "\u0001\u0000\u0000\u0000\u056c\u056d\u0006\u00b2\u000b\u0000\u056d\u0175"+
+    "\u0001\u0000\u0000\u0000\u056e\u056f\u0003>\u0017\u0000\u056f\u0570\u0001"+
+    "\u0000\u0000\u0000\u0570\u0571\u0006\u00b3\u000b\u0000\u0571\u0177\u0001"+
+    "\u0000\u0000\u0000\u0572\u0573\u0003@\u0018\u0000\u0573\u0574\u0001\u0000"+
+    "\u0000\u0000\u0574\u0575\u0006\u00b4\u000b\u0000\u0575\u0179\u0001\u0000"+
+    "\u0000\u0000\u0576\u0577\u0003B\u0019\u0000\u0577\u0578\u0001\u0000\u0000"+
+    "\u0000\u0578\u0579\u0006\u00b5\u0010\u0000\u0579\u057a\u0006\u00b5\f\u0000"+
+    "\u057a\u017b\u0001\u0000\u0000\u0000\u057b\u057c\u0003\u00d2a\u0000\u057c"+
+    "\u057d\u0001\u0000\u0000\u0000\u057d\u057e\u0006\u00b6\u0015\u0000\u057e"+
+    "\u057f\u0006\u00b6\f\u0000\u057f\u0580\u0006\u00b6!\u0000\u0580\u017d"+
+    "\u0001\u0000\u0000\u0000\u0581\u0582\u0003X$\u0000\u0582\u0583\u0001\u0000"+
+    "\u0000\u0000\u0583\u0584\u0006\u00b7\u0016\u0000\u0584\u0585\u0006\u00b7"+
+    "\f\u0000\u0585\u0586\u0006\u00b7!\u0000\u0586\u017f\u0001\u0000\u0000"+
+    "\u0000\u0587\u0588\u0003<\u0016\u0000\u0588\u0589\u0001\u0000\u0000\u0000"+
+    "\u0589\u058a\u0006\u00b8\u000b\u0000\u058a\u0181\u0001\u0000\u0000\u0000"+
+    "\u058b\u058c\u0003>\u0017\u0000\u058c\u058d\u0001\u0000\u0000\u0000\u058d"+
+    "\u058e\u0006\u00b9\u000b\u0000\u058e\u0183\u0001\u0000\u0000\u0000\u058f"+
+    "\u0590\u0003@\u0018\u0000\u0590\u0591\u0001\u0000\u0000\u0000\u0591\u0592"+
+    "\u0006\u00ba\u000b\u0000\u0592\u0185\u0001\u0000\u0000\u0000\u0593\u0594"+
+    "\u0003\u014e\u009f\u0000\u0594\u0595\u0001\u0000\u0000\u0000\u0595\u0596"+
+    "\u0006\u00bb\u0012\u0000\u0596\u0597\u0006\u00bb\f\u0000\u0597\u0598\u0006"+
+    "\u00bb\n\u0000\u0598\u0187\u0001\u0000\u0000\u0000\u0599\u059a\u0003h"+
+    ",\u0000\u059a\u059b\u0001\u0000\u0000\u0000\u059b\u059c\u0006\u00bc\u0013"+
+    "\u0000\u059c\u059d\u0006\u00bc\f\u0000\u059d\u059e\u0006\u00bc\n\u0000"+
+    "\u059e\u0189\u0001\u0000\u0000\u0000\u059f\u05a0\u0003<\u0016\u0000\u05a0"+
+    "\u05a1\u0001\u0000\u0000\u0000\u05a1\u05a2\u0006\u00bd\u000b\u0000\u05a2"+
+    "\u018b\u0001\u0000\u0000\u0000\u05a3\u05a4\u0003>\u0017\u0000\u05a4\u05a5"+
+    "\u0001\u0000\u0000\u0000\u05a5\u05a6\u0006\u00be\u000b\u0000\u05a6\u018d"+
+    "\u0001\u0000\u0000\u0000\u05a7\u05a8\u0003@\u0018\u0000\u05a8\u05a9\u0001"+
+    "\u0000\u0000\u0000\u05a9\u05aa\u0006\u00bf\u000b\u0000\u05aa\u018f\u0001"+
+    "\u0000\u0000\u0000\u05ab\u05ac\u0003\u00b0P\u0000\u05ac\u05ad\u0001\u0000"+
+    "\u0000\u0000\u05ad\u05ae\u0006\u00c0\f\u0000\u05ae\u05af\u0006\u00c0\u0000"+
+    "\u0000\u05af\u05b0\u0006\u00c0\u001d\u0000\u05b0\u0191\u0001\u0000\u0000"+
+    "\u0000\u05b1\u05b2\u0003\u00acN\u0000\u05b2\u05b3\u0001\u0000\u0000\u0000"+
+    "\u05b3\u05b4\u0006\u00c1\f\u0000\u05b4\u05b5\u0006\u00c1\u0000\u0000\u05b5"+
+    "\u05b6\u0006\u00c1\u001e\u0000\u05b6\u0193\u0001\u0000\u0000\u0000\u05b7"+
+    "\u05b8\u0003^\'\u0000\u05b8\u05b9\u0001\u0000\u0000\u0000\u05b9\u05ba"+
+    "\u0006\u00c2\f\u0000\u05ba\u05bb\u0006\u00c2\u0000\u0000\u05bb\u05bc\u0006"+
+    "\u00c2\"\u0000\u05bc\u0195\u0001\u0000\u0000\u0000\u05bd\u05be\u0003B"+
+    "\u0019\u0000\u05be\u05bf\u0001\u0000\u0000\u0000\u05bf\u05c0\u0006\u00c3"+
+    "\u0010\u0000\u05c0\u05c1\u0006\u00c3\f\u0000\u05c1\u0197\u0001\u0000\u0000"+
+    "\u0000B\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f"+
+    "\r\u000e\u000f\u0250\u025a\u025e\u0261\u026a\u026c\u0277\u028a\u028f\u0298"+
+    "\u029f\u02a4\u02a6\u02b1\u02b9\u02bc\u02be\u02c3\u02c8\u02ce\u02d5\u02da"+
+    "\u02e0\u02e3\u02eb\u02ef\u036e\u0373\u037a\u037c\u038c\u0391\u0396\u0398"+
+    "\u039e\u03eb\u03f0\u0417\u041b\u0420\u0425\u042a\u042c\u0430\u0432\u047f"+
+    "\u0483\u0488\u051e\u0520#\u0005\u0001\u0000\u0005\u0004\u0000\u0005\u0006"+
+    "\u0000\u0005\u0002\u0000\u0005\u0003\u0000\u0005\n\u0000\u0005\b\u0000"+
+    "\u0005\u0005\u0000\u0005\t\u0000\u0005\f\u0000\u0005\u000e\u0000\u0000"+
+    "\u0001\u0000\u0004\u0000\u0000\u0007\u0014\u0000\u0007B\u0000\u0005\u0000"+
+    "\u0000\u0007\u001a\u0000\u0007C\u0000\u0007m\u0000\u0007#\u0000\u0007"+
+    "!\u0000\u0007M\u0000\u0007\u001b\u0000\u0007%\u0000\u0007Q\u0000\u0005"+
+    "\u000b\u0000\u0005\u0007\u0000\u0007[\u0000\u0007Z\u0000\u0007E\u0000"+
+    "\u0007D\u0000\u0007Y\u0000\u0005\r\u0000\u0005\u000f\u0000\u0007\u001e"+
+    "\u0000";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
index 6c5edef9e98f0..d1d6aae8c3f52 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
@@ -7,12 +7,9 @@ null
 'explain'
 'from'
 'grok'
-'inlinestats'
 'keep'
 'limit'
-'lookup'
 'meta'
-'metrics'
 'mv_expand'
 'rename'
 'row'
@@ -42,11 +39,11 @@ null
 '.'
 'false'
 'first'
-'last'
-'('
 'in'
 'is'
+'last'
 'like'
+'('
 'not'
 'null'
 'nulls'
@@ -75,6 +72,9 @@ null
 null
 null
 null
+null
+null
+null
 'metadata'
 null
 null
@@ -83,6 +83,7 @@ null
 null
 null
 null
+null
 'as'
 null
 null
@@ -99,21 +100,21 @@ null
 null
 null
 null
+'info'
 null
 null
 null
+'functions'
 null
 null
 null
-'info'
+':'
 null
 null
 null
-'functions'
 null
 null
 null
-':'
 null
 null
 null
@@ -134,12 +135,9 @@ EVAL
 EXPLAIN
 FROM
 GROK
-INLINESTATS
 KEEP
 LIMIT
-LOOKUP
 META
-METRICS
 MV_EXPAND
 RENAME
 ROW
@@ -147,14 +145,14 @@ SHOW
 SORT
 STATS
 WHERE
+DEV_INLINESTATS
+DEV_LOOKUP
+DEV_MATCH
+DEV_METRICS
 UNKNOWN_CMD
 LINE_COMMENT
 MULTILINE_COMMENT
 WS
-UNQUOTED_SOURCE
-EXPLAIN_WS
-EXPLAIN_LINE_COMMENT
-EXPLAIN_MULTILINE_COMMENT
 PIPE
 QUOTED_STRING
 INTEGER_LITERAL
@@ -169,11 +167,11 @@ DESC
 DOT
 FALSE
 FIRST
-LAST
-LP
 IN
 IS
+LAST
 LIKE
+LP
 NOT
 NULL
 NULLS
@@ -202,7 +200,11 @@ QUOTED_IDENTIFIER
 EXPR_LINE_COMMENT
 EXPR_MULTILINE_COMMENT
 EXPR_WS
+EXPLAIN_WS
+EXPLAIN_LINE_COMMENT
+EXPLAIN_MULTILINE_COMMENT
 METADATA
+UNQUOTED_SOURCE
 FROM_LINE_COMMENT
 FROM_MULTILINE_COMMENT
 FROM_WS
@@ -223,12 +225,6 @@ ENRICH_WS
 ENRICH_FIELD_LINE_COMMENT
 ENRICH_FIELD_MULTILINE_COMMENT
 ENRICH_FIELD_WS
-LOOKUP_LINE_COMMENT
-LOOKUP_MULTILINE_COMMENT
-LOOKUP_WS
-LOOKUP_FIELD_LINE_COMMENT
-LOOKUP_FIELD_MULTILINE_COMMENT
-LOOKUP_FIELD_WS
 MVEXPAND_LINE_COMMENT
 MVEXPAND_MULTILINE_COMMENT
 MVEXPAND_WS
@@ -245,6 +241,12 @@ SETTING
 SETTING_LINE_COMMENT
 SETTTING_MULTILINE_COMMENT
 SETTING_WS
+LOOKUP_LINE_COMMENT
+LOOKUP_MULTILINE_COMMENT
+LOOKUP_WS
+LOOKUP_FIELD_LINE_COMMENT
+LOOKUP_FIELD_MULTILINE_COMMENT
+LOOKUP_FIELD_WS
 METRICS_LINE_COMMENT
 METRICS_MULTILINE_COMMENT
 METRICS_WS
@@ -260,6 +262,7 @@ processingCommand
 whereCommand
 booleanExpression
 regexBooleanExpression
+matchBooleanExpression
 valueExpression
 operatorExpression
 primaryExpression
@@ -278,7 +281,6 @@ deprecated_metadata
 metricsCommand
 evalCommand
 statsCommand
-inlinestatsCommand
 qualifiedName
 qualifiedNamePattern
 qualifiedNamePatterns
@@ -311,7 +313,10 @@ metaCommand
 enrichCommand
 enrichWithClause
 lookupCommand
+inlinestatsCommand
+matchCommand
+matchQuery
 
 
 atn:
-[4, 1, 124, 567, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 126, 8, 1, 10, 1, 12, 1, 129, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 137, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 153, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 165, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 172, 8, 5, 10, 5, 12, 5, 175, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 182, 8, 5, 1, 5, 1, 5, 3, 5, 186, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 194, 8, 5, 10, 5, 12, 5, 197, 9, 5, 1, 6, 1, 6, 3, 6, 201, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 208, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 213, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 3, 7, 220, 8, 7, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 226, 8, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 5, 8, 234, 8, 8, 10, 8, 12, 8, 237, 9, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 247, 8, 9, 1, 9, 1, 9, 1, 9, 5, 9, 252, 8, 9, 10, 9, 12, 9, 255, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 5, 10, 263, 8, 10, 10, 10, 12, 10, 266, 9, 10, 3, 10, 268, 8, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 5, 13, 280, 8, 13, 10, 13, 12, 13, 283, 9, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 3, 14, 290, 8, 14, 1, 15, 1, 15, 1, 15, 1, 15, 5, 15, 296, 8, 15, 10, 15, 12, 15, 299, 9, 15, 1, 15, 3, 15, 302, 8, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 3, 16, 309, 8, 16, 1, 17, 1, 17, 1, 18, 1, 18, 1, 19, 1, 19, 3, 19, 317, 8, 19, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 323, 8, 20, 10, 20, 12, 20, 326, 9, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 336, 8, 22, 10, 22, 12, 22, 339, 9, 22, 1, 22, 3, 22, 342, 8, 22, 1, 22, 1, 22, 3, 22, 346, 8, 22, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 3, 24, 353, 8, 24, 1, 24, 1, 24, 3, 24, 357, 8, 24, 1, 25, 1, 25, 1, 25, 1, 25, 3, 25, 363, 8, 25, 1, 26, 1, 26, 1, 26, 5, 26, 368, 8, 26, 10, 26, 12, 26, 371, 9, 26, 1, 27, 1, 27, 1, 27, 5, 27, 376, 8, 27, 10, 27, 12, 27, 379, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 384, 8, 28, 10, 28, 12, 28, 387, 9, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 406, 8, 31, 10, 31, 12, 31, 409, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 417, 8, 31, 10, 31, 12, 31, 420, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 428, 8, 31, 10, 31, 12, 31, 431, 9, 31, 1, 31, 1, 31, 3, 31, 435, 8, 31, 1, 32, 1, 32, 3, 32, 439, 8, 32, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 448, 8, 34, 10, 34, 12, 34, 451, 9, 34, 1, 35, 1, 35, 3, 35, 455, 8, 35, 1, 35, 1, 35, 3, 35, 459, 8, 35, 1, 36, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 471, 8, 38, 10, 38, 12, 38, 474, 9, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 3, 40, 484, 8, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 5, 43, 496, 8, 43, 10, 43, 12, 43, 499, 9, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 3, 46, 509, 8, 46, 1, 47, 3, 47, 512, 8, 47, 1, 47, 1, 47, 1, 48, 3, 48, 517, 8, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 3, 55, 542, 8, 55, 1, 55, 1, 55, 1, 55, 1, 55, 5, 55, 548, 8, 55, 10, 55, 12, 55, 551, 9, 55, 3, 55, 553, 8, 55, 1, 56, 1, 56, 1, 56, 3, 56, 558, 8, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 0, 4, 2, 10, 16, 18, 58, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 0, 8, 1, 0, 63, 64, 1, 0, 65, 67, 2, 0, 25, 25, 30, 30, 1, 0, 71, 72, 2, 0, 35, 35, 39, 39, 1, 0, 42, 43, 2, 0, 41, 41, 55, 55, 2, 0, 56, 56, 58, 62, 592, 0, 116, 1, 0, 0, 0, 2, 119, 1, 0, 0, 0, 4, 136, 1, 0, 0, 0, 6, 152, 1, 0, 0, 0, 8, 154, 1, 0, 0, 0, 10, 185, 1, 0, 0, 0, 12, 212, 1, 0, 0, 0, 14, 219, 1, 0, 0, 0, 16, 225, 1, 0, 0, 0, 18, 246, 1, 0, 0, 0, 20, 256, 1, 0, 0, 0, 22, 271, 1, 0, 0, 0, 24, 273, 1, 0, 0, 0, 26, 276, 1, 0, 0, 0, 28, 289, 1, 0, 0, 0, 30, 291, 1, 0, 0, 0, 32, 308, 1, 0, 0, 0, 34, 310, 1, 0, 0, 0, 36, 312, 1, 0, 0, 0, 38, 316, 1, 0, 0, 0, 40, 318, 1, 0, 0, 0, 42, 327, 1, 0, 0, 0, 44, 331, 1, 0, 0, 0, 46, 347, 1, 0, 0, 0, 48, 350, 1, 0, 0, 0, 50, 358, 1, 0, 0, 0, 52, 364, 1, 0, 0, 0, 54, 372, 1, 0, 0, 0, 56, 380, 1, 0, 0, 0, 58, 388, 1, 0, 0, 0, 60, 390, 1, 0, 0, 0, 62, 434, 1, 0, 0, 0, 64, 438, 1, 0, 0, 0, 66, 440, 1, 0, 0, 0, 68, 443, 1, 0, 0, 0, 70, 452, 1, 0, 0, 0, 72, 460, 1, 0, 0, 0, 74, 463, 1, 0, 0, 0, 76, 466, 1, 0, 0, 0, 78, 475, 1, 0, 0, 0, 80, 479, 1, 0, 0, 0, 82, 485, 1, 0, 0, 0, 84, 489, 1, 0, 0, 0, 86, 492, 1, 0, 0, 0, 88, 500, 1, 0, 0, 0, 90, 504, 1, 0, 0, 0, 92, 508, 1, 0, 0, 0, 94, 511, 1, 0, 0, 0, 96, 516, 1, 0, 0, 0, 98, 520, 1, 0, 0, 0, 100, 522, 1, 0, 0, 0, 102, 524, 1, 0, 0, 0, 104, 527, 1, 0, 0, 0, 106, 531, 1, 0, 0, 0, 108, 534, 1, 0, 0, 0, 110, 537, 1, 0, 0, 0, 112, 557, 1, 0, 0, 0, 114, 561, 1, 0, 0, 0, 116, 117, 3, 2, 1, 0, 117, 118, 5, 0, 0, 1, 118, 1, 1, 0, 0, 0, 119, 120, 6, 1, -1, 0, 120, 121, 3, 4, 2, 0, 121, 127, 1, 0, 0, 0, 122, 123, 10, 1, 0, 0, 123, 124, 5, 29, 0, 0, 124, 126, 3, 6, 3, 0, 125, 122, 1, 0, 0, 0, 126, 129, 1, 0, 0, 0, 127, 125, 1, 0, 0, 0, 127, 128, 1, 0, 0, 0, 128, 3, 1, 0, 0, 0, 129, 127, 1, 0, 0, 0, 130, 137, 3, 102, 51, 0, 131, 137, 3, 30, 15, 0, 132, 137, 3, 24, 12, 0, 133, 137, 3, 44, 22, 0, 134, 137, 3, 106, 53, 0, 135, 137, 3, 108, 54, 0, 136, 130, 1, 0, 0, 0, 136, 131, 1, 0, 0, 0, 136, 132, 1, 0, 0, 0, 136, 133, 1, 0, 0, 0, 136, 134, 1, 0, 0, 0, 136, 135, 1, 0, 0, 0, 137, 5, 1, 0, 0, 0, 138, 153, 3, 46, 23, 0, 139, 153, 3, 50, 25, 0, 140, 153, 3, 66, 33, 0, 141, 153, 3, 114, 57, 0, 142, 153, 3, 72, 36, 0, 143, 153, 3, 68, 34, 0, 144, 153, 3, 48, 24, 0, 145, 153, 3, 8, 4, 0, 146, 153, 3, 74, 37, 0, 147, 153, 3, 76, 38, 0, 148, 153, 3, 80, 40, 0, 149, 153, 3, 82, 41, 0, 150, 153, 3, 110, 55, 0, 151, 153, 3, 84, 42, 0, 152, 138, 1, 0, 0, 0, 152, 139, 1, 0, 0, 0, 152, 140, 1, 0, 0, 0, 152, 141, 1, 0, 0, 0, 152, 142, 1, 0, 0, 0, 152, 143, 1, 0, 0, 0, 152, 144, 1, 0, 0, 0, 152, 145, 1, 0, 0, 0, 152, 146, 1, 0, 0, 0, 152, 147, 1, 0, 0, 0, 152, 148, 1, 0, 0, 0, 152, 149, 1, 0, 0, 0, 152, 150, 1, 0, 0, 0, 152, 151, 1, 0, 0, 0, 153, 7, 1, 0, 0, 0, 154, 155, 5, 20, 0, 0, 155, 156, 3, 10, 5, 0, 156, 9, 1, 0, 0, 0, 157, 158, 6, 5, -1, 0, 158, 159, 5, 48, 0, 0, 159, 186, 3, 10, 5, 7, 160, 186, 3, 14, 7, 0, 161, 186, 3, 12, 6, 0, 162, 164, 3, 14, 7, 0, 163, 165, 5, 48, 0, 0, 164, 163, 1, 0, 0, 0, 164, 165, 1, 0, 0, 0, 165, 166, 1, 0, 0, 0, 166, 167, 5, 45, 0, 0, 167, 168, 5, 44, 0, 0, 168, 173, 3, 14, 7, 0, 169, 170, 5, 38, 0, 0, 170, 172, 3, 14, 7, 0, 171, 169, 1, 0, 0, 0, 172, 175, 1, 0, 0, 0, 173, 171, 1, 0, 0, 0, 173, 174, 1, 0, 0, 0, 174, 176, 1, 0, 0, 0, 175, 173, 1, 0, 0, 0, 176, 177, 5, 54, 0, 0, 177, 186, 1, 0, 0, 0, 178, 179, 3, 14, 7, 0, 179, 181, 5, 46, 0, 0, 180, 182, 5, 48, 0, 0, 181, 180, 1, 0, 0, 0, 181, 182, 1, 0, 0, 0, 182, 183, 1, 0, 0, 0, 183, 184, 5, 49, 0, 0, 184, 186, 1, 0, 0, 0, 185, 157, 1, 0, 0, 0, 185, 160, 1, 0, 0, 0, 185, 161, 1, 0, 0, 0, 185, 162, 1, 0, 0, 0, 185, 178, 1, 0, 0, 0, 186, 195, 1, 0, 0, 0, 187, 188, 10, 4, 0, 0, 188, 189, 5, 34, 0, 0, 189, 194, 3, 10, 5, 5, 190, 191, 10, 3, 0, 0, 191, 192, 5, 51, 0, 0, 192, 194, 3, 10, 5, 4, 193, 187, 1, 0, 0, 0, 193, 190, 1, 0, 0, 0, 194, 197, 1, 0, 0, 0, 195, 193, 1, 0, 0, 0, 195, 196, 1, 0, 0, 0, 196, 11, 1, 0, 0, 0, 197, 195, 1, 0, 0, 0, 198, 200, 3, 14, 7, 0, 199, 201, 5, 48, 0, 0, 200, 199, 1, 0, 0, 0, 200, 201, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 203, 5, 47, 0, 0, 203, 204, 3, 98, 49, 0, 204, 213, 1, 0, 0, 0, 205, 207, 3, 14, 7, 0, 206, 208, 5, 48, 0, 0, 207, 206, 1, 0, 0, 0, 207, 208, 1, 0, 0, 0, 208, 209, 1, 0, 0, 0, 209, 210, 5, 53, 0, 0, 210, 211, 3, 98, 49, 0, 211, 213, 1, 0, 0, 0, 212, 198, 1, 0, 0, 0, 212, 205, 1, 0, 0, 0, 213, 13, 1, 0, 0, 0, 214, 220, 3, 16, 8, 0, 215, 216, 3, 16, 8, 0, 216, 217, 3, 100, 50, 0, 217, 218, 3, 16, 8, 0, 218, 220, 1, 0, 0, 0, 219, 214, 1, 0, 0, 0, 219, 215, 1, 0, 0, 0, 220, 15, 1, 0, 0, 0, 221, 222, 6, 8, -1, 0, 222, 226, 3, 18, 9, 0, 223, 224, 7, 0, 0, 0, 224, 226, 3, 16, 8, 3, 225, 221, 1, 0, 0, 0, 225, 223, 1, 0, 0, 0, 226, 235, 1, 0, 0, 0, 227, 228, 10, 2, 0, 0, 228, 229, 7, 1, 0, 0, 229, 234, 3, 16, 8, 3, 230, 231, 10, 1, 0, 0, 231, 232, 7, 0, 0, 0, 232, 234, 3, 16, 8, 2, 233, 227, 1, 0, 0, 0, 233, 230, 1, 0, 0, 0, 234, 237, 1, 0, 0, 0, 235, 233, 1, 0, 0, 0, 235, 236, 1, 0, 0, 0, 236, 17, 1, 0, 0, 0, 237, 235, 1, 0, 0, 0, 238, 239, 6, 9, -1, 0, 239, 247, 3, 62, 31, 0, 240, 247, 3, 52, 26, 0, 241, 247, 3, 20, 10, 0, 242, 243, 5, 44, 0, 0, 243, 244, 3, 10, 5, 0, 244, 245, 5, 54, 0, 0, 245, 247, 1, 0, 0, 0, 246, 238, 1, 0, 0, 0, 246, 240, 1, 0, 0, 0, 246, 241, 1, 0, 0, 0, 246, 242, 1, 0, 0, 0, 247, 253, 1, 0, 0, 0, 248, 249, 10, 1, 0, 0, 249, 250, 5, 37, 0, 0, 250, 252, 3, 22, 11, 0, 251, 248, 1, 0, 0, 0, 252, 255, 1, 0, 0, 0, 253, 251, 1, 0, 0, 0, 253, 254, 1, 0, 0, 0, 254, 19, 1, 0, 0, 0, 255, 253, 1, 0, 0, 0, 256, 257, 3, 58, 29, 0, 257, 267, 5, 44, 0, 0, 258, 268, 5, 65, 0, 0, 259, 264, 3, 10, 5, 0, 260, 261, 5, 38, 0, 0, 261, 263, 3, 10, 5, 0, 262, 260, 1, 0, 0, 0, 263, 266, 1, 0, 0, 0, 264, 262, 1, 0, 0, 0, 264, 265, 1, 0, 0, 0, 265, 268, 1, 0, 0, 0, 266, 264, 1, 0, 0, 0, 267, 258, 1, 0, 0, 0, 267, 259, 1, 0, 0, 0, 267, 268, 1, 0, 0, 0, 268, 269, 1, 0, 0, 0, 269, 270, 5, 54, 0, 0, 270, 21, 1, 0, 0, 0, 271, 272, 3, 58, 29, 0, 272, 23, 1, 0, 0, 0, 273, 274, 5, 16, 0, 0, 274, 275, 3, 26, 13, 0, 275, 25, 1, 0, 0, 0, 276, 281, 3, 28, 14, 0, 277, 278, 5, 38, 0, 0, 278, 280, 3, 28, 14, 0, 279, 277, 1, 0, 0, 0, 280, 283, 1, 0, 0, 0, 281, 279, 1, 0, 0, 0, 281, 282, 1, 0, 0, 0, 282, 27, 1, 0, 0, 0, 283, 281, 1, 0, 0, 0, 284, 290, 3, 10, 5, 0, 285, 286, 3, 52, 26, 0, 286, 287, 5, 36, 0, 0, 287, 288, 3, 10, 5, 0, 288, 290, 1, 0, 0, 0, 289, 284, 1, 0, 0, 0, 289, 285, 1, 0, 0, 0, 290, 29, 1, 0, 0, 0, 291, 292, 5, 6, 0, 0, 292, 297, 3, 32, 16, 0, 293, 294, 5, 38, 0, 0, 294, 296, 3, 32, 16, 0, 295, 293, 1, 0, 0, 0, 296, 299, 1, 0, 0, 0, 297, 295, 1, 0, 0, 0, 297, 298, 1, 0, 0, 0, 298, 301, 1, 0, 0, 0, 299, 297, 1, 0, 0, 0, 300, 302, 3, 38, 19, 0, 301, 300, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 31, 1, 0, 0, 0, 303, 304, 3, 34, 17, 0, 304, 305, 5, 114, 0, 0, 305, 306, 3, 36, 18, 0, 306, 309, 1, 0, 0, 0, 307, 309, 3, 36, 18, 0, 308, 303, 1, 0, 0, 0, 308, 307, 1, 0, 0, 0, 309, 33, 1, 0, 0, 0, 310, 311, 5, 25, 0, 0, 311, 35, 1, 0, 0, 0, 312, 313, 7, 2, 0, 0, 313, 37, 1, 0, 0, 0, 314, 317, 3, 40, 20, 0, 315, 317, 3, 42, 21, 0, 316, 314, 1, 0, 0, 0, 316, 315, 1, 0, 0, 0, 317, 39, 1, 0, 0, 0, 318, 319, 5, 76, 0, 0, 319, 324, 5, 25, 0, 0, 320, 321, 5, 38, 0, 0, 321, 323, 5, 25, 0, 0, 322, 320, 1, 0, 0, 0, 323, 326, 1, 0, 0, 0, 324, 322, 1, 0, 0, 0, 324, 325, 1, 0, 0, 0, 325, 41, 1, 0, 0, 0, 326, 324, 1, 0, 0, 0, 327, 328, 5, 69, 0, 0, 328, 329, 3, 40, 20, 0, 329, 330, 5, 70, 0, 0, 330, 43, 1, 0, 0, 0, 331, 332, 5, 13, 0, 0, 332, 337, 3, 32, 16, 0, 333, 334, 5, 38, 0, 0, 334, 336, 3, 32, 16, 0, 335, 333, 1, 0, 0, 0, 336, 339, 1, 0, 0, 0, 337, 335, 1, 0, 0, 0, 337, 338, 1, 0, 0, 0, 338, 341, 1, 0, 0, 0, 339, 337, 1, 0, 0, 0, 340, 342, 3, 26, 13, 0, 341, 340, 1, 0, 0, 0, 341, 342, 1, 0, 0, 0, 342, 345, 1, 0, 0, 0, 343, 344, 5, 33, 0, 0, 344, 346, 3, 26, 13, 0, 345, 343, 1, 0, 0, 0, 345, 346, 1, 0, 0, 0, 346, 45, 1, 0, 0, 0, 347, 348, 5, 4, 0, 0, 348, 349, 3, 26, 13, 0, 349, 47, 1, 0, 0, 0, 350, 352, 5, 19, 0, 0, 351, 353, 3, 26, 13, 0, 352, 351, 1, 0, 0, 0, 352, 353, 1, 0, 0, 0, 353, 356, 1, 0, 0, 0, 354, 355, 5, 33, 0, 0, 355, 357, 3, 26, 13, 0, 356, 354, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 49, 1, 0, 0, 0, 358, 359, 5, 8, 0, 0, 359, 362, 3, 26, 13, 0, 360, 361, 5, 33, 0, 0, 361, 363, 3, 26, 13, 0, 362, 360, 1, 0, 0, 0, 362, 363, 1, 0, 0, 0, 363, 51, 1, 0, 0, 0, 364, 369, 3, 58, 29, 0, 365, 366, 5, 40, 0, 0, 366, 368, 3, 58, 29, 0, 367, 365, 1, 0, 0, 0, 368, 371, 1, 0, 0, 0, 369, 367, 1, 0, 0, 0, 369, 370, 1, 0, 0, 0, 370, 53, 1, 0, 0, 0, 371, 369, 1, 0, 0, 0, 372, 377, 3, 60, 30, 0, 373, 374, 5, 40, 0, 0, 374, 376, 3, 60, 30, 0, 375, 373, 1, 0, 0, 0, 376, 379, 1, 0, 0, 0, 377, 375, 1, 0, 0, 0, 377, 378, 1, 0, 0, 0, 378, 55, 1, 0, 0, 0, 379, 377, 1, 0, 0, 0, 380, 385, 3, 54, 27, 0, 381, 382, 5, 38, 0, 0, 382, 384, 3, 54, 27, 0, 383, 381, 1, 0, 0, 0, 384, 387, 1, 0, 0, 0, 385, 383, 1, 0, 0, 0, 385, 386, 1, 0, 0, 0, 386, 57, 1, 0, 0, 0, 387, 385, 1, 0, 0, 0, 388, 389, 7, 3, 0, 0, 389, 59, 1, 0, 0, 0, 390, 391, 5, 80, 0, 0, 391, 61, 1, 0, 0, 0, 392, 435, 5, 49, 0, 0, 393, 394, 3, 96, 48, 0, 394, 395, 5, 71, 0, 0, 395, 435, 1, 0, 0, 0, 396, 435, 3, 94, 47, 0, 397, 435, 3, 96, 48, 0, 398, 435, 3, 90, 45, 0, 399, 435, 3, 64, 32, 0, 400, 435, 3, 98, 49, 0, 401, 402, 5, 69, 0, 0, 402, 407, 3, 92, 46, 0, 403, 404, 5, 38, 0, 0, 404, 406, 3, 92, 46, 0, 405, 403, 1, 0, 0, 0, 406, 409, 1, 0, 0, 0, 407, 405, 1, 0, 0, 0, 407, 408, 1, 0, 0, 0, 408, 410, 1, 0, 0, 0, 409, 407, 1, 0, 0, 0, 410, 411, 5, 70, 0, 0, 411, 435, 1, 0, 0, 0, 412, 413, 5, 69, 0, 0, 413, 418, 3, 90, 45, 0, 414, 415, 5, 38, 0, 0, 415, 417, 3, 90, 45, 0, 416, 414, 1, 0, 0, 0, 417, 420, 1, 0, 0, 0, 418, 416, 1, 0, 0, 0, 418, 419, 1, 0, 0, 0, 419, 421, 1, 0, 0, 0, 420, 418, 1, 0, 0, 0, 421, 422, 5, 70, 0, 0, 422, 435, 1, 0, 0, 0, 423, 424, 5, 69, 0, 0, 424, 429, 3, 98, 49, 0, 425, 426, 5, 38, 0, 0, 426, 428, 3, 98, 49, 0, 427, 425, 1, 0, 0, 0, 428, 431, 1, 0, 0, 0, 429, 427, 1, 0, 0, 0, 429, 430, 1, 0, 0, 0, 430, 432, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 432, 433, 5, 70, 0, 0, 433, 435, 1, 0, 0, 0, 434, 392, 1, 0, 0, 0, 434, 393, 1, 0, 0, 0, 434, 396, 1, 0, 0, 0, 434, 397, 1, 0, 0, 0, 434, 398, 1, 0, 0, 0, 434, 399, 1, 0, 0, 0, 434, 400, 1, 0, 0, 0, 434, 401, 1, 0, 0, 0, 434, 412, 1, 0, 0, 0, 434, 423, 1, 0, 0, 0, 435, 63, 1, 0, 0, 0, 436, 439, 5, 52, 0, 0, 437, 439, 5, 68, 0, 0, 438, 436, 1, 0, 0, 0, 438, 437, 1, 0, 0, 0, 439, 65, 1, 0, 0, 0, 440, 441, 5, 10, 0, 0, 441, 442, 5, 31, 0, 0, 442, 67, 1, 0, 0, 0, 443, 444, 5, 18, 0, 0, 444, 449, 3, 70, 35, 0, 445, 446, 5, 38, 0, 0, 446, 448, 3, 70, 35, 0, 447, 445, 1, 0, 0, 0, 448, 451, 1, 0, 0, 0, 449, 447, 1, 0, 0, 0, 449, 450, 1, 0, 0, 0, 450, 69, 1, 0, 0, 0, 451, 449, 1, 0, 0, 0, 452, 454, 3, 10, 5, 0, 453, 455, 7, 4, 0, 0, 454, 453, 1, 0, 0, 0, 454, 455, 1, 0, 0, 0, 455, 458, 1, 0, 0, 0, 456, 457, 5, 50, 0, 0, 457, 459, 7, 5, 0, 0, 458, 456, 1, 0, 0, 0, 458, 459, 1, 0, 0, 0, 459, 71, 1, 0, 0, 0, 460, 461, 5, 9, 0, 0, 461, 462, 3, 56, 28, 0, 462, 73, 1, 0, 0, 0, 463, 464, 5, 2, 0, 0, 464, 465, 3, 56, 28, 0, 465, 75, 1, 0, 0, 0, 466, 467, 5, 15, 0, 0, 467, 472, 3, 78, 39, 0, 468, 469, 5, 38, 0, 0, 469, 471, 3, 78, 39, 0, 470, 468, 1, 0, 0, 0, 471, 474, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 472, 473, 1, 0, 0, 0, 473, 77, 1, 0, 0, 0, 474, 472, 1, 0, 0, 0, 475, 476, 3, 54, 27, 0, 476, 477, 5, 84, 0, 0, 477, 478, 3, 54, 27, 0, 478, 79, 1, 0, 0, 0, 479, 480, 5, 1, 0, 0, 480, 481, 3, 18, 9, 0, 481, 483, 3, 98, 49, 0, 482, 484, 3, 86, 43, 0, 483, 482, 1, 0, 0, 0, 483, 484, 1, 0, 0, 0, 484, 81, 1, 0, 0, 0, 485, 486, 5, 7, 0, 0, 486, 487, 3, 18, 9, 0, 487, 488, 3, 98, 49, 0, 488, 83, 1, 0, 0, 0, 489, 490, 5, 14, 0, 0, 490, 491, 3, 52, 26, 0, 491, 85, 1, 0, 0, 0, 492, 497, 3, 88, 44, 0, 493, 494, 5, 38, 0, 0, 494, 496, 3, 88, 44, 0, 495, 493, 1, 0, 0, 0, 496, 499, 1, 0, 0, 0, 497, 495, 1, 0, 0, 0, 497, 498, 1, 0, 0, 0, 498, 87, 1, 0, 0, 0, 499, 497, 1, 0, 0, 0, 500, 501, 3, 58, 29, 0, 501, 502, 5, 36, 0, 0, 502, 503, 3, 62, 31, 0, 503, 89, 1, 0, 0, 0, 504, 505, 7, 6, 0, 0, 505, 91, 1, 0, 0, 0, 506, 509, 3, 94, 47, 0, 507, 509, 3, 96, 48, 0, 508, 506, 1, 0, 0, 0, 508, 507, 1, 0, 0, 0, 509, 93, 1, 0, 0, 0, 510, 512, 7, 0, 0, 0, 511, 510, 1, 0, 0, 0, 511, 512, 1, 0, 0, 0, 512, 513, 1, 0, 0, 0, 513, 514, 5, 32, 0, 0, 514, 95, 1, 0, 0, 0, 515, 517, 7, 0, 0, 0, 516, 515, 1, 0, 0, 0, 516, 517, 1, 0, 0, 0, 517, 518, 1, 0, 0, 0, 518, 519, 5, 31, 0, 0, 519, 97, 1, 0, 0, 0, 520, 521, 5, 30, 0, 0, 521, 99, 1, 0, 0, 0, 522, 523, 7, 7, 0, 0, 523, 101, 1, 0, 0, 0, 524, 525, 5, 5, 0, 0, 525, 526, 3, 104, 52, 0, 526, 103, 1, 0, 0, 0, 527, 528, 5, 69, 0, 0, 528, 529, 3, 2, 1, 0, 529, 530, 5, 70, 0, 0, 530, 105, 1, 0, 0, 0, 531, 532, 5, 17, 0, 0, 532, 533, 5, 106, 0, 0, 533, 107, 1, 0, 0, 0, 534, 535, 5, 12, 0, 0, 535, 536, 5, 110, 0, 0, 536, 109, 1, 0, 0, 0, 537, 538, 5, 3, 0, 0, 538, 541, 5, 90, 0, 0, 539, 540, 5, 88, 0, 0, 540, 542, 3, 54, 27, 0, 541, 539, 1, 0, 0, 0, 541, 542, 1, 0, 0, 0, 542, 552, 1, 0, 0, 0, 543, 544, 5, 89, 0, 0, 544, 549, 3, 112, 56, 0, 545, 546, 5, 38, 0, 0, 546, 548, 3, 112, 56, 0, 547, 545, 1, 0, 0, 0, 548, 551, 1, 0, 0, 0, 549, 547, 1, 0, 0, 0, 549, 550, 1, 0, 0, 0, 550, 553, 1, 0, 0, 0, 551, 549, 1, 0, 0, 0, 552, 543, 1, 0, 0, 0, 552, 553, 1, 0, 0, 0, 553, 111, 1, 0, 0, 0, 554, 555, 3, 54, 27, 0, 555, 556, 5, 36, 0, 0, 556, 558, 1, 0, 0, 0, 557, 554, 1, 0, 0, 0, 557, 558, 1, 0, 0, 0, 558, 559, 1, 0, 0, 0, 559, 560, 3, 54, 27, 0, 560, 113, 1, 0, 0, 0, 561, 562, 5, 11, 0, 0, 562, 563, 3, 32, 16, 0, 563, 564, 5, 88, 0, 0, 564, 565, 3, 56, 28, 0, 565, 115, 1, 0, 0, 0, 54, 127, 136, 152, 164, 173, 181, 185, 193, 195, 200, 207, 212, 219, 225, 233, 235, 246, 253, 264, 267, 281, 289, 297, 301, 308, 316, 324, 337, 341, 345, 352, 356, 362, 369, 377, 385, 407, 418, 429, 434, 438, 449, 454, 458, 472, 483, 497, 508, 511, 516, 541, 549, 552, 557]
\ No newline at end of file
+[4, 1, 125, 589, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 132, 8, 1, 10, 1, 12, 1, 135, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 144, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 164, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 176, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 183, 8, 5, 10, 5, 12, 5, 186, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 193, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 199, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 207, 8, 5, 10, 5, 12, 5, 210, 9, 5, 1, 6, 1, 6, 3, 6, 214, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 221, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 226, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 237, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 243, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 251, 8, 9, 10, 9, 12, 9, 254, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 264, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 269, 8, 10, 10, 10, 12, 10, 272, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 280, 8, 11, 10, 11, 12, 11, 283, 9, 11, 3, 11, 285, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 5, 14, 297, 8, 14, 10, 14, 12, 14, 300, 9, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 3, 15, 307, 8, 15, 1, 16, 1, 16, 1, 16, 1, 16, 5, 16, 313, 8, 16, 10, 16, 12, 16, 316, 9, 16, 1, 16, 3, 16, 319, 8, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 3, 17, 326, 8, 17, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 3, 20, 334, 8, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 340, 8, 21, 10, 21, 12, 21, 343, 9, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 353, 8, 23, 10, 23, 12, 23, 356, 9, 23, 1, 23, 3, 23, 359, 8, 23, 1, 23, 1, 23, 3, 23, 363, 8, 23, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 3, 25, 370, 8, 25, 1, 25, 1, 25, 3, 25, 374, 8, 25, 1, 26, 1, 26, 1, 26, 5, 26, 379, 8, 26, 10, 26, 12, 26, 382, 9, 26, 1, 27, 1, 27, 1, 27, 5, 27, 387, 8, 27, 10, 27, 12, 27, 390, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 395, 8, 28, 10, 28, 12, 28, 398, 9, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 417, 8, 31, 10, 31, 12, 31, 420, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 428, 8, 31, 10, 31, 12, 31, 431, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 439, 8, 31, 10, 31, 12, 31, 442, 9, 31, 1, 31, 1, 31, 3, 31, 446, 8, 31, 1, 32, 1, 32, 3, 32, 450, 8, 32, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 459, 8, 34, 10, 34, 12, 34, 462, 9, 34, 1, 35, 1, 35, 3, 35, 466, 8, 35, 1, 35, 1, 35, 3, 35, 470, 8, 35, 1, 36, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 482, 8, 38, 10, 38, 12, 38, 485, 9, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 3, 40, 495, 8, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 5, 43, 507, 8, 43, 10, 43, 12, 43, 510, 9, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 3, 46, 520, 8, 46, 1, 47, 3, 47, 523, 8, 47, 1, 47, 1, 47, 1, 48, 3, 48, 528, 8, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 3, 55, 553, 8, 55, 1, 55, 1, 55, 1, 55, 1, 55, 5, 55, 559, 8, 55, 10, 55, 12, 55, 562, 9, 55, 3, 55, 564, 8, 55, 1, 56, 1, 56, 1, 56, 3, 56, 569, 8, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 582, 8, 58, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 0, 4, 2, 10, 18, 20, 61, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 0, 8, 1, 0, 60, 61, 1, 0, 62, 64, 2, 0, 27, 27, 77, 77, 1, 0, 68, 69, 2, 0, 32, 32, 36, 36, 2, 0, 39, 39, 42, 42, 2, 0, 38, 38, 52, 52, 2, 0, 53, 53, 55, 59, 613, 0, 122, 1, 0, 0, 0, 2, 125, 1, 0, 0, 0, 4, 143, 1, 0, 0, 0, 6, 163, 1, 0, 0, 0, 8, 165, 1, 0, 0, 0, 10, 198, 1, 0, 0, 0, 12, 225, 1, 0, 0, 0, 14, 227, 1, 0, 0, 0, 16, 236, 1, 0, 0, 0, 18, 242, 1, 0, 0, 0, 20, 263, 1, 0, 0, 0, 22, 273, 1, 0, 0, 0, 24, 288, 1, 0, 0, 0, 26, 290, 1, 0, 0, 0, 28, 293, 1, 0, 0, 0, 30, 306, 1, 0, 0, 0, 32, 308, 1, 0, 0, 0, 34, 325, 1, 0, 0, 0, 36, 327, 1, 0, 0, 0, 38, 329, 1, 0, 0, 0, 40, 333, 1, 0, 0, 0, 42, 335, 1, 0, 0, 0, 44, 344, 1, 0, 0, 0, 46, 348, 1, 0, 0, 0, 48, 364, 1, 0, 0, 0, 50, 367, 1, 0, 0, 0, 52, 375, 1, 0, 0, 0, 54, 383, 1, 0, 0, 0, 56, 391, 1, 0, 0, 0, 58, 399, 1, 0, 0, 0, 60, 401, 1, 0, 0, 0, 62, 445, 1, 0, 0, 0, 64, 449, 1, 0, 0, 0, 66, 451, 1, 0, 0, 0, 68, 454, 1, 0, 0, 0, 70, 463, 1, 0, 0, 0, 72, 471, 1, 0, 0, 0, 74, 474, 1, 0, 0, 0, 76, 477, 1, 0, 0, 0, 78, 486, 1, 0, 0, 0, 80, 490, 1, 0, 0, 0, 82, 496, 1, 0, 0, 0, 84, 500, 1, 0, 0, 0, 86, 503, 1, 0, 0, 0, 88, 511, 1, 0, 0, 0, 90, 515, 1, 0, 0, 0, 92, 519, 1, 0, 0, 0, 94, 522, 1, 0, 0, 0, 96, 527, 1, 0, 0, 0, 98, 531, 1, 0, 0, 0, 100, 533, 1, 0, 0, 0, 102, 535, 1, 0, 0, 0, 104, 538, 1, 0, 0, 0, 106, 542, 1, 0, 0, 0, 108, 545, 1, 0, 0, 0, 110, 548, 1, 0, 0, 0, 112, 568, 1, 0, 0, 0, 114, 572, 1, 0, 0, 0, 116, 577, 1, 0, 0, 0, 118, 583, 1, 0, 0, 0, 120, 586, 1, 0, 0, 0, 122, 123, 3, 2, 1, 0, 123, 124, 5, 0, 0, 1, 124, 1, 1, 0, 0, 0, 125, 126, 6, 1, -1, 0, 126, 127, 3, 4, 2, 0, 127, 133, 1, 0, 0, 0, 128, 129, 10, 1, 0, 0, 129, 130, 5, 26, 0, 0, 130, 132, 3, 6, 3, 0, 131, 128, 1, 0, 0, 0, 132, 135, 1, 0, 0, 0, 133, 131, 1, 0, 0, 0, 133, 134, 1, 0, 0, 0, 134, 3, 1, 0, 0, 0, 135, 133, 1, 0, 0, 0, 136, 144, 3, 102, 51, 0, 137, 144, 3, 32, 16, 0, 138, 144, 3, 108, 54, 0, 139, 144, 3, 26, 13, 0, 140, 144, 3, 106, 53, 0, 141, 142, 4, 2, 1, 0, 142, 144, 3, 46, 23, 0, 143, 136, 1, 0, 0, 0, 143, 137, 1, 0, 0, 0, 143, 138, 1, 0, 0, 0, 143, 139, 1, 0, 0, 0, 143, 140, 1, 0, 0, 0, 143, 141, 1, 0, 0, 0, 144, 5, 1, 0, 0, 0, 145, 164, 3, 48, 24, 0, 146, 164, 3, 8, 4, 0, 147, 164, 3, 72, 36, 0, 148, 164, 3, 66, 33, 0, 149, 164, 3, 50, 25, 0, 150, 164, 3, 68, 34, 0, 151, 164, 3, 74, 37, 0, 152, 164, 3, 76, 38, 0, 153, 164, 3, 80, 40, 0, 154, 164, 3, 82, 41, 0, 155, 164, 3, 110, 55, 0, 156, 164, 3, 84, 42, 0, 157, 158, 4, 3, 2, 0, 158, 164, 3, 116, 58, 0, 159, 160, 4, 3, 3, 0, 160, 164, 3, 114, 57, 0, 161, 162, 4, 3, 4, 0, 162, 164, 3, 118, 59, 0, 163, 145, 1, 0, 0, 0, 163, 146, 1, 0, 0, 0, 163, 147, 1, 0, 0, 0, 163, 148, 1, 0, 0, 0, 163, 149, 1, 0, 0, 0, 163, 150, 1, 0, 0, 0, 163, 151, 1, 0, 0, 0, 163, 152, 1, 0, 0, 0, 163, 153, 1, 0, 0, 0, 163, 154, 1, 0, 0, 0, 163, 155, 1, 0, 0, 0, 163, 156, 1, 0, 0, 0, 163, 157, 1, 0, 0, 0, 163, 159, 1, 0, 0, 0, 163, 161, 1, 0, 0, 0, 164, 7, 1, 0, 0, 0, 165, 166, 5, 17, 0, 0, 166, 167, 3, 10, 5, 0, 167, 9, 1, 0, 0, 0, 168, 169, 6, 5, -1, 0, 169, 170, 5, 45, 0, 0, 170, 199, 3, 10, 5, 8, 171, 199, 3, 16, 8, 0, 172, 199, 3, 12, 6, 0, 173, 175, 3, 16, 8, 0, 174, 176, 5, 45, 0, 0, 175, 174, 1, 0, 0, 0, 175, 176, 1, 0, 0, 0, 176, 177, 1, 0, 0, 0, 177, 178, 5, 40, 0, 0, 178, 179, 5, 44, 0, 0, 179, 184, 3, 16, 8, 0, 180, 181, 5, 35, 0, 0, 181, 183, 3, 16, 8, 0, 182, 180, 1, 0, 0, 0, 183, 186, 1, 0, 0, 0, 184, 182, 1, 0, 0, 0, 184, 185, 1, 0, 0, 0, 185, 187, 1, 0, 0, 0, 186, 184, 1, 0, 0, 0, 187, 188, 5, 51, 0, 0, 188, 199, 1, 0, 0, 0, 189, 190, 3, 16, 8, 0, 190, 192, 5, 41, 0, 0, 191, 193, 5, 45, 0, 0, 192, 191, 1, 0, 0, 0, 192, 193, 1, 0, 0, 0, 193, 194, 1, 0, 0, 0, 194, 195, 5, 46, 0, 0, 195, 199, 1, 0, 0, 0, 196, 197, 4, 5, 5, 0, 197, 199, 3, 14, 7, 0, 198, 168, 1, 0, 0, 0, 198, 171, 1, 0, 0, 0, 198, 172, 1, 0, 0, 0, 198, 173, 1, 0, 0, 0, 198, 189, 1, 0, 0, 0, 198, 196, 1, 0, 0, 0, 199, 208, 1, 0, 0, 0, 200, 201, 10, 5, 0, 0, 201, 202, 5, 31, 0, 0, 202, 207, 3, 10, 5, 6, 203, 204, 10, 4, 0, 0, 204, 205, 5, 48, 0, 0, 205, 207, 3, 10, 5, 5, 206, 200, 1, 0, 0, 0, 206, 203, 1, 0, 0, 0, 207, 210, 1, 0, 0, 0, 208, 206, 1, 0, 0, 0, 208, 209, 1, 0, 0, 0, 209, 11, 1, 0, 0, 0, 210, 208, 1, 0, 0, 0, 211, 213, 3, 16, 8, 0, 212, 214, 5, 45, 0, 0, 213, 212, 1, 0, 0, 0, 213, 214, 1, 0, 0, 0, 214, 215, 1, 0, 0, 0, 215, 216, 5, 43, 0, 0, 216, 217, 3, 98, 49, 0, 217, 226, 1, 0, 0, 0, 218, 220, 3, 16, 8, 0, 219, 221, 5, 45, 0, 0, 220, 219, 1, 0, 0, 0, 220, 221, 1, 0, 0, 0, 221, 222, 1, 0, 0, 0, 222, 223, 5, 50, 0, 0, 223, 224, 3, 98, 49, 0, 224, 226, 1, 0, 0, 0, 225, 211, 1, 0, 0, 0, 225, 218, 1, 0, 0, 0, 226, 13, 1, 0, 0, 0, 227, 228, 3, 16, 8, 0, 228, 229, 5, 20, 0, 0, 229, 230, 3, 98, 49, 0, 230, 15, 1, 0, 0, 0, 231, 237, 3, 18, 9, 0, 232, 233, 3, 18, 9, 0, 233, 234, 3, 100, 50, 0, 234, 235, 3, 18, 9, 0, 235, 237, 1, 0, 0, 0, 236, 231, 1, 0, 0, 0, 236, 232, 1, 0, 0, 0, 237, 17, 1, 0, 0, 0, 238, 239, 6, 9, -1, 0, 239, 243, 3, 20, 10, 0, 240, 241, 7, 0, 0, 0, 241, 243, 3, 18, 9, 3, 242, 238, 1, 0, 0, 0, 242, 240, 1, 0, 0, 0, 243, 252, 1, 0, 0, 0, 244, 245, 10, 2, 0, 0, 245, 246, 7, 1, 0, 0, 246, 251, 3, 18, 9, 3, 247, 248, 10, 1, 0, 0, 248, 249, 7, 0, 0, 0, 249, 251, 3, 18, 9, 2, 250, 244, 1, 0, 0, 0, 250, 247, 1, 0, 0, 0, 251, 254, 1, 0, 0, 0, 252, 250, 1, 0, 0, 0, 252, 253, 1, 0, 0, 0, 253, 19, 1, 0, 0, 0, 254, 252, 1, 0, 0, 0, 255, 256, 6, 10, -1, 0, 256, 264, 3, 62, 31, 0, 257, 264, 3, 52, 26, 0, 258, 264, 3, 22, 11, 0, 259, 260, 5, 44, 0, 0, 260, 261, 3, 10, 5, 0, 261, 262, 5, 51, 0, 0, 262, 264, 1, 0, 0, 0, 263, 255, 1, 0, 0, 0, 263, 257, 1, 0, 0, 0, 263, 258, 1, 0, 0, 0, 263, 259, 1, 0, 0, 0, 264, 270, 1, 0, 0, 0, 265, 266, 10, 1, 0, 0, 266, 267, 5, 34, 0, 0, 267, 269, 3, 24, 12, 0, 268, 265, 1, 0, 0, 0, 269, 272, 1, 0, 0, 0, 270, 268, 1, 0, 0, 0, 270, 271, 1, 0, 0, 0, 271, 21, 1, 0, 0, 0, 272, 270, 1, 0, 0, 0, 273, 274, 3, 58, 29, 0, 274, 284, 5, 44, 0, 0, 275, 285, 5, 62, 0, 0, 276, 281, 3, 10, 5, 0, 277, 278, 5, 35, 0, 0, 278, 280, 3, 10, 5, 0, 279, 277, 1, 0, 0, 0, 280, 283, 1, 0, 0, 0, 281, 279, 1, 0, 0, 0, 281, 282, 1, 0, 0, 0, 282, 285, 1, 0, 0, 0, 283, 281, 1, 0, 0, 0, 284, 275, 1, 0, 0, 0, 284, 276, 1, 0, 0, 0, 284, 285, 1, 0, 0, 0, 285, 286, 1, 0, 0, 0, 286, 287, 5, 51, 0, 0, 287, 23, 1, 0, 0, 0, 288, 289, 3, 58, 29, 0, 289, 25, 1, 0, 0, 0, 290, 291, 5, 13, 0, 0, 291, 292, 3, 28, 14, 0, 292, 27, 1, 0, 0, 0, 293, 298, 3, 30, 15, 0, 294, 295, 5, 35, 0, 0, 295, 297, 3, 30, 15, 0, 296, 294, 1, 0, 0, 0, 297, 300, 1, 0, 0, 0, 298, 296, 1, 0, 0, 0, 298, 299, 1, 0, 0, 0, 299, 29, 1, 0, 0, 0, 300, 298, 1, 0, 0, 0, 301, 307, 3, 10, 5, 0, 302, 303, 3, 52, 26, 0, 303, 304, 5, 33, 0, 0, 304, 305, 3, 10, 5, 0, 305, 307, 1, 0, 0, 0, 306, 301, 1, 0, 0, 0, 306, 302, 1, 0, 0, 0, 307, 31, 1, 0, 0, 0, 308, 309, 5, 6, 0, 0, 309, 314, 3, 34, 17, 0, 310, 311, 5, 35, 0, 0, 311, 313, 3, 34, 17, 0, 312, 310, 1, 0, 0, 0, 313, 316, 1, 0, 0, 0, 314, 312, 1, 0, 0, 0, 314, 315, 1, 0, 0, 0, 315, 318, 1, 0, 0, 0, 316, 314, 1, 0, 0, 0, 317, 319, 3, 40, 20, 0, 318, 317, 1, 0, 0, 0, 318, 319, 1, 0, 0, 0, 319, 33, 1, 0, 0, 0, 320, 321, 3, 36, 18, 0, 321, 322, 5, 109, 0, 0, 322, 323, 3, 38, 19, 0, 323, 326, 1, 0, 0, 0, 324, 326, 3, 38, 19, 0, 325, 320, 1, 0, 0, 0, 325, 324, 1, 0, 0, 0, 326, 35, 1, 0, 0, 0, 327, 328, 5, 77, 0, 0, 328, 37, 1, 0, 0, 0, 329, 330, 7, 2, 0, 0, 330, 39, 1, 0, 0, 0, 331, 334, 3, 42, 21, 0, 332, 334, 3, 44, 22, 0, 333, 331, 1, 0, 0, 0, 333, 332, 1, 0, 0, 0, 334, 41, 1, 0, 0, 0, 335, 336, 5, 76, 0, 0, 336, 341, 5, 77, 0, 0, 337, 338, 5, 35, 0, 0, 338, 340, 5, 77, 0, 0, 339, 337, 1, 0, 0, 0, 340, 343, 1, 0, 0, 0, 341, 339, 1, 0, 0, 0, 341, 342, 1, 0, 0, 0, 342, 43, 1, 0, 0, 0, 343, 341, 1, 0, 0, 0, 344, 345, 5, 66, 0, 0, 345, 346, 3, 42, 21, 0, 346, 347, 5, 67, 0, 0, 347, 45, 1, 0, 0, 0, 348, 349, 5, 21, 0, 0, 349, 354, 3, 34, 17, 0, 350, 351, 5, 35, 0, 0, 351, 353, 3, 34, 17, 0, 352, 350, 1, 0, 0, 0, 353, 356, 1, 0, 0, 0, 354, 352, 1, 0, 0, 0, 354, 355, 1, 0, 0, 0, 355, 358, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 357, 359, 3, 28, 14, 0, 358, 357, 1, 0, 0, 0, 358, 359, 1, 0, 0, 0, 359, 362, 1, 0, 0, 0, 360, 361, 5, 30, 0, 0, 361, 363, 3, 28, 14, 0, 362, 360, 1, 0, 0, 0, 362, 363, 1, 0, 0, 0, 363, 47, 1, 0, 0, 0, 364, 365, 5, 4, 0, 0, 365, 366, 3, 28, 14, 0, 366, 49, 1, 0, 0, 0, 367, 369, 5, 16, 0, 0, 368, 370, 3, 28, 14, 0, 369, 368, 1, 0, 0, 0, 369, 370, 1, 0, 0, 0, 370, 373, 1, 0, 0, 0, 371, 372, 5, 30, 0, 0, 372, 374, 3, 28, 14, 0, 373, 371, 1, 0, 0, 0, 373, 374, 1, 0, 0, 0, 374, 51, 1, 0, 0, 0, 375, 380, 3, 58, 29, 0, 376, 377, 5, 37, 0, 0, 377, 379, 3, 58, 29, 0, 378, 376, 1, 0, 0, 0, 379, 382, 1, 0, 0, 0, 380, 378, 1, 0, 0, 0, 380, 381, 1, 0, 0, 0, 381, 53, 1, 0, 0, 0, 382, 380, 1, 0, 0, 0, 383, 388, 3, 60, 30, 0, 384, 385, 5, 37, 0, 0, 385, 387, 3, 60, 30, 0, 386, 384, 1, 0, 0, 0, 387, 390, 1, 0, 0, 0, 388, 386, 1, 0, 0, 0, 388, 389, 1, 0, 0, 0, 389, 55, 1, 0, 0, 0, 390, 388, 1, 0, 0, 0, 391, 396, 3, 54, 27, 0, 392, 393, 5, 35, 0, 0, 393, 395, 3, 54, 27, 0, 394, 392, 1, 0, 0, 0, 395, 398, 1, 0, 0, 0, 396, 394, 1, 0, 0, 0, 396, 397, 1, 0, 0, 0, 397, 57, 1, 0, 0, 0, 398, 396, 1, 0, 0, 0, 399, 400, 7, 3, 0, 0, 400, 59, 1, 0, 0, 0, 401, 402, 5, 81, 0, 0, 402, 61, 1, 0, 0, 0, 403, 446, 5, 46, 0, 0, 404, 405, 3, 96, 48, 0, 405, 406, 5, 68, 0, 0, 406, 446, 1, 0, 0, 0, 407, 446, 3, 94, 47, 0, 408, 446, 3, 96, 48, 0, 409, 446, 3, 90, 45, 0, 410, 446, 3, 64, 32, 0, 411, 446, 3, 98, 49, 0, 412, 413, 5, 66, 0, 0, 413, 418, 3, 92, 46, 0, 414, 415, 5, 35, 0, 0, 415, 417, 3, 92, 46, 0, 416, 414, 1, 0, 0, 0, 417, 420, 1, 0, 0, 0, 418, 416, 1, 0, 0, 0, 418, 419, 1, 0, 0, 0, 419, 421, 1, 0, 0, 0, 420, 418, 1, 0, 0, 0, 421, 422, 5, 67, 0, 0, 422, 446, 1, 0, 0, 0, 423, 424, 5, 66, 0, 0, 424, 429, 3, 90, 45, 0, 425, 426, 5, 35, 0, 0, 426, 428, 3, 90, 45, 0, 427, 425, 1, 0, 0, 0, 428, 431, 1, 0, 0, 0, 429, 427, 1, 0, 0, 0, 429, 430, 1, 0, 0, 0, 430, 432, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 432, 433, 5, 67, 0, 0, 433, 446, 1, 0, 0, 0, 434, 435, 5, 66, 0, 0, 435, 440, 3, 98, 49, 0, 436, 437, 5, 35, 0, 0, 437, 439, 3, 98, 49, 0, 438, 436, 1, 0, 0, 0, 439, 442, 1, 0, 0, 0, 440, 438, 1, 0, 0, 0, 440, 441, 1, 0, 0, 0, 441, 443, 1, 0, 0, 0, 442, 440, 1, 0, 0, 0, 443, 444, 5, 67, 0, 0, 444, 446, 1, 0, 0, 0, 445, 403, 1, 0, 0, 0, 445, 404, 1, 0, 0, 0, 445, 407, 1, 0, 0, 0, 445, 408, 1, 0, 0, 0, 445, 409, 1, 0, 0, 0, 445, 410, 1, 0, 0, 0, 445, 411, 1, 0, 0, 0, 445, 412, 1, 0, 0, 0, 445, 423, 1, 0, 0, 0, 445, 434, 1, 0, 0, 0, 446, 63, 1, 0, 0, 0, 447, 450, 5, 49, 0, 0, 448, 450, 5, 65, 0, 0, 449, 447, 1, 0, 0, 0, 449, 448, 1, 0, 0, 0, 450, 65, 1, 0, 0, 0, 451, 452, 5, 9, 0, 0, 452, 453, 5, 28, 0, 0, 453, 67, 1, 0, 0, 0, 454, 455, 5, 15, 0, 0, 455, 460, 3, 70, 35, 0, 456, 457, 5, 35, 0, 0, 457, 459, 3, 70, 35, 0, 458, 456, 1, 0, 0, 0, 459, 462, 1, 0, 0, 0, 460, 458, 1, 0, 0, 0, 460, 461, 1, 0, 0, 0, 461, 69, 1, 0, 0, 0, 462, 460, 1, 0, 0, 0, 463, 465, 3, 10, 5, 0, 464, 466, 7, 4, 0, 0, 465, 464, 1, 0, 0, 0, 465, 466, 1, 0, 0, 0, 466, 469, 1, 0, 0, 0, 467, 468, 5, 47, 0, 0, 468, 470, 7, 5, 0, 0, 469, 467, 1, 0, 0, 0, 469, 470, 1, 0, 0, 0, 470, 71, 1, 0, 0, 0, 471, 472, 5, 8, 0, 0, 472, 473, 3, 56, 28, 0, 473, 73, 1, 0, 0, 0, 474, 475, 5, 2, 0, 0, 475, 476, 3, 56, 28, 0, 476, 75, 1, 0, 0, 0, 477, 478, 5, 12, 0, 0, 478, 483, 3, 78, 39, 0, 479, 480, 5, 35, 0, 0, 480, 482, 3, 78, 39, 0, 481, 479, 1, 0, 0, 0, 482, 485, 1, 0, 0, 0, 483, 481, 1, 0, 0, 0, 483, 484, 1, 0, 0, 0, 484, 77, 1, 0, 0, 0, 485, 483, 1, 0, 0, 0, 486, 487, 3, 54, 27, 0, 487, 488, 5, 85, 0, 0, 488, 489, 3, 54, 27, 0, 489, 79, 1, 0, 0, 0, 490, 491, 5, 1, 0, 0, 491, 492, 3, 20, 10, 0, 492, 494, 3, 98, 49, 0, 493, 495, 3, 86, 43, 0, 494, 493, 1, 0, 0, 0, 494, 495, 1, 0, 0, 0, 495, 81, 1, 0, 0, 0, 496, 497, 5, 7, 0, 0, 497, 498, 3, 20, 10, 0, 498, 499, 3, 98, 49, 0, 499, 83, 1, 0, 0, 0, 500, 501, 5, 11, 0, 0, 501, 502, 3, 52, 26, 0, 502, 85, 1, 0, 0, 0, 503, 508, 3, 88, 44, 0, 504, 505, 5, 35, 0, 0, 505, 507, 3, 88, 44, 0, 506, 504, 1, 0, 0, 0, 507, 510, 1, 0, 0, 0, 508, 506, 1, 0, 0, 0, 508, 509, 1, 0, 0, 0, 509, 87, 1, 0, 0, 0, 510, 508, 1, 0, 0, 0, 511, 512, 3, 58, 29, 0, 512, 513, 5, 33, 0, 0, 513, 514, 3, 62, 31, 0, 514, 89, 1, 0, 0, 0, 515, 516, 7, 6, 0, 0, 516, 91, 1, 0, 0, 0, 517, 520, 3, 94, 47, 0, 518, 520, 3, 96, 48, 0, 519, 517, 1, 0, 0, 0, 519, 518, 1, 0, 0, 0, 520, 93, 1, 0, 0, 0, 521, 523, 7, 0, 0, 0, 522, 521, 1, 0, 0, 0, 522, 523, 1, 0, 0, 0, 523, 524, 1, 0, 0, 0, 524, 525, 5, 29, 0, 0, 525, 95, 1, 0, 0, 0, 526, 528, 7, 0, 0, 0, 527, 526, 1, 0, 0, 0, 527, 528, 1, 0, 0, 0, 528, 529, 1, 0, 0, 0, 529, 530, 5, 28, 0, 0, 530, 97, 1, 0, 0, 0, 531, 532, 5, 27, 0, 0, 532, 99, 1, 0, 0, 0, 533, 534, 7, 7, 0, 0, 534, 101, 1, 0, 0, 0, 535, 536, 5, 5, 0, 0, 536, 537, 3, 104, 52, 0, 537, 103, 1, 0, 0, 0, 538, 539, 5, 66, 0, 0, 539, 540, 3, 2, 1, 0, 540, 541, 5, 67, 0, 0, 541, 105, 1, 0, 0, 0, 542, 543, 5, 14, 0, 0, 543, 544, 5, 101, 0, 0, 544, 107, 1, 0, 0, 0, 545, 546, 5, 10, 0, 0, 546, 547, 5, 105, 0, 0, 547, 109, 1, 0, 0, 0, 548, 549, 5, 3, 0, 0, 549, 552, 5, 91, 0, 0, 550, 551, 5, 89, 0, 0, 551, 553, 3, 54, 27, 0, 552, 550, 1, 0, 0, 0, 552, 553, 1, 0, 0, 0, 553, 563, 1, 0, 0, 0, 554, 555, 5, 90, 0, 0, 555, 560, 3, 112, 56, 0, 556, 557, 5, 35, 0, 0, 557, 559, 3, 112, 56, 0, 558, 556, 1, 0, 0, 0, 559, 562, 1, 0, 0, 0, 560, 558, 1, 0, 0, 0, 560, 561, 1, 0, 0, 0, 561, 564, 1, 0, 0, 0, 562, 560, 1, 0, 0, 0, 563, 554, 1, 0, 0, 0, 563, 564, 1, 0, 0, 0, 564, 111, 1, 0, 0, 0, 565, 566, 3, 54, 27, 0, 566, 567, 5, 33, 0, 0, 567, 569, 1, 0, 0, 0, 568, 565, 1, 0, 0, 0, 568, 569, 1, 0, 0, 0, 569, 570, 1, 0, 0, 0, 570, 571, 3, 54, 27, 0, 571, 113, 1, 0, 0, 0, 572, 573, 5, 19, 0, 0, 573, 574, 3, 34, 17, 0, 574, 575, 5, 89, 0, 0, 575, 576, 3, 56, 28, 0, 576, 115, 1, 0, 0, 0, 577, 578, 5, 18, 0, 0, 578, 581, 3, 28, 14, 0, 579, 580, 5, 30, 0, 0, 580, 582, 3, 28, 14, 0, 581, 579, 1, 0, 0, 0, 581, 582, 1, 0, 0, 0, 582, 117, 1, 0, 0, 0, 583, 584, 5, 20, 0, 0, 584, 585, 3, 120, 60, 0, 585, 119, 1, 0, 0, 0, 586, 587, 5, 27, 0, 0, 587, 121, 1, 0, 0, 0, 54, 133, 143, 163, 175, 184, 192, 198, 206, 208, 213, 220, 225, 236, 242, 250, 252, 263, 270, 281, 284, 298, 306, 314, 318, 325, 333, 341, 354, 358, 362, 369, 373, 380, 388, 396, 418, 429, 440, 445, 449, 460, 465, 469, 483, 494, 508, 519, 522, 527, 552, 560, 563, 568, 581]
\ No newline at end of file
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
index afaf57ba1d218..fb63e31a37c90 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
@@ -1,5 +1,13 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.xpack.esql.parser;
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
 import org.antlr.v4.runtime.atn.*;
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.*;
@@ -10,72 +18,76 @@
 import java.util.ArrayList;
 
 @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue"})
-public class EsqlBaseParser extends Parser {
+public class EsqlBaseParser extends ParserConfig {
   static { RuntimeMetaData.checkVersion("4.13.1", RuntimeMetaData.VERSION); }
 
   protected static final DFA[] _decisionToDFA;
   protected static final PredictionContextCache _sharedContextCache =
     new PredictionContextCache();
   public static final int
-    DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, INLINESTATS=8, 
-    KEEP=9, LIMIT=10, LOOKUP=11, META=12, METRICS=13, MV_EXPAND=14, RENAME=15, 
-    ROW=16, SHOW=17, SORT=18, STATS=19, WHERE=20, UNKNOWN_CMD=21, LINE_COMMENT=22, 
-    MULTILINE_COMMENT=23, WS=24, UNQUOTED_SOURCE=25, EXPLAIN_WS=26, EXPLAIN_LINE_COMMENT=27, 
-    EXPLAIN_MULTILINE_COMMENT=28, PIPE=29, QUOTED_STRING=30, INTEGER_LITERAL=31, 
-    DECIMAL_LITERAL=32, BY=33, AND=34, ASC=35, ASSIGN=36, CAST_OP=37, COMMA=38, 
-    DESC=39, DOT=40, FALSE=41, FIRST=42, LAST=43, LP=44, IN=45, IS=46, LIKE=47, 
-    NOT=48, NULL=49, NULLS=50, OR=51, PARAM=52, RLIKE=53, RP=54, TRUE=55, 
-    EQ=56, CIEQ=57, NEQ=58, LT=59, LTE=60, GT=61, GTE=62, PLUS=63, MINUS=64, 
-    ASTERISK=65, SLASH=66, PERCENT=67, NAMED_OR_POSITIONAL_PARAM=68, OPENING_BRACKET=69, 
-    CLOSING_BRACKET=70, UNQUOTED_IDENTIFIER=71, QUOTED_IDENTIFIER=72, EXPR_LINE_COMMENT=73, 
-    EXPR_MULTILINE_COMMENT=74, EXPR_WS=75, METADATA=76, FROM_LINE_COMMENT=77, 
-    FROM_MULTILINE_COMMENT=78, FROM_WS=79, ID_PATTERN=80, PROJECT_LINE_COMMENT=81, 
-    PROJECT_MULTILINE_COMMENT=82, PROJECT_WS=83, AS=84, RENAME_LINE_COMMENT=85, 
-    RENAME_MULTILINE_COMMENT=86, RENAME_WS=87, ON=88, WITH=89, ENRICH_POLICY_NAME=90, 
-    ENRICH_LINE_COMMENT=91, ENRICH_MULTILINE_COMMENT=92, ENRICH_WS=93, ENRICH_FIELD_LINE_COMMENT=94, 
-    ENRICH_FIELD_MULTILINE_COMMENT=95, ENRICH_FIELD_WS=96, LOOKUP_LINE_COMMENT=97, 
-    LOOKUP_MULTILINE_COMMENT=98, LOOKUP_WS=99, LOOKUP_FIELD_LINE_COMMENT=100, 
-    LOOKUP_FIELD_MULTILINE_COMMENT=101, LOOKUP_FIELD_WS=102, MVEXPAND_LINE_COMMENT=103, 
-    MVEXPAND_MULTILINE_COMMENT=104, MVEXPAND_WS=105, INFO=106, SHOW_LINE_COMMENT=107, 
-    SHOW_MULTILINE_COMMENT=108, SHOW_WS=109, FUNCTIONS=110, META_LINE_COMMENT=111, 
-    META_MULTILINE_COMMENT=112, META_WS=113, COLON=114, SETTING=115, SETTING_LINE_COMMENT=116, 
-    SETTTING_MULTILINE_COMMENT=117, SETTING_WS=118, METRICS_LINE_COMMENT=119, 
-    METRICS_MULTILINE_COMMENT=120, METRICS_WS=121, CLOSING_METRICS_LINE_COMMENT=122, 
-    CLOSING_METRICS_MULTILINE_COMMENT=123, CLOSING_METRICS_WS=124;
+    DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, 
+    LIMIT=9, META=10, MV_EXPAND=11, RENAME=12, ROW=13, SHOW=14, SORT=15, STATS=16, 
+    WHERE=17, DEV_INLINESTATS=18, DEV_LOOKUP=19, DEV_MATCH=20, DEV_METRICS=21, 
+    UNKNOWN_CMD=22, LINE_COMMENT=23, MULTILINE_COMMENT=24, WS=25, PIPE=26, 
+    QUOTED_STRING=27, INTEGER_LITERAL=28, DECIMAL_LITERAL=29, BY=30, AND=31, 
+    ASC=32, ASSIGN=33, CAST_OP=34, COMMA=35, DESC=36, DOT=37, FALSE=38, FIRST=39, 
+    IN=40, IS=41, LAST=42, LIKE=43, LP=44, NOT=45, NULL=46, NULLS=47, OR=48, 
+    PARAM=49, RLIKE=50, RP=51, TRUE=52, EQ=53, CIEQ=54, NEQ=55, LT=56, LTE=57, 
+    GT=58, GTE=59, PLUS=60, MINUS=61, ASTERISK=62, SLASH=63, PERCENT=64, NAMED_OR_POSITIONAL_PARAM=65, 
+    OPENING_BRACKET=66, CLOSING_BRACKET=67, UNQUOTED_IDENTIFIER=68, QUOTED_IDENTIFIER=69, 
+    EXPR_LINE_COMMENT=70, EXPR_MULTILINE_COMMENT=71, EXPR_WS=72, EXPLAIN_WS=73, 
+    EXPLAIN_LINE_COMMENT=74, EXPLAIN_MULTILINE_COMMENT=75, METADATA=76, UNQUOTED_SOURCE=77, 
+    FROM_LINE_COMMENT=78, FROM_MULTILINE_COMMENT=79, FROM_WS=80, ID_PATTERN=81, 
+    PROJECT_LINE_COMMENT=82, PROJECT_MULTILINE_COMMENT=83, PROJECT_WS=84, 
+    AS=85, RENAME_LINE_COMMENT=86, RENAME_MULTILINE_COMMENT=87, RENAME_WS=88, 
+    ON=89, WITH=90, ENRICH_POLICY_NAME=91, ENRICH_LINE_COMMENT=92, ENRICH_MULTILINE_COMMENT=93, 
+    ENRICH_WS=94, ENRICH_FIELD_LINE_COMMENT=95, ENRICH_FIELD_MULTILINE_COMMENT=96, 
+    ENRICH_FIELD_WS=97, MVEXPAND_LINE_COMMENT=98, MVEXPAND_MULTILINE_COMMENT=99, 
+    MVEXPAND_WS=100, INFO=101, SHOW_LINE_COMMENT=102, SHOW_MULTILINE_COMMENT=103, 
+    SHOW_WS=104, FUNCTIONS=105, META_LINE_COMMENT=106, META_MULTILINE_COMMENT=107, 
+    META_WS=108, COLON=109, SETTING=110, SETTING_LINE_COMMENT=111, SETTTING_MULTILINE_COMMENT=112, 
+    SETTING_WS=113, LOOKUP_LINE_COMMENT=114, LOOKUP_MULTILINE_COMMENT=115, 
+    LOOKUP_WS=116, LOOKUP_FIELD_LINE_COMMENT=117, LOOKUP_FIELD_MULTILINE_COMMENT=118, 
+    LOOKUP_FIELD_WS=119, METRICS_LINE_COMMENT=120, METRICS_MULTILINE_COMMENT=121, 
+    METRICS_WS=122, CLOSING_METRICS_LINE_COMMENT=123, CLOSING_METRICS_MULTILINE_COMMENT=124, 
+    CLOSING_METRICS_WS=125;
   public static final int
     RULE_singleStatement = 0, RULE_query = 1, RULE_sourceCommand = 2, RULE_processingCommand = 3, 
     RULE_whereCommand = 4, RULE_booleanExpression = 5, RULE_regexBooleanExpression = 6, 
-    RULE_valueExpression = 7, RULE_operatorExpression = 8, RULE_primaryExpression = 9, 
-    RULE_functionExpression = 10, RULE_dataType = 11, RULE_rowCommand = 12, 
-    RULE_fields = 13, RULE_field = 14, RULE_fromCommand = 15, RULE_indexPattern = 16, 
-    RULE_clusterString = 17, RULE_indexString = 18, RULE_metadata = 19, RULE_metadataOption = 20, 
-    RULE_deprecated_metadata = 21, RULE_metricsCommand = 22, RULE_evalCommand = 23, 
-    RULE_statsCommand = 24, RULE_inlinestatsCommand = 25, RULE_qualifiedName = 26, 
-    RULE_qualifiedNamePattern = 27, RULE_qualifiedNamePatterns = 28, RULE_identifier = 29, 
-    RULE_identifierPattern = 30, RULE_constant = 31, RULE_params = 32, RULE_limitCommand = 33, 
-    RULE_sortCommand = 34, RULE_orderExpression = 35, RULE_keepCommand = 36, 
-    RULE_dropCommand = 37, RULE_renameCommand = 38, RULE_renameClause = 39, 
-    RULE_dissectCommand = 40, RULE_grokCommand = 41, RULE_mvExpandCommand = 42, 
-    RULE_commandOptions = 43, RULE_commandOption = 44, RULE_booleanValue = 45, 
-    RULE_numericValue = 46, RULE_decimalValue = 47, RULE_integerValue = 48, 
-    RULE_string = 49, RULE_comparisonOperator = 50, RULE_explainCommand = 51, 
-    RULE_subqueryExpression = 52, RULE_showCommand = 53, RULE_metaCommand = 54, 
-    RULE_enrichCommand = 55, RULE_enrichWithClause = 56, RULE_lookupCommand = 57;
+    RULE_matchBooleanExpression = 7, RULE_valueExpression = 8, RULE_operatorExpression = 9, 
+    RULE_primaryExpression = 10, RULE_functionExpression = 11, RULE_dataType = 12, 
+    RULE_rowCommand = 13, RULE_fields = 14, RULE_field = 15, RULE_fromCommand = 16, 
+    RULE_indexPattern = 17, RULE_clusterString = 18, RULE_indexString = 19, 
+    RULE_metadata = 20, RULE_metadataOption = 21, RULE_deprecated_metadata = 22, 
+    RULE_metricsCommand = 23, RULE_evalCommand = 24, RULE_statsCommand = 25, 
+    RULE_qualifiedName = 26, RULE_qualifiedNamePattern = 27, RULE_qualifiedNamePatterns = 28, 
+    RULE_identifier = 29, RULE_identifierPattern = 30, RULE_constant = 31, 
+    RULE_params = 32, RULE_limitCommand = 33, RULE_sortCommand = 34, RULE_orderExpression = 35, 
+    RULE_keepCommand = 36, RULE_dropCommand = 37, RULE_renameCommand = 38, 
+    RULE_renameClause = 39, RULE_dissectCommand = 40, RULE_grokCommand = 41, 
+    RULE_mvExpandCommand = 42, RULE_commandOptions = 43, RULE_commandOption = 44, 
+    RULE_booleanValue = 45, RULE_numericValue = 46, RULE_decimalValue = 47, 
+    RULE_integerValue = 48, RULE_string = 49, RULE_comparisonOperator = 50, 
+    RULE_explainCommand = 51, RULE_subqueryExpression = 52, RULE_showCommand = 53, 
+    RULE_metaCommand = 54, RULE_enrichCommand = 55, RULE_enrichWithClause = 56, 
+    RULE_lookupCommand = 57, RULE_inlinestatsCommand = 58, RULE_matchCommand = 59, 
+    RULE_matchQuery = 60;
   private static String[] makeRuleNames() {
     return new String[] {
       "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", 
-      "booleanExpression", "regexBooleanExpression", "valueExpression", "operatorExpression", 
-      "primaryExpression", "functionExpression", "dataType", "rowCommand", 
-      "fields", "field", "fromCommand", "indexPattern", "clusterString", "indexString", 
-      "metadata", "metadataOption", "deprecated_metadata", "metricsCommand", 
-      "evalCommand", "statsCommand", "inlinestatsCommand", "qualifiedName", 
-      "qualifiedNamePattern", "qualifiedNamePatterns", "identifier", "identifierPattern", 
-      "constant", "params", "limitCommand", "sortCommand", "orderExpression", 
-      "keepCommand", "dropCommand", "renameCommand", "renameClause", "dissectCommand", 
-      "grokCommand", "mvExpandCommand", "commandOptions", "commandOption", 
-      "booleanValue", "numericValue", "decimalValue", "integerValue", "string", 
-      "comparisonOperator", "explainCommand", "subqueryExpression", "showCommand", 
-      "metaCommand", "enrichCommand", "enrichWithClause", "lookupCommand"
+      "booleanExpression", "regexBooleanExpression", "matchBooleanExpression", 
+      "valueExpression", "operatorExpression", "primaryExpression", "functionExpression", 
+      "dataType", "rowCommand", "fields", "field", "fromCommand", "indexPattern", 
+      "clusterString", "indexString", "metadata", "metadataOption", "deprecated_metadata", 
+      "metricsCommand", "evalCommand", "statsCommand", "qualifiedName", "qualifiedNamePattern", 
+      "qualifiedNamePatterns", "identifier", "identifierPattern", "constant", 
+      "params", "limitCommand", "sortCommand", "orderExpression", "keepCommand", 
+      "dropCommand", "renameCommand", "renameClause", "dissectCommand", "grokCommand", 
+      "mvExpandCommand", "commandOptions", "commandOption", "booleanValue", 
+      "numericValue", "decimalValue", "integerValue", "string", "comparisonOperator", 
+      "explainCommand", "subqueryExpression", "showCommand", "metaCommand", 
+      "enrichCommand", "enrichWithClause", "lookupCommand", "inlinestatsCommand", 
+      "matchCommand", "matchQuery"
     };
   }
   public static final String[] ruleNames = makeRuleNames();
@@ -83,46 +95,45 @@ private static String[] makeRuleNames() {
   private static String[] makeLiteralNames() {
     return new String[] {
       null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", 
-      "'grok'", "'inlinestats'", "'keep'", "'limit'", "'lookup'", "'meta'", 
-      "'metrics'", "'mv_expand'", "'rename'", "'row'", "'show'", "'sort'", 
-      "'stats'", "'where'", null, null, null, null, null, null, null, null, 
-      "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "','", 
-      "'desc'", "'.'", "'false'", "'first'", "'last'", "'('", "'in'", "'is'", 
-      "'like'", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", 
-      "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", 
-      "'-'", "'*'", "'/'", "'%'", null, null, "']'", null, null, null, null, 
-      null, "'metadata'", null, null, null, null, null, null, null, "'as'", 
-      null, null, null, "'on'", "'with'", null, null, null, null, null, null, 
-      null, null, null, null, null, null, null, null, null, null, "'info'", 
-      null, null, null, "'functions'", null, null, null, "':'"
+      "'grok'", "'keep'", "'limit'", "'meta'", "'mv_expand'", "'rename'", "'row'", 
+      "'show'", "'sort'", "'stats'", "'where'", null, null, null, null, null, 
+      null, null, null, "'|'", null, null, null, "'by'", "'and'", "'asc'", 
+      "'='", "'::'", "','", "'desc'", "'.'", "'false'", "'first'", "'in'", 
+      "'is'", "'last'", "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", 
+      "'?'", "'rlike'", "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", 
+      "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", 
+      null, null, null, null, null, null, null, null, "'metadata'", null, null, 
+      null, null, null, null, null, null, "'as'", null, null, null, "'on'", 
+      "'with'", null, null, null, null, null, null, null, null, null, null, 
+      "'info'", null, null, null, "'functions'", null, null, null, "':'"
     };
   }
   private static final String[] _LITERAL_NAMES = makeLiteralNames();
   private static String[] makeSymbolicNames() {
     return new String[] {
       null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", 
-      "INLINESTATS", "KEEP", "LIMIT", "LOOKUP", "META", "METRICS", "MV_EXPAND", 
-      "RENAME", "ROW", "SHOW", "SORT", "STATS", "WHERE", "UNKNOWN_CMD", "LINE_COMMENT", 
-      "MULTILINE_COMMENT", "WS", "UNQUOTED_SOURCE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", 
-      "EXPLAIN_MULTILINE_COMMENT", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", 
-      "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", 
-      "DESC", "DOT", "FALSE", "FIRST", "LAST", "LP", "IN", "IS", "LIKE", "NOT", 
-      "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", 
-      "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", 
-      "PERCENT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", 
+      "KEEP", "LIMIT", "META", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", 
+      "STATS", "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", 
+      "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", 
+      "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", 
+      "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", 
+      "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", 
+      "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", 
+      "SLASH", "PERCENT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", 
       "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", 
-      "EXPR_WS", "METADATA", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", 
+      "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", 
+      "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", 
       "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", 
       "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", 
       "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", "ENRICH_LINE_COMMENT", 
       "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", 
-      "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "LOOKUP_LINE_COMMENT", 
-      "LOOKUP_MULTILINE_COMMENT", "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", 
-      "LOOKUP_FIELD_MULTILINE_COMMENT", "LOOKUP_FIELD_WS", "MVEXPAND_LINE_COMMENT", 
+      "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", 
       "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", 
       "SHOW_MULTILINE_COMMENT", "SHOW_WS", "FUNCTIONS", "META_LINE_COMMENT", 
       "META_MULTILINE_COMMENT", "META_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", 
-      "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", 
+      "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", 
+      "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", 
+      "LOOKUP_FIELD_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", 
       "METRICS_WS", "CLOSING_METRICS_LINE_COMMENT", "CLOSING_METRICS_MULTILINE_COMMENT", 
       "CLOSING_METRICS_WS"
     };
@@ -211,9 +222,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(116);
+      setState(122);
       query(0);
-      setState(117);
+      setState(123);
       match(EOF);
       }
     }
@@ -309,11 +320,11 @@ private QueryContext query(int _p) throws RecognitionException {
       _ctx = _localctx;
       _prevctx = _localctx;
 
-      setState(120);
+      setState(126);
       sourceCommand();
       }
       _ctx.stop = _input.LT(-1);
-      setState(127);
+      setState(133);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -324,16 +335,16 @@ private QueryContext query(int _p) throws RecognitionException {
           {
           _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState));
           pushNewRecursionContext(_localctx, _startState, RULE_query);
-          setState(122);
+          setState(128);
           if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-          setState(123);
+          setState(129);
           match(PIPE);
-          setState(124);
+          setState(130);
           processingCommand();
           }
           } 
         }
-        setState(129);
+        setState(135);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       }
@@ -358,17 +369,17 @@ public ExplainCommandContext explainCommand() {
     public FromCommandContext fromCommand() {
       return getRuleContext(FromCommandContext.class,0);
     }
+    public MetaCommandContext metaCommand() {
+      return getRuleContext(MetaCommandContext.class,0);
+    }
     public RowCommandContext rowCommand() {
       return getRuleContext(RowCommandContext.class,0);
     }
-    public MetricsCommandContext metricsCommand() {
-      return getRuleContext(MetricsCommandContext.class,0);
-    }
     public ShowCommandContext showCommand() {
       return getRuleContext(ShowCommandContext.class,0);
     }
-    public MetaCommandContext metaCommand() {
-      return getRuleContext(MetaCommandContext.class,0);
+    public MetricsCommandContext metricsCommand() {
+      return getRuleContext(MetricsCommandContext.class,0);
     }
     @SuppressWarnings("this-escape")
     public SourceCommandContext(ParserRuleContext parent, int invokingState) {
@@ -394,53 +405,53 @@ public final SourceCommandContext sourceCommand() throws RecognitionException {
     SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState());
     enterRule(_localctx, 4, RULE_sourceCommand);
     try {
-      setState(136);
+      setState(143);
       _errHandler.sync(this);
-      switch (_input.LA(1)) {
-      case EXPLAIN:
+      switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) {
+      case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(130);
+        setState(136);
         explainCommand();
         }
         break;
-      case FROM:
+      case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(131);
+        setState(137);
         fromCommand();
         }
         break;
-      case ROW:
+      case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(132);
-        rowCommand();
+        setState(138);
+        metaCommand();
         }
         break;
-      case METRICS:
+      case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(133);
-        metricsCommand();
+        setState(139);
+        rowCommand();
         }
         break;
-      case SHOW:
+      case 5:
         enterOuterAlt(_localctx, 5);
         {
-        setState(134);
+        setState(140);
         showCommand();
         }
         break;
-      case META:
+      case 6:
         enterOuterAlt(_localctx, 6);
         {
-        setState(135);
-        metaCommand();
+        setState(141);
+        if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
+        setState(142);
+        metricsCommand();
         }
         break;
-      default:
-        throw new NoViableAltException(this);
       }
     }
     catch (RecognitionException re) {
@@ -459,26 +470,20 @@ public static class ProcessingCommandContext extends ParserRuleContext {
     public EvalCommandContext evalCommand() {
       return getRuleContext(EvalCommandContext.class,0);
     }
-    public InlinestatsCommandContext inlinestatsCommand() {
-      return getRuleContext(InlinestatsCommandContext.class,0);
-    }
-    public LimitCommandContext limitCommand() {
-      return getRuleContext(LimitCommandContext.class,0);
-    }
-    public LookupCommandContext lookupCommand() {
-      return getRuleContext(LookupCommandContext.class,0);
+    public WhereCommandContext whereCommand() {
+      return getRuleContext(WhereCommandContext.class,0);
     }
     public KeepCommandContext keepCommand() {
       return getRuleContext(KeepCommandContext.class,0);
     }
-    public SortCommandContext sortCommand() {
-      return getRuleContext(SortCommandContext.class,0);
+    public LimitCommandContext limitCommand() {
+      return getRuleContext(LimitCommandContext.class,0);
     }
     public StatsCommandContext statsCommand() {
       return getRuleContext(StatsCommandContext.class,0);
     }
-    public WhereCommandContext whereCommand() {
-      return getRuleContext(WhereCommandContext.class,0);
+    public SortCommandContext sortCommand() {
+      return getRuleContext(SortCommandContext.class,0);
     }
     public DropCommandContext dropCommand() {
       return getRuleContext(DropCommandContext.class,0);
@@ -498,6 +503,15 @@ public EnrichCommandContext enrichCommand() {
     public MvExpandCommandContext mvExpandCommand() {
       return getRuleContext(MvExpandCommandContext.class,0);
     }
+    public InlinestatsCommandContext inlinestatsCommand() {
+      return getRuleContext(InlinestatsCommandContext.class,0);
+    }
+    public LookupCommandContext lookupCommand() {
+      return getRuleContext(LookupCommandContext.class,0);
+    }
+    public MatchCommandContext matchCommand() {
+      return getRuleContext(MatchCommandContext.class,0);
+    }
     @SuppressWarnings("this-escape")
     public ProcessingCommandContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
@@ -522,109 +536,120 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce
     ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState());
     enterRule(_localctx, 6, RULE_processingCommand);
     try {
-      setState(152);
+      setState(163);
       _errHandler.sync(this);
-      switch (_input.LA(1)) {
-      case EVAL:
+      switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) {
+      case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(138);
+        setState(145);
         evalCommand();
         }
         break;
-      case INLINESTATS:
+      case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(139);
-        inlinestatsCommand();
+        setState(146);
+        whereCommand();
         }
         break;
-      case LIMIT:
+      case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(140);
-        limitCommand();
+        setState(147);
+        keepCommand();
         }
         break;
-      case LOOKUP:
+      case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(141);
-        lookupCommand();
+        setState(148);
+        limitCommand();
         }
         break;
-      case KEEP:
+      case 5:
         enterOuterAlt(_localctx, 5);
         {
-        setState(142);
-        keepCommand();
+        setState(149);
+        statsCommand();
         }
         break;
-      case SORT:
+      case 6:
         enterOuterAlt(_localctx, 6);
         {
-        setState(143);
+        setState(150);
         sortCommand();
         }
         break;
-      case STATS:
+      case 7:
         enterOuterAlt(_localctx, 7);
         {
-        setState(144);
-        statsCommand();
+        setState(151);
+        dropCommand();
         }
         break;
-      case WHERE:
+      case 8:
         enterOuterAlt(_localctx, 8);
         {
-        setState(145);
-        whereCommand();
+        setState(152);
+        renameCommand();
         }
         break;
-      case DROP:
+      case 9:
         enterOuterAlt(_localctx, 9);
         {
-        setState(146);
-        dropCommand();
+        setState(153);
+        dissectCommand();
         }
         break;
-      case RENAME:
+      case 10:
         enterOuterAlt(_localctx, 10);
         {
-        setState(147);
-        renameCommand();
+        setState(154);
+        grokCommand();
         }
         break;
-      case DISSECT:
+      case 11:
         enterOuterAlt(_localctx, 11);
         {
-        setState(148);
-        dissectCommand();
+        setState(155);
+        enrichCommand();
         }
         break;
-      case GROK:
+      case 12:
         enterOuterAlt(_localctx, 12);
         {
-        setState(149);
-        grokCommand();
+        setState(156);
+        mvExpandCommand();
         }
         break;
-      case ENRICH:
+      case 13:
         enterOuterAlt(_localctx, 13);
         {
-        setState(150);
-        enrichCommand();
+        setState(157);
+        if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
+        setState(158);
+        inlinestatsCommand();
         }
         break;
-      case MV_EXPAND:
+      case 14:
         enterOuterAlt(_localctx, 14);
         {
-        setState(151);
-        mvExpandCommand();
+        setState(159);
+        if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
+        setState(160);
+        lookupCommand();
+        }
+        break;
+      case 15:
+        enterOuterAlt(_localctx, 15);
+        {
+        setState(161);
+        if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
+        setState(162);
+        matchCommand();
         }
         break;
-      default:
-        throw new NoViableAltException(this);
       }
     }
     catch (RecognitionException re) {
@@ -670,9 +695,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(154);
+      setState(165);
       match(WHERE);
-      setState(155);
+      setState(166);
       booleanExpression(0);
       }
     }
@@ -702,6 +727,27 @@ public void copyFrom(BooleanExpressionContext ctx) {
     }
   }
   @SuppressWarnings("CheckReturnValue")
+  public static class MatchExpressionContext extends BooleanExpressionContext {
+    public MatchBooleanExpressionContext matchBooleanExpression() {
+      return getRuleContext(MatchBooleanExpressionContext.class,0);
+    }
+    @SuppressWarnings("this-escape")
+    public MatchExpressionContext(BooleanExpressionContext ctx) { copyFrom(ctx); }
+    @Override
+    public void enterRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterMatchExpression(this);
+    }
+    @Override
+    public void exitRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitMatchExpression(this);
+    }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitMatchExpression(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  @SuppressWarnings("CheckReturnValue")
   public static class LogicalNotContext extends BooleanExpressionContext {
     public TerminalNode NOT() { return getToken(EsqlBaseParser.NOT, 0); }
     public BooleanExpressionContext booleanExpression() {
@@ -867,7 +913,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(185);
+      setState(198);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
       case 1:
@@ -876,10 +922,10 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(158);
+        setState(169);
         match(NOT);
-        setState(159);
-        booleanExpression(7);
+        setState(170);
+        booleanExpression(8);
         }
         break;
       case 2:
@@ -887,7 +933,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new BooleanDefaultContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(160);
+        setState(171);
         valueExpression();
         }
         break;
@@ -896,7 +942,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new RegexExpressionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(161);
+        setState(172);
         regexBooleanExpression();
         }
         break;
@@ -905,41 +951,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new LogicalInContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(162);
+        setState(173);
         valueExpression();
-        setState(164);
+        setState(175);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(163);
+          setState(174);
           match(NOT);
           }
         }
 
-        setState(166);
+        setState(177);
         match(IN);
-        setState(167);
+        setState(178);
         match(LP);
-        setState(168);
+        setState(179);
         valueExpression();
-        setState(173);
+        setState(184);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(169);
+          setState(180);
           match(COMMA);
-          setState(170);
+          setState(181);
           valueExpression();
           }
           }
-          setState(175);
+          setState(186);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(176);
+        setState(187);
         match(RP);
         }
         break;
@@ -948,27 +994,38 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new IsNullContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(178);
+        setState(189);
         valueExpression();
-        setState(179);
+        setState(190);
         match(IS);
-        setState(181);
+        setState(192);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(180);
+          setState(191);
           match(NOT);
           }
         }
 
-        setState(183);
+        setState(194);
         match(NULL);
         }
         break;
+      case 6:
+        {
+        _localctx = new MatchExpressionContext(_localctx);
+        _ctx = _localctx;
+        _prevctx = _localctx;
+        setState(196);
+        if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()");
+        setState(197);
+        matchBooleanExpression();
+        }
+        break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(195);
+      setState(208);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,8,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -976,7 +1033,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(193);
+          setState(206);
           _errHandler.sync(this);
           switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) {
           case 1:
@@ -984,12 +1041,12 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
             _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState));
             ((LogicalBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression);
-            setState(187);
-            if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(188);
+            setState(200);
+            if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
+            setState(201);
             ((LogicalBinaryContext)_localctx).operator = match(AND);
-            setState(189);
-            ((LogicalBinaryContext)_localctx).right = booleanExpression(5);
+            setState(202);
+            ((LogicalBinaryContext)_localctx).right = booleanExpression(6);
             }
             break;
           case 2:
@@ -997,18 +1054,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
             _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState));
             ((LogicalBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression);
-            setState(190);
-            if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(191);
+            setState(203);
+            if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
+            setState(204);
             ((LogicalBinaryContext)_localctx).operator = match(OR);
-            setState(192);
-            ((LogicalBinaryContext)_localctx).right = booleanExpression(4);
+            setState(205);
+            ((LogicalBinaryContext)_localctx).right = booleanExpression(5);
             }
             break;
           }
           } 
         }
-        setState(197);
+        setState(210);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,8,_ctx);
       }
@@ -1063,48 +1120,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog
     enterRule(_localctx, 12, RULE_regexBooleanExpression);
     int _la;
     try {
-      setState(212);
+      setState(225);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(198);
+        setState(211);
         valueExpression();
-        setState(200);
+        setState(213);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(199);
+          setState(212);
           match(NOT);
           }
         }
 
-        setState(202);
+        setState(215);
         ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE);
-        setState(203);
+        setState(216);
         ((RegexBooleanExpressionContext)_localctx).pattern = string();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(205);
+        setState(218);
         valueExpression();
-        setState(207);
+        setState(220);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(206);
+          setState(219);
           match(NOT);
           }
         }
 
-        setState(209);
+        setState(222);
         ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE);
-        setState(210);
+        setState(223);
         ((RegexBooleanExpressionContext)_localctx).pattern = string();
         }
         break;
@@ -1121,6 +1178,61 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog
     return _localctx;
   }
 
+  @SuppressWarnings("CheckReturnValue")
+  public static class MatchBooleanExpressionContext extends ParserRuleContext {
+    public StringContext queryString;
+    public ValueExpressionContext valueExpression() {
+      return getRuleContext(ValueExpressionContext.class,0);
+    }
+    public TerminalNode DEV_MATCH() { return getToken(EsqlBaseParser.DEV_MATCH, 0); }
+    public StringContext string() {
+      return getRuleContext(StringContext.class,0);
+    }
+    @SuppressWarnings("this-escape")
+    public MatchBooleanExpressionContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_matchBooleanExpression; }
+    @Override
+    public void enterRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterMatchBooleanExpression(this);
+    }
+    @Override
+    public void exitRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitMatchBooleanExpression(this);
+    }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitMatchBooleanExpression(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final MatchBooleanExpressionContext matchBooleanExpression() throws RecognitionException {
+    MatchBooleanExpressionContext _localctx = new MatchBooleanExpressionContext(_ctx, getState());
+    enterRule(_localctx, 14, RULE_matchBooleanExpression);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(227);
+      valueExpression();
+      setState(228);
+      match(DEV_MATCH);
+      setState(229);
+      ((MatchBooleanExpressionContext)_localctx).queryString = string();
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   @SuppressWarnings("CheckReturnValue")
   public static class ValueExpressionContext extends ParserRuleContext {
     @SuppressWarnings("this-escape")
@@ -1188,16 +1300,16 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ValueExpressionContext valueExpression() throws RecognitionException {
     ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState());
-    enterRule(_localctx, 14, RULE_valueExpression);
+    enterRule(_localctx, 16, RULE_valueExpression);
     try {
-      setState(219);
+      setState(236);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) {
       case 1:
         _localctx = new ValueExpressionDefaultContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(214);
+        setState(231);
         operatorExpression(0);
         }
         break;
@@ -1205,11 +1317,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio
         _localctx = new ComparisonContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(215);
+        setState(232);
         ((ComparisonContext)_localctx).left = operatorExpression(0);
-        setState(216);
+        setState(233);
         comparisonOperator();
-        setState(217);
+        setState(234);
         ((ComparisonContext)_localctx).right = operatorExpression(0);
         }
         break;
@@ -1327,14 +1439,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
     int _parentState = getState();
     OperatorExpressionContext _localctx = new OperatorExpressionContext(_ctx, _parentState);
     OperatorExpressionContext _prevctx = _localctx;
-    int _startState = 16;
-    enterRecursionRule(_localctx, 16, RULE_operatorExpression, _p);
+    int _startState = 18;
+    enterRecursionRule(_localctx, 18, RULE_operatorExpression, _p);
     int _la;
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(225);
+      setState(242);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) {
       case 1:
@@ -1343,7 +1455,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(222);
+        setState(239);
         primaryExpression(0);
         }
         break;
@@ -1352,7 +1464,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
         _localctx = new ArithmeticUnaryContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(223);
+        setState(240);
         ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
@@ -1363,13 +1475,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
           _errHandler.reportMatch(this);
           consume();
         }
-        setState(224);
+        setState(241);
         operatorExpression(3);
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(235);
+      setState(252);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,15,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1377,7 +1489,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(233);
+          setState(250);
           _errHandler.sync(this);
           switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) {
           case 1:
@@ -1385,12 +1497,12 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
             _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState));
             ((ArithmeticBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression);
-            setState(227);
+            setState(244);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(228);
+            setState(245);
             ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1);
             _la = _input.LA(1);
-            if ( !(((((_la - 65)) & ~0x3f) == 0 && ((1L << (_la - 65)) & 7L) != 0)) ) {
+            if ( !(((((_la - 62)) & ~0x3f) == 0 && ((1L << (_la - 62)) & 7L) != 0)) ) {
               ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this);
             }
             else {
@@ -1398,7 +1510,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
               _errHandler.reportMatch(this);
               consume();
             }
-            setState(229);
+            setState(246);
             ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3);
             }
             break;
@@ -1407,9 +1519,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
             _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState));
             ((ArithmeticBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression);
-            setState(230);
+            setState(247);
             if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-            setState(231);
+            setState(248);
             ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1);
             _la = _input.LA(1);
             if ( !(_la==PLUS || _la==MINUS) ) {
@@ -1420,14 +1532,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
               _errHandler.reportMatch(this);
               consume();
             }
-            setState(232);
+            setState(249);
             ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2);
             }
             break;
           }
           } 
         }
-        setState(237);
+        setState(254);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,15,_ctx);
       }
@@ -1579,13 +1691,13 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
     int _parentState = getState();
     PrimaryExpressionContext _localctx = new PrimaryExpressionContext(_ctx, _parentState);
     PrimaryExpressionContext _prevctx = _localctx;
-    int _startState = 18;
-    enterRecursionRule(_localctx, 18, RULE_primaryExpression, _p);
+    int _startState = 20;
+    enterRecursionRule(_localctx, 20, RULE_primaryExpression, _p);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(246);
+      setState(263);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) {
       case 1:
@@ -1594,7 +1706,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(239);
+        setState(256);
         constant();
         }
         break;
@@ -1603,7 +1715,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new DereferenceContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(240);
+        setState(257);
         qualifiedName();
         }
         break;
@@ -1612,7 +1724,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new FunctionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(241);
+        setState(258);
         functionExpression();
         }
         break;
@@ -1621,17 +1733,17 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new ParenthesizedExpressionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(242);
+        setState(259);
         match(LP);
-        setState(243);
+        setState(260);
         booleanExpression(0);
-        setState(244);
+        setState(261);
         match(RP);
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(253);
+      setState(270);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1642,16 +1754,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
           {
           _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState));
           pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression);
-          setState(248);
+          setState(265);
           if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-          setState(249);
+          setState(266);
           match(CAST_OP);
-          setState(250);
+          setState(267);
           dataType();
           }
           } 
         }
-        setState(255);
+        setState(272);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       }
@@ -1708,68 +1820,50 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final FunctionExpressionContext functionExpression() throws RecognitionException {
     FunctionExpressionContext _localctx = new FunctionExpressionContext(_ctx, getState());
-    enterRule(_localctx, 20, RULE_functionExpression);
+    enterRule(_localctx, 22, RULE_functionExpression);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(256);
+      setState(273);
       identifier();
-      setState(257);
+      setState(274);
       match(LP);
-      setState(267);
+      setState(284);
       _errHandler.sync(this);
-      switch (_input.LA(1)) {
-      case ASTERISK:
+      switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
+      case 1:
         {
-        setState(258);
+        setState(275);
         match(ASTERISK);
         }
         break;
-      case QUOTED_STRING:
-      case INTEGER_LITERAL:
-      case DECIMAL_LITERAL:
-      case FALSE:
-      case LP:
-      case NOT:
-      case NULL:
-      case PARAM:
-      case TRUE:
-      case PLUS:
-      case MINUS:
-      case NAMED_OR_POSITIONAL_PARAM:
-      case OPENING_BRACKET:
-      case UNQUOTED_IDENTIFIER:
-      case QUOTED_IDENTIFIER:
+      case 2:
         {
         {
-        setState(259);
+        setState(276);
         booleanExpression(0);
-        setState(264);
+        setState(281);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(260);
+          setState(277);
           match(COMMA);
-          setState(261);
+          setState(278);
           booleanExpression(0);
           }
           }
-          setState(266);
+          setState(283);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         }
         break;
-      case RP:
-        break;
-      default:
-        break;
       }
-      setState(269);
+      setState(286);
       match(RP);
       }
     }
@@ -1822,12 +1916,12 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final DataTypeContext dataType() throws RecognitionException {
     DataTypeContext _localctx = new DataTypeContext(_ctx, getState());
-    enterRule(_localctx, 22, RULE_dataType);
+    enterRule(_localctx, 24, RULE_dataType);
     try {
       _localctx = new ToDataTypeContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(271);
+      setState(288);
       identifier();
       }
     }
@@ -1870,13 +1964,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final RowCommandContext rowCommand() throws RecognitionException {
     RowCommandContext _localctx = new RowCommandContext(_ctx, getState());
-    enterRule(_localctx, 24, RULE_rowCommand);
+    enterRule(_localctx, 26, RULE_rowCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(273);
+      setState(290);
       match(ROW);
-      setState(274);
+      setState(291);
       fields();
       }
     }
@@ -1925,28 +2019,28 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final FieldsContext fields() throws RecognitionException {
     FieldsContext _localctx = new FieldsContext(_ctx, getState());
-    enterRule(_localctx, 26, RULE_fields);
+    enterRule(_localctx, 28, RULE_fields);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(276);
+      setState(293);
       field();
-      setState(281);
+      setState(298);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(277);
+          setState(294);
           match(COMMA);
-          setState(278);
+          setState(295);
           field();
           }
           } 
         }
-        setState(283);
+        setState(300);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       }
@@ -1994,26 +2088,26 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final FieldContext field() throws RecognitionException {
     FieldContext _localctx = new FieldContext(_ctx, getState());
-    enterRule(_localctx, 28, RULE_field);
+    enterRule(_localctx, 30, RULE_field);
     try {
-      setState(289);
+      setState(306);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(284);
+        setState(301);
         booleanExpression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(285);
+        setState(302);
         qualifiedName();
-        setState(286);
+        setState(303);
         match(ASSIGN);
-        setState(287);
+        setState(304);
         booleanExpression(0);
         }
         break;
@@ -2068,39 +2162,39 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final FromCommandContext fromCommand() throws RecognitionException {
     FromCommandContext _localctx = new FromCommandContext(_ctx, getState());
-    enterRule(_localctx, 30, RULE_fromCommand);
+    enterRule(_localctx, 32, RULE_fromCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(291);
+      setState(308);
       match(FROM);
-      setState(292);
+      setState(309);
       indexPattern();
-      setState(297);
+      setState(314);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(293);
+          setState(310);
           match(COMMA);
-          setState(294);
+          setState(311);
           indexPattern();
           }
           } 
         }
-        setState(299);
+        setState(316);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
       }
-      setState(301);
+      setState(318);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
       case 1:
         {
-        setState(300);
+        setState(317);
         metadata();
         }
         break;
@@ -2149,26 +2243,26 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IndexPatternContext indexPattern() throws RecognitionException {
     IndexPatternContext _localctx = new IndexPatternContext(_ctx, getState());
-    enterRule(_localctx, 32, RULE_indexPattern);
+    enterRule(_localctx, 34, RULE_indexPattern);
     try {
-      setState(308);
+      setState(325);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(303);
+        setState(320);
         clusterString();
-        setState(304);
+        setState(321);
         match(COLON);
-        setState(305);
+        setState(322);
         indexString();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(307);
+        setState(324);
         indexString();
         }
         break;
@@ -2210,11 +2304,11 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ClusterStringContext clusterString() throws RecognitionException {
     ClusterStringContext _localctx = new ClusterStringContext(_ctx, getState());
-    enterRule(_localctx, 34, RULE_clusterString);
+    enterRule(_localctx, 36, RULE_clusterString);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(310);
+      setState(327);
       match(UNQUOTED_SOURCE);
       }
     }
@@ -2255,14 +2349,14 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IndexStringContext indexString() throws RecognitionException {
     IndexStringContext _localctx = new IndexStringContext(_ctx, getState());
-    enterRule(_localctx, 36, RULE_indexString);
+    enterRule(_localctx, 38, RULE_indexString);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(312);
+      setState(329);
       _la = _input.LA(1);
-      if ( !(_la==UNQUOTED_SOURCE || _la==QUOTED_STRING) ) {
+      if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) {
       _errHandler.recoverInline(this);
       }
       else {
@@ -2313,22 +2407,22 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final MetadataContext metadata() throws RecognitionException {
     MetadataContext _localctx = new MetadataContext(_ctx, getState());
-    enterRule(_localctx, 38, RULE_metadata);
+    enterRule(_localctx, 40, RULE_metadata);
     try {
-      setState(316);
+      setState(333);
       _errHandler.sync(this);
       switch (_input.LA(1)) {
       case METADATA:
         enterOuterAlt(_localctx, 1);
         {
-        setState(314);
+        setState(331);
         metadataOption();
         }
         break;
       case OPENING_BRACKET:
         enterOuterAlt(_localctx, 2);
         {
-        setState(315);
+        setState(332);
         deprecated_metadata();
         }
         break;
@@ -2380,30 +2474,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final MetadataOptionContext metadataOption() throws RecognitionException {
     MetadataOptionContext _localctx = new MetadataOptionContext(_ctx, getState());
-    enterRule(_localctx, 40, RULE_metadataOption);
+    enterRule(_localctx, 42, RULE_metadataOption);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(318);
+      setState(335);
       match(METADATA);
-      setState(319);
+      setState(336);
       match(UNQUOTED_SOURCE);
-      setState(324);
+      setState(341);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(320);
+          setState(337);
           match(COMMA);
-          setState(321);
+          setState(338);
           match(UNQUOTED_SOURCE);
           }
           } 
         }
-        setState(326);
+        setState(343);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
       }
@@ -2448,15 +2542,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final Deprecated_metadataContext deprecated_metadata() throws RecognitionException {
     Deprecated_metadataContext _localctx = new Deprecated_metadataContext(_ctx, getState());
-    enterRule(_localctx, 42, RULE_deprecated_metadata);
+    enterRule(_localctx, 44, RULE_deprecated_metadata);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(327);
+      setState(344);
       match(OPENING_BRACKET);
-      setState(328);
+      setState(345);
       metadataOption();
-      setState(329);
+      setState(346);
       match(CLOSING_BRACKET);
       }
     }
@@ -2475,7 +2569,7 @@ public final Deprecated_metadataContext deprecated_metadata() throws Recognition
   public static class MetricsCommandContext extends ParserRuleContext {
     public FieldsContext aggregates;
     public FieldsContext grouping;
-    public TerminalNode METRICS() { return getToken(EsqlBaseParser.METRICS, 0); }
+    public TerminalNode DEV_METRICS() { return getToken(EsqlBaseParser.DEV_METRICS, 0); }
     public List<IndexPatternContext> indexPattern() {
       return getRuleContexts(IndexPatternContext.class);
     }
@@ -2515,51 +2609,51 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final MetricsCommandContext metricsCommand() throws RecognitionException {
     MetricsCommandContext _localctx = new MetricsCommandContext(_ctx, getState());
-    enterRule(_localctx, 44, RULE_metricsCommand);
+    enterRule(_localctx, 46, RULE_metricsCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(331);
-      match(METRICS);
-      setState(332);
+      setState(348);
+      match(DEV_METRICS);
+      setState(349);
       indexPattern();
-      setState(337);
+      setState(354);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,27,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(333);
+          setState(350);
           match(COMMA);
-          setState(334);
+          setState(351);
           indexPattern();
           }
           } 
         }
-        setState(339);
+        setState(356);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,27,_ctx);
       }
-      setState(341);
+      setState(358);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
       case 1:
         {
-        setState(340);
+        setState(357);
         ((MetricsCommandContext)_localctx).aggregates = fields();
         }
         break;
       }
-      setState(345);
+      setState(362);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         {
-        setState(343);
+        setState(360);
         match(BY);
-        setState(344);
+        setState(361);
         ((MetricsCommandContext)_localctx).grouping = fields();
         }
         break;
@@ -2605,13 +2699,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final EvalCommandContext evalCommand() throws RecognitionException {
     EvalCommandContext _localctx = new EvalCommandContext(_ctx, getState());
-    enterRule(_localctx, 46, RULE_evalCommand);
+    enterRule(_localctx, 48, RULE_evalCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(347);
+      setState(364);
       match(EVAL);
-      setState(348);
+      setState(365);
       fields();
       }
     }
@@ -2660,30 +2754,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final StatsCommandContext statsCommand() throws RecognitionException {
     StatsCommandContext _localctx = new StatsCommandContext(_ctx, getState());
-    enterRule(_localctx, 48, RULE_statsCommand);
+    enterRule(_localctx, 50, RULE_statsCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(350);
+      setState(367);
       match(STATS);
-      setState(352);
+      setState(369);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         {
-        setState(351);
+        setState(368);
         ((StatsCommandContext)_localctx).stats = fields();
         }
         break;
       }
-      setState(356);
+      setState(373);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
       case 1:
         {
-        setState(354);
+        setState(371);
         match(BY);
-        setState(355);
+        setState(372);
         ((StatsCommandContext)_localctx).grouping = fields();
         }
         break;
@@ -2701,73 +2795,6 @@ public final StatsCommandContext statsCommand() throws RecognitionException {
     return _localctx;
   }
 
-  @SuppressWarnings("CheckReturnValue")
-  public static class InlinestatsCommandContext extends ParserRuleContext {
-    public FieldsContext stats;
-    public FieldsContext grouping;
-    public TerminalNode INLINESTATS() { return getToken(EsqlBaseParser.INLINESTATS, 0); }
-    public List<FieldsContext> fields() {
-      return getRuleContexts(FieldsContext.class);
-    }
-    public FieldsContext fields(int i) {
-      return getRuleContext(FieldsContext.class,i);
-    }
-    public TerminalNode BY() { return getToken(EsqlBaseParser.BY, 0); }
-    @SuppressWarnings("this-escape")
-    public InlinestatsCommandContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_inlinestatsCommand; }
-    @Override
-    public void enterRule(ParseTreeListener listener) {
-      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterInlinestatsCommand(this);
-    }
-    @Override
-    public void exitRule(ParseTreeListener listener) {
-      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitInlinestatsCommand(this);
-    }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitInlinestatsCommand(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-
-  public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionException {
-    InlinestatsCommandContext _localctx = new InlinestatsCommandContext(_ctx, getState());
-    enterRule(_localctx, 50, RULE_inlinestatsCommand);
-    try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(358);
-      match(INLINESTATS);
-      setState(359);
-      ((InlinestatsCommandContext)_localctx).stats = fields();
-      setState(362);
-      _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) {
-      case 1:
-        {
-        setState(360);
-        match(BY);
-        setState(361);
-        ((InlinestatsCommandContext)_localctx).grouping = fields();
-        }
-        break;
-      }
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
   @SuppressWarnings("CheckReturnValue")
   public static class QualifiedNameContext extends ParserRuleContext {
     public List<IdentifierContext> identifier() {
@@ -2807,25 +2834,25 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(364);
+      setState(375);
       identifier();
-      setState(369);
+      setState(380);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,32,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(365);
+          setState(376);
           match(DOT);
-          setState(366);
+          setState(377);
           identifier();
           }
           } 
         }
-        setState(371);
+        setState(382);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,32,_ctx);
       }
       }
     }
@@ -2879,25 +2906,25 @@ public final QualifiedNamePatternContext qualifiedNamePattern() throws Recogniti
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(372);
+      setState(383);
       identifierPattern();
-      setState(377);
+      setState(388);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,34,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(373);
+          setState(384);
           match(DOT);
-          setState(374);
+          setState(385);
           identifierPattern();
           }
           } 
         }
-        setState(379);
+        setState(390);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,34,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
       }
       }
     }
@@ -2951,25 +2978,25 @@ public final QualifiedNamePatternsContext qualifiedNamePatterns() throws Recogni
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(380);
+      setState(391);
       qualifiedNamePattern();
-      setState(385);
+      setState(396);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,35,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,34,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(381);
+          setState(392);
           match(COMMA);
-          setState(382);
+          setState(393);
           qualifiedNamePattern();
           }
           } 
         }
-        setState(387);
+        setState(398);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,35,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,34,_ctx);
       }
       }
     }
@@ -3015,7 +3042,7 @@ public final IdentifierContext identifier() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(388);
+      setState(399);
       _la = _input.LA(1);
       if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) {
       _errHandler.recoverInline(this);
@@ -3067,7 +3094,7 @@ public final IdentifierPatternContext identifierPattern() throws RecognitionExce
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(390);
+      setState(401);
       match(ID_PATTERN);
       }
     }
@@ -3338,14 +3365,14 @@ public final ConstantContext constant() throws RecognitionException {
     enterRule(_localctx, 62, RULE_constant);
     int _la;
     try {
-      setState(434);
+      setState(445);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) {
       case 1:
         _localctx = new NullLiteralContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(392);
+        setState(403);
         match(NULL);
         }
         break;
@@ -3353,9 +3380,9 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new QualifiedIntegerLiteralContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(393);
+        setState(404);
         integerValue();
-        setState(394);
+        setState(405);
         match(UNQUOTED_IDENTIFIER);
         }
         break;
@@ -3363,7 +3390,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new DecimalLiteralContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(396);
+        setState(407);
         decimalValue();
         }
         break;
@@ -3371,7 +3398,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new IntegerLiteralContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(397);
+        setState(408);
         integerValue();
         }
         break;
@@ -3379,7 +3406,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new BooleanLiteralContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(398);
+        setState(409);
         booleanValue();
         }
         break;
@@ -3387,7 +3414,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new InputParamsContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(399);
+        setState(410);
         params();
         }
         break;
@@ -3395,7 +3422,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new StringLiteralContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(400);
+        setState(411);
         string();
         }
         break;
@@ -3403,27 +3430,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new NumericArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(401);
+        setState(412);
         match(OPENING_BRACKET);
-        setState(402);
+        setState(413);
         numericValue();
-        setState(407);
+        setState(418);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(403);
+          setState(414);
           match(COMMA);
-          setState(404);
+          setState(415);
           numericValue();
           }
           }
-          setState(409);
+          setState(420);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(410);
+        setState(421);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3431,27 +3458,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new BooleanArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(412);
+        setState(423);
         match(OPENING_BRACKET);
-        setState(413);
+        setState(424);
         booleanValue();
-        setState(418);
+        setState(429);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(414);
+          setState(425);
           match(COMMA);
-          setState(415);
+          setState(426);
           booleanValue();
           }
           }
-          setState(420);
+          setState(431);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(421);
+        setState(432);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3459,27 +3486,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new StringArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(423);
+        setState(434);
         match(OPENING_BRACKET);
-        setState(424);
+        setState(435);
         string();
-        setState(429);
+        setState(440);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(425);
+          setState(436);
           match(COMMA);
-          setState(426);
+          setState(437);
           string();
           }
           }
-          setState(431);
+          setState(442);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(432);
+        setState(443);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3553,14 +3580,14 @@ public final ParamsContext params() throws RecognitionException {
     ParamsContext _localctx = new ParamsContext(_ctx, getState());
     enterRule(_localctx, 64, RULE_params);
     try {
-      setState(438);
+      setState(449);
       _errHandler.sync(this);
       switch (_input.LA(1)) {
       case PARAM:
         _localctx = new InputParamContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(436);
+        setState(447);
         match(PARAM);
         }
         break;
@@ -3568,7 +3595,7 @@ public final ParamsContext params() throws RecognitionException {
         _localctx = new InputNamedOrPositionalParamContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(437);
+        setState(448);
         match(NAMED_OR_POSITIONAL_PARAM);
         }
         break;
@@ -3617,9 +3644,9 @@ public final LimitCommandContext limitCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(440);
+      setState(451);
       match(LIMIT);
-      setState(441);
+      setState(452);
       match(INTEGER_LITERAL);
       }
     }
@@ -3674,27 +3701,27 @@ public final SortCommandContext sortCommand() throws RecognitionException {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(443);
+      setState(454);
       match(SORT);
-      setState(444);
+      setState(455);
       orderExpression();
-      setState(449);
+      setState(460);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,41,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,40,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(445);
+          setState(456);
           match(COMMA);
-          setState(446);
+          setState(457);
           orderExpression();
           }
           } 
         }
-        setState(451);
+        setState(462);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,41,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,40,_ctx);
       }
       }
     }
@@ -3748,14 +3775,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(452);
+      setState(463);
       booleanExpression(0);
-      setState(454);
+      setState(465);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) {
       case 1:
         {
-        setState(453);
+        setState(464);
         ((OrderExpressionContext)_localctx).ordering = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==ASC || _la==DESC) ) {
@@ -3769,14 +3796,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio
         }
         break;
       }
-      setState(458);
+      setState(469);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) {
       case 1:
         {
-        setState(456);
+        setState(467);
         match(NULLS);
-        setState(457);
+        setState(468);
         ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==FIRST || _la==LAST) ) {
@@ -3835,9 +3862,9 @@ public final KeepCommandContext keepCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(460);
+      setState(471);
       match(KEEP);
-      setState(461);
+      setState(472);
       qualifiedNamePatterns();
       }
     }
@@ -3884,9 +3911,9 @@ public final DropCommandContext dropCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(463);
+      setState(474);
       match(DROP);
-      setState(464);
+      setState(475);
       qualifiedNamePatterns();
       }
     }
@@ -3941,27 +3968,27 @@ public final RenameCommandContext renameCommand() throws RecognitionException {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(466);
+      setState(477);
       match(RENAME);
-      setState(467);
+      setState(478);
       renameClause();
-      setState(472);
+      setState(483);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,44,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,43,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(468);
+          setState(479);
           match(COMMA);
-          setState(469);
+          setState(480);
           renameClause();
           }
           } 
         }
-        setState(474);
+        setState(485);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,44,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,43,_ctx);
       }
       }
     }
@@ -4013,11 +4040,11 @@ public final RenameClauseContext renameClause() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(475);
+      setState(486);
       ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern();
-      setState(476);
+      setState(487);
       match(AS);
-      setState(477);
+      setState(488);
       ((RenameClauseContext)_localctx).newName = qualifiedNamePattern();
       }
     }
@@ -4070,18 +4097,18 @@ public final DissectCommandContext dissectCommand() throws RecognitionException
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(479);
+      setState(490);
       match(DISSECT);
-      setState(480);
+      setState(491);
       primaryExpression(0);
-      setState(481);
+      setState(492);
       string();
-      setState(483);
+      setState(494);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) {
       case 1:
         {
-        setState(482);
+        setState(493);
         commandOptions();
         }
         break;
@@ -4134,11 +4161,11 @@ public final GrokCommandContext grokCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(485);
+      setState(496);
       match(GROK);
-      setState(486);
+      setState(497);
       primaryExpression(0);
-      setState(487);
+      setState(498);
       string();
       }
     }
@@ -4185,9 +4212,9 @@ public final MvExpandCommandContext mvExpandCommand() throws RecognitionExceptio
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(489);
+      setState(500);
       match(MV_EXPAND);
-      setState(490);
+      setState(501);
       qualifiedName();
       }
     }
@@ -4241,25 +4268,25 @@ public final CommandOptionsContext commandOptions() throws RecognitionException
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(492);
+      setState(503);
       commandOption();
-      setState(497);
+      setState(508);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,46,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,45,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(493);
+          setState(504);
           match(COMMA);
-          setState(494);
+          setState(505);
           commandOption();
           }
           } 
         }
-        setState(499);
+        setState(510);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,46,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,45,_ctx);
       }
       }
     }
@@ -4309,11 +4336,11 @@ public final CommandOptionContext commandOption() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(500);
+      setState(511);
       identifier();
-      setState(501);
+      setState(512);
       match(ASSIGN);
-      setState(502);
+      setState(513);
       constant();
       }
     }
@@ -4359,7 +4386,7 @@ public final BooleanValueContext booleanValue() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(504);
+      setState(515);
       _la = _input.LA(1);
       if ( !(_la==FALSE || _la==TRUE) ) {
       _errHandler.recoverInline(this);
@@ -4414,20 +4441,20 @@ public final NumericValueContext numericValue() throws RecognitionException {
     NumericValueContext _localctx = new NumericValueContext(_ctx, getState());
     enterRule(_localctx, 92, RULE_numericValue);
     try {
-      setState(508);
+      setState(519);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(506);
+        setState(517);
         decimalValue();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(507);
+        setState(518);
         integerValue();
         }
         break;
@@ -4476,12 +4503,12 @@ public final DecimalValueContext decimalValue() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(511);
+      setState(522);
       _errHandler.sync(this);
       _la = _input.LA(1);
       if (_la==PLUS || _la==MINUS) {
         {
-        setState(510);
+        setState(521);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
         _errHandler.recoverInline(this);
@@ -4494,7 +4521,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException {
         }
       }
 
-      setState(513);
+      setState(524);
       match(DECIMAL_LITERAL);
       }
     }
@@ -4541,12 +4568,12 @@ public final IntegerValueContext integerValue() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(516);
+      setState(527);
       _errHandler.sync(this);
       _la = _input.LA(1);
       if (_la==PLUS || _la==MINUS) {
         {
-        setState(515);
+        setState(526);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
         _errHandler.recoverInline(this);
@@ -4559,7 +4586,7 @@ public final IntegerValueContext integerValue() throws RecognitionException {
         }
       }
 
-      setState(518);
+      setState(529);
       match(INTEGER_LITERAL);
       }
     }
@@ -4603,7 +4630,7 @@ public final StringContext string() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(520);
+      setState(531);
       match(QUOTED_STRING);
       }
     }
@@ -4653,9 +4680,9 @@ public final ComparisonOperatorContext comparisonOperator() throws RecognitionEx
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(522);
+      setState(533);
       _la = _input.LA(1);
-      if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 9007199254740992000L) != 0)) ) {
+      if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 1125899906842624000L) != 0)) ) {
       _errHandler.recoverInline(this);
       }
       else {
@@ -4708,9 +4735,9 @@ public final ExplainCommandContext explainCommand() throws RecognitionException
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(524);
+      setState(535);
       match(EXPLAIN);
-      setState(525);
+      setState(536);
       subqueryExpression();
       }
     }
@@ -4758,11 +4785,11 @@ public final SubqueryExpressionContext subqueryExpression() throws RecognitionEx
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(527);
+      setState(538);
       match(OPENING_BRACKET);
-      setState(528);
+      setState(539);
       query(0);
-      setState(529);
+      setState(540);
       match(CLOSING_BRACKET);
       }
     }
@@ -4819,9 +4846,9 @@ public final ShowCommandContext showCommand() throws RecognitionException {
       _localctx = new ShowInfoContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(531);
+      setState(542);
       match(SHOW);
-      setState(532);
+      setState(543);
       match(INFO);
       }
     }
@@ -4878,9 +4905,9 @@ public final MetaCommandContext metaCommand() throws RecognitionException {
       _localctx = new MetaFunctionsContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(534);
+      setState(545);
       match(META);
-      setState(535);
+      setState(546);
       match(FUNCTIONS);
       }
     }
@@ -4943,48 +4970,48 @@ public final EnrichCommandContext enrichCommand() throws RecognitionException {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(537);
+      setState(548);
       match(ENRICH);
-      setState(538);
+      setState(549);
       ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME);
-      setState(541);
+      setState(552);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) {
       case 1:
         {
-        setState(539);
+        setState(550);
         match(ON);
-        setState(540);
+        setState(551);
         ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern();
         }
         break;
       }
-      setState(552);
+      setState(563);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,52,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) {
       case 1:
         {
-        setState(543);
+        setState(554);
         match(WITH);
-        setState(544);
+        setState(555);
         enrichWithClause();
-        setState(549);
+        setState(560);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,51,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,50,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(545);
+            setState(556);
             match(COMMA);
-            setState(546);
+            setState(557);
             enrichWithClause();
             }
             } 
           }
-          setState(551);
+          setState(562);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,51,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,50,_ctx);
         }
         }
         break;
@@ -5039,19 +5066,19 @@ public final EnrichWithClauseContext enrichWithClause() throws RecognitionExcept
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(557);
+      setState(568);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,52,_ctx) ) {
       case 1:
         {
-        setState(554);
+        setState(565);
         ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern();
-        setState(555);
+        setState(566);
         match(ASSIGN);
         }
         break;
       }
-      setState(559);
+      setState(570);
       ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern();
       }
     }
@@ -5070,7 +5097,7 @@ public final EnrichWithClauseContext enrichWithClause() throws RecognitionExcept
   public static class LookupCommandContext extends ParserRuleContext {
     public IndexPatternContext tableName;
     public QualifiedNamePatternsContext matchFields;
-    public TerminalNode LOOKUP() { return getToken(EsqlBaseParser.LOOKUP, 0); }
+    public TerminalNode DEV_LOOKUP() { return getToken(EsqlBaseParser.DEV_LOOKUP, 0); }
     public TerminalNode ON() { return getToken(EsqlBaseParser.ON, 0); }
     public IndexPatternContext indexPattern() {
       return getRuleContext(IndexPatternContext.class,0);
@@ -5104,13 +5131,13 @@ public final LookupCommandContext lookupCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(561);
-      match(LOOKUP);
-      setState(562);
+      setState(572);
+      match(DEV_LOOKUP);
+      setState(573);
       ((LookupCommandContext)_localctx).tableName = indexPattern();
-      setState(563);
+      setState(574);
       match(ON);
-      setState(564);
+      setState(575);
       ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns();
       }
     }
@@ -5125,15 +5152,179 @@ public final LookupCommandContext lookupCommand() throws RecognitionException {
     return _localctx;
   }
 
+  @SuppressWarnings("CheckReturnValue")
+  public static class InlinestatsCommandContext extends ParserRuleContext {
+    public FieldsContext stats;
+    public FieldsContext grouping;
+    public TerminalNode DEV_INLINESTATS() { return getToken(EsqlBaseParser.DEV_INLINESTATS, 0); }
+    public List<FieldsContext> fields() {
+      return getRuleContexts(FieldsContext.class);
+    }
+    public FieldsContext fields(int i) {
+      return getRuleContext(FieldsContext.class,i);
+    }
+    public TerminalNode BY() { return getToken(EsqlBaseParser.BY, 0); }
+    @SuppressWarnings("this-escape")
+    public InlinestatsCommandContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_inlinestatsCommand; }
+    @Override
+    public void enterRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterInlinestatsCommand(this);
+    }
+    @Override
+    public void exitRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitInlinestatsCommand(this);
+    }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitInlinestatsCommand(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionException {
+    InlinestatsCommandContext _localctx = new InlinestatsCommandContext(_ctx, getState());
+    enterRule(_localctx, 116, RULE_inlinestatsCommand);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(577);
+      match(DEV_INLINESTATS);
+      setState(578);
+      ((InlinestatsCommandContext)_localctx).stats = fields();
+      setState(581);
+      _errHandler.sync(this);
+      switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) {
+      case 1:
+        {
+        setState(579);
+        match(BY);
+        setState(580);
+        ((InlinestatsCommandContext)_localctx).grouping = fields();
+        }
+        break;
+      }
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  @SuppressWarnings("CheckReturnValue")
+  public static class MatchCommandContext extends ParserRuleContext {
+    public TerminalNode DEV_MATCH() { return getToken(EsqlBaseParser.DEV_MATCH, 0); }
+    public MatchQueryContext matchQuery() {
+      return getRuleContext(MatchQueryContext.class,0);
+    }
+    @SuppressWarnings("this-escape")
+    public MatchCommandContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_matchCommand; }
+    @Override
+    public void enterRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterMatchCommand(this);
+    }
+    @Override
+    public void exitRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitMatchCommand(this);
+    }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitMatchCommand(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final MatchCommandContext matchCommand() throws RecognitionException {
+    MatchCommandContext _localctx = new MatchCommandContext(_ctx, getState());
+    enterRule(_localctx, 118, RULE_matchCommand);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(583);
+      match(DEV_MATCH);
+      setState(584);
+      matchQuery();
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  @SuppressWarnings("CheckReturnValue")
+  public static class MatchQueryContext extends ParserRuleContext {
+    public TerminalNode QUOTED_STRING() { return getToken(EsqlBaseParser.QUOTED_STRING, 0); }
+    @SuppressWarnings("this-escape")
+    public MatchQueryContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_matchQuery; }
+    @Override
+    public void enterRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterMatchQuery(this);
+    }
+    @Override
+    public void exitRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitMatchQuery(this);
+    }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitMatchQuery(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final MatchQueryContext matchQuery() throws RecognitionException {
+    MatchQueryContext _localctx = new MatchQueryContext(_ctx, getState());
+    enterRule(_localctx, 120, RULE_matchQuery);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(586);
+      match(QUOTED_STRING);
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
     case 1:
       return query_sempred((QueryContext)_localctx, predIndex);
+    case 2:
+      return sourceCommand_sempred((SourceCommandContext)_localctx, predIndex);
+    case 3:
+      return processingCommand_sempred((ProcessingCommandContext)_localctx, predIndex);
     case 5:
       return booleanExpression_sempred((BooleanExpressionContext)_localctx, predIndex);
-    case 8:
-      return operatorExpression_sempred((OperatorExpressionContext)_localctx, predIndex);
     case 9:
+      return operatorExpression_sempred((OperatorExpressionContext)_localctx, predIndex);
+    case 10:
       return primaryExpression_sempred((PrimaryExpressionContext)_localctx, predIndex);
     }
     return true;
@@ -5145,34 +5336,54 @@ private boolean query_sempred(QueryContext _localctx, int predIndex) {
     }
     return true;
   }
-  private boolean booleanExpression_sempred(BooleanExpressionContext _localctx, int predIndex) {
+  private boolean sourceCommand_sempred(SourceCommandContext _localctx, int predIndex) {
     switch (predIndex) {
     case 1:
-      return precpred(_ctx, 4);
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean processingCommand_sempred(ProcessingCommandContext _localctx, int predIndex) {
+    switch (predIndex) {
     case 2:
-      return precpred(_ctx, 3);
+      return this.isDevVersion();
+    case 3:
+      return this.isDevVersion();
+    case 4:
+      return this.isDevVersion();
+    }
+    return true;
+  }
+  private boolean booleanExpression_sempred(BooleanExpressionContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 5:
+      return this.isDevVersion();
+    case 6:
+      return precpred(_ctx, 5);
+    case 7:
+      return precpred(_ctx, 4);
     }
     return true;
   }
   private boolean operatorExpression_sempred(OperatorExpressionContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 3:
+    case 8:
       return precpred(_ctx, 2);
-    case 4:
+    case 9:
       return precpred(_ctx, 1);
     }
     return true;
   }
   private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 5:
+    case 10:
       return precpred(_ctx, 1);
     }
     return true;
   }
 
   public static final String _serializedATN =
-    "\u0004\u0001|\u0237\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+
+    "\u0004\u0001}\u024d\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+
     "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+
     "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+
     "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+
@@ -5187,351 +5398,365 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in
     "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+
     "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0002"+
     "2\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u00076\u0002"+
-    "7\u00077\u00028\u00078\u00029\u00079\u0001\u0000\u0001\u0000\u0001\u0000"+
-    "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+
-    "\u0005\u0001~\b\u0001\n\u0001\f\u0001\u0081\t\u0001\u0001\u0002\u0001"+
-    "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u0089"+
-    "\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+
+    "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007;\u0002"+
+    "<\u0007<\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001"+
+    "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001\u0084\b\u0001\n"+
+    "\u0001\f\u0001\u0087\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+
+    "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u0090\b\u0002\u0001"+
+    "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+
     "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+
-    "\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u0099\b\u0003\u0001\u0004\u0001"+
-    "\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+
-    "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00a5\b\u0005\u0001\u0005\u0001"+
-    "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00ac\b\u0005\n"+
-    "\u0005\f\u0005\u00af\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+
-    "\u0005\u0001\u0005\u0003\u0005\u00b6\b\u0005\u0001\u0005\u0001\u0005\u0003"+
-    "\u0005\u00ba\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+
-    "\u0005\u0001\u0005\u0005\u0005\u00c2\b\u0005\n\u0005\f\u0005\u00c5\t\u0005"+
-    "\u0001\u0006\u0001\u0006\u0003\u0006\u00c9\b\u0006\u0001\u0006\u0001\u0006"+
-    "\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00d0\b\u0006\u0001\u0006"+
-    "\u0001\u0006\u0001\u0006\u0003\u0006\u00d5\b\u0006\u0001\u0007\u0001\u0007"+
-    "\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00dc\b\u0007\u0001\b"+
-    "\u0001\b\u0001\b\u0001\b\u0003\b\u00e2\b\b\u0001\b\u0001\b\u0001\b\u0001"+
-    "\b\u0001\b\u0001\b\u0005\b\u00ea\b\b\n\b\f\b\u00ed\t\b\u0001\t\u0001\t"+
-    "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00f7\b\t\u0001"+
-    "\t\u0001\t\u0001\t\u0005\t\u00fc\b\t\n\t\f\t\u00ff\t\t\u0001\n\u0001\n"+
-    "\u0001\n\u0001\n\u0001\n\u0001\n\u0005\n\u0107\b\n\n\n\f\n\u010a\t\n\u0003"+
-    "\n\u010c\b\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001"+
-    "\f\u0001\r\u0001\r\u0001\r\u0005\r\u0118\b\r\n\r\f\r\u011b\t\r\u0001\u000e"+
-    "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0003\u000e\u0122\b\u000e"+
-    "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0005\u000f\u0128\b\u000f"+
-    "\n\u000f\f\u000f\u012b\t\u000f\u0001\u000f\u0003\u000f\u012e\b\u000f\u0001"+
-    "\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0003\u0010\u0135"+
-    "\b\u0010\u0001\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0013\u0001"+
-    "\u0013\u0003\u0013\u013d\b\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001"+
-    "\u0014\u0005\u0014\u0143\b\u0014\n\u0014\f\u0014\u0146\t\u0014\u0001\u0015"+
-    "\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016"+
-    "\u0001\u0016\u0005\u0016\u0150\b\u0016\n\u0016\f\u0016\u0153\t\u0016\u0001"+
-    "\u0016\u0003\u0016\u0156\b\u0016\u0001\u0016\u0001\u0016\u0003\u0016\u015a"+
-    "\b\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0003"+
-    "\u0018\u0161\b\u0018\u0001\u0018\u0001\u0018\u0003\u0018\u0165\b\u0018"+
-    "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0003\u0019\u016b\b\u0019"+
-    "\u0001\u001a\u0001\u001a\u0001\u001a\u0005\u001a\u0170\b\u001a\n\u001a"+
-    "\f\u001a\u0173\t\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b"+
-    "\u0178\b\u001b\n\u001b\f\u001b\u017b\t\u001b\u0001\u001c\u0001\u001c\u0001"+
-    "\u001c\u0005\u001c\u0180\b\u001c\n\u001c\f\u001c\u0183\t\u001c\u0001\u001d"+
-    "\u0001\u001d\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001\u001f"+
+    "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003"+
+    "\u0003\u00a4\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001"+
+    "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003"+
+    "\u0005\u00b0\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+
+    "\u0005\u0005\u0005\u00b7\b\u0005\n\u0005\f\u0005\u00ba\t\u0005\u0001\u0005"+
+    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00c1\b\u0005"+
+    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00c7\b\u0005"+
+    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+
+    "\u0005\u0005\u00cf\b\u0005\n\u0005\f\u0005\u00d2\t\u0005\u0001\u0006\u0001"+
+    "\u0006\u0003\u0006\u00d6\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+
+    "\u0006\u0001\u0006\u0003\u0006\u00dd\b\u0006\u0001\u0006\u0001\u0006\u0001"+
+    "\u0006\u0003\u0006\u00e2\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+
+    "\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00ed\b\b\u0001"+
+    "\t\u0001\t\u0001\t\u0001\t\u0003\t\u00f3\b\t\u0001\t\u0001\t\u0001\t\u0001"+
+    "\t\u0001\t\u0001\t\u0005\t\u00fb\b\t\n\t\f\t\u00fe\t\t\u0001\n\u0001\n"+
+    "\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0003\n\u0108\b\n\u0001"+
+    "\n\u0001\n\u0001\n\u0005\n\u010d\b\n\n\n\f\n\u0110\t\n\u0001\u000b\u0001"+
+    "\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0005\u000b\u0118"+
+    "\b\u000b\n\u000b\f\u000b\u011b\t\u000b\u0003\u000b\u011d\b\u000b\u0001"+
+    "\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\u000e"+
+    "\u0001\u000e\u0001\u000e\u0005\u000e\u0129\b\u000e\n\u000e\f\u000e\u012c"+
+    "\t\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0003"+
+    "\u000f\u0133\b\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0005"+
+    "\u0010\u0139\b\u0010\n\u0010\f\u0010\u013c\t\u0010\u0001\u0010\u0003\u0010"+
+    "\u013f\b\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+
+    "\u0003\u0011\u0146\b\u0011\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"+
+    "\u0001\u0014\u0001\u0014\u0003\u0014\u014e\b\u0014\u0001\u0015\u0001\u0015"+
+    "\u0001\u0015\u0001\u0015\u0005\u0015\u0154\b\u0015\n\u0015\f\u0015\u0157"+
+    "\t\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0001"+
+    "\u0017\u0001\u0017\u0001\u0017\u0005\u0017\u0161\b\u0017\n\u0017\f\u0017"+
+    "\u0164\t\u0017\u0001\u0017\u0003\u0017\u0167\b\u0017\u0001\u0017\u0001"+
+    "\u0017\u0003\u0017\u016b\b\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001"+
+    "\u0019\u0001\u0019\u0003\u0019\u0172\b\u0019\u0001\u0019\u0001\u0019\u0003"+
+    "\u0019\u0176\b\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0005\u001a\u017b"+
+    "\b\u001a\n\u001a\f\u001a\u017e\t\u001a\u0001\u001b\u0001\u001b\u0001\u001b"+
+    "\u0005\u001b\u0183\b\u001b\n\u001b\f\u001b\u0186\t\u001b\u0001\u001c\u0001"+
+    "\u001c\u0001\u001c\u0005\u001c\u018b\b\u001c\n\u001c\f\u001c\u018e\t\u001c"+
+    "\u0001\u001d\u0001\u001d\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f"+
     "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f"+
-    "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u0196\b\u001f"+
-    "\n\u001f\f\u001f\u0199\t\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001"+
-    "\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01a1\b\u001f\n\u001f\f\u001f"+
-    "\u01a4\t\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f"+
-    "\u0001\u001f\u0005\u001f\u01ac\b\u001f\n\u001f\f\u001f\u01af\t\u001f\u0001"+
-    "\u001f\u0001\u001f\u0003\u001f\u01b3\b\u001f\u0001 \u0001 \u0003 \u01b7"+
-    "\b \u0001!\u0001!\u0001!\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01c0"+
-    "\b\"\n\"\f\"\u01c3\t\"\u0001#\u0001#\u0003#\u01c7\b#\u0001#\u0001#\u0003"+
-    "#\u01cb\b#\u0001$\u0001$\u0001$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001"+
-    "&\u0001&\u0005&\u01d7\b&\n&\f&\u01da\t&\u0001\'\u0001\'\u0001\'\u0001"+
-    "\'\u0001(\u0001(\u0001(\u0001(\u0003(\u01e4\b(\u0001)\u0001)\u0001)\u0001"+
-    ")\u0001*\u0001*\u0001*\u0001+\u0001+\u0001+\u0005+\u01f0\b+\n+\f+\u01f3"+
-    "\t+\u0001,\u0001,\u0001,\u0001,\u0001-\u0001-\u0001.\u0001.\u0003.\u01fd"+
-    "\b.\u0001/\u0003/\u0200\b/\u0001/\u0001/\u00010\u00030\u0205\b0\u0001"+
-    "0\u00010\u00011\u00011\u00012\u00012\u00013\u00013\u00013\u00014\u0001"+
-    "4\u00014\u00014\u00015\u00015\u00015\u00016\u00016\u00016\u00017\u0001"+
-    "7\u00017\u00017\u00037\u021e\b7\u00017\u00017\u00017\u00017\u00057\u0224"+
-    "\b7\n7\f7\u0227\t7\u00037\u0229\b7\u00018\u00018\u00018\u00038\u022e\b"+
-    "8\u00018\u00018\u00019\u00019\u00019\u00019\u00019\u00019\u0000\u0004"+
-    "\u0002\n\u0010\u0012:\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012"+
-    "\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\"+
-    "^`bdfhjlnpr\u0000\b\u0001\u0000?@\u0001\u0000AC\u0002\u0000\u0019\u0019"+
-    "\u001e\u001e\u0001\u0000GH\u0002\u0000##\'\'\u0001\u0000*+\u0002\u0000"+
-    "))77\u0002\u000088:>\u0250\u0000t\u0001\u0000\u0000\u0000\u0002w\u0001"+
-    "\u0000\u0000\u0000\u0004\u0088\u0001\u0000\u0000\u0000\u0006\u0098\u0001"+
-    "\u0000\u0000\u0000\b\u009a\u0001\u0000\u0000\u0000\n\u00b9\u0001\u0000"+
-    "\u0000\u0000\f\u00d4\u0001\u0000\u0000\u0000\u000e\u00db\u0001\u0000\u0000"+
-    "\u0000\u0010\u00e1\u0001\u0000\u0000\u0000\u0012\u00f6\u0001\u0000\u0000"+
-    "\u0000\u0014\u0100\u0001\u0000\u0000\u0000\u0016\u010f\u0001\u0000\u0000"+
-    "\u0000\u0018\u0111\u0001\u0000\u0000\u0000\u001a\u0114\u0001\u0000\u0000"+
-    "\u0000\u001c\u0121\u0001\u0000\u0000\u0000\u001e\u0123\u0001\u0000\u0000"+
-    "\u0000 \u0134\u0001\u0000\u0000\u0000\"\u0136\u0001\u0000\u0000\u0000"+
-    "$\u0138\u0001\u0000\u0000\u0000&\u013c\u0001\u0000\u0000\u0000(\u013e"+
-    "\u0001\u0000\u0000\u0000*\u0147\u0001\u0000\u0000\u0000,\u014b\u0001\u0000"+
-    "\u0000\u0000.\u015b\u0001\u0000\u0000\u00000\u015e\u0001\u0000\u0000\u0000"+
-    "2\u0166\u0001\u0000\u0000\u00004\u016c\u0001\u0000\u0000\u00006\u0174"+
-    "\u0001\u0000\u0000\u00008\u017c\u0001\u0000\u0000\u0000:\u0184\u0001\u0000"+
-    "\u0000\u0000<\u0186\u0001\u0000\u0000\u0000>\u01b2\u0001\u0000\u0000\u0000"+
-    "@\u01b6\u0001\u0000\u0000\u0000B\u01b8\u0001\u0000\u0000\u0000D\u01bb"+
-    "\u0001\u0000\u0000\u0000F\u01c4\u0001\u0000\u0000\u0000H\u01cc\u0001\u0000"+
-    "\u0000\u0000J\u01cf\u0001\u0000\u0000\u0000L\u01d2\u0001\u0000\u0000\u0000"+
-    "N\u01db\u0001\u0000\u0000\u0000P\u01df\u0001\u0000\u0000\u0000R\u01e5"+
-    "\u0001\u0000\u0000\u0000T\u01e9\u0001\u0000\u0000\u0000V\u01ec\u0001\u0000"+
-    "\u0000\u0000X\u01f4\u0001\u0000\u0000\u0000Z\u01f8\u0001\u0000\u0000\u0000"+
-    "\\\u01fc\u0001\u0000\u0000\u0000^\u01ff\u0001\u0000\u0000\u0000`\u0204"+
-    "\u0001\u0000\u0000\u0000b\u0208\u0001\u0000\u0000\u0000d\u020a\u0001\u0000"+
-    "\u0000\u0000f\u020c\u0001\u0000\u0000\u0000h\u020f\u0001\u0000\u0000\u0000"+
-    "j\u0213\u0001\u0000\u0000\u0000l\u0216\u0001\u0000\u0000\u0000n\u0219"+
-    "\u0001\u0000\u0000\u0000p\u022d\u0001\u0000\u0000\u0000r\u0231\u0001\u0000"+
-    "\u0000\u0000tu\u0003\u0002\u0001\u0000uv\u0005\u0000\u0000\u0001v\u0001"+
-    "\u0001\u0000\u0000\u0000wx\u0006\u0001\uffff\uffff\u0000xy\u0003\u0004"+
-    "\u0002\u0000y\u007f\u0001\u0000\u0000\u0000z{\n\u0001\u0000\u0000{|\u0005"+
-    "\u001d\u0000\u0000|~\u0003\u0006\u0003\u0000}z\u0001\u0000\u0000\u0000"+
-    "~\u0081\u0001\u0000\u0000\u0000\u007f}\u0001\u0000\u0000\u0000\u007f\u0080"+
-    "\u0001\u0000\u0000\u0000\u0080\u0003\u0001\u0000\u0000\u0000\u0081\u007f"+
-    "\u0001\u0000\u0000\u0000\u0082\u0089\u0003f3\u0000\u0083\u0089\u0003\u001e"+
-    "\u000f\u0000\u0084\u0089\u0003\u0018\f\u0000\u0085\u0089\u0003,\u0016"+
-    "\u0000\u0086\u0089\u0003j5\u0000\u0087\u0089\u0003l6\u0000\u0088\u0082"+
-    "\u0001\u0000\u0000\u0000\u0088\u0083\u0001\u0000\u0000\u0000\u0088\u0084"+
-    "\u0001\u0000\u0000\u0000\u0088\u0085\u0001\u0000\u0000\u0000\u0088\u0086"+
-    "\u0001\u0000\u0000\u0000\u0088\u0087\u0001\u0000\u0000\u0000\u0089\u0005"+
-    "\u0001\u0000\u0000\u0000\u008a\u0099\u0003.\u0017\u0000\u008b\u0099\u0003"+
-    "2\u0019\u0000\u008c\u0099\u0003B!\u0000\u008d\u0099\u0003r9\u0000\u008e"+
-    "\u0099\u0003H$\u0000\u008f\u0099\u0003D\"\u0000\u0090\u0099\u00030\u0018"+
-    "\u0000\u0091\u0099\u0003\b\u0004\u0000\u0092\u0099\u0003J%\u0000\u0093"+
-    "\u0099\u0003L&\u0000\u0094\u0099\u0003P(\u0000\u0095\u0099\u0003R)\u0000"+
-    "\u0096\u0099\u0003n7\u0000\u0097\u0099\u0003T*\u0000\u0098\u008a\u0001"+
-    "\u0000\u0000\u0000\u0098\u008b\u0001\u0000\u0000\u0000\u0098\u008c\u0001"+
-    "\u0000\u0000\u0000\u0098\u008d\u0001\u0000\u0000\u0000\u0098\u008e\u0001"+
-    "\u0000\u0000\u0000\u0098\u008f\u0001\u0000\u0000\u0000\u0098\u0090\u0001"+
-    "\u0000\u0000\u0000\u0098\u0091\u0001\u0000\u0000\u0000\u0098\u0092\u0001"+
-    "\u0000\u0000\u0000\u0098\u0093\u0001\u0000\u0000\u0000\u0098\u0094\u0001"+
-    "\u0000\u0000\u0000\u0098\u0095\u0001\u0000\u0000\u0000\u0098\u0096\u0001"+
-    "\u0000\u0000\u0000\u0098\u0097\u0001\u0000\u0000\u0000\u0099\u0007\u0001"+
-    "\u0000\u0000\u0000\u009a\u009b\u0005\u0014\u0000\u0000\u009b\u009c\u0003"+
-    "\n\u0005\u0000\u009c\t\u0001\u0000\u0000\u0000\u009d\u009e\u0006\u0005"+
-    "\uffff\uffff\u0000\u009e\u009f\u00050\u0000\u0000\u009f\u00ba\u0003\n"+
-    "\u0005\u0007\u00a0\u00ba\u0003\u000e\u0007\u0000\u00a1\u00ba\u0003\f\u0006"+
-    "\u0000\u00a2\u00a4\u0003\u000e\u0007\u0000\u00a3\u00a5\u00050\u0000\u0000"+
-    "\u00a4\u00a3\u0001\u0000\u0000\u0000\u00a4\u00a5\u0001\u0000\u0000\u0000"+
-    "\u00a5\u00a6\u0001\u0000\u0000\u0000\u00a6\u00a7\u0005-\u0000\u0000\u00a7"+
-    "\u00a8\u0005,\u0000\u0000\u00a8\u00ad\u0003\u000e\u0007\u0000\u00a9\u00aa"+
-    "\u0005&\u0000\u0000\u00aa\u00ac\u0003\u000e\u0007\u0000\u00ab\u00a9\u0001"+
-    "\u0000\u0000\u0000\u00ac\u00af\u0001\u0000\u0000\u0000\u00ad\u00ab\u0001"+
-    "\u0000\u0000\u0000\u00ad\u00ae\u0001\u0000\u0000\u0000\u00ae\u00b0\u0001"+
-    "\u0000\u0000\u0000\u00af\u00ad\u0001\u0000\u0000\u0000\u00b0\u00b1\u0005"+
-    "6\u0000\u0000\u00b1\u00ba\u0001\u0000\u0000\u0000\u00b2\u00b3\u0003\u000e"+
-    "\u0007\u0000\u00b3\u00b5\u0005.\u0000\u0000\u00b4\u00b6\u00050\u0000\u0000"+
-    "\u00b5\u00b4\u0001\u0000\u0000\u0000\u00b5\u00b6\u0001\u0000\u0000\u0000"+
-    "\u00b6\u00b7\u0001\u0000\u0000\u0000\u00b7\u00b8\u00051\u0000\u0000\u00b8"+
-    "\u00ba\u0001\u0000\u0000\u0000\u00b9\u009d\u0001\u0000\u0000\u0000\u00b9"+
-    "\u00a0\u0001\u0000\u0000\u0000\u00b9\u00a1\u0001\u0000\u0000\u0000\u00b9"+
-    "\u00a2\u0001\u0000\u0000\u0000\u00b9\u00b2\u0001\u0000\u0000\u0000\u00ba"+
-    "\u00c3\u0001\u0000\u0000\u0000\u00bb\u00bc\n\u0004\u0000\u0000\u00bc\u00bd"+
-    "\u0005\"\u0000\u0000\u00bd\u00c2\u0003\n\u0005\u0005\u00be\u00bf\n\u0003"+
-    "\u0000\u0000\u00bf\u00c0\u00053\u0000\u0000\u00c0\u00c2\u0003\n\u0005"+
-    "\u0004\u00c1\u00bb\u0001\u0000\u0000\u0000\u00c1\u00be\u0001\u0000\u0000"+
-    "\u0000\u00c2\u00c5\u0001\u0000\u0000\u0000\u00c3\u00c1\u0001\u0000\u0000"+
-    "\u0000\u00c3\u00c4\u0001\u0000\u0000\u0000\u00c4\u000b\u0001\u0000\u0000"+
-    "\u0000\u00c5\u00c3\u0001\u0000\u0000\u0000\u00c6\u00c8\u0003\u000e\u0007"+
-    "\u0000\u00c7\u00c9\u00050\u0000\u0000\u00c8\u00c7\u0001\u0000\u0000\u0000"+
-    "\u00c8\u00c9\u0001\u0000\u0000\u0000\u00c9\u00ca\u0001\u0000\u0000\u0000"+
-    "\u00ca\u00cb\u0005/\u0000\u0000\u00cb\u00cc\u0003b1\u0000\u00cc\u00d5"+
-    "\u0001\u0000\u0000\u0000\u00cd\u00cf\u0003\u000e\u0007\u0000\u00ce\u00d0"+
-    "\u00050\u0000\u0000\u00cf\u00ce\u0001\u0000\u0000\u0000\u00cf\u00d0\u0001"+
-    "\u0000\u0000\u0000\u00d0\u00d1\u0001\u0000\u0000\u0000\u00d1\u00d2\u0005"+
-    "5\u0000\u0000\u00d2\u00d3\u0003b1\u0000\u00d3\u00d5\u0001\u0000\u0000"+
-    "\u0000\u00d4\u00c6\u0001\u0000\u0000\u0000\u00d4\u00cd\u0001\u0000\u0000"+
-    "\u0000\u00d5\r\u0001\u0000\u0000\u0000\u00d6\u00dc\u0003\u0010\b\u0000"+
-    "\u00d7\u00d8\u0003\u0010\b\u0000\u00d8\u00d9\u0003d2\u0000\u00d9\u00da"+
-    "\u0003\u0010\b\u0000\u00da\u00dc\u0001\u0000\u0000\u0000\u00db\u00d6\u0001"+
-    "\u0000\u0000\u0000\u00db\u00d7\u0001\u0000\u0000\u0000\u00dc\u000f\u0001"+
-    "\u0000\u0000\u0000\u00dd\u00de\u0006\b\uffff\uffff\u0000\u00de\u00e2\u0003"+
-    "\u0012\t\u0000\u00df\u00e0\u0007\u0000\u0000\u0000\u00e0\u00e2\u0003\u0010"+
-    "\b\u0003\u00e1\u00dd\u0001\u0000\u0000\u0000\u00e1\u00df\u0001\u0000\u0000"+
-    "\u0000\u00e2\u00eb\u0001\u0000\u0000\u0000\u00e3\u00e4\n\u0002\u0000\u0000"+
-    "\u00e4\u00e5\u0007\u0001\u0000\u0000\u00e5\u00ea\u0003\u0010\b\u0003\u00e6"+
-    "\u00e7\n\u0001\u0000\u0000\u00e7\u00e8\u0007\u0000\u0000\u0000\u00e8\u00ea"+
-    "\u0003\u0010\b\u0002\u00e9\u00e3\u0001\u0000\u0000\u0000\u00e9\u00e6\u0001"+
-    "\u0000\u0000\u0000\u00ea\u00ed\u0001\u0000\u0000\u0000\u00eb\u00e9\u0001"+
-    "\u0000\u0000\u0000\u00eb\u00ec\u0001\u0000\u0000\u0000\u00ec\u0011\u0001"+
-    "\u0000\u0000\u0000\u00ed\u00eb\u0001\u0000\u0000\u0000\u00ee\u00ef\u0006"+
-    "\t\uffff\uffff\u0000\u00ef\u00f7\u0003>\u001f\u0000\u00f0\u00f7\u0003"+
-    "4\u001a\u0000\u00f1\u00f7\u0003\u0014\n\u0000\u00f2\u00f3\u0005,\u0000"+
-    "\u0000\u00f3\u00f4\u0003\n\u0005\u0000\u00f4\u00f5\u00056\u0000\u0000"+
-    "\u00f5\u00f7\u0001\u0000\u0000\u0000\u00f6\u00ee\u0001\u0000\u0000\u0000"+
-    "\u00f6\u00f0\u0001\u0000\u0000\u0000\u00f6\u00f1\u0001\u0000\u0000\u0000"+
-    "\u00f6\u00f2\u0001\u0000\u0000\u0000\u00f7\u00fd\u0001\u0000\u0000\u0000"+
-    "\u00f8\u00f9\n\u0001\u0000\u0000\u00f9\u00fa\u0005%\u0000\u0000\u00fa"+
-    "\u00fc\u0003\u0016\u000b\u0000\u00fb\u00f8\u0001\u0000\u0000\u0000\u00fc"+
-    "\u00ff\u0001\u0000\u0000\u0000\u00fd\u00fb\u0001\u0000\u0000\u0000\u00fd"+
-    "\u00fe\u0001\u0000\u0000\u0000\u00fe\u0013\u0001\u0000\u0000\u0000\u00ff"+
-    "\u00fd\u0001\u0000\u0000\u0000\u0100\u0101\u0003:\u001d\u0000\u0101\u010b"+
-    "\u0005,\u0000\u0000\u0102\u010c\u0005A\u0000\u0000\u0103\u0108\u0003\n"+
-    "\u0005\u0000\u0104\u0105\u0005&\u0000\u0000\u0105\u0107\u0003\n\u0005"+
-    "\u0000\u0106\u0104\u0001\u0000\u0000\u0000\u0107\u010a\u0001\u0000\u0000"+
-    "\u0000\u0108\u0106\u0001\u0000\u0000\u0000\u0108\u0109\u0001\u0000\u0000"+
-    "\u0000\u0109\u010c\u0001\u0000\u0000\u0000\u010a\u0108\u0001\u0000\u0000"+
-    "\u0000\u010b\u0102\u0001\u0000\u0000\u0000\u010b\u0103\u0001\u0000\u0000"+
-    "\u0000\u010b\u010c\u0001\u0000\u0000\u0000\u010c\u010d\u0001\u0000\u0000"+
-    "\u0000\u010d\u010e\u00056\u0000\u0000\u010e\u0015\u0001\u0000\u0000\u0000"+
-    "\u010f\u0110\u0003:\u001d\u0000\u0110\u0017\u0001\u0000\u0000\u0000\u0111"+
-    "\u0112\u0005\u0010\u0000\u0000\u0112\u0113\u0003\u001a\r\u0000\u0113\u0019"+
-    "\u0001\u0000\u0000\u0000\u0114\u0119\u0003\u001c\u000e\u0000\u0115\u0116"+
-    "\u0005&\u0000\u0000\u0116\u0118\u0003\u001c\u000e\u0000\u0117\u0115\u0001"+
-    "\u0000\u0000\u0000\u0118\u011b\u0001\u0000\u0000\u0000\u0119\u0117\u0001"+
-    "\u0000\u0000\u0000\u0119\u011a\u0001\u0000\u0000\u0000\u011a\u001b\u0001"+
-    "\u0000\u0000\u0000\u011b\u0119\u0001\u0000\u0000\u0000\u011c\u0122\u0003"+
-    "\n\u0005\u0000\u011d\u011e\u00034\u001a\u0000\u011e\u011f\u0005$\u0000"+
-    "\u0000\u011f\u0120\u0003\n\u0005\u0000\u0120\u0122\u0001\u0000\u0000\u0000"+
-    "\u0121\u011c\u0001\u0000\u0000\u0000\u0121\u011d\u0001\u0000\u0000\u0000"+
-    "\u0122\u001d\u0001\u0000\u0000\u0000\u0123\u0124\u0005\u0006\u0000\u0000"+
-    "\u0124\u0129\u0003 \u0010\u0000\u0125\u0126\u0005&\u0000\u0000\u0126\u0128"+
-    "\u0003 \u0010\u0000\u0127\u0125\u0001\u0000\u0000\u0000\u0128\u012b\u0001"+
-    "\u0000\u0000\u0000\u0129\u0127\u0001\u0000\u0000\u0000\u0129\u012a\u0001"+
-    "\u0000\u0000\u0000\u012a\u012d\u0001\u0000\u0000\u0000\u012b\u0129\u0001"+
-    "\u0000\u0000\u0000\u012c\u012e\u0003&\u0013\u0000\u012d\u012c\u0001\u0000"+
-    "\u0000\u0000\u012d\u012e\u0001\u0000\u0000\u0000\u012e\u001f\u0001\u0000"+
-    "\u0000\u0000\u012f\u0130\u0003\"\u0011\u0000\u0130\u0131\u0005r\u0000"+
-    "\u0000\u0131\u0132\u0003$\u0012\u0000\u0132\u0135\u0001\u0000\u0000\u0000"+
-    "\u0133\u0135\u0003$\u0012\u0000\u0134\u012f\u0001\u0000\u0000\u0000\u0134"+
-    "\u0133\u0001\u0000\u0000\u0000\u0135!\u0001\u0000\u0000\u0000\u0136\u0137"+
-    "\u0005\u0019\u0000\u0000\u0137#\u0001\u0000\u0000\u0000\u0138\u0139\u0007"+
-    "\u0002\u0000\u0000\u0139%\u0001\u0000\u0000\u0000\u013a\u013d\u0003(\u0014"+
-    "\u0000\u013b\u013d\u0003*\u0015\u0000\u013c\u013a\u0001\u0000\u0000\u0000"+
-    "\u013c\u013b\u0001\u0000\u0000\u0000\u013d\'\u0001\u0000\u0000\u0000\u013e"+
-    "\u013f\u0005L\u0000\u0000\u013f\u0144\u0005\u0019\u0000\u0000\u0140\u0141"+
-    "\u0005&\u0000\u0000\u0141\u0143\u0005\u0019\u0000\u0000\u0142\u0140\u0001"+
-    "\u0000\u0000\u0000\u0143\u0146\u0001\u0000\u0000\u0000\u0144\u0142\u0001"+
-    "\u0000\u0000\u0000\u0144\u0145\u0001\u0000\u0000\u0000\u0145)\u0001\u0000"+
-    "\u0000\u0000\u0146\u0144\u0001\u0000\u0000\u0000\u0147\u0148\u0005E\u0000"+
-    "\u0000\u0148\u0149\u0003(\u0014\u0000\u0149\u014a\u0005F\u0000\u0000\u014a"+
-    "+\u0001\u0000\u0000\u0000\u014b\u014c\u0005\r\u0000\u0000\u014c\u0151"+
-    "\u0003 \u0010\u0000\u014d\u014e\u0005&\u0000\u0000\u014e\u0150\u0003 "+
-    "\u0010\u0000\u014f\u014d\u0001\u0000\u0000\u0000\u0150\u0153\u0001\u0000"+
-    "\u0000\u0000\u0151\u014f\u0001\u0000\u0000\u0000\u0151\u0152\u0001\u0000"+
-    "\u0000\u0000\u0152\u0155\u0001\u0000\u0000\u0000\u0153\u0151\u0001\u0000"+
-    "\u0000\u0000\u0154\u0156\u0003\u001a\r\u0000\u0155\u0154\u0001\u0000\u0000"+
-    "\u0000\u0155\u0156\u0001\u0000\u0000\u0000\u0156\u0159\u0001\u0000\u0000"+
-    "\u0000\u0157\u0158\u0005!\u0000\u0000\u0158\u015a\u0003\u001a\r\u0000"+
-    "\u0159\u0157\u0001\u0000\u0000\u0000\u0159\u015a\u0001\u0000\u0000\u0000"+
-    "\u015a-\u0001\u0000\u0000\u0000\u015b\u015c\u0005\u0004\u0000\u0000\u015c"+
-    "\u015d\u0003\u001a\r\u0000\u015d/\u0001\u0000\u0000\u0000\u015e\u0160"+
-    "\u0005\u0013\u0000\u0000\u015f\u0161\u0003\u001a\r\u0000\u0160\u015f\u0001"+
-    "\u0000\u0000\u0000\u0160\u0161\u0001\u0000\u0000\u0000\u0161\u0164\u0001"+
-    "\u0000\u0000\u0000\u0162\u0163\u0005!\u0000\u0000\u0163\u0165\u0003\u001a"+
-    "\r\u0000\u0164\u0162\u0001\u0000\u0000\u0000\u0164\u0165\u0001\u0000\u0000"+
-    "\u0000\u01651\u0001\u0000\u0000\u0000\u0166\u0167\u0005\b\u0000\u0000"+
-    "\u0167\u016a\u0003\u001a\r\u0000\u0168\u0169\u0005!\u0000\u0000\u0169"+
-    "\u016b\u0003\u001a\r\u0000\u016a\u0168\u0001\u0000\u0000\u0000\u016a\u016b"+
-    "\u0001\u0000\u0000\u0000\u016b3\u0001\u0000\u0000\u0000\u016c\u0171\u0003"+
-    ":\u001d\u0000\u016d\u016e\u0005(\u0000\u0000\u016e\u0170\u0003:\u001d"+
-    "\u0000\u016f\u016d\u0001\u0000\u0000\u0000\u0170\u0173\u0001\u0000\u0000"+
-    "\u0000\u0171\u016f\u0001\u0000\u0000\u0000\u0171\u0172\u0001\u0000\u0000"+
-    "\u0000\u01725\u0001\u0000\u0000\u0000\u0173\u0171\u0001\u0000\u0000\u0000"+
-    "\u0174\u0179\u0003<\u001e\u0000\u0175\u0176\u0005(\u0000\u0000\u0176\u0178"+
-    "\u0003<\u001e\u0000\u0177\u0175\u0001\u0000\u0000\u0000\u0178\u017b\u0001"+
-    "\u0000\u0000\u0000\u0179\u0177\u0001\u0000\u0000\u0000\u0179\u017a\u0001"+
-    "\u0000\u0000\u0000\u017a7\u0001\u0000\u0000\u0000\u017b\u0179\u0001\u0000"+
-    "\u0000\u0000\u017c\u0181\u00036\u001b\u0000\u017d\u017e\u0005&\u0000\u0000"+
-    "\u017e\u0180\u00036\u001b\u0000\u017f\u017d\u0001\u0000\u0000\u0000\u0180"+
-    "\u0183\u0001\u0000\u0000\u0000\u0181\u017f\u0001\u0000\u0000\u0000\u0181"+
-    "\u0182\u0001\u0000\u0000\u0000\u01829\u0001\u0000\u0000\u0000\u0183\u0181"+
-    "\u0001\u0000\u0000\u0000\u0184\u0185\u0007\u0003\u0000\u0000\u0185;\u0001"+
-    "\u0000\u0000\u0000\u0186\u0187\u0005P\u0000\u0000\u0187=\u0001\u0000\u0000"+
-    "\u0000\u0188\u01b3\u00051\u0000\u0000\u0189\u018a\u0003`0\u0000\u018a"+
-    "\u018b\u0005G\u0000\u0000\u018b\u01b3\u0001\u0000\u0000\u0000\u018c\u01b3"+
-    "\u0003^/\u0000\u018d\u01b3\u0003`0\u0000\u018e\u01b3\u0003Z-\u0000\u018f"+
-    "\u01b3\u0003@ \u0000\u0190\u01b3\u0003b1\u0000\u0191\u0192\u0005E\u0000"+
-    "\u0000\u0192\u0197\u0003\\.\u0000\u0193\u0194\u0005&\u0000\u0000\u0194"+
-    "\u0196\u0003\\.\u0000\u0195\u0193\u0001\u0000\u0000\u0000\u0196\u0199"+
-    "\u0001\u0000\u0000\u0000\u0197\u0195\u0001\u0000\u0000\u0000\u0197\u0198"+
-    "\u0001\u0000\u0000\u0000\u0198\u019a\u0001\u0000\u0000\u0000\u0199\u0197"+
-    "\u0001\u0000\u0000\u0000\u019a\u019b\u0005F\u0000\u0000\u019b\u01b3\u0001"+
-    "\u0000\u0000\u0000\u019c\u019d\u0005E\u0000\u0000\u019d\u01a2\u0003Z-"+
-    "\u0000\u019e\u019f\u0005&\u0000\u0000\u019f\u01a1\u0003Z-\u0000\u01a0"+
-    "\u019e\u0001\u0000\u0000\u0000\u01a1\u01a4\u0001\u0000\u0000\u0000\u01a2"+
-    "\u01a0\u0001\u0000\u0000\u0000\u01a2\u01a3\u0001\u0000\u0000\u0000\u01a3"+
-    "\u01a5\u0001\u0000\u0000\u0000\u01a4\u01a2\u0001\u0000\u0000\u0000\u01a5"+
-    "\u01a6\u0005F\u0000\u0000\u01a6\u01b3\u0001\u0000\u0000\u0000\u01a7\u01a8"+
-    "\u0005E\u0000\u0000\u01a8\u01ad\u0003b1\u0000\u01a9\u01aa\u0005&\u0000"+
-    "\u0000\u01aa\u01ac\u0003b1\u0000\u01ab\u01a9\u0001\u0000\u0000\u0000\u01ac"+
-    "\u01af\u0001\u0000\u0000\u0000\u01ad\u01ab\u0001\u0000\u0000\u0000\u01ad"+
-    "\u01ae\u0001\u0000\u0000\u0000\u01ae\u01b0\u0001\u0000\u0000\u0000\u01af"+
-    "\u01ad\u0001\u0000\u0000\u0000\u01b0\u01b1\u0005F\u0000\u0000\u01b1\u01b3"+
-    "\u0001\u0000\u0000\u0000\u01b2\u0188\u0001\u0000\u0000\u0000\u01b2\u0189"+
-    "\u0001\u0000\u0000\u0000\u01b2\u018c\u0001\u0000\u0000\u0000\u01b2\u018d"+
-    "\u0001\u0000\u0000\u0000\u01b2\u018e\u0001\u0000\u0000\u0000\u01b2\u018f"+
-    "\u0001\u0000\u0000\u0000\u01b2\u0190\u0001\u0000\u0000\u0000\u01b2\u0191"+
-    "\u0001\u0000\u0000\u0000\u01b2\u019c\u0001\u0000\u0000\u0000\u01b2\u01a7"+
-    "\u0001\u0000\u0000\u0000\u01b3?\u0001\u0000\u0000\u0000\u01b4\u01b7\u0005"+
-    "4\u0000\u0000\u01b5\u01b7\u0005D\u0000\u0000\u01b6\u01b4\u0001\u0000\u0000"+
-    "\u0000\u01b6\u01b5\u0001\u0000\u0000\u0000\u01b7A\u0001\u0000\u0000\u0000"+
-    "\u01b8\u01b9\u0005\n\u0000\u0000\u01b9\u01ba\u0005\u001f\u0000\u0000\u01ba"+
-    "C\u0001\u0000\u0000\u0000\u01bb\u01bc\u0005\u0012\u0000\u0000\u01bc\u01c1"+
-    "\u0003F#\u0000\u01bd\u01be\u0005&\u0000\u0000\u01be\u01c0\u0003F#\u0000"+
-    "\u01bf\u01bd\u0001\u0000\u0000\u0000\u01c0\u01c3\u0001\u0000\u0000\u0000"+
-    "\u01c1\u01bf\u0001\u0000\u0000\u0000\u01c1\u01c2\u0001\u0000\u0000\u0000"+
-    "\u01c2E\u0001\u0000\u0000\u0000\u01c3\u01c1\u0001\u0000\u0000\u0000\u01c4"+
-    "\u01c6\u0003\n\u0005\u0000\u01c5\u01c7\u0007\u0004\u0000\u0000\u01c6\u01c5"+
-    "\u0001\u0000\u0000\u0000\u01c6\u01c7\u0001\u0000\u0000\u0000\u01c7\u01ca"+
-    "\u0001\u0000\u0000\u0000\u01c8\u01c9\u00052\u0000\u0000\u01c9\u01cb\u0007"+
-    "\u0005\u0000\u0000\u01ca\u01c8\u0001\u0000\u0000\u0000\u01ca\u01cb\u0001"+
-    "\u0000\u0000\u0000\u01cbG\u0001\u0000\u0000\u0000\u01cc\u01cd\u0005\t"+
-    "\u0000\u0000\u01cd\u01ce\u00038\u001c\u0000\u01ceI\u0001\u0000\u0000\u0000"+
-    "\u01cf\u01d0\u0005\u0002\u0000\u0000\u01d0\u01d1\u00038\u001c\u0000\u01d1"+
-    "K\u0001\u0000\u0000\u0000\u01d2\u01d3\u0005\u000f\u0000\u0000\u01d3\u01d8"+
-    "\u0003N\'\u0000\u01d4\u01d5\u0005&\u0000\u0000\u01d5\u01d7\u0003N\'\u0000"+
-    "\u01d6\u01d4\u0001\u0000\u0000\u0000\u01d7\u01da\u0001\u0000\u0000\u0000"+
-    "\u01d8\u01d6\u0001\u0000\u0000\u0000\u01d8\u01d9\u0001\u0000\u0000\u0000"+
-    "\u01d9M\u0001\u0000\u0000\u0000\u01da\u01d8\u0001\u0000\u0000\u0000\u01db"+
-    "\u01dc\u00036\u001b\u0000\u01dc\u01dd\u0005T\u0000\u0000\u01dd\u01de\u0003"+
-    "6\u001b\u0000\u01deO\u0001\u0000\u0000\u0000\u01df\u01e0\u0005\u0001\u0000"+
-    "\u0000\u01e0\u01e1\u0003\u0012\t\u0000\u01e1\u01e3\u0003b1\u0000\u01e2"+
-    "\u01e4\u0003V+\u0000\u01e3\u01e2\u0001\u0000\u0000\u0000\u01e3\u01e4\u0001"+
-    "\u0000\u0000\u0000\u01e4Q\u0001\u0000\u0000\u0000\u01e5\u01e6\u0005\u0007"+
-    "\u0000\u0000\u01e6\u01e7\u0003\u0012\t\u0000\u01e7\u01e8\u0003b1\u0000"+
-    "\u01e8S\u0001\u0000\u0000\u0000\u01e9\u01ea\u0005\u000e\u0000\u0000\u01ea"+
-    "\u01eb\u00034\u001a\u0000\u01ebU\u0001\u0000\u0000\u0000\u01ec\u01f1\u0003"+
-    "X,\u0000\u01ed\u01ee\u0005&\u0000\u0000\u01ee\u01f0\u0003X,\u0000\u01ef"+
-    "\u01ed\u0001\u0000\u0000\u0000\u01f0\u01f3\u0001\u0000\u0000\u0000\u01f1"+
-    "\u01ef\u0001\u0000\u0000\u0000\u01f1\u01f2\u0001\u0000\u0000\u0000\u01f2"+
-    "W\u0001\u0000\u0000\u0000\u01f3\u01f1\u0001\u0000\u0000\u0000\u01f4\u01f5"+
-    "\u0003:\u001d\u0000\u01f5\u01f6\u0005$\u0000\u0000\u01f6\u01f7\u0003>"+
-    "\u001f\u0000\u01f7Y\u0001\u0000\u0000\u0000\u01f8\u01f9\u0007\u0006\u0000"+
-    "\u0000\u01f9[\u0001\u0000\u0000\u0000\u01fa\u01fd\u0003^/\u0000\u01fb"+
-    "\u01fd\u0003`0\u0000\u01fc\u01fa\u0001\u0000\u0000\u0000\u01fc\u01fb\u0001"+
-    "\u0000\u0000\u0000\u01fd]\u0001\u0000\u0000\u0000\u01fe\u0200\u0007\u0000"+
-    "\u0000\u0000\u01ff\u01fe\u0001\u0000\u0000\u0000\u01ff\u0200\u0001\u0000"+
-    "\u0000\u0000\u0200\u0201\u0001\u0000\u0000\u0000\u0201\u0202\u0005 \u0000"+
-    "\u0000\u0202_\u0001\u0000\u0000\u0000\u0203\u0205\u0007\u0000\u0000\u0000"+
-    "\u0204\u0203\u0001\u0000\u0000\u0000\u0204\u0205\u0001\u0000\u0000\u0000"+
-    "\u0205\u0206\u0001\u0000\u0000\u0000\u0206\u0207\u0005\u001f\u0000\u0000"+
-    "\u0207a\u0001\u0000\u0000\u0000\u0208\u0209\u0005\u001e\u0000\u0000\u0209"+
-    "c\u0001\u0000\u0000\u0000\u020a\u020b\u0007\u0007\u0000\u0000\u020be\u0001"+
-    "\u0000\u0000\u0000\u020c\u020d\u0005\u0005\u0000\u0000\u020d\u020e\u0003"+
-    "h4\u0000\u020eg\u0001\u0000\u0000\u0000\u020f\u0210\u0005E\u0000\u0000"+
-    "\u0210\u0211\u0003\u0002\u0001\u0000\u0211\u0212\u0005F\u0000\u0000\u0212"+
-    "i\u0001\u0000\u0000\u0000\u0213\u0214\u0005\u0011\u0000\u0000\u0214\u0215"+
-    "\u0005j\u0000\u0000\u0215k\u0001\u0000\u0000\u0000\u0216\u0217\u0005\f"+
-    "\u0000\u0000\u0217\u0218\u0005n\u0000\u0000\u0218m\u0001\u0000\u0000\u0000"+
-    "\u0219\u021a\u0005\u0003\u0000\u0000\u021a\u021d\u0005Z\u0000\u0000\u021b"+
-    "\u021c\u0005X\u0000\u0000\u021c\u021e\u00036\u001b\u0000\u021d\u021b\u0001"+
-    "\u0000\u0000\u0000\u021d\u021e\u0001\u0000\u0000\u0000\u021e\u0228\u0001"+
-    "\u0000\u0000\u0000\u021f\u0220\u0005Y\u0000\u0000\u0220\u0225\u0003p8"+
-    "\u0000\u0221\u0222\u0005&\u0000\u0000\u0222\u0224\u0003p8\u0000\u0223"+
-    "\u0221\u0001\u0000\u0000\u0000\u0224\u0227\u0001\u0000\u0000\u0000\u0225"+
-    "\u0223\u0001\u0000\u0000\u0000\u0225\u0226\u0001\u0000\u0000\u0000\u0226"+
-    "\u0229\u0001\u0000\u0000\u0000\u0227\u0225\u0001\u0000\u0000\u0000\u0228"+
-    "\u021f\u0001\u0000\u0000\u0000\u0228\u0229\u0001\u0000\u0000\u0000\u0229"+
-    "o\u0001\u0000\u0000\u0000\u022a\u022b\u00036\u001b\u0000\u022b\u022c\u0005"+
-    "$\u0000\u0000\u022c\u022e\u0001\u0000\u0000\u0000\u022d\u022a\u0001\u0000"+
-    "\u0000\u0000\u022d\u022e\u0001\u0000\u0000\u0000\u022e\u022f\u0001\u0000"+
-    "\u0000\u0000\u022f\u0230\u00036\u001b\u0000\u0230q\u0001\u0000\u0000\u0000"+
-    "\u0231\u0232\u0005\u000b\u0000\u0000\u0232\u0233\u0003 \u0010\u0000\u0233"+
-    "\u0234\u0005X\u0000\u0000\u0234\u0235\u00038\u001c\u0000\u0235s\u0001"+
-    "\u0000\u0000\u00006\u007f\u0088\u0098\u00a4\u00ad\u00b5\u00b9\u00c1\u00c3"+
-    "\u00c8\u00cf\u00d4\u00db\u00e1\u00e9\u00eb\u00f6\u00fd\u0108\u010b\u0119"+
-    "\u0121\u0129\u012d\u0134\u013c\u0144\u0151\u0155\u0159\u0160\u0164\u016a"+
-    "\u0171\u0179\u0181\u0197\u01a2\u01ad\u01b2\u01b6\u01c1\u01c6\u01ca\u01d8"+
-    "\u01e3\u01f1\u01fc\u01ff\u0204\u021d\u0225\u0228\u022d";
+    "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f"+
+    "\u01a1\b\u001f\n\u001f\f\u001f\u01a4\t\u001f\u0001\u001f\u0001\u001f\u0001"+
+    "\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01ac\b\u001f\n"+
+    "\u001f\f\u001f\u01af\t\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001"+
+    "\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01b7\b\u001f\n\u001f\f\u001f"+
+    "\u01ba\t\u001f\u0001\u001f\u0001\u001f\u0003\u001f\u01be\b\u001f\u0001"+
+    " \u0001 \u0003 \u01c2\b \u0001!\u0001!\u0001!\u0001\"\u0001\"\u0001\""+
+    "\u0001\"\u0005\"\u01cb\b\"\n\"\f\"\u01ce\t\"\u0001#\u0001#\u0003#\u01d2"+
+    "\b#\u0001#\u0001#\u0003#\u01d6\b#\u0001$\u0001$\u0001$\u0001%\u0001%\u0001"+
+    "%\u0001&\u0001&\u0001&\u0001&\u0005&\u01e2\b&\n&\f&\u01e5\t&\u0001\'\u0001"+
+    "\'\u0001\'\u0001\'\u0001(\u0001(\u0001(\u0001(\u0003(\u01ef\b(\u0001)"+
+    "\u0001)\u0001)\u0001)\u0001*\u0001*\u0001*\u0001+\u0001+\u0001+\u0005"+
+    "+\u01fb\b+\n+\f+\u01fe\t+\u0001,\u0001,\u0001,\u0001,\u0001-\u0001-\u0001"+
+    ".\u0001.\u0003.\u0208\b.\u0001/\u0003/\u020b\b/\u0001/\u0001/\u00010\u0003"+
+    "0\u0210\b0\u00010\u00010\u00011\u00011\u00012\u00012\u00013\u00013\u0001"+
+    "3\u00014\u00014\u00014\u00014\u00015\u00015\u00015\u00016\u00016\u0001"+
+    "6\u00017\u00017\u00017\u00017\u00037\u0229\b7\u00017\u00017\u00017\u0001"+
+    "7\u00057\u022f\b7\n7\f7\u0232\t7\u00037\u0234\b7\u00018\u00018\u00018"+
+    "\u00038\u0239\b8\u00018\u00018\u00019\u00019\u00019\u00019\u00019\u0001"+
+    ":\u0001:\u0001:\u0001:\u0003:\u0246\b:\u0001;\u0001;\u0001;\u0001<\u0001"+
+    "<\u0001<\u0000\u0004\u0002\n\u0012\u0014=\u0000\u0002\u0004\u0006\b\n"+
+    "\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.0246"+
+    "8:<>@BDFHJLNPRTVXZ\\^`bdfhjlnprtvx\u0000\b\u0001\u0000<=\u0001\u0000>"+
+    "@\u0002\u0000\u001b\u001bMM\u0001\u0000DE\u0002\u0000  $$\u0002\u0000"+
+    "\'\'**\u0002\u0000&&44\u0002\u0000557;\u0265\u0000z\u0001\u0000\u0000"+
+    "\u0000\u0002}\u0001\u0000\u0000\u0000\u0004\u008f\u0001\u0000\u0000\u0000"+
+    "\u0006\u00a3\u0001\u0000\u0000\u0000\b\u00a5\u0001\u0000\u0000\u0000\n"+
+    "\u00c6\u0001\u0000\u0000\u0000\f\u00e1\u0001\u0000\u0000\u0000\u000e\u00e3"+
+    "\u0001\u0000\u0000\u0000\u0010\u00ec\u0001\u0000\u0000\u0000\u0012\u00f2"+
+    "\u0001\u0000\u0000\u0000\u0014\u0107\u0001\u0000\u0000\u0000\u0016\u0111"+
+    "\u0001\u0000\u0000\u0000\u0018\u0120\u0001\u0000\u0000\u0000\u001a\u0122"+
+    "\u0001\u0000\u0000\u0000\u001c\u0125\u0001\u0000\u0000\u0000\u001e\u0132"+
+    "\u0001\u0000\u0000\u0000 \u0134\u0001\u0000\u0000\u0000\"\u0145\u0001"+
+    "\u0000\u0000\u0000$\u0147\u0001\u0000\u0000\u0000&\u0149\u0001\u0000\u0000"+
+    "\u0000(\u014d\u0001\u0000\u0000\u0000*\u014f\u0001\u0000\u0000\u0000,"+
+    "\u0158\u0001\u0000\u0000\u0000.\u015c\u0001\u0000\u0000\u00000\u016c\u0001"+
+    "\u0000\u0000\u00002\u016f\u0001\u0000\u0000\u00004\u0177\u0001\u0000\u0000"+
+    "\u00006\u017f\u0001\u0000\u0000\u00008\u0187\u0001\u0000\u0000\u0000:"+
+    "\u018f\u0001\u0000\u0000\u0000<\u0191\u0001\u0000\u0000\u0000>\u01bd\u0001"+
+    "\u0000\u0000\u0000@\u01c1\u0001\u0000\u0000\u0000B\u01c3\u0001\u0000\u0000"+
+    "\u0000D\u01c6\u0001\u0000\u0000\u0000F\u01cf\u0001\u0000\u0000\u0000H"+
+    "\u01d7\u0001\u0000\u0000\u0000J\u01da\u0001\u0000\u0000\u0000L\u01dd\u0001"+
+    "\u0000\u0000\u0000N\u01e6\u0001\u0000\u0000\u0000P\u01ea\u0001\u0000\u0000"+
+    "\u0000R\u01f0\u0001\u0000\u0000\u0000T\u01f4\u0001\u0000\u0000\u0000V"+
+    "\u01f7\u0001\u0000\u0000\u0000X\u01ff\u0001\u0000\u0000\u0000Z\u0203\u0001"+
+    "\u0000\u0000\u0000\\\u0207\u0001\u0000\u0000\u0000^\u020a\u0001\u0000"+
+    "\u0000\u0000`\u020f\u0001\u0000\u0000\u0000b\u0213\u0001\u0000\u0000\u0000"+
+    "d\u0215\u0001\u0000\u0000\u0000f\u0217\u0001\u0000\u0000\u0000h\u021a"+
+    "\u0001\u0000\u0000\u0000j\u021e\u0001\u0000\u0000\u0000l\u0221\u0001\u0000"+
+    "\u0000\u0000n\u0224\u0001\u0000\u0000\u0000p\u0238\u0001\u0000\u0000\u0000"+
+    "r\u023c\u0001\u0000\u0000\u0000t\u0241\u0001\u0000\u0000\u0000v\u0247"+
+    "\u0001\u0000\u0000\u0000x\u024a\u0001\u0000\u0000\u0000z{\u0003\u0002"+
+    "\u0001\u0000{|\u0005\u0000\u0000\u0001|\u0001\u0001\u0000\u0000\u0000"+
+    "}~\u0006\u0001\uffff\uffff\u0000~\u007f\u0003\u0004\u0002\u0000\u007f"+
+    "\u0085\u0001\u0000\u0000\u0000\u0080\u0081\n\u0001\u0000\u0000\u0081\u0082"+
+    "\u0005\u001a\u0000\u0000\u0082\u0084\u0003\u0006\u0003\u0000\u0083\u0080"+
+    "\u0001\u0000\u0000\u0000\u0084\u0087\u0001\u0000\u0000\u0000\u0085\u0083"+
+    "\u0001\u0000\u0000\u0000\u0085\u0086\u0001\u0000\u0000\u0000\u0086\u0003"+
+    "\u0001\u0000\u0000\u0000\u0087\u0085\u0001\u0000\u0000\u0000\u0088\u0090"+
+    "\u0003f3\u0000\u0089\u0090\u0003 \u0010\u0000\u008a\u0090\u0003l6\u0000"+
+    "\u008b\u0090\u0003\u001a\r\u0000\u008c\u0090\u0003j5\u0000\u008d\u008e"+
+    "\u0004\u0002\u0001\u0000\u008e\u0090\u0003.\u0017\u0000\u008f\u0088\u0001"+
+    "\u0000\u0000\u0000\u008f\u0089\u0001\u0000\u0000\u0000\u008f\u008a\u0001"+
+    "\u0000\u0000\u0000\u008f\u008b\u0001\u0000\u0000\u0000\u008f\u008c\u0001"+
+    "\u0000\u0000\u0000\u008f\u008d\u0001\u0000\u0000\u0000\u0090\u0005\u0001"+
+    "\u0000\u0000\u0000\u0091\u00a4\u00030\u0018\u0000\u0092\u00a4\u0003\b"+
+    "\u0004\u0000\u0093\u00a4\u0003H$\u0000\u0094\u00a4\u0003B!\u0000\u0095"+
+    "\u00a4\u00032\u0019\u0000\u0096\u00a4\u0003D\"\u0000\u0097\u00a4\u0003"+
+    "J%\u0000\u0098\u00a4\u0003L&\u0000\u0099\u00a4\u0003P(\u0000\u009a\u00a4"+
+    "\u0003R)\u0000\u009b\u00a4\u0003n7\u0000\u009c\u00a4\u0003T*\u0000\u009d"+
+    "\u009e\u0004\u0003\u0002\u0000\u009e\u00a4\u0003t:\u0000\u009f\u00a0\u0004"+
+    "\u0003\u0003\u0000\u00a0\u00a4\u0003r9\u0000\u00a1\u00a2\u0004\u0003\u0004"+
+    "\u0000\u00a2\u00a4\u0003v;\u0000\u00a3\u0091\u0001\u0000\u0000\u0000\u00a3"+
+    "\u0092\u0001\u0000\u0000\u0000\u00a3\u0093\u0001\u0000\u0000\u0000\u00a3"+
+    "\u0094\u0001\u0000\u0000\u0000\u00a3\u0095\u0001\u0000\u0000\u0000\u00a3"+
+    "\u0096\u0001\u0000\u0000\u0000\u00a3\u0097\u0001\u0000\u0000\u0000\u00a3"+
+    "\u0098\u0001\u0000\u0000\u0000\u00a3\u0099\u0001\u0000\u0000\u0000\u00a3"+
+    "\u009a\u0001\u0000\u0000\u0000\u00a3\u009b\u0001\u0000\u0000\u0000\u00a3"+
+    "\u009c\u0001\u0000\u0000\u0000\u00a3\u009d\u0001\u0000\u0000\u0000\u00a3"+
+    "\u009f\u0001\u0000\u0000\u0000\u00a3\u00a1\u0001\u0000\u0000\u0000\u00a4"+
+    "\u0007\u0001\u0000\u0000\u0000\u00a5\u00a6\u0005\u0011\u0000\u0000\u00a6"+
+    "\u00a7\u0003\n\u0005\u0000\u00a7\t\u0001\u0000\u0000\u0000\u00a8\u00a9"+
+    "\u0006\u0005\uffff\uffff\u0000\u00a9\u00aa\u0005-\u0000\u0000\u00aa\u00c7"+
+    "\u0003\n\u0005\b\u00ab\u00c7\u0003\u0010\b\u0000\u00ac\u00c7\u0003\f\u0006"+
+    "\u0000\u00ad\u00af\u0003\u0010\b\u0000\u00ae\u00b0\u0005-\u0000\u0000"+
+    "\u00af\u00ae\u0001\u0000\u0000\u0000\u00af\u00b0\u0001\u0000\u0000\u0000"+
+    "\u00b0\u00b1\u0001\u0000\u0000\u0000\u00b1\u00b2\u0005(\u0000\u0000\u00b2"+
+    "\u00b3\u0005,\u0000\u0000\u00b3\u00b8\u0003\u0010\b\u0000\u00b4\u00b5"+
+    "\u0005#\u0000\u0000\u00b5\u00b7\u0003\u0010\b\u0000\u00b6\u00b4\u0001"+
+    "\u0000\u0000\u0000\u00b7\u00ba\u0001\u0000\u0000\u0000\u00b8\u00b6\u0001"+
+    "\u0000\u0000\u0000\u00b8\u00b9\u0001\u0000\u0000\u0000\u00b9\u00bb\u0001"+
+    "\u0000\u0000\u0000\u00ba\u00b8\u0001\u0000\u0000\u0000\u00bb\u00bc\u0005"+
+    "3\u0000\u0000\u00bc\u00c7\u0001\u0000\u0000\u0000\u00bd\u00be\u0003\u0010"+
+    "\b\u0000\u00be\u00c0\u0005)\u0000\u0000\u00bf\u00c1\u0005-\u0000\u0000"+
+    "\u00c0\u00bf\u0001\u0000\u0000\u0000\u00c0\u00c1\u0001\u0000\u0000\u0000"+
+    "\u00c1\u00c2\u0001\u0000\u0000\u0000\u00c2\u00c3\u0005.\u0000\u0000\u00c3"+
+    "\u00c7\u0001\u0000\u0000\u0000\u00c4\u00c5\u0004\u0005\u0005\u0000\u00c5"+
+    "\u00c7\u0003\u000e\u0007\u0000\u00c6\u00a8\u0001\u0000\u0000\u0000\u00c6"+
+    "\u00ab\u0001\u0000\u0000\u0000\u00c6\u00ac\u0001\u0000\u0000\u0000\u00c6"+
+    "\u00ad\u0001\u0000\u0000\u0000\u00c6\u00bd\u0001\u0000\u0000\u0000\u00c6"+
+    "\u00c4\u0001\u0000\u0000\u0000\u00c7\u00d0\u0001\u0000\u0000\u0000\u00c8"+
+    "\u00c9\n\u0005\u0000\u0000\u00c9\u00ca\u0005\u001f\u0000\u0000\u00ca\u00cf"+
+    "\u0003\n\u0005\u0006\u00cb\u00cc\n\u0004\u0000\u0000\u00cc\u00cd\u0005"+
+    "0\u0000\u0000\u00cd\u00cf\u0003\n\u0005\u0005\u00ce\u00c8\u0001\u0000"+
+    "\u0000\u0000\u00ce\u00cb\u0001\u0000\u0000\u0000\u00cf\u00d2\u0001\u0000"+
+    "\u0000\u0000\u00d0\u00ce\u0001\u0000\u0000\u0000\u00d0\u00d1\u0001\u0000"+
+    "\u0000\u0000\u00d1\u000b\u0001\u0000\u0000\u0000\u00d2\u00d0\u0001\u0000"+
+    "\u0000\u0000\u00d3\u00d5\u0003\u0010\b\u0000\u00d4\u00d6\u0005-\u0000"+
+    "\u0000\u00d5\u00d4\u0001\u0000\u0000\u0000\u00d5\u00d6\u0001\u0000\u0000"+
+    "\u0000\u00d6\u00d7\u0001\u0000\u0000\u0000\u00d7\u00d8\u0005+\u0000\u0000"+
+    "\u00d8\u00d9\u0003b1\u0000\u00d9\u00e2\u0001\u0000\u0000\u0000\u00da\u00dc"+
+    "\u0003\u0010\b\u0000\u00db\u00dd\u0005-\u0000\u0000\u00dc\u00db\u0001"+
+    "\u0000\u0000\u0000\u00dc\u00dd\u0001\u0000\u0000\u0000\u00dd\u00de\u0001"+
+    "\u0000\u0000\u0000\u00de\u00df\u00052\u0000\u0000\u00df\u00e0\u0003b1"+
+    "\u0000\u00e0\u00e2\u0001\u0000\u0000\u0000\u00e1\u00d3\u0001\u0000\u0000"+
+    "\u0000\u00e1\u00da\u0001\u0000\u0000\u0000\u00e2\r\u0001\u0000\u0000\u0000"+
+    "\u00e3\u00e4\u0003\u0010\b\u0000\u00e4\u00e5\u0005\u0014\u0000\u0000\u00e5"+
+    "\u00e6\u0003b1\u0000\u00e6\u000f\u0001\u0000\u0000\u0000\u00e7\u00ed\u0003"+
+    "\u0012\t\u0000\u00e8\u00e9\u0003\u0012\t\u0000\u00e9\u00ea\u0003d2\u0000"+
+    "\u00ea\u00eb\u0003\u0012\t\u0000\u00eb\u00ed\u0001\u0000\u0000\u0000\u00ec"+
+    "\u00e7\u0001\u0000\u0000\u0000\u00ec\u00e8\u0001\u0000\u0000\u0000\u00ed"+
+    "\u0011\u0001\u0000\u0000\u0000\u00ee\u00ef\u0006\t\uffff\uffff\u0000\u00ef"+
+    "\u00f3\u0003\u0014\n\u0000\u00f0\u00f1\u0007\u0000\u0000\u0000\u00f1\u00f3"+
+    "\u0003\u0012\t\u0003\u00f2\u00ee\u0001\u0000\u0000\u0000\u00f2\u00f0\u0001"+
+    "\u0000\u0000\u0000\u00f3\u00fc\u0001\u0000\u0000\u0000\u00f4\u00f5\n\u0002"+
+    "\u0000\u0000\u00f5\u00f6\u0007\u0001\u0000\u0000\u00f6\u00fb\u0003\u0012"+
+    "\t\u0003\u00f7\u00f8\n\u0001\u0000\u0000\u00f8\u00f9\u0007\u0000\u0000"+
+    "\u0000\u00f9\u00fb\u0003\u0012\t\u0002\u00fa\u00f4\u0001\u0000\u0000\u0000"+
+    "\u00fa\u00f7\u0001\u0000\u0000\u0000\u00fb\u00fe\u0001\u0000\u0000\u0000"+
+    "\u00fc\u00fa\u0001\u0000\u0000\u0000\u00fc\u00fd\u0001\u0000\u0000\u0000"+
+    "\u00fd\u0013\u0001\u0000\u0000\u0000\u00fe\u00fc\u0001\u0000\u0000\u0000"+
+    "\u00ff\u0100\u0006\n\uffff\uffff\u0000\u0100\u0108\u0003>\u001f\u0000"+
+    "\u0101\u0108\u00034\u001a\u0000\u0102\u0108\u0003\u0016\u000b\u0000\u0103"+
+    "\u0104\u0005,\u0000\u0000\u0104\u0105\u0003\n\u0005\u0000\u0105\u0106"+
+    "\u00053\u0000\u0000\u0106\u0108\u0001\u0000\u0000\u0000\u0107\u00ff\u0001"+
+    "\u0000\u0000\u0000\u0107\u0101\u0001\u0000\u0000\u0000\u0107\u0102\u0001"+
+    "\u0000\u0000\u0000\u0107\u0103\u0001\u0000\u0000\u0000\u0108\u010e\u0001"+
+    "\u0000\u0000\u0000\u0109\u010a\n\u0001\u0000\u0000\u010a\u010b\u0005\""+
+    "\u0000\u0000\u010b\u010d\u0003\u0018\f\u0000\u010c\u0109\u0001\u0000\u0000"+
+    "\u0000\u010d\u0110\u0001\u0000\u0000\u0000\u010e\u010c\u0001\u0000\u0000"+
+    "\u0000\u010e\u010f\u0001\u0000\u0000\u0000\u010f\u0015\u0001\u0000\u0000"+
+    "\u0000\u0110\u010e\u0001\u0000\u0000\u0000\u0111\u0112\u0003:\u001d\u0000"+
+    "\u0112\u011c\u0005,\u0000\u0000\u0113\u011d\u0005>\u0000\u0000\u0114\u0119"+
+    "\u0003\n\u0005\u0000\u0115\u0116\u0005#\u0000\u0000\u0116\u0118\u0003"+
+    "\n\u0005\u0000\u0117\u0115\u0001\u0000\u0000\u0000\u0118\u011b\u0001\u0000"+
+    "\u0000\u0000\u0119\u0117\u0001\u0000\u0000\u0000\u0119\u011a\u0001\u0000"+
+    "\u0000\u0000\u011a\u011d\u0001\u0000\u0000\u0000\u011b\u0119\u0001\u0000"+
+    "\u0000\u0000\u011c\u0113\u0001\u0000\u0000\u0000\u011c\u0114\u0001\u0000"+
+    "\u0000\u0000\u011c\u011d\u0001\u0000\u0000\u0000\u011d\u011e\u0001\u0000"+
+    "\u0000\u0000\u011e\u011f\u00053\u0000\u0000\u011f\u0017\u0001\u0000\u0000"+
+    "\u0000\u0120\u0121\u0003:\u001d\u0000\u0121\u0019\u0001\u0000\u0000\u0000"+
+    "\u0122\u0123\u0005\r\u0000\u0000\u0123\u0124\u0003\u001c\u000e\u0000\u0124"+
+    "\u001b\u0001\u0000\u0000\u0000\u0125\u012a\u0003\u001e\u000f\u0000\u0126"+
+    "\u0127\u0005#\u0000\u0000\u0127\u0129\u0003\u001e\u000f\u0000\u0128\u0126"+
+    "\u0001\u0000\u0000\u0000\u0129\u012c\u0001\u0000\u0000\u0000\u012a\u0128"+
+    "\u0001\u0000\u0000\u0000\u012a\u012b\u0001\u0000\u0000\u0000\u012b\u001d"+
+    "\u0001\u0000\u0000\u0000\u012c\u012a\u0001\u0000\u0000\u0000\u012d\u0133"+
+    "\u0003\n\u0005\u0000\u012e\u012f\u00034\u001a\u0000\u012f\u0130\u0005"+
+    "!\u0000\u0000\u0130\u0131\u0003\n\u0005\u0000\u0131\u0133\u0001\u0000"+
+    "\u0000\u0000\u0132\u012d\u0001\u0000\u0000\u0000\u0132\u012e\u0001\u0000"+
+    "\u0000\u0000\u0133\u001f\u0001\u0000\u0000\u0000\u0134\u0135\u0005\u0006"+
+    "\u0000\u0000\u0135\u013a\u0003\"\u0011\u0000\u0136\u0137\u0005#\u0000"+
+    "\u0000\u0137\u0139\u0003\"\u0011\u0000\u0138\u0136\u0001\u0000\u0000\u0000"+
+    "\u0139\u013c\u0001\u0000\u0000\u0000\u013a\u0138\u0001\u0000\u0000\u0000"+
+    "\u013a\u013b\u0001\u0000\u0000\u0000\u013b\u013e\u0001\u0000\u0000\u0000"+
+    "\u013c\u013a\u0001\u0000\u0000\u0000\u013d\u013f\u0003(\u0014\u0000\u013e"+
+    "\u013d\u0001\u0000\u0000\u0000\u013e\u013f\u0001\u0000\u0000\u0000\u013f"+
+    "!\u0001\u0000\u0000\u0000\u0140\u0141\u0003$\u0012\u0000\u0141\u0142\u0005"+
+    "m\u0000\u0000\u0142\u0143\u0003&\u0013\u0000\u0143\u0146\u0001\u0000\u0000"+
+    "\u0000\u0144\u0146\u0003&\u0013\u0000\u0145\u0140\u0001\u0000\u0000\u0000"+
+    "\u0145\u0144\u0001\u0000\u0000\u0000\u0146#\u0001\u0000\u0000\u0000\u0147"+
+    "\u0148\u0005M\u0000\u0000\u0148%\u0001\u0000\u0000\u0000\u0149\u014a\u0007"+
+    "\u0002\u0000\u0000\u014a\'\u0001\u0000\u0000\u0000\u014b\u014e\u0003*"+
+    "\u0015\u0000\u014c\u014e\u0003,\u0016\u0000\u014d\u014b\u0001\u0000\u0000"+
+    "\u0000\u014d\u014c\u0001\u0000\u0000\u0000\u014e)\u0001\u0000\u0000\u0000"+
+    "\u014f\u0150\u0005L\u0000\u0000\u0150\u0155\u0005M\u0000\u0000\u0151\u0152"+
+    "\u0005#\u0000\u0000\u0152\u0154\u0005M\u0000\u0000\u0153\u0151\u0001\u0000"+
+    "\u0000\u0000\u0154\u0157\u0001\u0000\u0000\u0000\u0155\u0153\u0001\u0000"+
+    "\u0000\u0000\u0155\u0156\u0001\u0000\u0000\u0000\u0156+\u0001\u0000\u0000"+
+    "\u0000\u0157\u0155\u0001\u0000\u0000\u0000\u0158\u0159\u0005B\u0000\u0000"+
+    "\u0159\u015a\u0003*\u0015\u0000\u015a\u015b\u0005C\u0000\u0000\u015b-"+
+    "\u0001\u0000\u0000\u0000\u015c\u015d\u0005\u0015\u0000\u0000\u015d\u0162"+
+    "\u0003\"\u0011\u0000\u015e\u015f\u0005#\u0000\u0000\u015f\u0161\u0003"+
+    "\"\u0011\u0000\u0160\u015e\u0001\u0000\u0000\u0000\u0161\u0164\u0001\u0000"+
+    "\u0000\u0000\u0162\u0160\u0001\u0000\u0000\u0000\u0162\u0163\u0001\u0000"+
+    "\u0000\u0000\u0163\u0166\u0001\u0000\u0000\u0000\u0164\u0162\u0001\u0000"+
+    "\u0000\u0000\u0165\u0167\u0003\u001c\u000e\u0000\u0166\u0165\u0001\u0000"+
+    "\u0000\u0000\u0166\u0167\u0001\u0000\u0000\u0000\u0167\u016a\u0001\u0000"+
+    "\u0000\u0000\u0168\u0169\u0005\u001e\u0000\u0000\u0169\u016b\u0003\u001c"+
+    "\u000e\u0000\u016a\u0168\u0001\u0000\u0000\u0000\u016a\u016b\u0001\u0000"+
+    "\u0000\u0000\u016b/\u0001\u0000\u0000\u0000\u016c\u016d\u0005\u0004\u0000"+
+    "\u0000\u016d\u016e\u0003\u001c\u000e\u0000\u016e1\u0001\u0000\u0000\u0000"+
+    "\u016f\u0171\u0005\u0010\u0000\u0000\u0170\u0172\u0003\u001c\u000e\u0000"+
+    "\u0171\u0170\u0001\u0000\u0000\u0000\u0171\u0172\u0001\u0000\u0000\u0000"+
+    "\u0172\u0175\u0001\u0000\u0000\u0000\u0173\u0174\u0005\u001e\u0000\u0000"+
+    "\u0174\u0176\u0003\u001c\u000e\u0000\u0175\u0173\u0001\u0000\u0000\u0000"+
+    "\u0175\u0176\u0001\u0000\u0000\u0000\u01763\u0001\u0000\u0000\u0000\u0177"+
+    "\u017c\u0003:\u001d\u0000\u0178\u0179\u0005%\u0000\u0000\u0179\u017b\u0003"+
+    ":\u001d\u0000\u017a\u0178\u0001\u0000\u0000\u0000\u017b\u017e\u0001\u0000"+
+    "\u0000\u0000\u017c\u017a\u0001\u0000\u0000\u0000\u017c\u017d\u0001\u0000"+
+    "\u0000\u0000\u017d5\u0001\u0000\u0000\u0000\u017e\u017c\u0001\u0000\u0000"+
+    "\u0000\u017f\u0184\u0003<\u001e\u0000\u0180\u0181\u0005%\u0000\u0000\u0181"+
+    "\u0183\u0003<\u001e\u0000\u0182\u0180\u0001\u0000\u0000\u0000\u0183\u0186"+
+    "\u0001\u0000\u0000\u0000\u0184\u0182\u0001\u0000\u0000\u0000\u0184\u0185"+
+    "\u0001\u0000\u0000\u0000\u01857\u0001\u0000\u0000\u0000\u0186\u0184\u0001"+
+    "\u0000\u0000\u0000\u0187\u018c\u00036\u001b\u0000\u0188\u0189\u0005#\u0000"+
+    "\u0000\u0189\u018b\u00036\u001b\u0000\u018a\u0188\u0001\u0000\u0000\u0000"+
+    "\u018b\u018e\u0001\u0000\u0000\u0000\u018c\u018a\u0001\u0000\u0000\u0000"+
+    "\u018c\u018d\u0001\u0000\u0000\u0000\u018d9\u0001\u0000\u0000\u0000\u018e"+
+    "\u018c\u0001\u0000\u0000\u0000\u018f\u0190\u0007\u0003\u0000\u0000\u0190"+
+    ";\u0001\u0000\u0000\u0000\u0191\u0192\u0005Q\u0000\u0000\u0192=\u0001"+
+    "\u0000\u0000\u0000\u0193\u01be\u0005.\u0000\u0000\u0194\u0195\u0003`0"+
+    "\u0000\u0195\u0196\u0005D\u0000\u0000\u0196\u01be\u0001\u0000\u0000\u0000"+
+    "\u0197\u01be\u0003^/\u0000\u0198\u01be\u0003`0\u0000\u0199\u01be\u0003"+
+    "Z-\u0000\u019a\u01be\u0003@ \u0000\u019b\u01be\u0003b1\u0000\u019c\u019d"+
+    "\u0005B\u0000\u0000\u019d\u01a2\u0003\\.\u0000\u019e\u019f\u0005#\u0000"+
+    "\u0000\u019f\u01a1\u0003\\.\u0000\u01a0\u019e\u0001\u0000\u0000\u0000"+
+    "\u01a1\u01a4\u0001\u0000\u0000\u0000\u01a2\u01a0\u0001\u0000\u0000\u0000"+
+    "\u01a2\u01a3\u0001\u0000\u0000\u0000\u01a3\u01a5\u0001\u0000\u0000\u0000"+
+    "\u01a4\u01a2\u0001\u0000\u0000\u0000\u01a5\u01a6\u0005C\u0000\u0000\u01a6"+
+    "\u01be\u0001\u0000\u0000\u0000\u01a7\u01a8\u0005B\u0000\u0000\u01a8\u01ad"+
+    "\u0003Z-\u0000\u01a9\u01aa\u0005#\u0000\u0000\u01aa\u01ac\u0003Z-\u0000"+
+    "\u01ab\u01a9\u0001\u0000\u0000\u0000\u01ac\u01af\u0001\u0000\u0000\u0000"+
+    "\u01ad\u01ab\u0001\u0000\u0000\u0000\u01ad\u01ae\u0001\u0000\u0000\u0000"+
+    "\u01ae\u01b0\u0001\u0000\u0000\u0000\u01af\u01ad\u0001\u0000\u0000\u0000"+
+    "\u01b0\u01b1\u0005C\u0000\u0000\u01b1\u01be\u0001\u0000\u0000\u0000\u01b2"+
+    "\u01b3\u0005B\u0000\u0000\u01b3\u01b8\u0003b1\u0000\u01b4\u01b5\u0005"+
+    "#\u0000\u0000\u01b5\u01b7\u0003b1\u0000\u01b6\u01b4\u0001\u0000\u0000"+
+    "\u0000\u01b7\u01ba\u0001\u0000\u0000\u0000\u01b8\u01b6\u0001\u0000\u0000"+
+    "\u0000\u01b8\u01b9\u0001\u0000\u0000\u0000\u01b9\u01bb\u0001\u0000\u0000"+
+    "\u0000\u01ba\u01b8\u0001\u0000\u0000\u0000\u01bb\u01bc\u0005C\u0000\u0000"+
+    "\u01bc\u01be\u0001\u0000\u0000\u0000\u01bd\u0193\u0001\u0000\u0000\u0000"+
+    "\u01bd\u0194\u0001\u0000\u0000\u0000\u01bd\u0197\u0001\u0000\u0000\u0000"+
+    "\u01bd\u0198\u0001\u0000\u0000\u0000\u01bd\u0199\u0001\u0000\u0000\u0000"+
+    "\u01bd\u019a\u0001\u0000\u0000\u0000\u01bd\u019b\u0001\u0000\u0000\u0000"+
+    "\u01bd\u019c\u0001\u0000\u0000\u0000\u01bd\u01a7\u0001\u0000\u0000\u0000"+
+    "\u01bd\u01b2\u0001\u0000\u0000\u0000\u01be?\u0001\u0000\u0000\u0000\u01bf"+
+    "\u01c2\u00051\u0000\u0000\u01c0\u01c2\u0005A\u0000\u0000\u01c1\u01bf\u0001"+
+    "\u0000\u0000\u0000\u01c1\u01c0\u0001\u0000\u0000\u0000\u01c2A\u0001\u0000"+
+    "\u0000\u0000\u01c3\u01c4\u0005\t\u0000\u0000\u01c4\u01c5\u0005\u001c\u0000"+
+    "\u0000\u01c5C\u0001\u0000\u0000\u0000\u01c6\u01c7\u0005\u000f\u0000\u0000"+
+    "\u01c7\u01cc\u0003F#\u0000\u01c8\u01c9\u0005#\u0000\u0000\u01c9\u01cb"+
+    "\u0003F#\u0000\u01ca\u01c8\u0001\u0000\u0000\u0000\u01cb\u01ce\u0001\u0000"+
+    "\u0000\u0000\u01cc\u01ca\u0001\u0000\u0000\u0000\u01cc\u01cd\u0001\u0000"+
+    "\u0000\u0000\u01cdE\u0001\u0000\u0000\u0000\u01ce\u01cc\u0001\u0000\u0000"+
+    "\u0000\u01cf\u01d1\u0003\n\u0005\u0000\u01d0\u01d2\u0007\u0004\u0000\u0000"+
+    "\u01d1\u01d0\u0001\u0000\u0000\u0000\u01d1\u01d2\u0001\u0000\u0000\u0000"+
+    "\u01d2\u01d5\u0001\u0000\u0000\u0000\u01d3\u01d4\u0005/\u0000\u0000\u01d4"+
+    "\u01d6\u0007\u0005\u0000\u0000\u01d5\u01d3\u0001\u0000\u0000\u0000\u01d5"+
+    "\u01d6\u0001\u0000\u0000\u0000\u01d6G\u0001\u0000\u0000\u0000\u01d7\u01d8"+
+    "\u0005\b\u0000\u0000\u01d8\u01d9\u00038\u001c\u0000\u01d9I\u0001\u0000"+
+    "\u0000\u0000\u01da\u01db\u0005\u0002\u0000\u0000\u01db\u01dc\u00038\u001c"+
+    "\u0000\u01dcK\u0001\u0000\u0000\u0000\u01dd\u01de\u0005\f\u0000\u0000"+
+    "\u01de\u01e3\u0003N\'\u0000\u01df\u01e0\u0005#\u0000\u0000\u01e0\u01e2"+
+    "\u0003N\'\u0000\u01e1\u01df\u0001\u0000\u0000\u0000\u01e2\u01e5\u0001"+
+    "\u0000\u0000\u0000\u01e3\u01e1\u0001\u0000\u0000\u0000\u01e3\u01e4\u0001"+
+    "\u0000\u0000\u0000\u01e4M\u0001\u0000\u0000\u0000\u01e5\u01e3\u0001\u0000"+
+    "\u0000\u0000\u01e6\u01e7\u00036\u001b\u0000\u01e7\u01e8\u0005U\u0000\u0000"+
+    "\u01e8\u01e9\u00036\u001b\u0000\u01e9O\u0001\u0000\u0000\u0000\u01ea\u01eb"+
+    "\u0005\u0001\u0000\u0000\u01eb\u01ec\u0003\u0014\n\u0000\u01ec\u01ee\u0003"+
+    "b1\u0000\u01ed\u01ef\u0003V+\u0000\u01ee\u01ed\u0001\u0000\u0000\u0000"+
+    "\u01ee\u01ef\u0001\u0000\u0000\u0000\u01efQ\u0001\u0000\u0000\u0000\u01f0"+
+    "\u01f1\u0005\u0007\u0000\u0000\u01f1\u01f2\u0003\u0014\n\u0000\u01f2\u01f3"+
+    "\u0003b1\u0000\u01f3S\u0001\u0000\u0000\u0000\u01f4\u01f5\u0005\u000b"+
+    "\u0000\u0000\u01f5\u01f6\u00034\u001a\u0000\u01f6U\u0001\u0000\u0000\u0000"+
+    "\u01f7\u01fc\u0003X,\u0000\u01f8\u01f9\u0005#\u0000\u0000\u01f9\u01fb"+
+    "\u0003X,\u0000\u01fa\u01f8\u0001\u0000\u0000\u0000\u01fb\u01fe\u0001\u0000"+
+    "\u0000\u0000\u01fc\u01fa\u0001\u0000\u0000\u0000\u01fc\u01fd\u0001\u0000"+
+    "\u0000\u0000\u01fdW\u0001\u0000\u0000\u0000\u01fe\u01fc\u0001\u0000\u0000"+
+    "\u0000\u01ff\u0200\u0003:\u001d\u0000\u0200\u0201\u0005!\u0000\u0000\u0201"+
+    "\u0202\u0003>\u001f\u0000\u0202Y\u0001\u0000\u0000\u0000\u0203\u0204\u0007"+
+    "\u0006\u0000\u0000\u0204[\u0001\u0000\u0000\u0000\u0205\u0208\u0003^/"+
+    "\u0000\u0206\u0208\u0003`0\u0000\u0207\u0205\u0001\u0000\u0000\u0000\u0207"+
+    "\u0206\u0001\u0000\u0000\u0000\u0208]\u0001\u0000\u0000\u0000\u0209\u020b"+
+    "\u0007\u0000\u0000\u0000\u020a\u0209\u0001\u0000\u0000\u0000\u020a\u020b"+
+    "\u0001\u0000\u0000\u0000\u020b\u020c\u0001\u0000\u0000\u0000\u020c\u020d"+
+    "\u0005\u001d\u0000\u0000\u020d_\u0001\u0000\u0000\u0000\u020e\u0210\u0007"+
+    "\u0000\u0000\u0000\u020f\u020e\u0001\u0000\u0000\u0000\u020f\u0210\u0001"+
+    "\u0000\u0000\u0000\u0210\u0211\u0001\u0000\u0000\u0000\u0211\u0212\u0005"+
+    "\u001c\u0000\u0000\u0212a\u0001\u0000\u0000\u0000\u0213\u0214\u0005\u001b"+
+    "\u0000\u0000\u0214c\u0001\u0000\u0000\u0000\u0215\u0216\u0007\u0007\u0000"+
+    "\u0000\u0216e\u0001\u0000\u0000\u0000\u0217\u0218\u0005\u0005\u0000\u0000"+
+    "\u0218\u0219\u0003h4\u0000\u0219g\u0001\u0000\u0000\u0000\u021a\u021b"+
+    "\u0005B\u0000\u0000\u021b\u021c\u0003\u0002\u0001\u0000\u021c\u021d\u0005"+
+    "C\u0000\u0000\u021di\u0001\u0000\u0000\u0000\u021e\u021f\u0005\u000e\u0000"+
+    "\u0000\u021f\u0220\u0005e\u0000\u0000\u0220k\u0001\u0000\u0000\u0000\u0221"+
+    "\u0222\u0005\n\u0000\u0000\u0222\u0223\u0005i\u0000\u0000\u0223m\u0001"+
+    "\u0000\u0000\u0000\u0224\u0225\u0005\u0003\u0000\u0000\u0225\u0228\u0005"+
+    "[\u0000\u0000\u0226\u0227\u0005Y\u0000\u0000\u0227\u0229\u00036\u001b"+
+    "\u0000\u0228\u0226\u0001\u0000\u0000\u0000\u0228\u0229\u0001\u0000\u0000"+
+    "\u0000\u0229\u0233\u0001\u0000\u0000\u0000\u022a\u022b\u0005Z\u0000\u0000"+
+    "\u022b\u0230\u0003p8\u0000\u022c\u022d\u0005#\u0000\u0000\u022d\u022f"+
+    "\u0003p8\u0000\u022e\u022c\u0001\u0000\u0000\u0000\u022f\u0232\u0001\u0000"+
+    "\u0000\u0000\u0230\u022e\u0001\u0000\u0000\u0000\u0230\u0231\u0001\u0000"+
+    "\u0000\u0000\u0231\u0234\u0001\u0000\u0000\u0000\u0232\u0230\u0001\u0000"+
+    "\u0000\u0000\u0233\u022a\u0001\u0000\u0000\u0000\u0233\u0234\u0001\u0000"+
+    "\u0000\u0000\u0234o\u0001\u0000\u0000\u0000\u0235\u0236\u00036\u001b\u0000"+
+    "\u0236\u0237\u0005!\u0000\u0000\u0237\u0239\u0001\u0000\u0000\u0000\u0238"+
+    "\u0235\u0001\u0000\u0000\u0000\u0238\u0239\u0001\u0000\u0000\u0000\u0239"+
+    "\u023a\u0001\u0000\u0000\u0000\u023a\u023b\u00036\u001b\u0000\u023bq\u0001"+
+    "\u0000\u0000\u0000\u023c\u023d\u0005\u0013\u0000\u0000\u023d\u023e\u0003"+
+    "\"\u0011\u0000\u023e\u023f\u0005Y\u0000\u0000\u023f\u0240\u00038\u001c"+
+    "\u0000\u0240s\u0001\u0000\u0000\u0000\u0241\u0242\u0005\u0012\u0000\u0000"+
+    "\u0242\u0245\u0003\u001c\u000e\u0000\u0243\u0244\u0005\u001e\u0000\u0000"+
+    "\u0244\u0246\u0003\u001c\u000e\u0000\u0245\u0243\u0001\u0000\u0000\u0000"+
+    "\u0245\u0246\u0001\u0000\u0000\u0000\u0246u\u0001\u0000\u0000\u0000\u0247"+
+    "\u0248\u0005\u0014\u0000\u0000\u0248\u0249\u0003x<\u0000\u0249w\u0001"+
+    "\u0000\u0000\u0000\u024a\u024b\u0005\u001b\u0000\u0000\u024by\u0001\u0000"+
+    "\u0000\u00006\u0085\u008f\u00a3\u00af\u00b8\u00c0\u00c6\u00ce\u00d0\u00d5"+
+    "\u00dc\u00e1\u00ec\u00f2\u00fa\u00fc\u0107\u010e\u0119\u011c\u012a\u0132"+
+    "\u013a\u013e\u0145\u014d\u0155\u0162\u0166\u016a\u0171\u0175\u017c\u0184"+
+    "\u018c\u01a2\u01ad\u01b8\u01bd\u01c1\u01cc\u01d1\u01d5\u01e3\u01ee\u01fc"+
+    "\u0207\u020a\u020f\u0228\u0230\u0233\u0238\u0245";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
index c2c682e0eea17..7db53cb7713c6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
@@ -1,6 +1,14 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.xpack.esql.parser;
 
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.tree.ErrorNode;
 import org.antlr.v4.runtime.tree.TerminalNode;
@@ -84,6 +92,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitWhereCommand(EsqlBaseParser.WhereCommandContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterMatchExpression(EsqlBaseParser.MatchExpressionContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitMatchExpression(EsqlBaseParser.MatchExpressionContext ctx) { }
   /**
    * {@inheritDoc}
    *
@@ -168,6 +188,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitRegexBooleanExpression(EsqlBaseParser.RegexBooleanExpressionContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterMatchBooleanExpression(EsqlBaseParser.MatchBooleanExpressionContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitMatchBooleanExpression(EsqlBaseParser.MatchBooleanExpressionContext ctx) { }
   /**
    * {@inheritDoc}
    *
@@ -468,18 +500,6 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitStatsCommand(EsqlBaseParser.StatsCommandContext ctx) { }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation does nothing.</p>
-   */
-  @Override public void enterInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx) { }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation does nothing.</p>
-   */
-  @Override public void exitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx) { }
   /**
    * {@inheritDoc}
    *
@@ -984,6 +1004,42 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitLookupCommand(EsqlBaseParser.LookupCommandContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterMatchCommand(EsqlBaseParser.MatchCommandContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitMatchCommand(EsqlBaseParser.MatchCommandContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterMatchQuery(EsqlBaseParser.MatchQueryContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitMatchQuery(EsqlBaseParser.MatchQueryContext ctx) { }
 
   /**
    * {@inheritDoc}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
index 3b2675d3490a2..446cdd4cd7834 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
@@ -1,5 +1,13 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.xpack.esql.parser;
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
 import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor;
 
 /**
@@ -54,6 +62,13 @@ public class EsqlBaseParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitWhereCommand(EsqlBaseParser.WhereCommandContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitMatchExpression(EsqlBaseParser.MatchExpressionContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -103,6 +118,13 @@ public class EsqlBaseParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitRegexBooleanExpression(EsqlBaseParser.RegexBooleanExpressionContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitMatchBooleanExpression(EsqlBaseParser.MatchBooleanExpressionContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -278,13 +300,6 @@ public class EsqlBaseParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitStatsCommand(EsqlBaseParser.StatsCommandContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -579,4 +594,25 @@ public class EsqlBaseParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitLookupCommand(EsqlBaseParser.LookupCommandContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitMatchCommand(EsqlBaseParser.MatchCommandContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitMatchQuery(EsqlBaseParser.MatchQueryContext ctx) { return visitChildren(ctx); }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
index a6420e6fadebd..0c39b3ea83fa9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
@@ -1,5 +1,13 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.xpack.esql.parser;
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
 import org.antlr.v4.runtime.tree.ParseTreeListener;
 
 /**
@@ -71,6 +79,18 @@ public interface EsqlBaseParserListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitWhereCommand(EsqlBaseParser.WhereCommandContext ctx);
+  /**
+   * Enter a parse tree produced by the {@code matchExpression}
+   * labeled alternative in {@link EsqlBaseParser#booleanExpression}.
+   * @param ctx the parse tree
+   */
+  void enterMatchExpression(EsqlBaseParser.MatchExpressionContext ctx);
+  /**
+   * Exit a parse tree produced by the {@code matchExpression}
+   * labeled alternative in {@link EsqlBaseParser#booleanExpression}.
+   * @param ctx the parse tree
+   */
+  void exitMatchExpression(EsqlBaseParser.MatchExpressionContext ctx);
   /**
    * Enter a parse tree produced by the {@code logicalNot}
    * labeled alternative in {@link EsqlBaseParser#booleanExpression}.
@@ -153,6 +173,16 @@ public interface EsqlBaseParserListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitRegexBooleanExpression(EsqlBaseParser.RegexBooleanExpressionContext ctx);
+  /**
+   * Enter a parse tree produced by {@link EsqlBaseParser#matchBooleanExpression}.
+   * @param ctx the parse tree
+   */
+  void enterMatchBooleanExpression(EsqlBaseParser.MatchBooleanExpressionContext ctx);
+  /**
+   * Exit a parse tree produced by {@link EsqlBaseParser#matchBooleanExpression}.
+   * @param ctx the parse tree
+   */
+  void exitMatchBooleanExpression(EsqlBaseParser.MatchBooleanExpressionContext ctx);
   /**
    * Enter a parse tree produced by the {@code valueExpressionDefault}
    * labeled alternative in {@link EsqlBaseParser#valueExpression}.
@@ -425,16 +455,6 @@ public interface EsqlBaseParserListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitStatsCommand(EsqlBaseParser.StatsCommandContext ctx);
-  /**
-   * Enter a parse tree produced by {@link EsqlBaseParser#inlinestatsCommand}.
-   * @param ctx the parse tree
-   */
-  void enterInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx);
-  /**
-   * Exit a parse tree produced by {@link EsqlBaseParser#inlinestatsCommand}.
-   * @param ctx the parse tree
-   */
-  void exitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx);
   /**
    * Enter a parse tree produced by {@link EsqlBaseParser#qualifiedName}.
    * @param ctx the parse tree
@@ -883,4 +903,34 @@ public interface EsqlBaseParserListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitLookupCommand(EsqlBaseParser.LookupCommandContext ctx);
+  /**
+   * Enter a parse tree produced by {@link EsqlBaseParser#inlinestatsCommand}.
+   * @param ctx the parse tree
+   */
+  void enterInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx);
+  /**
+   * Exit a parse tree produced by {@link EsqlBaseParser#inlinestatsCommand}.
+   * @param ctx the parse tree
+   */
+  void exitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx);
+  /**
+   * Enter a parse tree produced by {@link EsqlBaseParser#matchCommand}.
+   * @param ctx the parse tree
+   */
+  void enterMatchCommand(EsqlBaseParser.MatchCommandContext ctx);
+  /**
+   * Exit a parse tree produced by {@link EsqlBaseParser#matchCommand}.
+   * @param ctx the parse tree
+   */
+  void exitMatchCommand(EsqlBaseParser.MatchCommandContext ctx);
+  /**
+   * Enter a parse tree produced by {@link EsqlBaseParser#matchQuery}.
+   * @param ctx the parse tree
+   */
+  void enterMatchQuery(EsqlBaseParser.MatchQueryContext ctx);
+  /**
+   * Exit a parse tree produced by {@link EsqlBaseParser#matchQuery}.
+   * @param ctx the parse tree
+   */
+  void exitMatchQuery(EsqlBaseParser.MatchQueryContext ctx);
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
index ec84b7234d67e..31c9371b9f806 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
@@ -1,5 +1,13 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.xpack.esql.parser;
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
 import org.antlr.v4.runtime.tree.ParseTreeVisitor;
 
 /**
@@ -48,6 +56,13 @@ public interface EsqlBaseParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitWhereCommand(EsqlBaseParser.WhereCommandContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code matchExpression}
+   * labeled alternative in {@link EsqlBaseParser#booleanExpression}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitMatchExpression(EsqlBaseParser.MatchExpressionContext ctx);
   /**
    * Visit a parse tree produced by the {@code logicalNot}
    * labeled alternative in {@link EsqlBaseParser#booleanExpression}.
@@ -96,6 +111,12 @@ public interface EsqlBaseParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitRegexBooleanExpression(EsqlBaseParser.RegexBooleanExpressionContext ctx);
+  /**
+   * Visit a parse tree produced by {@link EsqlBaseParser#matchBooleanExpression}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitMatchBooleanExpression(EsqlBaseParser.MatchBooleanExpressionContext ctx);
   /**
    * Visit a parse tree produced by the {@code valueExpressionDefault}
    * labeled alternative in {@link EsqlBaseParser#valueExpression}.
@@ -257,12 +278,6 @@ public interface EsqlBaseParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitStatsCommand(EsqlBaseParser.StatsCommandContext ctx);
-  /**
-   * Visit a parse tree produced by {@link EsqlBaseParser#inlinestatsCommand}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx);
   /**
    * Visit a parse tree produced by {@link EsqlBaseParser#qualifiedName}.
    * @param ctx the parse tree
@@ -529,4 +544,22 @@ public interface EsqlBaseParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitLookupCommand(EsqlBaseParser.LookupCommandContext ctx);
+  /**
+   * Visit a parse tree produced by {@link EsqlBaseParser#inlinestatsCommand}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx);
+  /**
+   * Visit a parse tree produced by {@link EsqlBaseParser#matchCommand}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitMatchCommand(EsqlBaseParser.MatchCommandContext ctx);
+  /**
+   * Visit a parse tree produced by {@link EsqlBaseParser#matchQuery}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitMatchQuery(EsqlBaseParser.MatchQueryContext ctx);
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlConfig.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlConfig.java
new file mode 100644
index 0000000000000..8e5aa28d25c00
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlConfig.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.parser;
+
+import org.elasticsearch.Build;
+
+class EsqlConfig {
+
+    // versioning information
+    boolean devVersion = Build.current().isSnapshot();
+
+    public boolean isDevVersion() {
+        return devVersion;
+    }
+
+    boolean isReleaseVersion() {
+        return isDevVersion() == false;
+    }
+
+    public void setDevVersion(boolean dev) {
+        this.devVersion = dev;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java
index ebbcfa3b2863b..4be80af958e36 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java
@@ -7,32 +7,42 @@
 package org.elasticsearch.xpack.esql.parser;
 
 import org.antlr.v4.runtime.BaseErrorListener;
-import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.RecognitionException;
 import org.antlr.v4.runtime.Recognizer;
 import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.TokenFactory;
 import org.antlr.v4.runtime.TokenSource;
 import org.antlr.v4.runtime.atn.PredictionMode;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
-import org.elasticsearch.xpack.esql.core.parser.CaseChangingCharStream;
+import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 
 import java.util.BitSet;
 import java.util.function.BiFunction;
 import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
-import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.source;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.isInteger;
+import static org.elasticsearch.xpack.esql.parser.ParserUtils.source;
 
 public class EsqlParser {
 
     private static final Logger log = LogManager.getLogger(EsqlParser.class);
 
+    private EsqlConfig config = new EsqlConfig();
+
+    public EsqlConfig config() {
+        return config;
+    }
+
+    public void setEsqlConfig(EsqlConfig config) {
+        this.config = config;
+    }
+
     public LogicalPlan createStatement(String query) {
         return createStatement(query, new QueryParams());
     }
@@ -51,11 +61,14 @@ private <T> T invokeParser(
         BiFunction<AstBuilder, ParserRuleContext, T> result
     ) {
         try {
-            EsqlBaseLexer lexer = new EsqlBaseLexer(new CaseChangingCharStream(CharStreams.fromString(query)));
+            // new CaseChangingCharStream()
+            EsqlBaseLexer lexer = new EsqlBaseLexer(CharStreams.fromString(query));
 
             lexer.removeErrorListeners();
             lexer.addErrorListener(ERROR_LISTENER);
 
+            lexer.setEsqlConfig(config);
+
             TokenSource tokenSource = new ParametrizedTokenSource(lexer, params);
             CommonTokenStream tokenStream = new CommonTokenStream(tokenSource);
             EsqlBaseParser parser = new EsqlBaseParser(tokenStream);
@@ -67,6 +80,8 @@ private <T> T invokeParser(
 
             parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
 
+            parser.setEsqlConfig(config);
+
             ParserRuleContext tree = parseFunction.apply(parser);
 
             if (log.isTraceEnabled()) {
@@ -94,6 +109,9 @@ public void exitFunctionExpression(EsqlBaseParser.FunctionExpressionContext ctx)
     }
 
     private static final BaseErrorListener ERROR_LISTENER = new BaseErrorListener() {
+        // replace entries that start with <comma?><space?>DEV_<space?>
+        private final Pattern REPLACE_DEV = Pattern.compile(",*\\s*DEV_\\w+\\s*");
+
         @Override
         public void syntaxError(
             Recognizer<?, ?> recognizer,
@@ -103,27 +121,31 @@ public void syntaxError(
             String message,
             RecognitionException e
         ) {
+            if (recognizer instanceof EsqlBaseParser parser && parser.isDevVersion() == false) {
+                Matcher m = REPLACE_DEV.matcher(message);
+                message = m.replaceAll(StringUtils.EMPTY);
+            }
+
             throw new ParsingException(message, e, line, charPositionInLine);
         }
     };
 
     /**
-     * Finds all parameter tokens (?) and associates them with actual parameter values
+     * Finds all parameter tokens (?) and associates them with actual parameter values.
      * <p>
      * Parameters are positional and we know where parameters occurred in the original stream in order to associate them
      * with actual values.
      */
-    private static class ParametrizedTokenSource implements TokenSource {
+    private static class ParametrizedTokenSource extends DelegatingTokenSource {
         private static String message = "Inconsistent parameter declaration, "
             + "use one of positional, named or anonymous params but not a combination of ";
 
-        private TokenSource delegate;
         private QueryParams params;
         private BitSet paramTypes = new BitSet(3);
         private int param = 1;
 
         ParametrizedTokenSource(TokenSource delegate, QueryParams params) {
-            this.delegate = delegate;
+            super(delegate);
             this.params = params;
         }
 
@@ -149,36 +171,6 @@ public Token nextToken() {
             return token;
         }
 
-        @Override
-        public int getLine() {
-            return delegate.getLine();
-        }
-
-        @Override
-        public int getCharPositionInLine() {
-            return delegate.getCharPositionInLine();
-        }
-
-        @Override
-        public CharStream getInputStream() {
-            return delegate.getInputStream();
-        }
-
-        @Override
-        public String getSourceName() {
-            return delegate.getSourceName();
-        }
-
-        @Override
-        public void setTokenFactory(TokenFactory<?> factory) {
-            delegate.setTokenFactory(factory);
-        }
-
-        @Override
-        public TokenFactory<?> getTokenFactory() {
-            return delegate.getTokenFactory();
-        }
-
         private void checkAnonymousParam(Token token) {
             paramTypes.set(0);
             if (paramTypes.cardinality() > 1) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
index 9769d286b484d..0352afdee4622 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
+import org.elasticsearch.Build;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
@@ -25,6 +26,7 @@
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar;
+import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
@@ -35,7 +37,7 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DateUtils;
+import org.elasticsearch.xpack.esql.core.util.DateUtils;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern;
@@ -71,15 +73,15 @@
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
-import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.source;
-import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.typedParsing;
-import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.visitList;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
 import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.isInteger;
+import static org.elasticsearch.xpack.esql.parser.ParserUtils.source;
+import static org.elasticsearch.xpack.esql.parser.ParserUtils.typedParsing;
+import static org.elasticsearch.xpack.esql.parser.ParserUtils.visitList;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.bigIntegerToUnsignedLong;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.parseTemporalAmout;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToIntegral;
@@ -89,9 +91,22 @@ public abstract class ExpressionBuilder extends IdentifierBuilder {
     private int expressionDepth = 0;
 
     /**
-     * Maximum depth for nested expressions
+     * Maximum depth for nested expressions.
+     * Avoids StackOverflowErrors at parse time with very convoluted expressions,
+     * eg. EVAL x = sin(sin(sin(sin(sin(sin(sin(sin(sin(....sin(x)....)
+     * ANTLR parser is recursive, so the only way to prevent a StackOverflow is to detect how
+     * deep we are in the expression parsing and abort the query execution after a threshold
+     *
+     * This value is defined empirically, but the actual stack limit is highly
+     * dependent on the JVM and on the JIT.
+     *
+     * A value of 500 proved to be right below the stack limit, but it still triggered
+     * some CI failures (once every ~2000 iterations). see https://github.com/elastic/elasticsearch/issues/109846
+     * Even though we didn't manage to reproduce the problem in real conditions, we decided
+     * to reduce the max allowed depth to 400 (that is still a pretty reasonable limit for real use cases) and be more safe.
+     *
      */
-    public static final int MAX_EXPRESSION_DEPTH = 500;
+    public static final int MAX_EXPRESSION_DEPTH = 400;
 
     protected final QueryParams params;
 
@@ -642,7 +657,7 @@ public Alias visitField(EsqlBaseParser.FieldContext ctx) {
         UnresolvedAttribute id = visitQualifiedName(ctx.qualifiedName());
         Expression value = expression(ctx.booleanExpression());
         var source = source(ctx);
-        String name = id == null ? source.text() : id.qualifiedName();
+        String name = id == null ? source.text() : id.name();
         return new Alias(source, name, value);
     }
 
@@ -673,7 +688,7 @@ public List<NamedExpression> visitGrouping(EsqlBaseParser.FieldsContext ctx) {
                         name = source(field).text();
                     }
                 } else {
-                    name = id.qualifiedName();
+                    name = id.name();
                 }
                 // wrap when necessary - no alias and no underlying attribute
                 if (ne == null) {
@@ -749,4 +764,17 @@ QueryParam paramByNameOrPosition(TerminalNode node) {
             return params.get(nameOrPosition);
         }
     }
+
+    @Override
+    public Expression visitMatchBooleanExpression(EsqlBaseParser.MatchBooleanExpressionContext ctx) {
+        if (Build.current().isSnapshot() == false) {
+            throw new ParsingException(source(ctx), "MATCH operator currently requires a snapshot build");
+        }
+        return new MatchQueryPredicate(
+            source(ctx),
+            expression(ctx.valueExpression()),
+            visitString(ctx.queryString).fold().toString(),
+            null
+        );
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LexerConfig.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LexerConfig.java
new file mode 100644
index 0000000000000..adcdba2f2eb4d
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LexerConfig.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.parser;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.Lexer;
+
+/**
+ * Base class for hooking versioning information into the ANTLR parser.
+ */
+public abstract class LexerConfig extends Lexer {
+
+    // is null when running inside the IDEA plugin
+    EsqlConfig config;
+
+    public LexerConfig() {}
+
+    public LexerConfig(CharStream input) {
+        super(input);
+    }
+
+    boolean isDevVersion() {
+        return config == null || config.isDevVersion();
+    }
+
+    void setEsqlConfig(EsqlConfig config) {
+        this.config = config;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
index fe2ae34e75ed2..ffd2375a688ad 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
@@ -25,18 +25,17 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.expression.Order;
-import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar;
-import org.elasticsearch.xpack.esql.core.parser.ParserUtils;
-import org.elasticsearch.xpack.esql.core.plan.TableIdentifier;
+import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.StringQueryPredicate;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.Holder;
+import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern;
 import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
 import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.MetadataOptionContext;
+import org.elasticsearch.xpack.esql.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Dissect;
 import org.elasticsearch.xpack.esql.plan.logical.Drop;
@@ -57,6 +56,8 @@
 import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation;
 import org.elasticsearch.xpack.esql.plan.logical.meta.MetaFunctions;
 import org.elasticsearch.xpack.esql.plan.logical.show.ShowInfo;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
+import org.joni.exception.SyntaxException;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -70,10 +71,10 @@
 import java.util.function.Function;
 
 import static org.elasticsearch.common.logging.HeaderWarning.addWarning;
-import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.source;
-import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.typedParsing;
-import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.visitList;
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions;
+import static org.elasticsearch.xpack.esql.parser.ParserUtils.source;
+import static org.elasticsearch.xpack.esql.parser.ParserUtils.typedParsing;
+import static org.elasticsearch.xpack.esql.parser.ParserUtils.visitList;
 import static org.elasticsearch.xpack.esql.plan.logical.Enrich.Mode;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToInt;
 
@@ -153,7 +154,12 @@ public PlanFactory visitGrokCommand(EsqlBaseParser.GrokCommandContext ctx) {
         return p -> {
             Source source = source(ctx);
             String pattern = visitString(ctx.string()).fold().toString();
-            Grok.Parser grokParser = Grok.pattern(source, pattern);
+            Grok.Parser grokParser;
+            try {
+                grokParser = Grok.pattern(source, pattern);
+            } catch (SyntaxException e) {
+                throw new ParsingException(source, "Invalid grok pattern [{}]: [{}]", pattern, e.getMessage());
+            }
             validateGrokPattern(source, grokParser, pattern);
             Grok result = new Grok(source(ctx), p, expression(ctx.primaryExpression()), grokParser);
             return result;
@@ -198,21 +204,20 @@ public PlanFactory visitDissectCommand(EsqlBaseParser.DissectCommandContext ctx)
 
             try {
                 DissectParser parser = new DissectParser(pattern, appendSeparator);
+
                 Set<String> referenceKeys = parser.referenceKeys();
-                if (referenceKeys.size() > 0) {
+                if (referenceKeys.isEmpty() == false) {
                     throw new ParsingException(
                         src,
                         "Reference keys not supported in dissect patterns: [%{*{}}]",
                         referenceKeys.iterator().next()
                     );
                 }
-                List<Attribute> keys = new ArrayList<>();
-                for (var x : parser.outputKeys()) {
-                    if (x.isEmpty() == false) {
-                        keys.add(new ReferenceAttribute(src, x, DataType.KEYWORD));
-                    }
-                }
-                return new Dissect(src, p, expression(ctx.primaryExpression()), new Dissect.Parser(pattern, appendSeparator, parser), keys);
+
+                Dissect.Parser esqlDissectParser = new Dissect.Parser(pattern, appendSeparator, parser);
+                List<Attribute> keys = esqlDissectParser.keyAttributes(src);
+
+                return new Dissect(src, p, expression(ctx.primaryExpression()), esqlDissectParser, keys);
             } catch (DissectException e) {
                 throw new ParsingException(src, "Invalid pattern for dissect: [{}]", pattern);
             }
@@ -333,6 +338,9 @@ private void fail(Expression exp, String message, Object... args) {
 
     @Override
     public PlanFactory visitInlinestatsCommand(EsqlBaseParser.InlinestatsCommandContext ctx) {
+        if (false == EsqlPlugin.INLINESTATS_FEATURE_FLAG.isEnabled()) {
+            throw new ParsingException(source(ctx), "INLINESTATS command currently requires a snapshot build");
+        }
         List<NamedExpression> aggregates = new ArrayList<>(visitFields(ctx.stats));
         List<NamedExpression> groupings = visitGrouping(ctx.grouping);
         aggregates.addAll(groupings);
@@ -345,6 +353,23 @@ public PlanFactory visitWhereCommand(EsqlBaseParser.WhereCommandContext ctx) {
         return input -> new Filter(source(ctx), input, expression);
     }
 
+    @Override
+    public PlanFactory visitMatchCommand(EsqlBaseParser.MatchCommandContext ctx) {
+        if (Build.current().isSnapshot() == false) {
+            throw new ParsingException(source(ctx), "MATCH command currently requires a snapshot build");
+        }
+
+        StringQueryPredicate stringQueryPredicate = visitMatchQuery(ctx.matchQuery());
+        return input -> new Filter(source(ctx), input, stringQueryPredicate);
+    }
+
+    @Override
+    public StringQueryPredicate visitMatchQuery(EsqlBaseParser.MatchQueryContext ctx) {
+        Source source = source(ctx);
+        String queryString = unquote(ctx.QUOTED_STRING().getText());
+        return new StringQueryPredicate(source, queryString, null);
+    }
+
     @Override
     public PlanFactory visitLimitCommand(EsqlBaseParser.LimitCommandContext ctx) {
         Source source = source(ctx);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParserConfig.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParserConfig.java
new file mode 100644
index 0000000000000..c6d4d3efa4a5a
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParserConfig.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.parser;
+
+import org.antlr.v4.runtime.Parser;
+import org.antlr.v4.runtime.TokenStream;
+
+public abstract class ParserConfig extends Parser {
+
+    // is null when running inside the IDEA plugin
+    private EsqlConfig config;
+
+    public ParserConfig(TokenStream input) {
+        super(input);
+    }
+
+    boolean isDevVersion() {
+        return config == null || config.isDevVersion();
+    }
+
+    void setEsqlConfig(EsqlConfig config) {
+        this.config = config;
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/parser/ParserUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParserUtils.java
similarity index 98%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/parser/ParserUtils.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParserUtils.java
index be8b08a05d2a2..ddc7496202704 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/parser/ParserUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ParserUtils.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core.parser;
+package org.elasticsearch.xpack.esql.parser;
 
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.Token;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/GeneratingPlan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/GeneratingPlan.java
new file mode 100644
index 0000000000000..49bab1f577b33
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/GeneratingPlan.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+
+import java.util.List;
+
+/**
+ * A plan that creates new {@link Attribute}s and appends them to the child {@link org.elasticsearch.xpack.esql.plan.logical.UnaryPlan}'s
+ * attributes.
+ * Attributes are appended on the right hand side of the child's input. In case of name conflicts, the rightmost attribute with
+ * a given name shadows any attributes left of it
+ * (c.f. {@link org.elasticsearch.xpack.esql.expression.NamedExpressions#mergeOutputAttributes(List, List)}).
+ */
+public interface GeneratingPlan<PlanType extends GeneratingPlan<PlanType>> {
+    List<Attribute> generatedAttributes();
+
+    /**
+     * Create a new instance of this node with new output {@link Attribute}s using the given names.
+     * If an output attribute already has the desired name, we continue using it; otherwise, we
+     * create a new attribute with a new {@link org.elasticsearch.xpack.esql.core.expression.NameId}.
+     */
+    // TODO: the generated attributes should probably become synthetic once renamed
+    // blocked on https://github.com/elastic/elasticsearch/issues/98703
+    PlanType withGeneratedNames(List<String> newNames);
+
+    default void checkNumberOfNewNames(List<String> newNames) {
+        if (newNames.size() != generatedAttributes().size()) {
+            throw new IllegalArgumentException(
+                "Number of new names is [" + newNames.size() + "] but there are [" + generatedAttributes().size() + "] existing names."
+            );
+        }
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plan/QueryPlan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QueryPlan.java
similarity index 99%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plan/QueryPlan.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QueryPlan.java
index 0129ad423b0f9..3ee56771bc3f7 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plan/QueryPlan.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QueryPlan.java
@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.plan;
+package org.elasticsearch.xpack.esql.plan;
 
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plan/TableIdentifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/TableIdentifier.java
similarity index 97%
rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plan/TableIdentifier.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/TableIdentifier.java
index 4acbf3c92b8b1..ceefe4e254557 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plan/TableIdentifier.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/TableIdentifier.java
@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.plan;
+package org.elasticsearch.xpack.esql.plan;
 
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java
index 78d77baa57aac..5b6fe8c0112c6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.esql.plan.logical;
 
 import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
@@ -17,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 
 import java.io.IOException;
 import java.util.List;
@@ -26,8 +26,12 @@
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
 
-public class Aggregate extends UnaryPlan {
-    private List<Attribute> lazyOutput;
+public class Aggregate extends UnaryPlan implements Stats {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        LogicalPlan.class,
+        "Aggregate",
+        Aggregate::new
+    );
 
     public enum AggregateType {
         STANDARD,
@@ -54,6 +58,7 @@ static AggregateType readType(StreamInput in) throws IOException {
     private final AggregateType aggregateType;
     private final List<Expression> groupings;
     private final List<? extends NamedExpression> aggregates;
+    private List<Attribute> lazyOutput;
 
     public Aggregate(
         Source source,
@@ -68,22 +73,28 @@ public Aggregate(
         this.aggregates = aggregates;
     }
 
-    public Aggregate(PlanStreamInput in) throws IOException {
+    public Aggregate(StreamInput in) throws IOException {
         this(
-            Source.readFrom(in),
-            in.readLogicalPlanNode(),
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(LogicalPlan.class),
             AggregateType.readType(in),
             in.readNamedWriteableCollectionAsList(Expression.class),
             in.readNamedWriteableCollectionAsList(NamedExpression.class)
         );
     }
 
-    public static void writeAggregate(PlanStreamOutput out, Aggregate aggregate) throws IOException {
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
         Source.EMPTY.writeTo(out);
-        out.writeLogicalPlanNode(aggregate.child());
-        AggregateType.writeType(out, aggregate.aggregateType());
-        out.writeNamedWriteableCollection(aggregate.groupings);
-        out.writeNamedWriteableCollection(aggregate.aggregates());
+        out.writeNamedWriteable(child());
+        AggregateType.writeType(out, aggregateType());
+        out.writeNamedWriteableCollection(groupings);
+        out.writeNamedWriteableCollection(aggregates());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
     }
 
     @Override
@@ -96,6 +107,11 @@ public Aggregate replaceChild(LogicalPlan newChild) {
         return new Aggregate(source(), newChild, aggregateType, groupings, aggregates);
     }
 
+    @Override
+    public Aggregate with(LogicalPlan child, List<Expression> newGroupings, List<? extends NamedExpression> newAggregates) {
+        return new Aggregate(source(), child, aggregateType(), newGroupings, newAggregates);
+    }
+
     public AggregateType aggregateType() {
         return aggregateType;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Dissect.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Dissect.java
index c0c564b1b36eb..ae9805124890a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Dissect.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Dissect.java
@@ -7,19 +7,52 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.dissect.DissectParser;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
 public class Dissect extends RegexExtract {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Dissect", Dissect::new);
+
     private final Parser parser;
 
-    public record Parser(String pattern, String appendSeparator, DissectParser parser) {
+    public record Parser(String pattern, String appendSeparator, DissectParser parser) implements Writeable {
+        public static Parser readFrom(StreamInput in) throws IOException {
+            String pattern = in.readString();
+            String appendSeparator = in.readString();
+            return new Parser(pattern, appendSeparator, new DissectParser(pattern, appendSeparator));
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(pattern());
+            out.writeString(appendSeparator());
+        }
+
+        public List<Attribute> keyAttributes(Source src) {
+            List<Attribute> keys = new ArrayList<>();
+            for (var x : parser.outputKeys()) {
+                if (x.isEmpty() == false) {
+                    keys.add(new ReferenceAttribute(src, x, DataType.KEYWORD));
+                }
+            }
+
+            return keys;
+        }
 
         // Override hashCode and equals since the parser is considered equal if its pattern and
         // appendSeparator are equal ( and DissectParser uses reference equality )
@@ -42,6 +75,30 @@ public Dissect(Source source, LogicalPlan child, Expression input, Parser parser
         this.parser = parser;
     }
 
+    private Dissect(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(LogicalPlan.class),
+            in.readNamedWriteable(Expression.class),
+            Parser.readFrom(in),
+            in.readNamedWriteableCollectionAsList(Attribute.class)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(child());
+        out.writeNamedWriteable(input());
+        parser.writeTo(out);
+        out.writeNamedWriteableCollection(extractedFields());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     public UnaryPlan replaceChild(LogicalPlan newChild) {
         return new Dissect(source(), newChild, input, parser, extractedFields);
@@ -52,6 +109,11 @@ protected NodeInfo<? extends LogicalPlan> info() {
         return NodeInfo.create(this, Dissect::new, child(), input, parser, extractedFields);
     }
 
+    @Override
+    public Dissect withGeneratedNames(List<String> newNames) {
+        return new Dissect(source(), child(), input, parser, renameExtractedFields(newNames));
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Drop.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Drop.java
index d1c5d70018d91..2df8727cf0e65 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Drop.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Drop.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -23,6 +24,16 @@ public Drop(Source source, LogicalPlan child, List<NamedExpression> removals) {
         this.removals = removals;
     }
 
+    @Override
+    public void writeTo(StreamOutput out) {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
     public List<NamedExpression> removals() {
         return removals;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java
index a4d553eae4749..e0e40b0242fa7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java
@@ -7,28 +7,52 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
+import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.plan.GeneratingPlan;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 
+import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes;
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
 
-public class Enrich extends UnaryPlan {
+public class Enrich extends UnaryPlan implements GeneratingPlan<Enrich> {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        LogicalPlan.class,
+        "Enrich",
+        Enrich::readFrom
+    );
+
     private final Expression policyName;
     private final NamedExpression matchField;
     private final EnrichPolicy policy;
     private final Map<String, String> concreteIndices; // cluster -> enrich indices
+    // This could be simplified by just always using an Alias.
     private final List<NamedExpression> enrichFields;
     private List<Attribute> output;
 
@@ -73,6 +97,75 @@ public Enrich(
         this.enrichFields = enrichFields;
     }
 
+    private static Enrich readFrom(StreamInput in) throws IOException {
+        Enrich.Mode mode = Enrich.Mode.ANY;
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
+            mode = in.readEnum(Enrich.Mode.class);
+        }
+        final Source source = Source.readFrom((PlanStreamInput) in);
+        final LogicalPlan child = in.readNamedWriteable(LogicalPlan.class);
+        final Expression policyName = in.readNamedWriteable(Expression.class);
+        final NamedExpression matchField = in.readNamedWriteable(NamedExpression.class);
+        if (in.getTransportVersion().before(TransportVersions.V_8_13_0)) {
+            in.readString(); // discard the old policy name
+        }
+        final EnrichPolicy policy = new EnrichPolicy(in);
+        final Map<String, String> concreteIndices;
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
+            concreteIndices = in.readMap(StreamInput::readString, StreamInput::readString);
+        } else {
+            EsIndex esIndex = new EsIndex(in);
+            if (esIndex.concreteIndices().size() > 1) {
+                throw new IllegalStateException("expected a single enrich index; got " + esIndex);
+            }
+            concreteIndices = Map.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, Iterables.get(esIndex.concreteIndices(), 0));
+        }
+        return new Enrich(
+            source,
+            child,
+            mode,
+            policyName,
+            matchField,
+            policy,
+            concreteIndices,
+            in.readNamedWriteableCollectionAsList(NamedExpression.class)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
+            out.writeEnum(mode());
+        }
+
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(child());
+        out.writeNamedWriteable(policyName());
+        out.writeNamedWriteable(matchField());
+        if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) {
+            out.writeString(BytesRefs.toString(policyName().fold())); // old policy name
+        }
+        policy().writeTo(out);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
+            out.writeMap(concreteIndices(), StreamOutput::writeString, StreamOutput::writeString);
+        } else {
+            Map<String, String> concreteIndices = concreteIndices();
+            if (concreteIndices.keySet().equals(Set.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY))) {
+                String enrichIndex = concreteIndices.get(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
+                EsIndex esIndex = new EsIndex(enrichIndex, Map.of(), Set.of(enrichIndex));
+                esIndex.writeTo(out);
+            } else {
+                throw new IllegalStateException("expected a single enrich index; got " + concreteIndices);
+            }
+        }
+        out.writeNamedWriteableCollection(enrichFields());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     public NamedExpression matchField() {
         return matchField;
     }
@@ -126,6 +219,32 @@ public List<Attribute> output() {
         return output;
     }
 
+    @Override
+    public List<Attribute> generatedAttributes() {
+        return asAttributes(enrichFields);
+    }
+
+    @Override
+    public Enrich withGeneratedNames(List<String> newNames) {
+        checkNumberOfNewNames(newNames);
+
+        List<NamedExpression> newEnrichFields = new ArrayList<>(enrichFields.size());
+        for (int i = 0; i < enrichFields.size(); i++) {
+            NamedExpression enrichField = enrichFields.get(i);
+            String newName = newNames.get(i);
+            if (enrichField.name().equals(newName)) {
+                newEnrichFields.add(enrichField);
+            } else if (enrichField instanceof ReferenceAttribute ra) {
+                newEnrichFields.add(new Alias(ra.source(), newName, ra, new NameId(), ra.synthetic()));
+            } else if (enrichField instanceof Alias a) {
+                newEnrichFields.add(new Alias(a.source(), newName, a.child(), new NameId(), a.synthetic()));
+            } else {
+                throw new IllegalArgumentException("Enrich field must be Alias or ReferenceAttribute");
+            }
+        }
+        return new Enrich(source(), child(), mode(), policyName(), matchField(), policy(), concreteIndices(), newEnrichFields);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java
index 866385e6c7c28..b080c425d2312 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java
@@ -6,15 +6,22 @@
  */
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.NodeUtils;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -22,6 +29,11 @@
 import java.util.Objects;
 
 public class EsRelation extends LeafPlan {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        LogicalPlan.class,
+        "EsRelation",
+        EsRelation::readFrom
+    );
 
     private final EsIndex index;
     private final List<Attribute> attrs;
@@ -44,6 +56,45 @@ public EsRelation(Source source, EsIndex index, List<Attribute> attributes, Inde
         this.frozen = frozen;
     }
 
+    private static EsRelation readFrom(StreamInput in) throws IOException {
+        Source source = Source.readFrom((PlanStreamInput) in);
+        EsIndex esIndex = new EsIndex(in);
+        List<Attribute> attributes = in.readNamedWriteableCollectionAsList(Attribute.class);
+        if (supportingEsSourceOptions(in.getTransportVersion())) {
+            // We don't do anything with these strings
+            in.readOptionalString();
+            in.readOptionalString();
+            in.readOptionalString();
+        }
+        IndexMode indexMode = readIndexMode(in);
+        boolean frozen = in.readBoolean();
+        return new EsRelation(source, esIndex, attributes, indexMode, frozen);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        index().writeTo(out);
+        out.writeNamedWriteableCollection(output());
+        if (supportingEsSourceOptions(out.getTransportVersion())) {
+            // write (null) string fillers expected by remote
+            out.writeOptionalString(null);
+            out.writeOptionalString(null);
+            out.writeOptionalString(null);
+        }
+        writeIndexMode(out, indexMode());
+        out.writeBoolean(frozen());
+    }
+
+    private static boolean supportingEsSourceOptions(TransportVersion version) {
+        return version.onOrAfter(TransportVersions.V_8_14_0) && version.before(TransportVersions.ESQL_REMOVE_ES_SOURCE_OPTIONS);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected NodeInfo<EsRelation> info() {
         return NodeInfo.create(this, EsRelation::new, index, attrs, indexMode, frozen);
@@ -122,4 +173,20 @@ public boolean equals(Object obj) {
     public String nodeString() {
         return nodeName() + "[" + index + "]" + NodeUtils.limitedToString(attrs);
     }
+
+    public static IndexMode readIndexMode(StreamInput in) throws IOException {
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_TO_SOURCE)) {
+            return IndexMode.fromString(in.readString());
+        } else {
+            return IndexMode.STANDARD;
+        }
+    }
+
+    public static void writeIndexMode(StreamOutput out, IndexMode indexMode) throws IOException {
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ADD_INDEX_MODE_TO_SOURCE)) {
+            out.writeString(indexMode.getName());
+        } else if (indexMode != IndexMode.STANDARD) {
+            throw new IllegalStateException("not ready to support index mode [" + indexMode + "]");
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java
index 20117a873c143..2cecef42a42ac 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java
@@ -7,18 +7,30 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
+import org.elasticsearch.xpack.esql.core.expression.NameId;
+import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.plan.GeneratingPlan;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes;
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
 
-public class Eval extends UnaryPlan {
+public class Eval extends UnaryPlan implements GeneratingPlan<Eval> {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Eval", Eval::new);
 
     private final List<Alias> fields;
     private List<Attribute> lazyOutput;
@@ -28,6 +40,22 @@ public Eval(Source source, LogicalPlan child, List<Alias> fields) {
         this.fields = fields;
     }
 
+    private Eval(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(LogicalPlan.class), in.readCollectionAsList(Alias::new));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(child());
+        out.writeCollection(fields());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     public List<Alias> fields() {
         return fields;
     }
@@ -41,6 +69,50 @@ public List<Attribute> output() {
         return lazyOutput;
     }
 
+    @Override
+    public List<Attribute> generatedAttributes() {
+        return asAttributes(fields);
+    }
+
+    @Override
+    public Eval withGeneratedNames(List<String> newNames) {
+        checkNumberOfNewNames(newNames);
+
+        return new Eval(source(), child(), renameAliases(fields, newNames));
+    }
+
+    private List<Alias> renameAliases(List<Alias> originalAttributes, List<String> newNames) {
+        AttributeMap.Builder<Attribute> aliasReplacedByBuilder = AttributeMap.builder();
+        List<Alias> newFields = new ArrayList<>(originalAttributes.size());
+        for (int i = 0; i < originalAttributes.size(); i++) {
+            Alias field = originalAttributes.get(i);
+            String newName = newNames.get(i);
+            if (field.name().equals(newName)) {
+                newFields.add(field);
+            } else {
+                Alias newField = new Alias(field.source(), newName, field.child(), new NameId(), field.synthetic());
+                newFields.add(newField);
+                aliasReplacedByBuilder.put(field.toAttribute(), newField.toAttribute());
+            }
+        }
+        AttributeMap<Attribute> aliasReplacedBy = aliasReplacedByBuilder.build();
+
+        // We need to also update any references to the old attributes in the new attributes; e.g.
+        // EVAL x = 1, y = x + 1
+        // renaming x, y to x1, y1
+        // so far became
+        // EVAL x1 = 1, y1 = x + 1
+        // - but x doesn't exist anymore, so replace it by x1 to obtain
+        // EVAL x1 = 1, y1 = x1 + 1
+
+        List<Alias> newFieldsWithUpdatedRefs = new ArrayList<>(originalAttributes.size());
+        for (Alias newField : newFields) {
+            newFieldsWithUpdatedRefs.add((Alias) newField.transformUp(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r)));
+        }
+
+        return newFieldsWithUpdatedRefs;
+    }
+
     @Override
     public boolean expressionsResolved() {
         return Resolvables.resolved(fields);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Explain.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Explain.java
index 8d2640a43f38c..009c9519a9fe5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Explain.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Explain.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -30,6 +31,16 @@ public Explain(Source source, LogicalPlan query) {
         this.query = query;
     }
 
+    @Override
+    public void writeTo(StreamOutput out) {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
     // TODO: implement again
     // @Override
     // public void execute(EsqlSession session, ActionListener<Result> listener) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java
index 46fafe57e7d26..aca1a83344f32 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Filter.java
@@ -6,10 +6,15 @@
  */
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
+import java.io.IOException;
 import java.util.Objects;
 
 /**
@@ -18,6 +23,7 @@
  * {@code Filter} has a "condition" Expression that does the filtering.
  */
 public class Filter extends UnaryPlan {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Filter", Filter::new);
 
     private final Expression condition;
 
@@ -26,6 +32,22 @@ public Filter(Source source, LogicalPlan child, Expression condition) {
         this.condition = condition;
     }
 
+    private Filter(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(LogicalPlan.class), in.readNamedWriteable(Expression.class));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(child());
+        out.writeNamedWriteable(condition());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected NodeInfo<Filter> info() {
         return NodeInfo.create(this, Filter::new, child(), condition);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java
index 963fd318f814c..2f1c468415aa3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java
@@ -7,6 +7,9 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.grok.GrokBuiltinPatterns;
 import org.elasticsearch.grok.GrokCaptureConfig;
 import org.elasticsearch.grok.GrokCaptureType;
@@ -19,14 +22,17 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.NamedExpressions;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.parser.ParsingException;
 
+import java.io.IOException;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
 
 public class Grok extends RegexExtract {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Grok", Grok::readFrom);
 
     public record Parser(String pattern, org.elasticsearch.grok.Grok grok) {
 
@@ -88,6 +94,31 @@ public Grok(Source source, LogicalPlan child, Expression inputExpr, Parser parse
 
     }
 
+    private static Grok readFrom(StreamInput in) throws IOException {
+        Source source = Source.readFrom((PlanStreamInput) in);
+        return new Grok(
+            source,
+            in.readNamedWriteable(LogicalPlan.class),
+            in.readNamedWriteable(Expression.class),
+            Grok.pattern(source, in.readString()),
+            in.readNamedWriteableCollectionAsList(Attribute.class)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(child());
+        out.writeNamedWriteable(input());
+        out.writeString(parser().pattern());
+        out.writeNamedWriteableCollection(extractedFields());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     public UnaryPlan replaceChild(LogicalPlan newChild) {
         return new Grok(source(), newChild, input, parser, extractedFields);
@@ -103,6 +134,11 @@ public List<Attribute> output() {
         return NamedExpressions.mergeOutputAttributes(extractedFields, child().output());
     }
 
+    @Override
+    public Grok withGeneratedNames(List<String> newNames) {
+        return new Grok(source(), child(), input, parser, renameExtractedFields(newNames));
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java
index 46ec56223384c..b37976c00ad06 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/InlineStats.java
@@ -7,21 +7,57 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.core.Releasables;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
+import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinType;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
+import org.elasticsearch.xpack.esql.planner.PlannerUtils;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
-public class InlineStats extends UnaryPlan {
+import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
+
+/**
+ * Enriches the stream of data with the results of running a {@link Aggregate STATS}.
+ * <p>
+ *     This is a {@link Phased} operation that doesn't have a "native" implementation.
+ *     Instead, it's implemented as first running a {@link Aggregate STATS} and then
+ *     a {@link Join}.
+ * </p>
+ */
+public class InlineStats extends UnaryPlan implements NamedWriteable, Phased, Stats {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        LogicalPlan.class,
+        "InlineStats",
+        InlineStats::new
+    );
 
     private final List<Expression> groupings;
     private final List<? extends NamedExpression> aggregates;
+    private List<Attribute> lazyOutput;
 
     public InlineStats(Source source, LogicalPlan child, List<Expression> groupings, List<? extends NamedExpression> aggregates) {
         super(source, child);
@@ -29,6 +65,28 @@ public InlineStats(Source source, LogicalPlan child, List<Expression> groupings,
         this.aggregates = aggregates;
     }
 
+    public InlineStats(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(LogicalPlan.class),
+            in.readNamedWriteableCollectionAsList(Expression.class),
+            in.readNamedWriteableCollectionAsList(NamedExpression.class)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeNamedWriteable(child());
+        out.writeNamedWriteableCollection(groupings);
+        out.writeNamedWriteableCollection(aggregates);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected NodeInfo<InlineStats> info() {
         return NodeInfo.create(this, InlineStats::new, child(), groupings, aggregates);
@@ -39,10 +97,17 @@ public InlineStats replaceChild(LogicalPlan newChild) {
         return new InlineStats(source(), newChild, groupings, aggregates);
     }
 
+    @Override
+    public InlineStats with(LogicalPlan child, List<Expression> newGroupings, List<? extends NamedExpression> newAggregates) {
+        return new InlineStats(source(), child, newGroupings, newAggregates);
+    }
+
+    @Override
     public List<Expression> groupings() {
         return groupings;
     }
 
+    @Override
     public List<? extends NamedExpression> aggregates() {
         return aggregates;
     }
@@ -54,7 +119,21 @@ public boolean expressionsResolved() {
 
     @Override
     public List<Attribute> output() {
-        return Expressions.asAttributes(aggregates);
+        if (this.lazyOutput == null) {
+            List<NamedExpression> addedFields = new ArrayList<>();
+            AttributeSet set = child().outputSet();
+
+            for (NamedExpression agg : aggregates) {
+                Attribute att = agg.toAttribute();
+                if (set.contains(att) == false) {
+                    addedFields.add(agg);
+                    set.add(att);
+                }
+            }
+
+            this.lazyOutput = mergeOutputAttributes(addedFields, child().output());
+        }
+        return lazyOutput;
     }
 
     @Override
@@ -77,4 +156,102 @@ public boolean equals(Object obj) {
             && Objects.equals(aggregates, other.aggregates)
             && Objects.equals(child(), other.child());
     }
+
+    @Override
+    public LogicalPlan firstPhase() {
+        return new Aggregate(source(), child(), Aggregate.AggregateType.STANDARD, groupings, aggregates);
+    }
+
+    @Override
+    public LogicalPlan nextPhase(List<Attribute> schema, List<Page> firstPhaseResult) {
+        if (equalsAndSemanticEquals(firstPhase().output(), schema) == false) {
+            throw new IllegalStateException("Unexpected first phase outputs: " + firstPhase().output() + " vs " + schema);
+        }
+        if (groupings.isEmpty()) {
+            return ungroupedNextPhase(schema, firstPhaseResult);
+        }
+        return groupedNextPhase(schema, firstPhaseResult);
+    }
+
+    private LogicalPlan ungroupedNextPhase(List<Attribute> schema, List<Page> firstPhaseResult) {
+        if (firstPhaseResult.size() != 1) {
+            throw new IllegalArgumentException("expected single row");
+        }
+        Page p = firstPhaseResult.get(0);
+        if (p.getPositionCount() != 1) {
+            throw new IllegalArgumentException("expected single row");
+        }
+        List<Alias> values = new ArrayList<>(schema.size());
+        for (int i = 0; i < schema.size(); i++) {
+            Attribute s = schema.get(i);
+            Object value = BlockUtils.toJavaObject(p.getBlock(i), 0);
+            values.add(new Alias(source(), s.name(), new Literal(source(), value, s.dataType()), aggregates.get(i).id()));
+        }
+        return new Eval(source(), child(), values);
+    }
+
+    private static boolean equalsAndSemanticEquals(List<Attribute> left, List<Attribute> right) {
+        if (left.equals(right) == false) {
+            return false;
+        }
+        for (int i = 0; i < left.size(); i++) {
+            if (left.get(i).semanticEquals(right.get(i)) == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private LogicalPlan groupedNextPhase(List<Attribute> schema, List<Page> firstPhaseResult) {
+        LocalRelation local = firstPhaseResultsToLocalRelation(schema, firstPhaseResult);
+        List<Attribute> groupingAttributes = new ArrayList<>(groupings.size());
+        for (Expression g : groupings) {
+            if (g instanceof Attribute a) {
+                groupingAttributes.add(a);
+            } else {
+                throw new IllegalStateException("optimized plans should only have attributes in groups, but got [" + g + "]");
+            }
+        }
+        List<Attribute> leftFields = new ArrayList<>(groupingAttributes.size());
+        List<Attribute> rightFields = new ArrayList<>(groupingAttributes.size());
+        List<Attribute> rhsOutput = Join.makeReference(local.output());
+        for (Attribute lhs : groupingAttributes) {
+            for (Attribute rhs : rhsOutput) {
+                if (lhs.name().equals(rhs.name())) {
+                    leftFields.add(lhs);
+                    rightFields.add(rhs);
+                    break;
+                }
+            }
+        }
+        JoinConfig config = new JoinConfig(JoinType.LEFT, groupingAttributes, leftFields, rightFields);
+        return new Join(source(), child(), local, config);
+    }
+
+    private LocalRelation firstPhaseResultsToLocalRelation(List<Attribute> schema, List<Page> firstPhaseResult) {
+        // Limit ourselves to 1mb of results similar to LOOKUP for now.
+        long bytesUsed = firstPhaseResult.stream().mapToLong(Page::ramBytesUsedByBlocks).sum();
+        if (bytesUsed > ByteSizeValue.ofMb(1).getBytes()) {
+            throw new IllegalArgumentException("first phase result too large [" + ByteSizeValue.ofBytes(bytesUsed) + "] > 1mb");
+        }
+        int positionCount = firstPhaseResult.stream().mapToInt(Page::getPositionCount).sum();
+        Block.Builder[] builders = new Block.Builder[schema.size()];
+        Block[] blocks;
+        try {
+            for (int b = 0; b < builders.length; b++) {
+                builders[b] = PlannerUtils.toElementType(schema.get(b).dataType())
+                    .newBlockBuilder(positionCount, PlannerUtils.NON_BREAKING_BLOCK_FACTORY);
+            }
+            for (Page p : firstPhaseResult) {
+                for (int b = 0; b < builders.length; b++) {
+                    builders[b].copyFrom(p.getBlock(b), 0, p.getPositionCount());
+                }
+            }
+            blocks = Block.Builder.buildAll(builders);
+        } finally {
+            Releasables.closeExpectNoException(builders);
+        }
+        return new LocalRelation(source(), schema, LocalSupplier.of(blocks));
+    }
+
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Limit.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Limit.java
index df5e1cf23275c..00b9ad3c2b6a2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Limit.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Limit.java
@@ -6,13 +6,19 @@
  */
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
+import java.io.IOException;
 import java.util.Objects;
 
 public class Limit extends UnaryPlan {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Limit", Limit::new);
 
     private final Expression limit;
 
@@ -21,6 +27,22 @@ public Limit(Source source, Expression limit, LogicalPlan child) {
         this.limit = limit;
     }
 
+    private Limit(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(LogicalPlan.class));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(limit());
+        out.writeNamedWriteable(child());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected NodeInfo<Limit> info() {
         return NodeInfo.create(this, Limit::new, limit, child());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/LogicalPlan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/LogicalPlan.java
index 0397183c6a6c3..b94d60a9face2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/LogicalPlan.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/LogicalPlan.java
@@ -6,10 +6,14 @@
  */
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvable;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
-import org.elasticsearch.xpack.esql.core.plan.QueryPlan;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.plan.QueryPlan;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
+import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 
 import java.util.List;
 
@@ -18,6 +22,27 @@
  * For example, a logical plan in English would be: "I want to get from DEN to SFO".
  */
 public abstract class LogicalPlan extends QueryPlan<LogicalPlan> implements Resolvable {
+    public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        return List.of(
+            Aggregate.ENTRY,
+            Dissect.ENTRY,
+            Enrich.ENTRY,
+            EsRelation.ENTRY,
+            EsqlProject.ENTRY,
+            Eval.ENTRY,
+            Filter.ENTRY,
+            Grok.ENTRY,
+            InlineStats.ENTRY,
+            LocalRelation.ENTRY,
+            Limit.ENTRY,
+            Lookup.ENTRY,
+            MvExpand.ENTRY,
+            Join.ENTRY,
+            OrderBy.ENTRY,
+            Project.ENTRY,
+            TopN.ENTRY
+        );
+    }
 
     /**
      * Order is important in the enum; any values should be added at the end.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java
index 6893935f20b5b..c0eaaa96b86d9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java
@@ -7,6 +7,9 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
@@ -14,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 import org.elasticsearch.xpack.esql.plan.logical.join.Join;
 import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig;
 import org.elasticsearch.xpack.esql.plan.logical.join.JoinType;
@@ -30,6 +32,8 @@
  * The class is supposed to be substituted by a {@link Join}.
  */
 public class Lookup extends UnaryPlan {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Lookup", Lookup::new);
+
     private final Expression tableName;
     /**
      * References to the input fields to match against the {@link #localRelation}.
@@ -53,16 +57,17 @@ public Lookup(
         this.localRelation = localRelation;
     }
 
-    public Lookup(PlanStreamInput in) throws IOException {
-        super(Source.readFrom(in), in.readLogicalPlanNode());
+    public Lookup(StreamInput in) throws IOException {
+        super(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(LogicalPlan.class));
         this.tableName = in.readNamedWriteable(Expression.class);
         this.matchFields = in.readNamedWriteableCollectionAsList(Attribute.class);
         this.localRelation = in.readBoolean() ? new LocalRelation(in) : null;
     }
 
-    public void writeTo(PlanStreamOutput out) throws IOException {
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
         source().writeTo(out);
-        out.writeLogicalPlanNode(child());
+        out.writeNamedWriteable(child());
         out.writeNamedWriteable(tableName);
         out.writeNamedWriteableCollection(matchFields);
         if (localRelation == null) {
@@ -73,6 +78,11 @@ public void writeTo(PlanStreamOutput out) throws IOException {
         }
     }
 
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     public Expression tableName() {
         return tableName;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java
index 5e9dca26a6863..98eb5d3fcc451 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java
@@ -7,16 +7,23 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
 public class MvExpand extends UnaryPlan {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "MvExpand", MvExpand::new);
+
     private final NamedExpression target;
     private final Attribute expanded;
 
@@ -28,6 +35,28 @@ public MvExpand(Source source, LogicalPlan child, NamedExpression target, Attrib
         this.expanded = expanded;
     }
 
+    private MvExpand(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(LogicalPlan.class),
+            in.readNamedWriteable(NamedExpression.class),
+            in.readNamedWriteable(Attribute.class)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(child());
+        out.writeNamedWriteable(target());
+        out.writeNamedWriteable(expanded());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     public static List<Attribute> calculateOutput(List<Attribute> input, NamedExpression target, Attribute expanded) {
         List<Attribute> result = new ArrayList<>();
         for (Attribute attribute : input) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java
index 68d089980074c..9b79f9510feaf 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/OrderBy.java
@@ -6,15 +6,21 @@
  */
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
-import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.Order;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 public class OrderBy extends UnaryPlan {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "OrderBy", OrderBy::new);
 
     private final List<Order> order;
 
@@ -23,6 +29,26 @@ public OrderBy(Source source, LogicalPlan child, List<Order> order) {
         this.order = order;
     }
 
+    private OrderBy(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(LogicalPlan.class),
+            in.readCollectionAsList(org.elasticsearch.xpack.esql.expression.Order::new)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(child());
+        out.writeCollection(order());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected NodeInfo<OrderBy> info() {
         return NodeInfo.create(this, OrderBy::new, child(), order);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Phased.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Phased.java
new file mode 100644
index 0000000000000..6923f9e137eab
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Phased.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.xpack.esql.analysis.Analyzer;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.util.Holder;
+
+import java.util.List;
+
+/**
+ * Marks a {@link LogicalPlan} node as requiring multiple ESQL executions to run.
+ * All logical plans are now run by:
+ * <ol>
+ *     <li>{@link Analyzer analyzing} the entire query</li>
+ *     <li>{@link Phased#extractFirstPhase extracting} the first phase from the
+ *         logical plan</li>
+ *     <li>if there isn't a first phase, run the entire logical plan and return the
+ *         results. you are done.</li>
+ *     <li>if there is first phase, run that</li>
+ *     <li>{@link Phased#applyResultsFromFirstPhase applying} the results from the
+ *         first phase into the logical plan</li>
+ *     <li>start over from step 2 using the new logical plan</li>
+ * </ol>
+ * <p>For example, {@code INLINESTATS} is written like this:</p>
+ * <pre>{@code
+ * FROM foo
+ * | EVAL bar = a * b
+ * | INLINESTATS m = MAX(bar) BY b
+ * | WHERE m = bar
+ * | LIMIT 1
+ * }</pre>
+ * <p>And it's split into:</p>
+ * <pre>{@code
+ * FROM foo
+ * | EVAL bar = a * b
+ * | STATS m = MAX(bar) BY b
+ * }</pre>
+ * <p>and</p>
+ * <pre>{@code
+ * FROM foo
+ * | EVAL bar = a * b
+ * | LOOKUP (results of m = MAX(bar) BY b) ON b
+ * | WHERE m = bar
+ * | LIMIT 1
+ * }</pre>
+ * <p>If there are multiple {@linkplain Phased} nodes in the plan we always
+ * operate on the lowest one first, counting from the data source "upwards".
+ * Generally that'll read left to right in the query. So:</p>
+ * <pre>{@code
+ * FROM foo | INLINESTATS | INLINESTATS
+ * }</pre>
+ * becomes
+ * <pre>{@code
+ * FROM foo | STATS
+ * }</pre>
+ * and
+ * <pre>{@code
+ * FROM foo | HASHJOIN | INLINESTATS
+ * }</pre>
+ * which is further broken into
+ * <pre>{@code
+ * FROM foo | HASHJOIN | STATS
+ * }</pre>
+ * and finally:
+ * <pre>{@code
+ * FROM foo | HASHJOIN | HASHJOIN
+ * }</pre>
+ */
+public interface Phased {
+    /**
+     * Return a {@link LogicalPlan} for the first "phase" of this operation.
+     * The result of this phase will be provided to {@link #nextPhase}.
+     */
+    LogicalPlan firstPhase();
+
+    /**
+     * Use the results of plan provided from {@link #firstPhase} to produce the
+     * next phase of the query.
+     */
+    LogicalPlan nextPhase(List<Attribute> schema, List<Page> firstPhaseResult);
+
+    /**
+     * Find the first {@link Phased} operation and return it's {@link #firstPhase}.
+     * Or {@code null} if there aren't any {@linkplain Phased} operations.
+     */
+    static LogicalPlan extractFirstPhase(LogicalPlan plan) {
+        if (false == plan.optimized()) {
+            throw new IllegalArgumentException("plan must be optimized");
+        }
+        var firstPhase = new Holder<LogicalPlan>();
+        plan.forEachUp(t -> {
+            if (firstPhase.get() == null && t instanceof Phased phased) {
+                firstPhase.set(phased.firstPhase());
+            }
+        });
+        LogicalPlan firstPhasePlan = firstPhase.get();
+        if (firstPhasePlan != null) {
+            firstPhasePlan.setAnalyzed();
+        }
+        return firstPhasePlan;
+    }
+
+    /**
+     * Merge the results of {@link #extractFirstPhase} into a {@link LogicalPlan}
+     * and produce a new {@linkplain LogicalPlan} that will execute the rest of the
+     * query. This plan <strong>may</strong> contain <strong>another</strong>
+     * {@link #firstPhase}. If it does then it will also need to be
+     * {@link #extractFirstPhase extracted} and the results will need to be applied
+     * again by calling this method. Eventually this will produce a plan which
+     * does not have a {@link #firstPhase} and <strong>that</strong> is the "final"
+     * phase of the plan.
+     */
+    static LogicalPlan applyResultsFromFirstPhase(LogicalPlan plan, List<Attribute> schema, List<Page> result) {
+        if (false == plan.analyzed()) {
+            throw new IllegalArgumentException("plan must be analyzed");
+        }
+        Holder<Boolean> seen = new Holder<>(false);
+        LogicalPlan applied = plan.transformUp(logicalPlan -> {
+            if (seen.get() == false && logicalPlan instanceof Phased phased) {
+                seen.set(true);
+                return phased.nextPhase(schema, result);
+            }
+            return logicalPlan;
+        });
+        applied.setAnalyzed();
+        return applied;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Project.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Project.java
index d3896b1dfc844..e12a8cb557fde 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Project.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Project.java
@@ -6,6 +6,9 @@
  */
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
@@ -13,7 +16,9 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.function.Functions;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
@@ -21,6 +26,8 @@
  * A {@code Project} is a {@code Plan} with one child. In {@code SELECT x FROM y}, the "SELECT" statement is a Project.
  */
 public class Project extends UnaryPlan {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Project", Project::new);
+
     private final List<? extends NamedExpression> projections;
 
     public Project(Source source, LogicalPlan child, List<? extends NamedExpression> projections) {
@@ -28,6 +35,26 @@ public Project(Source source, LogicalPlan child, List<? extends NamedExpression>
         this.projections = projections;
     }
 
+    private Project(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(LogicalPlan.class),
+            in.readNamedWriteableCollectionAsList(NamedExpression.class)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(child());
+        out.writeNamedWriteableCollection(projections());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected NodeInfo<Project> info() {
         return NodeInfo.create(this, Project::new, child(), projections);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java
index 649173f11dfaf..f95bee92d4e1a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/RegexExtract.java
@@ -9,14 +9,17 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.plan.GeneratingPlan;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
 
-public abstract class RegexExtract extends UnaryPlan {
+public abstract class RegexExtract extends UnaryPlan implements GeneratingPlan<RegexExtract> {
     protected final Expression input;
     protected final List<Attribute> extractedFields;
 
@@ -40,10 +43,36 @@ public Expression input() {
         return input;
     }
 
+    /**
+     * Upon parsing, these are named according to the {@link Dissect} or {@link Grok} pattern, but can be renamed without changing the
+     * pattern.
+     */
     public List<Attribute> extractedFields() {
         return extractedFields;
     }
 
+    @Override
+    public List<Attribute> generatedAttributes() {
+        return extractedFields;
+    }
+
+    List<Attribute> renameExtractedFields(List<String> newNames) {
+        checkNumberOfNewNames(newNames);
+
+        List<Attribute> renamedExtractedFields = new ArrayList<>(extractedFields.size());
+        for (int i = 0; i < newNames.size(); i++) {
+            Attribute extractedField = extractedFields.get(i);
+            String newName = newNames.get(i);
+            if (extractedField.name().equals(newName)) {
+                renamedExtractedFields.add(extractedField);
+            } else {
+                renamedExtractedFields.add(extractedFields.get(i).withName(newNames.get(i)).withId(new NameId()));
+            }
+        }
+
+        return renamedExtractedFields;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java
index 29ee7f0504c70..e587d7ad94d0e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Rename.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.analysis.Analyzer.ResolveRefs;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
@@ -28,6 +29,16 @@ public Rename(Source source, LogicalPlan child, List<Alias> renamings) {
         this.renamings = renamings;
     }
 
+    @Override
+    public void writeTo(StreamOutput out) {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
     public List<Alias> renamings() {
         return renamings;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Row.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Row.java
index 30e16d9e1b227..0fe45e9182f0a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Row.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Row.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
@@ -26,6 +27,16 @@ public Row(Source source, List<Alias> fields) {
         this.fields = fields;
     }
 
+    @Override
+    public void writeTo(StreamOutput out) {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
     public List<Alias> fields() {
         return fields;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Stats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Stats.java
new file mode 100644
index 0000000000000..c46c735e7482e
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Stats.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.List;
+
+/**
+ * STATS-like operations. Like {@link Aggregate} and {@link InlineStats}.
+ */
+public interface Stats {
+    /**
+     * The user supplied text in the query for this command.
+     */
+    Source source();
+
+    /**
+     * Rebuild this plan with new groupings and new aggregates.
+     */
+    Stats with(LogicalPlan child, List<Expression> newGroupings, List<? extends NamedExpression> newAggregates);
+
+    /**
+     * Have all the expressions in this plan been resolved?
+     */
+    boolean expressionsResolved();
+
+    /**
+     * The operation directly before this one in the plan.
+     */
+    LogicalPlan child();
+
+    /**
+     * List containing both the aggregate expressions and grouping expressions.
+     */
+    List<? extends NamedExpression> aggregates();
+
+    /**
+     * List containing just the grouping expressions.
+     */
+    List<Expression> groupings();
+
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/TopN.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/TopN.java
index 227d7785804d4..a9a5dbddc544f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/TopN.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/TopN.java
@@ -7,16 +7,22 @@
 
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.Order;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 public class TopN extends UnaryPlan {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "TopN", TopN::new);
 
     private final List<Order> order;
     private final Expression limit;
@@ -27,6 +33,28 @@ public TopN(Source source, LogicalPlan child, List<Order> order, Expression limi
         this.limit = limit;
     }
 
+    private TopN(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(LogicalPlan.class),
+            in.readCollectionAsList(Order::new),
+            in.readNamedWriteable(Expression.class)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(child());
+        out.writeCollection(order);
+        out.writeNamedWriteable(limit);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     public boolean expressionsResolved() {
         return limit.resolved() && Resolvables.resolved(order);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java
index cd1817367167d..8414419529c47 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedRelation.java
@@ -6,15 +6,16 @@
  */
 package org.elasticsearch.xpack.esql.plan.logical;
 
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
-import org.elasticsearch.xpack.esql.core.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.plan.TableIdentifier;
 
 import java.util.Collections;
 import java.util.List;
@@ -46,6 +47,16 @@ public UnresolvedRelation(
         this.unresolvedMsg = unresolvedMessage == null ? "Unknown index [" + table.index() + "]" : unresolvedMessage;
     }
 
+    @Override
+    public void writeTo(StreamOutput out) {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
     @Override
     protected NodeInfo<UnresolvedRelation> info() {
         return NodeInfo.create(this, UnresolvedRelation::new, table, frozen, metadataFields, indexMode, unresolvedMsg);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java
index 79278995b29bd..7ad1bcad2d9d0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java
@@ -7,6 +7,9 @@
 
 package org.elasticsearch.xpack.esql.plan.logical.join;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
@@ -15,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 import org.elasticsearch.xpack.esql.plan.logical.BinaryPlan;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 
@@ -29,6 +31,7 @@
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
 
 public class Join extends BinaryPlan {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Join", Join::new);
 
     private final JoinConfig config;
     private List<Attribute> lazyOutput;
@@ -51,18 +54,24 @@ public Join(
         this.config = new JoinConfig(type, matchFields, leftFields, rightFields);
     }
 
-    public Join(PlanStreamInput in) throws IOException {
-        super(Source.readFrom(in), in.readLogicalPlanNode(), in.readLogicalPlanNode());
+    public Join(StreamInput in) throws IOException {
+        super(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(LogicalPlan.class), in.readNamedWriteable(LogicalPlan.class));
         this.config = new JoinConfig(in);
     }
 
-    public void writeTo(PlanStreamOutput out) throws IOException {
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
         source().writeTo(out);
-        out.writeLogicalPlanNode(left());
-        out.writeLogicalPlanNode(right());
+        out.writeNamedWriteable(left());
+        out.writeNamedWriteable(right());
         config.writeTo(out);
     }
 
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     public JoinConfig config() {
         return config;
     }
@@ -147,7 +156,7 @@ public static List<Attribute> makeReference(List<Attribute> output) {
         List<Attribute> out = new ArrayList<>(output.size());
         for (Attribute a : output) {
             if (a.resolved() && a instanceof ReferenceAttribute == false) {
-                out.add(new ReferenceAttribute(a.source(), a.name(), a.dataType(), a.qualifier(), a.nullable(), a.id(), a.synthetic()));
+                out.add(new ReferenceAttribute(a.source(), a.name(), a.dataType(), a.nullable(), a.id(), a.synthetic()));
             } else {
                 out.add(a);
             }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinType.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinType.java
index bd3ba43c25016..c3095efc9e623 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinType.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinType.java
@@ -44,8 +44,8 @@ public static JoinType readFrom(StreamInput in) throws IOException {
             case 0 -> INNER;
             case 1 -> LEFT;
             case 2 -> RIGHT;
-            case 4 -> FULL;
-            case 5 -> CROSS;
+            case 3 -> FULL;
+            case 4 -> CROSS;
             default -> throw new IllegalArgumentException("unsupported join [" + id + "]");
         };
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/EsqlProject.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/EsqlProject.java
index e359c6f928f7c..91796b237262c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/EsqlProject.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/EsqlProject.java
@@ -7,21 +7,51 @@
 
 package org.elasticsearch.xpack.esql.plan.logical.local;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 
+import java.io.IOException;
 import java.util.List;
 
 public class EsqlProject extends Project {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        LogicalPlan.class,
+        "EsqlProject",
+        EsqlProject::new
+    );
 
     public EsqlProject(Source source, LogicalPlan child, List<? extends NamedExpression> projections) {
         super(source, child, projections);
     }
 
+    public EsqlProject(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteable(LogicalPlan.class),
+            in.readNamedWriteableCollectionAsList(NamedExpression.class)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteable(child());
+        out.writeNamedWriteableCollection(projections());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected NodeInfo<Project> info() {
         return NodeInfo.create(this, EsqlProject::new, child(), projections());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java
index 195eb3b6304e4..d6106bae6b6b8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java
@@ -6,18 +6,26 @@
  */
 package org.elasticsearch.xpack.esql.plan.logical.local;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 import org.elasticsearch.xpack.esql.plan.logical.LeafPlan;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 public class LocalRelation extends LeafPlan {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        LogicalPlan.class,
+        "LocalRelation",
+        LocalRelation::new
+    );
 
     private final List<Attribute> output;
     private final LocalSupplier supplier;
@@ -28,18 +36,24 @@ public LocalRelation(Source source, List<Attribute> output, LocalSupplier suppli
         this.supplier = supplier;
     }
 
-    public LocalRelation(PlanStreamInput in) throws IOException {
-        super(Source.readFrom(in));
+    public LocalRelation(StreamInput in) throws IOException {
+        super(Source.readFrom((PlanStreamInput) in));
         this.output = in.readNamedWriteableCollectionAsList(Attribute.class);
-        this.supplier = LocalSupplier.readFrom(in);
+        this.supplier = LocalSupplier.readFrom((PlanStreamInput) in);
     }
 
-    public void writeTo(PlanStreamOutput out) throws IOException {
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
         source().writeTo(out);
         out.writeNamedWriteableCollection(output);
         supplier.writeTo(out);
     }
 
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected NodeInfo<LocalRelation> info() {
         return NodeInfo.create(this, LocalRelation::new, output, supplier);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java
index 9ac9ccdf2a876..58c344219e6a9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java
@@ -9,6 +9,7 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -43,6 +44,16 @@ public MetaFunctions(Source source) {
         }
     }
 
+    @Override
+    public void writeTo(StreamOutput out) {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
     @Override
     public List<Attribute> output() {
         return attributes;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/show/ShowInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/show/ShowInfo.java
index 6e98df32580ae..0fc30c96c809d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/show/ShowInfo.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/show/ShowInfo.java
@@ -9,6 +9,7 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Build;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -34,6 +35,16 @@ public ShowInfo(Source source) {
         }
     }
 
+    @Override
+    public void writeTo(StreamOutput out) {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
     @Override
     public List<Attribute> output() {
         return attributes;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java
index c3a7f065cc803..42fb0ab0bdf3e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExec.java
@@ -7,17 +7,29 @@
 
 package org.elasticsearch.xpack.esql.plan.physical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 public class AggregateExec extends UnaryExec implements EstimatesRowSize {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        PhysicalPlan.class,
+        "AggregateExec",
+        AggregateExec::new
+    );
+
     private final List<? extends Expression> groupings;
     private final List<? extends NamedExpression> aggregates;
 
@@ -50,6 +62,32 @@ public AggregateExec(
         this.estimatedRowSize = estimatedRowSize;
     }
 
+    private AggregateExec(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            ((PlanStreamInput) in).readPhysicalPlanNode(),
+            in.readNamedWriteableCollectionAsList(Expression.class),
+            in.readNamedWriteableCollectionAsList(NamedExpression.class),
+            in.readEnum(AggregateExec.Mode.class),
+            in.readOptionalVInt()
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        ((PlanStreamOutput) out).writePhysicalPlanNode(child());
+        out.writeNamedWriteableCollection(groupings());
+        out.writeNamedWriteableCollection(aggregates());
+        out.writeEnum(getMode());
+        out.writeOptionalVInt(estimatedRowSize());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected NodeInfo<AggregateExec> info() {
         return NodeInfo.create(this, AggregateExec::new, child(), groupings, aggregates, mode, estimatedRowSize);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/DissectExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/DissectExec.java
index 339b8e8bb4d86..35a364126a66b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/DissectExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/DissectExec.java
@@ -7,16 +7,27 @@
 
 package org.elasticsearch.xpack.esql.plan.physical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 import org.elasticsearch.xpack.esql.plan.logical.Dissect;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 public class DissectExec extends RegexExtractExec {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        PhysicalPlan.class,
+        "DissectExec",
+        DissectExec::new
+    );
 
     private final Dissect.Parser parser;
 
@@ -31,6 +42,30 @@ public DissectExec(
         this.parser = parser;
     }
 
+    private DissectExec(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            ((PlanStreamInput) in).readPhysicalPlanNode(),
+            in.readNamedWriteable(Expression.class),
+            Dissect.Parser.readFrom(in),
+            in.readNamedWriteableCollectionAsList(Attribute.class)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        ((PlanStreamOutput) out).writePhysicalPlanNode(child());
+        out.writeNamedWriteable(inputExpression());
+        parser().writeTo(out);
+        out.writeNamedWriteableCollection(extractedFields());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     public UnaryExec replaceChild(PhysicalPlan newChild) {
         return new DissectExec(source(), newChild, inputExpression, parser, extractedFields);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
index b8f96709a583f..21aa2cb7d1860 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
@@ -8,26 +8,39 @@
 package org.elasticsearch.xpack.esql.plan.physical;
 
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
-import org.elasticsearch.xpack.esql.core.expression.Order;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.querydsl.container.Sort;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.NodeUtils;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.expression.Order;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
 public class EsQueryExec extends LeafExec implements EstimatesRowSize {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        PhysicalPlan.class,
+        "EsQueryExec",
+        EsQueryExec::new
+    );
+
     public static final EsField DOC_ID_FIELD = new EsField("_doc", DataType.DOC_DATA_TYPE, Map.of(), false);
 
     private final EsIndex index;
@@ -43,14 +56,29 @@ public class EsQueryExec extends LeafExec implements EstimatesRowSize {
      */
     private final Integer estimatedRowSize;
 
-    public record FieldSort(FieldAttribute field, Order.OrderDirection direction, Order.NullsPosition nulls) {
+    public record FieldSort(FieldAttribute field, Order.OrderDirection direction, Order.NullsPosition nulls) implements Writeable {
         public FieldSortBuilder fieldSortBuilder() {
             FieldSortBuilder builder = new FieldSortBuilder(field.name());
-            builder.order(Sort.Direction.from(direction).asOrder());
-            builder.missing(Sort.Missing.from(nulls).searchOrder());
+            builder.order(Direction.from(direction).asOrder());
+            builder.missing(Missing.from(nulls).searchOrder());
             builder.unmappedType(field.dataType().esType());
             return builder;
         }
+
+        private static FieldSort readFrom(StreamInput in) throws IOException {
+            return new EsQueryExec.FieldSort(
+                FieldAttribute.readFrom(in),
+                in.readEnum(Order.OrderDirection.class),
+                in.readEnum(Order.NullsPosition.class)
+            );
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            field().writeTo(out);
+            out.writeEnum(direction());
+            out.writeEnum(nulls());
+        }
     }
 
     public EsQueryExec(Source source, EsIndex index, IndexMode indexMode, List<Attribute> attributes, QueryBuilder query) {
@@ -77,6 +105,36 @@ public EsQueryExec(
         this.estimatedRowSize = estimatedRowSize;
     }
 
+    private EsQueryExec(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            new EsIndex(in),
+            EsRelation.readIndexMode(in),
+            in.readNamedWriteableCollectionAsList(Attribute.class),
+            in.readOptionalNamedWriteable(QueryBuilder.class),
+            in.readOptionalNamedWriteable(Expression.class),
+            in.readOptionalCollectionAsList(FieldSort::readFrom),
+            in.readOptionalVInt()
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        index().writeTo(out);
+        EsRelation.writeIndexMode(out, indexMode());
+        out.writeNamedWriteableCollection(output());
+        out.writeOptionalNamedWriteable(query());
+        out.writeOptionalNamedWriteable(limit());
+        out.writeOptionalCollection(sorts());
+        out.writeOptionalVInt(estimatedRowSize());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     public static boolean isSourceAttribute(Attribute attr) {
         return DOC_ID_FIELD.getName().equals(attr.name());
     }
@@ -206,4 +264,47 @@ public String nodeString() {
             + estimatedRowSize
             + "]";
     }
+
+    public enum Direction {
+        ASC,
+        DESC;
+
+        public static Direction from(Order.OrderDirection dir) {
+            return dir == null || dir == Order.OrderDirection.ASC ? ASC : DESC;
+        }
+
+        public SortOrder asOrder() {
+            return this == Direction.ASC ? SortOrder.ASC : SortOrder.DESC;
+        }
+    }
+
+    public enum Missing {
+        FIRST("_first"),
+        LAST("_last"),
+        /**
+         * Nulls position has not been specified by the user and an appropriate default will be used.
+         *
+         * The default values are chosen such that it stays compatible with previous behavior. Unfortunately, this results in
+         * inconsistencies across different types of queries (see https://github.com/elastic/elasticsearch/issues/77068).
+         */
+        ANY(null);
+
+        private final String searchOrder;
+
+        Missing(String searchOrder) {
+            this.searchOrder = searchOrder;
+        }
+
+        public static Missing from(Order.NullsPosition pos) {
+            return switch (pos) {
+                case FIRST -> FIRST;
+                case LAST -> LAST;
+                default -> ANY;
+            };
+        }
+
+        public String searchOrder() {
+            return searchOrder;
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java
index 7e3bb44bc40e0..cd167b4683493 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExec.java
@@ -7,19 +7,29 @@
 
 package org.elasticsearch.xpack.esql.plan.physical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.NodeUtils;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 public class EsSourceExec extends LeafExec {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        PhysicalPlan.class,
+        "EsSourceExec",
+        EsSourceExec::new
+    );
 
     private final EsIndex index;
     private final List<Attribute> attributes;
@@ -38,6 +48,30 @@ public EsSourceExec(Source source, EsIndex index, List<Attribute> attributes, Qu
         this.indexMode = indexMode;
     }
 
+    private EsSourceExec(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            new EsIndex(in),
+            in.readNamedWriteableCollectionAsList(Attribute.class),
+            in.readOptionalNamedWriteable(QueryBuilder.class),
+            EsRelation.readIndexMode(in)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        index().writeTo(out);
+        out.writeNamedWriteableCollection(output());
+        out.writeOptionalNamedWriteable(query());
+        EsRelation.writeIndexMode(out, indexMode());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     public EsIndex index() {
         return index;
     }
@@ -62,7 +96,7 @@ protected NodeInfo<? extends PhysicalPlan> info() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(index);
+        return Objects.hash(index, attributes, query, indexMode);
     }
 
     @Override
@@ -76,7 +110,10 @@ public boolean equals(Object obj) {
         }
 
         EsSourceExec other = (EsSourceExec) obj;
-        return Objects.equals(index, other.index) && Objects.equals(query, other.query);
+        return Objects.equals(index, other.index)
+            && Objects.equals(attributes, other.attributes)
+            && Objects.equals(query, other.query)
+            && Objects.equals(indexMode, other.indexMode);
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java
index 7f5d15f907a43..dbcdf3378028a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java
@@ -11,11 +11,11 @@
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.NodeUtils;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.Queries;
+import org.elasticsearch.xpack.esql.index.EsIndex;
 
 import java.util.List;
 import java.util.Objects;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EvalExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EvalExec.java
index 3876891b27752..97b81914f8889 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EvalExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EvalExec.java
@@ -7,17 +7,29 @@
 
 package org.elasticsearch.xpack.esql.plan.physical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes;
 
 public class EvalExec extends UnaryExec implements EstimatesRowSize {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        PhysicalPlan.class,
+        "EvalExec",
+        EvalExec::new
+    );
+
     private final List<Alias> fields;
 
     public EvalExec(Source source, PhysicalPlan child, List<Alias> fields) {
@@ -25,6 +37,22 @@ public EvalExec(Source source, PhysicalPlan child, List<Alias> fields) {
         this.fields = fields;
     }
 
+    private EvalExec(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), ((PlanStreamInput) in).readPhysicalPlanNode(), in.readCollectionAsList(Alias::new));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        ((PlanStreamOutput) out).writePhysicalPlanNode(child());
+        out.writeCollection(fields());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     public List<Alias> fields() {
         return fields;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeExec.java
index 61c65c484059e..f20b218f28efb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeExec.java
@@ -7,15 +7,27 @@
 
 package org.elasticsearch.xpack.esql.plan.physical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 
+import java.io.IOException;
 import java.util.List;
+import java.util.Objects;
 
 import static java.util.Collections.emptyList;
 
 public class ExchangeExec extends UnaryExec {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        PhysicalPlan.class,
+        "ExchangeExec",
+        ExchangeExec::new
+    );
 
     private final List<Attribute> output;
     private final boolean inBetweenAggs;
@@ -30,12 +42,34 @@ public ExchangeExec(Source source, List<Attribute> output, boolean inBetweenAggs
         this.inBetweenAggs = inBetweenAggs;
     }
 
+    private ExchangeExec(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteableCollectionAsList(Attribute.class),
+            in.readBoolean(),
+            ((PlanStreamInput) in).readPhysicalPlanNode()
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteableCollection(output);
+        out.writeBoolean(inBetweenAggs());
+        ((PlanStreamOutput) out).writePhysicalPlanNode(child());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     public List<Attribute> output() {
         return output.isEmpty() ? super.output() : output;
     }
 
-    public boolean isInBetweenAggs() {
+    public boolean inBetweenAggs() {
         return inBetweenAggs;
     }
 
@@ -48,4 +82,18 @@ public UnaryExec replaceChild(PhysicalPlan newChild) {
     protected NodeInfo<? extends PhysicalPlan> info() {
         return NodeInfo.create(this, ExchangeExec::new, output, inBetweenAggs, child());
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (super.equals(obj) == false) {
+            return false;
+        }
+        ExchangeExec other = (ExchangeExec) obj;
+        return output.equals(other.output) && inBetweenAggs == other.inBetweenAggs;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), output, inBetweenAggs);
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExec.java
index 2f7c4a93eec71..2992619da75ef 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExec.java
@@ -7,14 +7,25 @@
 
 package org.elasticsearch.xpack.esql.plan.physical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 public class ExchangeSinkExec extends UnaryExec {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        PhysicalPlan.class,
+        "ExchangeSinkExec",
+        ExchangeSinkExec::new
+    );
 
     private final List<Attribute> output;
     private final boolean intermediateAgg;
@@ -25,6 +36,28 @@ public ExchangeSinkExec(Source source, List<Attribute> output, boolean intermedi
         this.intermediateAgg = intermediateAgg;
     }
 
+    private ExchangeSinkExec(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readNamedWriteableCollectionAsList(Attribute.class),
+            in.readBoolean(),
+            ((PlanStreamInput) in).readPhysicalPlanNode()
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeNamedWriteableCollection(output());
+        out.writeBoolean(isIntermediateAgg());
+        ((PlanStreamOutput) out).writePhysicalPlanNode(child());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     public List<Attribute> output() {
         return output;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSourceExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSourceExec.java
index 44c9b38feee48..e679a9b131d1d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSourceExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSourceExec.java
@@ -7,14 +7,24 @@
 
 package org.elasticsearch.xpack.esql.plan.physical;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 public class ExchangeSourceExec extends LeafExec {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        PhysicalPlan.class,
+        "ExchangeSourceExec",
+        ExchangeSourceExec::new
+    );
 
     private final List<Attribute> output;
     private final boolean intermediateAgg;
@@ -25,6 +35,22 @@ public ExchangeSourceExec(Source source, List<Attribute> output, boolean interme
         this.intermediateAgg = intermediateAgg;
     }
 
+    private ExchangeSourceExec(StreamInput in) throws IOException {
+        this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteableCollectionAsList(Attribute.class), in.readBoolean());
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeNamedWriteableCollection(output());
+        out.writeBoolean(isIntermediateAgg());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     public List<Attribute> output() {
         return output;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/OrderExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/OrderExec.java
index 08c16ce8cebbc..63b838e068f1e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/OrderExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/OrderExec.java
@@ -7,9 +7,9 @@
 
 package org.elasticsearch.xpack.esql.plan.physical;
 
-import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.Order;
 
 import java.util.List;
 import java.util.Objects;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/PhysicalPlan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/PhysicalPlan.java
index 66f571e24b95a..6fcdf573564ff 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/PhysicalPlan.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/PhysicalPlan.java
@@ -7,9 +7,12 @@
 
 package org.elasticsearch.xpack.esql.plan.physical;
 
-import org.elasticsearch.xpack.esql.core.plan.QueryPlan;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.plan.QueryPlan;
 
+import java.io.IOException;
 import java.util.List;
 
 /**
@@ -19,11 +22,34 @@
  * PhysicalPlan = take Delta, DEN to SJC, then SJC to SFO
  */
 public abstract class PhysicalPlan extends QueryPlan<PhysicalPlan> {
+    public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        return List.of(
+            AggregateExec.ENTRY,
+            DissectExec.ENTRY,
+            EsQueryExec.ENTRY,
+            EsSourceExec.ENTRY,
+            EvalExec.ENTRY,
+            ExchangeExec.ENTRY,
+            ExchangeSinkExec.ENTRY,
+            ExchangeSourceExec.ENTRY
+        );
+    }
 
     public PhysicalPlan(Source source, List<PhysicalPlan> children) {
         super(source, children);
     }
 
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        // TODO remove when all PhysicalPlans are migrated to NamedWriteable
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public abstract int hashCode();
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java
index 22785e62cebcb..a671e85b3a754 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/TopNExec.java
@@ -8,9 +8,9 @@
 package org.elasticsearch.xpack.esql.plan.physical;
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.Order;
 
 import java.util.List;
 import java.util.Objects;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
index 41440384dd871..60bf4be1d2b03 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
@@ -160,7 +160,7 @@ private static Stream<Tuple<Class<?>, Tuple<String, String>>> typeAndNames(Class
         if (NumericAggregate.class.isAssignableFrom(clazz)) {
             types = NUMERIC;
         } else if (Max.class.isAssignableFrom(clazz) || Min.class.isAssignableFrom(clazz)) {
-            types = List.of("Boolean", "Int", "Long", "Double", "Ip");
+            types = List.of("Boolean", "Int", "Long", "Double", "Ip", "BytesRef");
         } else if (clazz == Count.class) {
             types = List.of(""); // no extra type distinction
         } else if (SpatialAggregateFunction.class.isAssignableFrom(clazz)) {
@@ -170,7 +170,7 @@ private static Stream<Tuple<Class<?>, Tuple<String, String>>> typeAndNames(Class
             // TODO can't we figure this out from the function itself?
             types = List.of("Int", "Long", "Double", "Boolean", "BytesRef");
         } else if (Top.class.isAssignableFrom(clazz)) {
-            types = List.of("Boolean", "Int", "Long", "Double");
+            types = List.of("Boolean", "Int", "Long", "Double", "Ip");
         } else if (Rate.class.isAssignableFrom(clazz)) {
             types = List.of("Int", "Long", "Double");
         } else if (FromPartial.class.isAssignableFrom(clazz) || ToPartial.class.isAssignableFrom(clazz)) {
@@ -294,6 +294,9 @@ private static String dataTypeToString(DataType type, Class<?> aggClass) {
         if ((aggClass == Max.class || aggClass == Min.class) && type.equals(DataType.IP)) {
             return "Ip";
         }
+        if (aggClass == Top.class && type.equals(DataType.IP)) {
+            return "Ip";
+        }
         if (type.equals(DataType.BOOLEAN)) {
             return "Boolean";
         } else if (type.equals(DataType.INTEGER) || type.equals(DataType.COUNTER_INTEGER)) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java
index 45989b4f563ce..04be731484267 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java
@@ -52,6 +52,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
@@ -60,7 +61,6 @@
 import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.DriverParallelism;
 import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.LocalExecutionPlannerContext;
 import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.PhysicalOperation;
-import org.elasticsearch.xpack.esql.type.MultiTypeEsField;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -138,7 +138,9 @@ private BlockLoader getBlockLoaderFor(
         if (unionTypes != null) {
             String indexName = shardContext.ctx.index().getName();
             Expression conversion = unionTypes.getConversionExpressionForIndex(indexName);
-            return new TypeConvertingBlockLoader(blockLoader, (AbstractConvertFunction) conversion);
+            return conversion == null
+                ? BlockLoader.CONSTANT_NULLS
+                : new TypeConvertingBlockLoader(blockLoader, (AbstractConvertFunction) conversion);
         }
         return blockLoader;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
index ef5eac444a2db..b508dc6556456 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
@@ -7,8 +7,8 @@
 
 package org.elasticsearch.xpack.esql.planner;
 
-import org.apache.lucene.document.ShapeField;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.geometry.Geometry;
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.TypedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator;
 import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslators;
@@ -55,6 +56,7 @@
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf;
 import static org.elasticsearch.xpack.esql.core.planner.ExpressionTranslators.or;
 import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
@@ -67,14 +69,13 @@
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString;
 
 public final class EsqlExpressionTranslators {
+
     public static final List<ExpressionTranslator<?>> QUERY_TRANSLATORS = List.of(
         new EqualsIgnoreCaseTranslator(),
         new BinaryComparisons(),
         new SpatialRelatesTranslator(),
         new InComparisons(),
-        // Ranges is redundant until we start combining binary comparisons (see CombineBinaryComparisons in ql's OptimizerRules)
-        // or introduce a BETWEEN keyword.
-        new ExpressionTranslators.Ranges(),
+        new Ranges(), // Create Range in PushFiltersToSource for qualified pushable filters on the same field.
         new ExpressionTranslators.BinaryLogic(),
         new ExpressionTranslators.IsNulls(),
         new ExpressionTranslators.IsNotNulls(),
@@ -124,7 +125,7 @@ public static void checkInsensitiveComparison(InsensitiveEquals bc) {
         static Query translate(InsensitiveEquals bc) {
             TypedAttribute attribute = checkIsPushableAttribute(bc.left());
             Source source = bc.source();
-            BytesRef value = BytesRefs.toBytesRef(ExpressionTranslators.valueOf(bc.right()));
+            BytesRef value = BytesRefs.toBytesRef(valueOf(bc.right()));
             String name = pushableAttributeName(attribute);
             return new TermQuery(source, name, value.utf8ToString(), true);
         }
@@ -249,7 +250,7 @@ private static Query translateOutOfRangeComparisons(BinaryComparison bc) {
                 return null;
             }
             Source source = bc.source();
-            Object value = ExpressionTranslators.valueOf(bc.right());
+            Object value = valueOf(bc.right());
 
             // Comparisons with multi-values always return null in ESQL.
             if (value instanceof List<?>) {
@@ -369,7 +370,7 @@ protected Query asQuery(SpatialRelatesFunction bc, TranslatorHandler handler) {
             return doTranslate(bc, handler);
         }
 
-        public static void checkSpatialRelatesFunction(Expression constantExpression, ShapeField.QueryRelation queryRelation) {
+        public static void checkSpatialRelatesFunction(Expression constantExpression, ShapeRelation queryRelation) {
             Check.isTrue(
                 constantExpression.foldable(),
                 "Line {}:{}: Comparisons against fields are not (currently) supported; offender [{}] in [ST_{}]",
@@ -453,12 +454,69 @@ private static Query translate(In in, TranslatorHandler handler) {
 
             return queries.stream().reduce((q1, q2) -> or(in.source(), q1, q2)).get();
         }
+    }
+
+    public static class Ranges extends ExpressionTranslator<Range> {
+
+        @Override
+        protected Query asQuery(Range r, TranslatorHandler handler) {
+            return doTranslate(r, handler);
+        }
+
+        public static Query doTranslate(Range r, TranslatorHandler handler) {
+            return handler.wrapFunctionQuery(r, r.value(), () -> translate(r, handler));
+        }
+
+        private static RangeQuery translate(Range r, TranslatorHandler handler) {
+            Object lower = valueOf(r.lower());
+            Object upper = valueOf(r.upper());
+            String format = null;
+
+            DataType dataType = r.value().dataType();
+            if (DataType.isDateTime(dataType) && DataType.isDateTime(r.lower().dataType()) && DataType.isDateTime(r.upper().dataType())) {
+                lower = dateTimeToString((Long) lower);
+                upper = dateTimeToString((Long) upper);
+                format = DEFAULT_DATE_TIME_FORMATTER.pattern();
+            }
 
-        public static Object valueOf(Expression e) {
-            if (e.foldable()) {
-                return e.fold();
+            if (dataType == IP) {
+                if (lower instanceof BytesRef bytesRef) {
+                    lower = ipToString(bytesRef);
+                }
+                if (upper instanceof BytesRef bytesRef) {
+                    upper = ipToString(bytesRef);
+                }
+            } else if (dataType == VERSION) {
+                // VersionStringFieldMapper#indexedValueForSearch() only accepts as input String or BytesRef with the String (i.e. not
+                // encoded) representation of the version as it'll do the encoding itself.
+                if (lower instanceof BytesRef bytesRef) {
+                    lower = versionToString(bytesRef);
+                } else if (lower instanceof Version version) {
+                    lower = versionToString(version);
+                }
+                if (upper instanceof BytesRef bytesRef) {
+                    upper = versionToString(bytesRef);
+                } else if (upper instanceof Version version) {
+                    upper = versionToString(version);
+                }
+            } else if (dataType == UNSIGNED_LONG) {
+                if (lower instanceof Long ul) {
+                    lower = unsignedLongAsNumber(ul);
+                }
+                if (upper instanceof Long ul) {
+                    upper = unsignedLongAsNumber(ul);
+                }
             }
-            throw new QlIllegalArgumentException("Cannot determine value for {}", e);
+            return new RangeQuery(
+                r.source(),
+                handler.nameOf(r.value()),
+                lower,
+                r.includeLower(),
+                upper,
+                r.includeUpper(),
+                format,
+                r.zoneId()
+            );
         }
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
index ddf5fa6eaf8a3..60d76f1a8593a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
@@ -57,12 +57,14 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.expression.Order;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupOperator;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupService;
 import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
 import org.elasticsearch.xpack.esql.evaluator.command.GrokEvaluatorExtracter;
+import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.esql.plan.physical.DissectExec;
 import org.elasticsearch.xpack.esql.plan.physical.EnrichExec;
@@ -86,7 +88,7 @@
 import org.elasticsearch.xpack.esql.plan.physical.ShowExec;
 import org.elasticsearch.xpack.esql.plan.physical.TopNExec;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -118,7 +120,7 @@ public class LocalExecutionPlanner {
     private final BigArrays bigArrays;
     private final BlockFactory blockFactory;
     private final Settings settings;
-    private final EsqlConfiguration configuration;
+    private final Configuration configuration;
     private final ExchangeSourceHandler exchangeSourceHandler;
     private final ExchangeSinkHandler exchangeSinkHandler;
     private final EnrichLookupService enrichLookupService;
@@ -131,7 +133,7 @@ public LocalExecutionPlanner(
         BigArrays bigArrays,
         BlockFactory blockFactory,
         Settings settings,
-        EsqlConfiguration configuration,
+        Configuration configuration,
         ExchangeSourceHandler exchangeSourceHandler,
         ExchangeSinkHandler exchangeSinkHandler,
         EnrichLookupService enrichLookupService,
@@ -348,8 +350,8 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte
                 case IP -> TopNEncoder.IP;
                 case TEXT, KEYWORD -> TopNEncoder.UTF8;
                 case VERSION -> TopNEncoder.VERSION;
-                case BOOLEAN, NULL, BYTE, SHORT, INTEGER, LONG, DOUBLE, FLOAT, HALF_FLOAT, DATETIME, DATE_PERIOD, TIME_DURATION, OBJECT,
-                    NESTED, SCALED_FLOAT, UNSIGNED_LONG, DOC_DATA_TYPE, TSID_DATA_TYPE -> TopNEncoder.DEFAULT_SORTABLE;
+                case BOOLEAN, NULL, BYTE, SHORT, INTEGER, LONG, DOUBLE, FLOAT, HALF_FLOAT, DATETIME, DATE_NANOS, DATE_PERIOD, TIME_DURATION,
+                    OBJECT, SCALED_FLOAT, UNSIGNED_LONG, DOC_DATA_TYPE, TSID_DATA_TYPE -> TopNEncoder.DEFAULT_SORTABLE;
                 case GEO_POINT, CARTESIAN_POINT, GEO_SHAPE, CARTESIAN_SHAPE, COUNTER_LONG, COUNTER_INTEGER, COUNTER_DOUBLE ->
                     TopNEncoder.DEFAULT_UNSORTABLE;
                 // unsupported fields are encoded as BytesRef, we'll use the same encoder; all values should be null at this point
@@ -417,12 +419,14 @@ private PhysicalOperation planDissect(DissectExec dissect, LocalExecutionPlanner
         Layout.Builder layoutBuilder = source.layout.builder();
         layoutBuilder.append(dissect.extractedFields());
         final Expression expr = dissect.inputExpression();
-        String[] attributeNames = Expressions.names(dissect.extractedFields()).toArray(new String[0]);
+        // Names in the pattern and layout can differ.
+        // Attributes need to be rename-able to avoid problems with shadowing - see GeneratingPlan resp. PushDownRegexExtract.
+        String[] patternNames = Expressions.names(dissect.parser().keyAttributes(Source.EMPTY)).toArray(new String[0]);
 
         Layout layout = layoutBuilder.build();
         source = source.with(
             new StringExtractOperator.StringExtractOperatorFactory(
-                attributeNames,
+                patternNames,
                 EvalMapper.toEvaluator(expr, layout),
                 () -> (input) -> dissect.parser().parser().parse(input)
             ),
@@ -439,11 +443,15 @@ private PhysicalOperation planGrok(GrokExec grok, LocalExecutionPlannerContext c
         Map<String, Integer> fieldToPos = new HashMap<>(extractedFields.size());
         Map<String, ElementType> fieldToType = new HashMap<>(extractedFields.size());
         ElementType[] types = new ElementType[extractedFields.size()];
+        List<Attribute> extractedFieldsFromPattern = grok.pattern().extractedFields();
         for (int i = 0; i < extractedFields.size(); i++) {
-            Attribute extractedField = extractedFields.get(i);
-            ElementType type = PlannerUtils.toElementType(extractedField.dataType());
-            fieldToPos.put(extractedField.name(), i);
-            fieldToType.put(extractedField.name(), type);
+            DataType extractedFieldType = extractedFields.get(i).dataType();
+            // Names in pattern and layout can differ.
+            // Attributes need to be rename-able to avoid problems with shadowing - see GeneratingPlan resp. PushDownRegexExtract.
+            String patternName = extractedFieldsFromPattern.get(i).name();
+            ElementType type = PlannerUtils.toElementType(extractedFieldType);
+            fieldToPos.put(patternName, i);
+            fieldToType.put(patternName, type);
             types[i] = type;
         }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
index d9f073d952a37..29be49e60ad37 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
@@ -48,9 +48,8 @@
 import org.elasticsearch.xpack.esql.plan.physical.OrderExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.plan.physical.TopNExec;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.LinkedHashSet;
@@ -67,16 +66,16 @@
 
 public class PlannerUtils {
 
-    public static Tuple<PhysicalPlan, PhysicalPlan> breakPlanBetweenCoordinatorAndDataNode(PhysicalPlan plan, EsqlConfiguration config) {
+    public static Tuple<PhysicalPlan, PhysicalPlan> breakPlanBetweenCoordinatorAndDataNode(PhysicalPlan plan, Configuration config) {
         var dataNodePlan = new Holder<PhysicalPlan>();
 
         // split the given plan when encountering the exchange
         PhysicalPlan coordinatorPlan = plan.transformUp(ExchangeExec.class, e -> {
             // remember the datanode subplan and wire it to a sink
             var subplan = e.child();
-            dataNodePlan.set(new ExchangeSinkExec(e.source(), e.output(), e.isInBetweenAggs(), subplan));
+            dataNodePlan.set(new ExchangeSinkExec(e.source(), e.output(), e.inBetweenAggs(), subplan));
 
-            return new ExchangeSourceExec(e.source(), e.output(), e.isInBetweenAggs());
+            return new ExchangeSourceExec(e.source(), e.output(), e.inBetweenAggs());
         });
         return new Tuple<>(coordinatorPlan, dataNodePlan.get());
     }
@@ -134,11 +133,11 @@ public static String[] planOriginalIndices(PhysicalPlan plan) {
         return indices.toArray(String[]::new);
     }
 
-    public static PhysicalPlan localPlan(List<SearchExecutionContext> searchContexts, EsqlConfiguration configuration, PhysicalPlan plan) {
+    public static PhysicalPlan localPlan(List<SearchExecutionContext> searchContexts, Configuration configuration, PhysicalPlan plan) {
         return localPlan(configuration, plan, new SearchStats(searchContexts));
     }
 
-    public static PhysicalPlan localPlan(EsqlConfiguration configuration, PhysicalPlan plan, SearchStats searchStats) {
+    public static PhysicalPlan localPlan(Configuration configuration, PhysicalPlan plan, SearchStats searchStats) {
         final var logicalOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, searchStats));
         var physicalOptimizer = new LocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, searchStats));
 
@@ -221,7 +220,7 @@ static QueryBuilder detectFilter(PhysicalPlan plan, String fieldName, Predicate<
      * This specifically excludes spatial data types, which are not themselves sortable.
      */
     public static ElementType toSortableElementType(DataType dataType) {
-        if (EsqlDataTypes.isSpatial(dataType)) {
+        if (DataType.isSpatial(dataType)) {
             return ElementType.UNKNOWN;
         }
         return toElementType(dataType);
@@ -242,7 +241,7 @@ public static ElementType toElementType(DataType dataType) {
     public static ElementType toElementType(DataType dataType, MappedFieldType.FieldExtractPreference fieldExtractPreference) {
 
         return switch (dataType) {
-            case LONG, DATETIME, UNSIGNED_LONG, COUNTER_LONG -> ElementType.LONG;
+            case LONG, DATETIME, DATE_NANOS, UNSIGNED_LONG, COUNTER_LONG -> ElementType.LONG;
             case INTEGER, COUNTER_INTEGER -> ElementType.INT;
             case DOUBLE, COUNTER_DOUBLE -> ElementType.DOUBLE;
             // unsupported fields are passed through as a BytesRef
@@ -254,8 +253,8 @@ public static ElementType toElementType(DataType dataType, MappedFieldType.Field
             case GEO_POINT, CARTESIAN_POINT -> fieldExtractPreference == DOC_VALUES ? ElementType.LONG : ElementType.BYTES_REF;
             case GEO_SHAPE, CARTESIAN_SHAPE -> ElementType.BYTES_REF;
             case PARTIAL_AGG -> ElementType.COMPOSITE;
-            case SHORT, BYTE, DATE_PERIOD, TIME_DURATION, OBJECT, NESTED, FLOAT, HALF_FLOAT, SCALED_FLOAT ->
-                throw EsqlIllegalArgumentException.illegalDataType(dataType);
+            case SHORT, BYTE, DATE_PERIOD, TIME_DURATION, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT -> throw EsqlIllegalArgumentException
+                .illegalDataType(dataType);
         };
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeRequest.java
index bedbd517f1184..f82060419f73a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeRequest.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeRequest.java
@@ -24,11 +24,9 @@
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
-import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Map;
 import java.util.Objects;
 
@@ -38,53 +36,41 @@
  * via {@link ExchangeService#openExchange} before sending this request to the remote cluster. The coordinator on the main cluster
  * will poll pages from this sink. Internally, this compute will trigger sub-computes on data nodes via {@link DataNodeRequest}.
  */
-final class ClusterComputeRequest extends TransportRequest implements IndicesRequest {
+final class ClusterComputeRequest extends TransportRequest implements IndicesRequest.Replaceable {
     private static final PlanNameRegistry planNameRegistry = new PlanNameRegistry();
     private final String clusterAlias;
     private final String sessionId;
-    private final EsqlConfiguration configuration;
-    private final PhysicalPlan plan;
+    private final Configuration configuration;
+    private final RemoteClusterPlan plan;
 
-    private final String[] originalIndices;
-    private final String[] indices;
+    private transient String[] indices;
 
     /**
      * A request to start a compute on a remote cluster.
      *
-     * @param clusterAlias the cluster alias of this remote cluster
-     * @param sessionId the sessionId in which the output pages will be placed in the exchange sink specified by this id
-     * @param configuration the configuration for this compute
+     * @param clusterAlias      the cluster alias of this remote cluster
+     * @param sessionId         the sessionId in which the output pages will be placed in the exchange sink specified by this id
+     * @param configuration     the configuration for this compute
      * @param plan the physical plan to be executed
-     * @param indices the target indices
-     * @param originalIndices the original indices - needed to resolve alias filters
      */
-    ClusterComputeRequest(
-        String clusterAlias,
-        String sessionId,
-        EsqlConfiguration configuration,
-        PhysicalPlan plan,
-        String[] indices,
-        String[] originalIndices
-    ) {
+    ClusterComputeRequest(String clusterAlias, String sessionId, Configuration configuration, RemoteClusterPlan plan) {
         this.clusterAlias = clusterAlias;
         this.sessionId = sessionId;
         this.configuration = configuration;
         this.plan = plan;
-        this.indices = indices;
-        this.originalIndices = originalIndices;
+        this.indices = plan.originalIndices().indices();
     }
 
     ClusterComputeRequest(StreamInput in) throws IOException {
         super(in);
         this.clusterAlias = in.readString();
         this.sessionId = in.readString();
-        this.configuration = new EsqlConfiguration(
+        this.configuration = new Configuration(
             // TODO make EsqlConfiguration Releasable
             new BlockStreamInput(in, new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE))
         );
-        this.plan = new PlanStreamInput(in, planNameRegistry, in.namedWriteableRegistry(), configuration).readPhysicalPlanNode();
-        this.indices = in.readStringArray();
-        this.originalIndices = in.readStringArray();
+        this.plan = RemoteClusterPlan.from(new PlanStreamInput(in, planNameRegistry, in.namedWriteableRegistry(), configuration));
+        this.indices = plan.originalIndices().indices();
     }
 
     @Override
@@ -93,9 +79,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(clusterAlias);
         out.writeString(sessionId);
         configuration.writeTo(out);
-        new PlanStreamOutput(out, planNameRegistry, configuration).writePhysicalPlanNode(plan);
-        out.writeStringArray(indices);
-        out.writeStringArray(originalIndices);
+        plan.writeTo(new PlanStreamOutput(out, planNameRegistry, configuration));
     }
 
     @Override
@@ -103,9 +87,15 @@ public String[] indices() {
         return indices;
     }
 
+    @Override
+    public IndicesRequest indices(String... indices) {
+        this.indices = indices;
+        return this;
+    }
+
     @Override
     public IndicesOptions indicesOptions() {
-        return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
+        return plan.originalIndices().indicesOptions();
     }
 
     @Override
@@ -130,21 +120,17 @@ String sessionId() {
         return sessionId;
     }
 
-    EsqlConfiguration configuration() {
+    Configuration configuration() {
         return configuration;
     }
 
-    String[] originalIndices() {
-        return originalIndices;
-    }
-
-    PhysicalPlan plan() {
+    RemoteClusterPlan remoteClusterPlan() {
         return plan;
     }
 
     @Override
     public String getDescription() {
-        return "indices=" + Arrays.toString(indices) + " plan=" + plan;
+        return "plan=" + plan;
     }
 
     @Override
@@ -160,14 +146,12 @@ public boolean equals(Object o) {
         return clusterAlias.equals(request.clusterAlias)
             && sessionId.equals(request.sessionId)
             && configuration.equals(request.configuration)
-            && Arrays.equals(indices, request.indices)
-            && Arrays.equals(originalIndices, request.originalIndices)
             && plan.equals(request.plan)
             && getParentTask().equals(request.getParentTask());
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(sessionId, configuration, Arrays.hashCode(indices), Arrays.hashCode(originalIndices), plan);
+        return Objects.hash(sessionId, configuration, plan);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
index 673e320e5106b..fa8a5693c59bb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
@@ -15,15 +15,12 @@
 import org.elasticsearch.action.search.SearchShardsGroup;
 import org.elasticsearch.action.search.SearchShardsRequest;
 import org.elasticsearch.action.search.SearchShardsResponse;
-import org.elasticsearch.action.search.TransportSearchShardsAction;
 import org.elasticsearch.action.support.ChannelActionListener;
-import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.action.support.RefCountingListener;
 import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.Driver;
@@ -60,6 +57,7 @@
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.esql.action.EsqlQueryAction;
+import org.elasticsearch.xpack.esql.action.EsqlSearchShardsAction;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupService;
 import org.elasticsearch.xpack.esql.plan.physical.ExchangeSinkExec;
 import org.elasticsearch.xpack.esql.plan.physical.ExchangeSourceExec;
@@ -69,7 +67,7 @@
 import org.elasticsearch.xpack.esql.planner.EsPhysicalOperationProviders;
 import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.session.Result;
 
 import java.util.ArrayList;
@@ -131,7 +129,7 @@ public void execute(
         String sessionId,
         CancellableTask rootTask,
         PhysicalPlan physicalPlan,
-        EsqlConfiguration configuration,
+        Configuration configuration,
         ActionListener<Result> listener
     ) {
         Tuple<PhysicalPlan, PhysicalPlan> coordinatorAndDataNodePlan = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode(
@@ -219,7 +217,7 @@ public void execute(
                     configuration,
                     dataNodePlan,
                     Set.of(localConcreteIndices.indices()),
-                    localOriginalIndices.indices(),
+                    localOriginalIndices,
                     exchangeSource,
                     computeListener
                 );
@@ -253,7 +251,7 @@ private List<RemoteCluster> getRemoteClusters(
             }
             if (concreteIndices.indices().length > 0) {
                 Transport.Connection connection = remoteClusterService.getConnection(clusterAlias);
-                remoteClusters.add(new RemoteCluster(clusterAlias, connection, concreteIndices.indices(), originalIndices.indices()));
+                remoteClusters.add(new RemoteCluster(clusterAlias, connection, concreteIndices.indices(), originalIndices));
             }
         }
         return remoteClusters;
@@ -263,10 +261,10 @@ private void startComputeOnDataNodes(
         String sessionId,
         String clusterAlias,
         CancellableTask parentTask,
-        EsqlConfiguration configuration,
+        Configuration configuration,
         PhysicalPlan dataNodePlan,
         Set<String> concreteIndices,
-        String[] originalIndices,
+        OriginalIndices originalIndices,
         ExchangeSourceHandler exchangeSource,
         ComputeListener computeListener
     ) {
@@ -308,7 +306,9 @@ private void startComputeOnDataNodes(
                                     clusterAlias,
                                     node.shardIds,
                                     node.aliasFilters,
-                                    planWithReducer
+                                    planWithReducer,
+                                    originalIndices.indices(),
+                                    originalIndices.indicesOptions()
                                 ),
                                 parentTask,
                                 TransportRequestOptions.EMPTY,
@@ -324,7 +324,7 @@ private void startComputeOnDataNodes(
     private void startComputeOnRemoteClusters(
         String sessionId,
         CancellableTask rootTask,
-        EsqlConfiguration configuration,
+        Configuration configuration,
         PhysicalPlan plan,
         ExchangeSourceHandler exchangeSource,
         List<RemoteCluster> clusters,
@@ -343,14 +343,8 @@ private void startComputeOnRemoteClusters(
                     refs.acquire().delegateFailureAndWrap((l, unused) -> {
                         var remoteSink = exchangeService.newRemoteSink(rootTask, sessionId, transportService, cluster.connection);
                         exchangeSource.addRemoteSink(remoteSink, queryPragmas.concurrentExchangeClients());
-                        var clusterRequest = new ClusterComputeRequest(
-                            cluster.clusterAlias,
-                            sessionId,
-                            configuration,
-                            plan,
-                            cluster.concreteIndices,
-                            cluster.originalIndices
-                        );
+                        var remotePlan = new RemoteClusterPlan(plan, cluster.concreteIndices, cluster.originalIndices);
+                        var clusterRequest = new ClusterComputeRequest(cluster.clusterAlias, sessionId, configuration, remotePlan);
                         var clusterListener = ActionListener.runBefore(computeListener.acquireCompute(), () -> l.onResponse(null));
                         transportService.sendChildRequest(
                             cluster.connection,
@@ -433,7 +427,7 @@ void runCompute(CancellableTask task, ComputeContext context, PhysicalPlan plan,
     private void acquireSearchContexts(
         String clusterAlias,
         List<ShardId> shardIds,
-        EsqlConfiguration configuration,
+        Configuration configuration,
         Map<Index, AliasFilter> aliasFilters,
         ActionListener<List<SearchContext>> listener
     ) {
@@ -500,7 +494,7 @@ record DataNode(Transport.Connection connection, List<ShardId> shardIds, Map<Ind
 
     }
 
-    record RemoteCluster(String clusterAlias, Transport.Connection connection, String[] concreteIndices, String[] originalIndices) {
+    record RemoteCluster(String clusterAlias, Transport.Connection connection, String[] concreteIndices, OriginalIndices originalIndices) {
 
     }
 
@@ -515,66 +509,59 @@ private void lookupDataNodes(
         String clusterAlias,
         QueryBuilder filter,
         Set<String> concreteIndices,
-        String[] originalIndices,
+        OriginalIndices originalIndices,
         ActionListener<List<DataNode>> listener
     ) {
-        ThreadContext threadContext = transportService.getThreadPool().getThreadContext();
-        ActionListener<SearchShardsResponse> preservingContextListener = ContextPreservingActionListener.wrapPreservingContext(
-            listener.map(resp -> {
-                Map<String, DiscoveryNode> nodes = new HashMap<>();
-                for (DiscoveryNode node : resp.getNodes()) {
-                    nodes.put(node.getId(), node);
+        ActionListener<SearchShardsResponse> searchShardsListener = listener.map(resp -> {
+            Map<String, DiscoveryNode> nodes = new HashMap<>();
+            for (DiscoveryNode node : resp.getNodes()) {
+                nodes.put(node.getId(), node);
+            }
+            Map<String, List<ShardId>> nodeToShards = new HashMap<>();
+            Map<String, Map<Index, AliasFilter>> nodeToAliasFilters = new HashMap<>();
+            for (SearchShardsGroup group : resp.getGroups()) {
+                var shardId = group.shardId();
+                if (group.skipped()) {
+                    continue;
                 }
-                Map<String, List<ShardId>> nodeToShards = new HashMap<>();
-                Map<String, Map<Index, AliasFilter>> nodeToAliasFilters = new HashMap<>();
-                for (SearchShardsGroup group : resp.getGroups()) {
-                    var shardId = group.shardId();
-                    if (group.skipped()) {
-                        continue;
-                    }
-                    if (group.allocatedNodes().isEmpty()) {
-                        throw new ShardNotFoundException(group.shardId(), "no shard copies found {}", group.shardId());
-                    }
-                    if (concreteIndices.contains(shardId.getIndexName()) == false) {
-                        continue;
-                    }
-                    String targetNode = group.allocatedNodes().get(0);
-                    nodeToShards.computeIfAbsent(targetNode, k -> new ArrayList<>()).add(shardId);
-                    AliasFilter aliasFilter = resp.getAliasFilters().get(shardId.getIndex().getUUID());
-                    if (aliasFilter != null) {
-                        nodeToAliasFilters.computeIfAbsent(targetNode, k -> new HashMap<>()).put(shardId.getIndex(), aliasFilter);
-                    }
+                if (group.allocatedNodes().isEmpty()) {
+                    throw new ShardNotFoundException(group.shardId(), "no shard copies found {}", group.shardId());
+                }
+                if (concreteIndices.contains(shardId.getIndexName()) == false) {
+                    continue;
                 }
-                List<DataNode> dataNodes = new ArrayList<>(nodeToShards.size());
-                for (Map.Entry<String, List<ShardId>> e : nodeToShards.entrySet()) {
-                    DiscoveryNode node = nodes.get(e.getKey());
-                    Map<Index, AliasFilter> aliasFilters = nodeToAliasFilters.getOrDefault(e.getKey(), Map.of());
-                    dataNodes.add(new DataNode(transportService.getConnection(node), e.getValue(), aliasFilters));
+                String targetNode = group.allocatedNodes().get(0);
+                nodeToShards.computeIfAbsent(targetNode, k -> new ArrayList<>()).add(shardId);
+                AliasFilter aliasFilter = resp.getAliasFilters().get(shardId.getIndex().getUUID());
+                if (aliasFilter != null) {
+                    nodeToAliasFilters.computeIfAbsent(targetNode, k -> new HashMap<>()).put(shardId.getIndex(), aliasFilter);
                 }
-                return dataNodes;
-            }),
-            threadContext
+            }
+            List<DataNode> dataNodes = new ArrayList<>(nodeToShards.size());
+            for (Map.Entry<String, List<ShardId>> e : nodeToShards.entrySet()) {
+                DiscoveryNode node = nodes.get(e.getKey());
+                Map<Index, AliasFilter> aliasFilters = nodeToAliasFilters.getOrDefault(e.getKey(), Map.of());
+                dataNodes.add(new DataNode(transportService.getConnection(node), e.getValue(), aliasFilters));
+            }
+            return dataNodes;
+        });
+        SearchShardsRequest searchShardsRequest = new SearchShardsRequest(
+            originalIndices.indices(),
+            originalIndices.indicesOptions(),
+            filter,
+            null,
+            null,
+            false,
+            clusterAlias
+        );
+        transportService.sendChildRequest(
+            transportService.getLocalNode(),
+            EsqlSearchShardsAction.TYPE.name(),
+            searchShardsRequest,
+            parentTask,
+            TransportRequestOptions.EMPTY,
+            new ActionListenerResponseHandler<>(searchShardsListener, SearchShardsResponse::new, esqlExecutor)
         );
-        try (ThreadContext.StoredContext ignored = threadContext.newStoredContextPreservingResponseHeaders()) {
-            threadContext.markAsSystemContext();
-            SearchShardsRequest searchShardsRequest = new SearchShardsRequest(
-                originalIndices,
-                SearchRequest.DEFAULT_INDICES_OPTIONS,
-                filter,
-                null,
-                null,
-                false,
-                clusterAlias
-            );
-            transportService.sendChildRequest(
-                transportService.getLocalNode(),
-                TransportSearchShardsAction.TYPE.name(),
-                searchShardsRequest,
-                parentTask,
-                TransportRequestOptions.EMPTY,
-                new ActionListenerResponseHandler<>(preservingContextListener, SearchShardsResponse::new, esqlExecutor)
-            );
-        }
     }
 
     // TODO: Use an internal action here
@@ -611,7 +598,7 @@ void start() {
         }
 
         private void runBatch(int startBatchIndex) {
-            final EsqlConfiguration configuration = request.configuration();
+            final Configuration configuration = request.configuration();
             final String clusterAlias = request.clusterAlias();
             final var sessionId = request.sessionId();
             final int endBatchIndex = Math.min(startBatchIndex + maxConcurrentShards, request.shardIds().size());
@@ -745,7 +732,9 @@ public void messageReceived(DataNodeRequest request, TransportChannel channel, T
                 request.clusterAlias(),
                 request.shardIds(),
                 request.aliasFilters(),
-                request.plan()
+                request.plan(),
+                request.indices(),
+                request.indicesOptions()
             );
             try (var computeListener = new ComputeListener(transportService, (CancellableTask) task, listener)) {
                 runComputeOnDataNode((CancellableTask) task, sessionId, reducePlan, request, computeListener);
@@ -759,8 +748,10 @@ private class ClusterRequestHandler implements TransportRequestHandler<ClusterCo
         @Override
         public void messageReceived(ClusterComputeRequest request, TransportChannel channel, Task task) {
             ChannelActionListener<ComputeResponse> listener = new ChannelActionListener<>(channel);
-            if (request.plan() instanceof ExchangeSinkExec == false) {
-                listener.onFailure(new IllegalStateException("expected exchange sink for a remote compute; got " + request.plan()));
+            RemoteClusterPlan remoteClusterPlan = request.remoteClusterPlan();
+            var plan = remoteClusterPlan.plan();
+            if (plan instanceof ExchangeSinkExec == false) {
+                listener.onFailure(new IllegalStateException("expected exchange sink for a remote compute; got " + plan));
                 return;
             }
             try (var computeListener = new ComputeListener(transportService, (CancellableTask) task, listener)) {
@@ -769,9 +760,9 @@ public void messageReceived(ClusterComputeRequest request, TransportChannel chan
                     request.sessionId(),
                     (CancellableTask) task,
                     request.configuration(),
-                    (ExchangeSinkExec) request.plan(),
-                    Set.of(request.indices()),
-                    request.originalIndices(),
+                    (ExchangeSinkExec) plan,
+                    Set.of(remoteClusterPlan.targetIndices()),
+                    remoteClusterPlan.originalIndices(),
                     computeListener
                 );
             }
@@ -791,10 +782,10 @@ void runComputeOnRemoteCluster(
         String clusterAlias,
         String globalSessionId,
         CancellableTask parentTask,
-        EsqlConfiguration configuration,
+        Configuration configuration,
         ExchangeSinkExec plan,
         Set<String> concreteIndices,
-        String[] originalIndices,
+        OriginalIndices originalIndices,
         ComputeListener computeListener
     ) {
         final var exchangeSink = exchangeService.getSinkHandler(globalSessionId);
@@ -839,7 +830,7 @@ record ComputeContext(
         String sessionId,
         String clusterAlias,
         List<SearchContext> searchContexts,
-        EsqlConfiguration configuration,
+        Configuration configuration,
         ExchangeSourceHandler exchangeSource,
         ExchangeSinkHandler exchangeSink
     ) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java
index ab2df4a2ba6a9..4174cc25552f5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequest.java
@@ -29,31 +29,34 @@
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-final class DataNodeRequest extends TransportRequest implements IndicesRequest {
+final class DataNodeRequest extends TransportRequest implements IndicesRequest.Replaceable {
     private static final PlanNameRegistry planNameRegistry = new PlanNameRegistry();
     private final String sessionId;
-    private final EsqlConfiguration configuration;
+    private final Configuration configuration;
     private final String clusterAlias;
     private final List<ShardId> shardIds;
     private final Map<Index, AliasFilter> aliasFilters;
     private final PhysicalPlan plan;
-
-    private String[] indices; // lazily computed
+    private String[] indices;
+    private final IndicesOptions indicesOptions;
 
     DataNodeRequest(
         String sessionId,
-        EsqlConfiguration configuration,
+        Configuration configuration,
         String clusterAlias,
         List<ShardId> shardIds,
         Map<Index, AliasFilter> aliasFilters,
-        PhysicalPlan plan
+        PhysicalPlan plan,
+        String[] indices,
+        IndicesOptions indicesOptions
     ) {
         this.sessionId = sessionId;
         this.configuration = configuration;
@@ -61,12 +64,14 @@ final class DataNodeRequest extends TransportRequest implements IndicesRequest {
         this.shardIds = shardIds;
         this.aliasFilters = aliasFilters;
         this.plan = plan;
+        this.indices = indices;
+        this.indicesOptions = indicesOptions;
     }
 
     DataNodeRequest(StreamInput in) throws IOException {
         super(in);
         this.sessionId = in.readString();
-        this.configuration = new EsqlConfiguration(
+        this.configuration = new Configuration(
             // TODO make EsqlConfiguration Releasable
             new BlockStreamInput(in, new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE))
         );
@@ -78,6 +83,13 @@ final class DataNodeRequest extends TransportRequest implements IndicesRequest {
         this.shardIds = in.readCollectionAsList(ShardId::new);
         this.aliasFilters = in.readMap(Index::new, AliasFilter::readFrom);
         this.plan = new PlanStreamInput(in, planNameRegistry, in.namedWriteableRegistry(), configuration).readPhysicalPlanNode();
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_ORIGINAL_INDICES)) {
+            this.indices = in.readStringArray();
+            this.indicesOptions = IndicesOptions.readIndicesOptions(in);
+        } else {
+            this.indices = shardIds.stream().map(ShardId::getIndexName).distinct().toArray(String[]::new);
+            this.indicesOptions = IndicesOptions.strictSingleIndexNoExpandForbidClosed();
+        }
     }
 
     @Override
@@ -91,19 +103,26 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(shardIds);
         out.writeMap(aliasFilters);
         new PlanStreamOutput(out, planNameRegistry, configuration).writePhysicalPlanNode(plan);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ORIGINAL_INDICES)) {
+            out.writeStringArray(indices);
+            indicesOptions.writeIndicesOptions(out);
+        }
     }
 
     @Override
     public String[] indices() {
-        if (indices == null) {
-            indices = shardIds.stream().map(ShardId::getIndexName).distinct().toArray(String[]::new);
-        }
         return indices;
     }
 
+    @Override
+    public IndicesRequest indices(String... indices) {
+        this.indices = indices;
+        return this;
+    }
+
     @Override
     public IndicesOptions indicesOptions() {
-        return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
+        return indicesOptions;
     }
 
     @Override
@@ -124,7 +143,7 @@ String sessionId() {
         return sessionId;
     }
 
-    EsqlConfiguration configuration() {
+    Configuration configuration() {
         return configuration;
     }
 
@@ -172,11 +191,13 @@ public boolean equals(Object o) {
             && shardIds.equals(request.shardIds)
             && aliasFilters.equals(request.aliasFilters)
             && plan.equals(request.plan)
-            && getParentTask().equals(request.getParentTask());
+            && getParentTask().equals(request.getParentTask())
+            && Arrays.equals(indices, request.indices)
+            && indicesOptions.equals(request.indicesOptions);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(sessionId, configuration, clusterAlias, shardIds, aliasFilters, plan);
+        return Objects.hash(sessionId, configuration, clusterAlias, shardIds, aliasFilters, plan, Arrays.hashCode(indices), indicesOptions);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
index 0d9f3cd460af5..f0686baf68f6f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
@@ -52,6 +53,7 @@
 import org.elasticsearch.xpack.esql.action.EsqlQueryAction;
 import org.elasticsearch.xpack.esql.action.EsqlQueryRequestBuilder;
 import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction;
+import org.elasticsearch.xpack.esql.action.EsqlSearchShardsAction;
 import org.elasticsearch.xpack.esql.action.RestEsqlAsyncQueryAction;
 import org.elasticsearch.xpack.esql.action.RestEsqlDeleteAsyncResultAction;
 import org.elasticsearch.xpack.esql.action.RestEsqlGetAsyncResultAction;
@@ -59,16 +61,14 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupOperator;
 import org.elasticsearch.xpack.esql.execution.PlanExecutor;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery;
 import org.elasticsearch.xpack.esql.session.IndexResolver;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
-import org.elasticsearch.xpack.esql.type.MultiTypeEsField;
 
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
@@ -79,6 +79,7 @@
 import java.util.function.Supplier;
 
 public class EsqlPlugin extends Plugin implements ActionPlugin {
+    public static final FeatureFlag INLINESTATS_FEATURE_FLAG = new FeatureFlag("esql_inlinestats");
 
     public static final String ESQL_WORKER_THREAD_POOL_NAME = "esql_worker";
 
@@ -113,7 +114,7 @@ public Collection<?> createComponents(PluginServices services) {
         BlockFactory blockFactory = new BlockFactory(circuitBreaker, bigArrays, maxPrimitiveArrayBlockSize);
         setupSharedSecrets();
         return List.of(
-            new PlanExecutor(new IndexResolver(services.client(), EsqlDataTypeRegistry.INSTANCE)),
+            new PlanExecutor(new IndexResolver(services.client())),
             new ExchangeService(services.clusterService().getSettings(), services.threadPool(), ThreadPool.Names.SEARCH, blockFactory),
             blockFactory
         );
@@ -146,7 +147,8 @@ public List<Setting<?>> getSettings() {
             new ActionHandler<>(EsqlStatsAction.INSTANCE, TransportEsqlStatsAction.class),
             new ActionHandler<>(XPackUsageFeatureAction.ESQL, EsqlUsageTransportAction.class),
             new ActionHandler<>(XPackInfoFeatureAction.ESQL, EsqlInfoTransportAction.class),
-            new ActionHandler<>(EsqlResolveFieldsAction.TYPE, EsqlResolveFieldsAction.class)
+            new ActionHandler<>(EsqlResolveFieldsAction.TYPE, EsqlResolveFieldsAction.class),
+            new ActionHandler<>(EsqlSearchShardsAction.TYPE, EsqlSearchShardsAction.class)
         );
     }
 
@@ -189,16 +191,15 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(AsyncOperator.Status.ENTRY);
         entries.add(EnrichLookupOperator.Status.ENTRY);
         entries.addAll(Block.getNamedWriteables());
-        entries.addAll(EsField.getNamedWriteables());
         entries.addAll(Attribute.getNamedWriteables());
         entries.add(UnsupportedAttribute.ENTRY);  // TODO combine with above once these are in the same project
         entries.addAll(NamedExpression.getNamedWriteables());
         entries.add(UnsupportedAttribute.NAMED_EXPRESSION_ENTRY); // TODO combine with above once these are in the same project
         entries.addAll(Expression.getNamedWriteables());
         entries.add(UnsupportedAttribute.EXPRESSION_ENTRY); // TODO combine with above once these are in the same project
-        entries.add(MultiTypeEsField.ENTRY); // TODO combine with EsField.getNamedWriteables() once these are in the same module
         entries.addAll(EsqlScalarFunction.getNamedWriteables());
         entries.addAll(AggregateFunction.getNamedWriteables());
+        entries.addAll(LogicalPlan.getNamedWriteables());
         return entries;
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/RemoteClusterPlan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/RemoteClusterPlan.java
new file mode 100644
index 0000000000000..3fd8b17f01778
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/RemoteClusterPlan.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plugin;
+
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
+
+import java.io.IOException;
+
+record RemoteClusterPlan(PhysicalPlan plan, String[] targetIndices, OriginalIndices originalIndices) {
+    static RemoteClusterPlan from(PlanStreamInput planIn) throws IOException {
+        var plan = planIn.readPhysicalPlanNode();
+        var targetIndices = planIn.readStringArray();
+        final OriginalIndices originalIndices;
+        if (planIn.getTransportVersion().onOrAfter(TransportVersions.ESQL_ORIGINAL_INDICES)) {
+            originalIndices = OriginalIndices.readOriginalIndices(planIn);
+        } else {
+            originalIndices = new OriginalIndices(planIn.readStringArray(), IndicesOptions.strictSingleIndexNoExpandForbidClosed());
+        }
+        return new RemoteClusterPlan(plan, targetIndices, originalIndices);
+    }
+
+    public void writeTo(PlanStreamOutput out) throws IOException {
+        out.writePhysicalPlanNode(plan);
+        out.writeStringArray(targetIndices);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_ORIGINAL_INDICES)) {
+            OriginalIndices.writeOriginalIndices(originalIndices, out);
+        } else {
+            out.writeStringArray(originalIndices.indices());
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncGetResultsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncGetResultsAction.java
index 1a4a5433a1571..4bcebcfe64cb9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncGetResultsAction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlAsyncGetResultsAction.java
@@ -13,12 +13,12 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
index 5a6812c969757..561baa76a01a9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java
@@ -13,13 +13,13 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.operator.exchange.ExchangeService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
@@ -38,7 +38,7 @@
 import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver;
 import org.elasticsearch.xpack.esql.execution.PlanExecutor;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.session.Result;
 
 import java.io.IOException;
@@ -55,6 +55,7 @@ public class TransportEsqlQueryAction extends HandledTransportAction<EsqlQueryRe
     implements
         AsyncTaskManagementService.AsyncOperation<EsqlQueryRequest, EsqlQueryResponse, EsqlQueryTask> {
 
+    private final ThreadPool threadPool;
     private final PlanExecutor planExecutor;
     private final ComputeService computeService;
     private final ExchangeService exchangeService;
@@ -82,6 +83,7 @@ public TransportEsqlQueryAction(
     ) {
         // TODO replace SAME when removing workaround for https://github.com/elastic/elasticsearch/issues/97916
         super(EsqlQueryAction.NAME, transportService, actionFilters, EsqlQueryRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
+        this.threadPool = threadPool;
         this.planExecutor = planExecutor;
         this.clusterService = clusterService;
         this.requestExecutor = threadPool.executor(ThreadPool.Names.SEARCH);
@@ -146,7 +148,7 @@ public void execute(EsqlQueryRequest request, EsqlQueryTask task, ActionListener
     }
 
     private void innerExecute(Task task, EsqlQueryRequest request, ActionListener<EsqlQueryResponse> listener) {
-        EsqlConfiguration configuration = new EsqlConfiguration(
+        Configuration configuration = new Configuration(
             ZoneOffset.UTC,
             request.locale() != null ? request.locale() : Locale.US,
             // TODO: plug-in security
@@ -178,15 +180,14 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener<Es
         );
     }
 
-    private EsqlQueryResponse toResponse(Task task, EsqlQueryRequest request, EsqlConfiguration configuration, Result result) {
-        List<ColumnInfoImpl> columns = result.schema()
-            .stream()
-            .map(c -> new ColumnInfoImpl(c.qualifiedName(), c.dataType().outputType()))
-            .toList();
+    private EsqlQueryResponse toResponse(Task task, EsqlQueryRequest request, Configuration configuration, Result result) {
+        List<ColumnInfoImpl> columns = result.schema().stream().map(c -> new ColumnInfoImpl(c.name(), c.dataType().outputType())).toList();
         EsqlQueryResponse.Profile profile = configuration.profile() ? new EsqlQueryResponse.Profile(result.profiles()) : null;
+        threadPool.getThreadContext().addResponseHeader(AsyncExecutionId.ASYNC_EXECUTION_IS_RUNNING_HEADER, "?0");
         if (task instanceof EsqlQueryTask asyncTask && request.keepOnCompletion()) {
-            String id = asyncTask.getExecutionId().getEncoded();
-            return new EsqlQueryResponse(columns, result.pages(), profile, request.columnar(), id, false, request.async());
+            String asyncExecutionId = asyncTask.getExecutionId().getEncoded();
+            threadPool.getThreadContext().addResponseHeader(AsyncExecutionId.ASYNC_EXECUTION_ID_HEADER, asyncExecutionId);
+            return new EsqlQueryResponse(columns, result.pages(), profile, request.columnar(), asyncExecutionId, false, request.async());
         }
         return new EsqlQueryResponse(columns, result.pages(), profile, request.columnar(), request.async());
     }
@@ -234,12 +235,15 @@ public EsqlQueryTask createTask(
 
     @Override
     public EsqlQueryResponse initialResponse(EsqlQueryTask task) {
+        var asyncExecutionId = task.getExecutionId().getEncoded();
+        threadPool.getThreadContext().addResponseHeader(AsyncExecutionId.ASYNC_EXECUTION_ID_HEADER, asyncExecutionId);
+        threadPool.getThreadContext().addResponseHeader(AsyncExecutionId.ASYNC_EXECUTION_IS_RUNNING_HEADER, "?1");
         return new EsqlQueryResponse(
             List.of(),
             List.of(),
             null,
             false,
-            task.getExecutionId().getEncoded(),
+            asyncExecutionId,
             true, // is_running
             true // isAsync
         );
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java
index 223cdf6f3c9be..7ed027436bbcd 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java
@@ -12,10 +12,10 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java
index 07db69e6c5e51..ad17c97e3d9ca 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java
@@ -24,15 +24,14 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
+import org.elasticsearch.xpack.esql.core.tree.Location;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.function.Warnings;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.xpack.esql.core.util.SourceUtils.readSource;
-import static org.elasticsearch.xpack.esql.core.util.SourceUtils.writeSource;
-
 /**
  * Lucene query that wraps another query and only selects documents that match
  * the wrapped query <strong>and</strong> have a single field value.
@@ -107,11 +106,20 @@ public static class Builder extends AbstractQueryBuilder<Builder> {
             super(in);
             this.next = in.readNamedWriteable(QueryBuilder.class);
             this.field = in.readString();
-            if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
-                this.source = readSource(in);
+            if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_SINGLE_VALUE_QUERY_SOURCE)) {
+                if (in instanceof PlanStreamInput psi) {
+                    this.source = Source.readFrom(psi);
+                } else {
+                    /*
+                     * For things like CanMatchNodeRequest we serialize without the Source. But we
+                     * don't use it, so that's ok.
+                     */
+                    this.source = Source.readEmpty(in);
+                }
+            } else if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
+                this.source = readOldSource(in);
             } else {
                 this.source = Source.EMPTY;
-
             }
         }
 
@@ -119,8 +127,10 @@ public static class Builder extends AbstractQueryBuilder<Builder> {
         protected void doWriteTo(StreamOutput out) throws IOException {
             out.writeNamedWriteable(next);
             out.writeString(field);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
-                writeSource(out, source);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_SINGLE_VALUE_QUERY_SOURCE)) {
+                source.writeTo(out);
+            } else if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
+                writeOldSource(out, source);
             }
         }
 
@@ -199,4 +209,24 @@ protected int doHashCode() {
         }
     }
 
+    /**
+     * Write a {@link Source} including the text in it.
+     */
+    static void writeOldSource(StreamOutput out, Source source) throws IOException {
+        out.writeInt(source.source().getLineNumber());
+        out.writeInt(source.source().getColumnNumber());
+        out.writeString(source.text());
+    }
+
+    /**
+     * Read a {@link Source} including the text in it.
+     */
+    static Source readOldSource(StreamInput in) throws IOException {
+        int line = in.readInt();
+        int column = in.readInt();
+        int charPositionInLine = column - 1;
+
+        String text = in.readString();
+        return new Source(new Location(line, charPositionInLine), text);
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
index 23de36d6d3d77..4f0bcbb43e260 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
@@ -7,35 +7,22 @@
 
 package org.elasticsearch.xpack.esql.querydsl.query;
 
-import org.apache.lucene.document.ShapeField;
-import org.apache.lucene.document.XYDocValuesField;
-import org.apache.lucene.document.XYPointField;
-import org.apache.lucene.document.XYShape;
-import org.apache.lucene.geo.XYGeometry;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.IndexOrDocValuesQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.common.geo.LuceneGeometriesUtils;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.geometry.Geometry;
-import org.elasticsearch.geometry.ShapeType;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.GeoShapeQueryable;
 import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.query.ExistsQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
-import org.elasticsearch.lucene.spatial.CartesianShapeDocValuesQuery;
+import org.elasticsearch.lucene.spatial.XYQueriesUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -44,11 +31,11 @@
 
 public class SpatialRelatesQuery extends Query {
     private final String field;
-    private final ShapeField.QueryRelation queryRelation;
+    private final ShapeRelation queryRelation;
     private final Geometry shape;
     private final DataType dataType;
 
-    public SpatialRelatesQuery(Source source, String field, ShapeField.QueryRelation queryRelation, Geometry shape, DataType dataType) {
+    public SpatialRelatesQuery(Source source, String field, ShapeRelation queryRelation, Geometry shape, DataType dataType) {
         super(source);
         this.field = field;
         this.queryRelation = queryRelation;
@@ -58,7 +45,7 @@ public SpatialRelatesQuery(Source source, String field, ShapeField.QueryRelation
 
     @Override
     public QueryBuilder asBuilder() {
-        return EsqlDataTypes.isSpatialGeo(dataType) ? new GeoShapeQueryBuilder() : new CartesianShapeQueryBuilder();
+        return DataType.isSpatialGeo(dataType) ? new GeoShapeQueryBuilder() : new CartesianShapeQueryBuilder();
     }
 
     @Override
@@ -207,83 +194,36 @@ org.apache.lucene.search.Query buildShapeQuery(SearchExecutionContext context, M
             return new ConstantScoreQuery(innerQuery);
         }
 
-        /**
-         * This code is based on the ShapeQueryPointProcessor.shapeQuery() method, with additional support for two special cases:
-         * <ul>
-         *     <li>
-         *         DISJOINT queries (using {@code EXISTS && !INTERSECTS}, similar to {@code LegacyGeoShapeQueryProcessor.geoShapeQuery()})
-         *     </li>
-         *     <li>
-         *         CONTAINS queries (if the shape is a point, INTERSECTS is used, otherwise a MatchNoDocsQuery is built,
-         *         similar to {@code LatLonPoint.makeContainsGeometryQuery()})
-         *     </li>
-         * </ul>
-         */
         private static org.apache.lucene.search.Query pointShapeQuery(
             Geometry geometry,
             String fieldName,
-            ShapeField.QueryRelation relation,
+            ShapeRelation relation,
             SearchExecutionContext context
         ) {
-            final boolean hasDocValues = context.getFieldType(fieldName).hasDocValues();
-            if (geometry == null || geometry.isEmpty()) {
-                throw new QueryShardException(context, "Invalid/empty geometry");
-            }
-            if (geometry.type() != ShapeType.POINT && relation == ShapeField.QueryRelation.CONTAINS) {
-                return new MatchNoDocsQuery("A point field can never contain a non-point geometry");
-            }
-            final XYGeometry[] luceneGeometries = LuceneGeometriesUtils.toXYGeometry(geometry, t -> {});
-            org.apache.lucene.search.Query intersects = XYPointField.newGeometryQuery(fieldName, luceneGeometries);
-            if (relation == ShapeField.QueryRelation.DISJOINT) {
-                // XYPointField does not support DISJOINT queries, so we build one as EXISTS && !INTERSECTS
-                BooleanQuery.Builder bool = new BooleanQuery.Builder();
-                org.apache.lucene.search.Query exists = ExistsQueryBuilder.newFilter(context, fieldName, false);
-                bool.add(exists, BooleanClause.Occur.MUST);
-                bool.add(intersects, BooleanClause.Occur.MUST_NOT);
-                return bool.build();
-            }
-
-            // Point-Intersects works for all cases except CONTAINS(shape) and DISJOINT, which are handled separately above
-            if (hasDocValues) {
-                final org.apache.lucene.search.Query queryDocValues = XYDocValuesField.newSlowGeometryQuery(fieldName, luceneGeometries);
-                intersects = new IndexOrDocValuesQuery(intersects, queryDocValues);
+            final MappedFieldType fieldType = context.getFieldType(fieldName);
+            try {
+                return XYQueriesUtils.toXYPointQuery(geometry, fieldName, relation, fieldType.isIndexed(), fieldType.hasDocValues());
+            } catch (IllegalArgumentException e) {
+                throw new QueryShardException(context, "Exception creating query on Field [" + fieldName + "] " + e.getMessage(), e);
             }
-            return intersects;
         }
 
-        /**
-         * This code is based on the ShapeQueryProcessor.shapeQuery() method
-         */
         private static org.apache.lucene.search.Query shapeShapeQuery(
             Geometry geometry,
             String fieldName,
-            ShapeField.QueryRelation relation,
+            ShapeRelation relation,
             SearchExecutionContext context
         ) {
-            final boolean hasDocValues = context.getFieldType(fieldName).hasDocValues();
             // CONTAINS queries are not supported by VECTOR strategy for indices created before version 7.5.0 (Lucene 8.3.0);
-            if (relation == ShapeField.QueryRelation.CONTAINS && context.indexVersionCreated().before(IndexVersions.V_7_5_0)) {
+            if (relation == ShapeRelation.CONTAINS && context.indexVersionCreated().before(IndexVersions.V_7_5_0)) {
                 throw new QueryShardException(context, relation + " query relation not supported for Field [" + fieldName + "].");
             }
-            if (geometry == null || geometry.isEmpty()) {
-                throw new QueryShardException(context, "Invalid/empty geometry");
-            }
-            final XYGeometry[] luceneGeometries;
+            final MappedFieldType fieldType = context.getFieldType(fieldName);
             try {
-                luceneGeometries = LuceneGeometriesUtils.toXYGeometry(geometry, t -> {});
+                return XYQueriesUtils.toXYShapeQuery(geometry, fieldName, relation, fieldType.isIndexed(), fieldType.hasDocValues());
             } catch (IllegalArgumentException e) {
                 throw new QueryShardException(context, "Exception creating query on Field [" + fieldName + "] " + e.getMessage(), e);
             }
-            org.apache.lucene.search.Query query = XYShape.newGeometryQuery(fieldName, relation, luceneGeometries);
-            if (hasDocValues) {
-                final org.apache.lucene.search.Query queryDocValues = new CartesianShapeDocValuesQuery(
-                    fieldName,
-                    relation,
-                    luceneGeometries
-                );
-                query = new IndexOrDocValuesQuery(query, queryDocValues);
-            }
-            return query;
         }
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java
new file mode 100644
index 0000000000000..33a48d2e7df05
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java
@@ -0,0 +1,262 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.session;
+
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.compress.CompressorFactory;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.compute.data.BlockStreamInput;
+import org.elasticsearch.xpack.esql.Column;
+import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.common.unit.ByteSizeUnit.KB;
+
+public class Configuration implements Writeable {
+
+    public static final int QUERY_COMPRESS_THRESHOLD_CHARS = KB.toIntBytes(5);
+
+    private final String clusterName;
+    private final String username;
+    private final ZonedDateTime now;
+    private final ZoneId zoneId;
+
+    private final QueryPragmas pragmas;
+
+    private final int resultTruncationMaxSize;
+    private final int resultTruncationDefaultSize;
+
+    private final Locale locale;
+
+    private final String query;
+
+    private final boolean profile;
+
+    private final Map<String, Map<String, Column>> tables;
+
+    public Configuration(
+        ZoneId zi,
+        Locale locale,
+        String username,
+        String clusterName,
+        QueryPragmas pragmas,
+        int resultTruncationMaxSize,
+        int resultTruncationDefaultSize,
+        String query,
+        boolean profile,
+        Map<String, Map<String, Column>> tables
+    ) {
+        this.zoneId = zi.normalized();
+        this.now = ZonedDateTime.now(Clock.tick(Clock.system(zoneId), Duration.ofNanos(1)));
+        this.username = username;
+        this.clusterName = clusterName;
+        this.locale = locale;
+        this.pragmas = pragmas;
+        this.resultTruncationMaxSize = resultTruncationMaxSize;
+        this.resultTruncationDefaultSize = resultTruncationDefaultSize;
+        this.query = query;
+        this.profile = profile;
+        this.tables = tables;
+        assert tables != null;
+    }
+
+    public Configuration(BlockStreamInput in) throws IOException {
+        this.zoneId = in.readZoneId();
+        this.now = Instant.ofEpochSecond(in.readVLong(), in.readVInt()).atZone(zoneId);
+        this.username = in.readOptionalString();
+        this.clusterName = in.readOptionalString();
+        locale = Locale.forLanguageTag(in.readString());
+        this.pragmas = new QueryPragmas(in);
+        this.resultTruncationMaxSize = in.readVInt();
+        this.resultTruncationDefaultSize = in.readVInt();
+        this.query = readQuery(in);
+        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
+            this.profile = in.readBoolean();
+        } else {
+            this.profile = false;
+        }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_REQUEST_TABLES)) {
+            this.tables = in.readImmutableMap(i1 -> i1.readImmutableMap(i2 -> new Column((BlockStreamInput) i2)));
+        } else {
+            this.tables = Map.of();
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeZoneId(zoneId);
+        var instant = now.toInstant();
+        out.writeVLong(instant.getEpochSecond());
+        out.writeVInt(instant.getNano());
+        out.writeOptionalString(username);    // TODO this one is always null
+        out.writeOptionalString(clusterName); // TODO this one is never null so maybe not optional
+        out.writeString(locale.toLanguageTag());
+        pragmas.writeTo(out);
+        out.writeVInt(resultTruncationMaxSize);
+        out.writeVInt(resultTruncationDefaultSize);
+        writeQuery(out, query);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
+            out.writeBoolean(profile);
+        }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_REQUEST_TABLES)) {
+            out.writeMap(tables, (o1, columns) -> o1.writeMap(columns, StreamOutput::writeWriteable));
+        }
+    }
+
+    public ZoneId zoneId() {
+        return zoneId;
+    }
+
+    public ZonedDateTime now() {
+        return now;
+    }
+
+    public String clusterName() {
+        return clusterName;
+    }
+
+    public String username() {
+        return username;
+    }
+
+    public QueryPragmas pragmas() {
+        return pragmas;
+    }
+
+    public int resultTruncationMaxSize() {
+        return resultTruncationMaxSize;
+    }
+
+    public int resultTruncationDefaultSize() {
+        return resultTruncationDefaultSize;
+    }
+
+    public Locale locale() {
+        return locale;
+    }
+
+    public String query() {
+        return query;
+    }
+
+    /**
+     * Returns the current time in milliseconds from the time epoch for the execution of this request.
+     * It ensures consistency by using the same value on all nodes involved in the search request.
+     * Note: Currently, it returns {@link System#currentTimeMillis()}, but this value will be serialized between nodes.
+     */
+    public long absoluteStartedTimeInMillis() {
+        return System.currentTimeMillis();
+    }
+
+    /**
+     * Tables specified in the request.
+     */
+    public Map<String, Map<String, Column>> tables() {
+        return tables;
+    }
+
+    /**
+     * Enable profiling, sacrificing performance to return information about
+     * what operations are taking the most time.
+     */
+    public boolean profile() {
+        return profile;
+    }
+
+    private static void writeQuery(StreamOutput out, String query) throws IOException {
+        if (query.length() > QUERY_COMPRESS_THRESHOLD_CHARS) { // compare on chars to avoid UTF-8 encoding unless actually required
+            out.writeBoolean(true);
+            var bytesArray = new BytesArray(query.getBytes(StandardCharsets.UTF_8));
+            var bytesRef = CompressorFactory.COMPRESSOR.compress(bytesArray);
+            out.writeByteArray(bytesRef.array());
+        } else {
+            out.writeBoolean(false);
+            out.writeString(query);
+        }
+    }
+
+    private static String readQuery(StreamInput in) throws IOException {
+        boolean compressed = in.readBoolean();
+        if (compressed) {
+            byte[] bytes = in.readByteArray();
+            var bytesRef = CompressorFactory.uncompress(new BytesArray(bytes));
+            return new String(bytesRef.array(), StandardCharsets.UTF_8);
+        } else {
+            return in.readString();
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Configuration that = (Configuration) o;
+        return Objects.equals(zoneId, that.zoneId)
+            && Objects.equals(now, that.now)
+            && Objects.equals(username, that.username)
+            && Objects.equals(clusterName, that.clusterName)
+            && resultTruncationMaxSize == that.resultTruncationMaxSize
+            && resultTruncationDefaultSize == that.resultTruncationDefaultSize
+            && Objects.equals(pragmas, that.pragmas)
+            && Objects.equals(locale, that.locale)
+            && Objects.equals(that.query, query)
+            && profile == that.profile
+            && tables.equals(that.tables);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            zoneId,
+            now,
+            username,
+            clusterName,
+            pragmas,
+            resultTruncationMaxSize,
+            resultTruncationDefaultSize,
+            locale,
+            query,
+            profile,
+            tables
+        );
+    }
+
+    @Override
+    public String toString() {
+        return "EsqlConfiguration{"
+            + "pragmas="
+            + pragmas
+            + ", resultTruncationMaxSize="
+            + resultTruncationMaxSize
+            + ", resultTruncationDefaultSize="
+            + resultTruncationDefaultSize
+            + ", locale="
+            + locale
+            + ", query='"
+            + query
+            + '\''
+            + ", profile="
+            + profile
+            + ", tables="
+            + tables
+            + '}';
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java
deleted file mode 100644
index 0cfc0cf002b97..0000000000000
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlConfiguration.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.session;
-
-import org.elasticsearch.TransportVersions;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.compress.CompressorFactory;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.compute.data.BlockStreamInput;
-import org.elasticsearch.xpack.esql.Column;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
-import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.elasticsearch.common.unit.ByteSizeUnit.KB;
-
-public class EsqlConfiguration extends Configuration implements Writeable {
-
-    static final int QUERY_COMPRESS_THRESHOLD_CHARS = KB.toIntBytes(5);
-
-    private final QueryPragmas pragmas;
-
-    private final int resultTruncationMaxSize;
-    private final int resultTruncationDefaultSize;
-
-    private final Locale locale;
-
-    private final String query;
-
-    private final boolean profile;
-
-    private final Map<String, Map<String, Column>> tables;
-
-    public EsqlConfiguration(
-        ZoneId zi,
-        Locale locale,
-        String username,
-        String clusterName,
-        QueryPragmas pragmas,
-        int resultTruncationMaxSize,
-        int resultTruncationDefaultSize,
-        String query,
-        boolean profile,
-        Map<String, Map<String, Column>> tables
-    ) {
-        super(zi, username, clusterName);
-        this.locale = locale;
-        this.pragmas = pragmas;
-        this.resultTruncationMaxSize = resultTruncationMaxSize;
-        this.resultTruncationDefaultSize = resultTruncationDefaultSize;
-        this.query = query;
-        this.profile = profile;
-        this.tables = tables;
-        assert tables != null;
-    }
-
-    public EsqlConfiguration(BlockStreamInput in) throws IOException {
-        super(in.readZoneId(), Instant.ofEpochSecond(in.readVLong(), in.readVInt()), in.readOptionalString(), in.readOptionalString());
-        locale = Locale.forLanguageTag(in.readString());
-        this.pragmas = new QueryPragmas(in);
-        this.resultTruncationMaxSize = in.readVInt();
-        this.resultTruncationDefaultSize = in.readVInt();
-        this.query = readQuery(in);
-        if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
-            this.profile = in.readBoolean();
-        } else {
-            this.profile = false;
-        }
-        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_REQUEST_TABLES)) {
-            this.tables = in.readImmutableMap(i1 -> i1.readImmutableMap(i2 -> new Column((BlockStreamInput) i2)));
-        } else {
-            this.tables = Map.of();
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeZoneId(zoneId);
-        var instant = now.toInstant();
-        out.writeVLong(instant.getEpochSecond());
-        out.writeVInt(instant.getNano());
-        out.writeOptionalString(username);
-        out.writeOptionalString(clusterName);
-        out.writeString(locale.toLanguageTag());
-        pragmas.writeTo(out);
-        out.writeVInt(resultTruncationMaxSize);
-        out.writeVInt(resultTruncationDefaultSize);
-        writeQuery(out, query);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
-            out.writeBoolean(profile);
-        }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_REQUEST_TABLES)) {
-            out.writeMap(tables, (o1, columns) -> o1.writeMap(columns, (o2, column) -> column.writeTo(o2)));
-        }
-    }
-
-    public QueryPragmas pragmas() {
-        return pragmas;
-    }
-
-    public int resultTruncationMaxSize() {
-        return resultTruncationMaxSize;
-    }
-
-    public int resultTruncationDefaultSize() {
-        return resultTruncationDefaultSize;
-    }
-
-    public Locale locale() {
-        return locale;
-    }
-
-    public String query() {
-        return query;
-    }
-
-    /**
-     * Returns the current time in milliseconds from the time epoch for the execution of this request.
-     * It ensures consistency by using the same value on all nodes involved in the search request.
-     * Note: Currently, it returns {@link System#currentTimeMillis()}, but this value will be serialized between nodes.
-     */
-    public long absoluteStartedTimeInMillis() {
-        return System.currentTimeMillis();
-    }
-
-    /**
-     * Tables specified in the request.
-     */
-    public Map<String, Map<String, Column>> tables() {
-        return tables;
-    }
-
-    /**
-     * Enable profiling, sacrificing performance to return information about
-     * what operations are taking the most time.
-     */
-    public boolean profile() {
-        return profile;
-    }
-
-    private static void writeQuery(StreamOutput out, String query) throws IOException {
-        if (query.length() > QUERY_COMPRESS_THRESHOLD_CHARS) { // compare on chars to avoid UTF-8 encoding unless actually required
-            out.writeBoolean(true);
-            var bytesArray = new BytesArray(query.getBytes(StandardCharsets.UTF_8));
-            var bytesRef = CompressorFactory.COMPRESSOR.compress(bytesArray);
-            out.writeByteArray(bytesRef.array());
-        } else {
-            out.writeBoolean(false);
-            out.writeString(query);
-        }
-    }
-
-    private static String readQuery(StreamInput in) throws IOException {
-        boolean compressed = in.readBoolean();
-        if (compressed) {
-            byte[] bytes = in.readByteArray();
-            var bytesRef = CompressorFactory.uncompress(new BytesArray(bytes));
-            return new String(bytesRef.array(), StandardCharsets.UTF_8);
-        } else {
-            return in.readString();
-        }
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (super.equals(o)) {
-            EsqlConfiguration that = (EsqlConfiguration) o;
-            return resultTruncationMaxSize == that.resultTruncationMaxSize
-                && resultTruncationDefaultSize == that.resultTruncationDefaultSize
-                && Objects.equals(pragmas, that.pragmas)
-                && Objects.equals(locale, that.locale)
-                && Objects.equals(that.query, query)
-                && profile == that.profile
-                && tables.equals(that.tables);
-        }
-        return false;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(
-            super.hashCode(),
-            pragmas,
-            resultTruncationMaxSize,
-            resultTruncationDefaultSize,
-            locale,
-            query,
-            profile,
-            tables
-        );
-    }
-
-    @Override
-    public String toString() {
-        return "EsqlConfiguration{"
-            + "pragmas="
-            + pragmas
-            + ", resultTruncationMaxSize="
-            + resultTruncationMaxSize
-            + ", resultTruncationDefaultSize="
-            + resultTruncationDefaultSize
-            + ", locale="
-            + locale
-            + ", query='"
-            + query
-            + '\''
-            + ", profile="
-            + profile
-            + ", tables="
-            + tables
-            + '}';
-    }
-}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
index 8c831cc260e03..25d155ccfde07 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
@@ -10,7 +10,10 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.compute.operator.DriverProfile;
+import org.elasticsearch.core.Releasables;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
@@ -19,8 +22,8 @@
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
 import org.elasticsearch.xpack.esql.analysis.PreAnalyzer;
+import org.elasticsearch.xpack.esql.analysis.TableInfo;
 import org.elasticsearch.xpack.esql.analysis.Verifier;
-import org.elasticsearch.xpack.esql.core.analyzer.TableInfo;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
@@ -28,24 +31,25 @@
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
-import org.elasticsearch.xpack.esql.core.index.MappingException;
-import org.elasticsearch.xpack.esql.core.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
 import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
+import org.elasticsearch.xpack.esql.index.MappingException;
 import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.parser.QueryParams;
+import org.elasticsearch.xpack.esql.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.Keep;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.plan.logical.Phased;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.RegexExtract;
 import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize;
@@ -71,7 +75,7 @@ public class EsqlSession {
     private static final Logger LOGGER = LogManager.getLogger(EsqlSession.class);
 
     private final String sessionId;
-    private final EsqlConfiguration configuration;
+    private final Configuration configuration;
     private final IndexResolver indexResolver;
     private final EnrichPolicyResolver enrichPolicyResolver;
 
@@ -85,7 +89,7 @@ public class EsqlSession {
 
     public EsqlSession(
         String sessionId,
-        EsqlConfiguration configuration,
+        Configuration configuration,
         IndexResolver indexResolver,
         EnrichPolicyResolver enrichPolicyResolver,
         PreAnalyzer preAnalyzer,
@@ -121,18 +125,57 @@ public void execute(
         LOGGER.debug("ESQL query:\n{}", request.query());
         analyzedPlan(
             parse(request.query(), request.params()),
-            listener.delegateFailureAndWrap((next, analyzedPlan) -> executeAnalyzedPlan(request, runPhase, analyzedPlan, next))
+            listener.delegateFailureAndWrap(
+                (next, analyzedPlan) -> executeOptimizedPlan(request, runPhase, optimizedPlan(analyzedPlan), next)
+            )
         );
     }
 
-    public void executeAnalyzedPlan(
+    /**
+     * Execute an analyzed plan. Most code should prefer calling {@link #execute} but
+     * this is public for testing. See {@link Phased} for the sequence of operations.
+     */
+    public void executeOptimizedPlan(
+        EsqlQueryRequest request,
+        BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase,
+        LogicalPlan optimizedPlan,
+        ActionListener<Result> listener
+    ) {
+        LogicalPlan firstPhase = Phased.extractFirstPhase(optimizedPlan);
+        if (firstPhase == null) {
+            runPhase.accept(logicalPlanToPhysicalPlan(optimizedPlan, request), listener);
+        } else {
+            executePhased(new ArrayList<>(), optimizedPlan, request, firstPhase, runPhase, listener);
+        }
+    }
+
+    private void executePhased(
+        List<DriverProfile> profileAccumulator,
+        LogicalPlan mainPlan,
         EsqlQueryRequest request,
+        LogicalPlan firstPhase,
         BiConsumer<PhysicalPlan, ActionListener<Result>> runPhase,
-        LogicalPlan analyzedPlan,
         ActionListener<Result> listener
     ) {
-        // TODO phased execution lands here.
-        runPhase.accept(logicalPlanToPhysicalPlan(analyzedPlan, request), listener);
+        PhysicalPlan physicalPlan = logicalPlanToPhysicalPlan(optimizedPlan(firstPhase), request);
+        runPhase.accept(physicalPlan, listener.delegateFailureAndWrap((next, result) -> {
+            try {
+                profileAccumulator.addAll(result.profiles());
+                LogicalPlan newMainPlan = optimizedPlan(Phased.applyResultsFromFirstPhase(mainPlan, physicalPlan.output(), result.pages()));
+                LogicalPlan newFirstPhase = Phased.extractFirstPhase(newMainPlan);
+                if (newFirstPhase == null) {
+                    PhysicalPlan finalPhysicalPlan = logicalPlanToPhysicalPlan(newMainPlan, request);
+                    runPhase.accept(finalPhysicalPlan, next.delegateFailureAndWrap((finalListener, finalResult) -> {
+                        profileAccumulator.addAll(finalResult.profiles());
+                        finalListener.onResponse(new Result(finalResult.schema(), finalResult.pages(), profileAccumulator));
+                    }));
+                } else {
+                    executePhased(profileAccumulator, newMainPlan, request, newFirstPhase, runPhase, next);
+                }
+            } finally {
+                Releasables.closeExpectNoException(Releasables.wrap(Iterators.map(result.pages().iterator(), p -> p::releaseBlocks)));
+            }
+        }));
     }
 
     private LogicalPlan parse(String query, QueryParams params) {
@@ -194,7 +237,7 @@ private <T> void preAnalyze(LogicalPlan parsed, BiFunction<IndexResolution, Enri
         }));
     }
 
-    private <T> void preAnalyzeIndices(LogicalPlan parsed, ActionListener<IndexResolution> listener, Set<String> enrichPolicyMatchFields) {
+    private void preAnalyzeIndices(LogicalPlan parsed, ActionListener<IndexResolution> listener, Set<String> enrichPolicyMatchFields) {
         PreAnalyzer.PreAnalysis preAnalysis = new PreAnalyzer().preAnalyze(parsed);
         // TODO we plan to support joins in the future when possible, but for now we'll just fail early if we see one
         if (preAnalysis.indices.size() > 1) {
@@ -243,7 +286,7 @@ static Set<String> fieldNames(LogicalPlan parsed, Set<String> enrichPolicyMatchF
             if (p instanceof RegexExtract re) { // for Grok and Dissect
                 // remove other down-the-tree references to the extracted fields
                 for (Attribute extracted : re.extractedFields()) {
-                    references.removeIf(attr -> matchByName(attr, extracted.qualifiedName(), false));
+                    references.removeIf(attr -> matchByName(attr, extracted.name(), false));
                 }
                 // but keep the inputs needed by Grok/Dissect
                 references.addAll(re.input().references());
@@ -275,16 +318,16 @@ static Set<String> fieldNames(LogicalPlan parsed, Set<String> enrichPolicyMatchF
             p.forEachExpressionDown(Alias.class, alias -> {
                 // do not remove the UnresolvedAttribute that has the same name as its alias, ie "rename id = id"
                 // or the UnresolvedAttributes that are used in Functions that have aliases "STATS id = MAX(id)"
-                if (p.references().names().contains(alias.qualifiedName())) {
+                if (p.references().names().contains(alias.name())) {
                     return;
                 }
-                references.removeIf(attr -> matchByName(attr, alias.qualifiedName(), keepCommandReferences.contains(attr)));
+                references.removeIf(attr -> matchByName(attr, alias.name(), keepCommandReferences.contains(attr)));
             });
         });
 
         // remove valid metadata attributes because they will be filtered out by the IndexResolver anyway
         // otherwise, in some edge cases, we will fail to ask for "*" (all fields) instead
-        references.removeIf(a -> a instanceof MetadataAttribute || MetadataAttribute.isSupported(a.qualifiedName()));
+        references.removeIf(a -> a instanceof MetadataAttribute || MetadataAttribute.isSupported(a.name()));
         Set<String> fieldNames = references.names();
 
         if (fieldNames.isEmpty() && enrichPolicyMatchFields.isEmpty()) {
@@ -299,11 +342,11 @@ static Set<String> fieldNames(LogicalPlan parsed, Set<String> enrichPolicyMatchF
     }
 
     private static boolean matchByName(Attribute attr, String other, boolean skipIfPattern) {
-        boolean isPattern = Regex.isSimpleMatchPattern(attr.qualifiedName());
+        boolean isPattern = Regex.isSimpleMatchPattern(attr.name());
         if (skipIfPattern && isPattern) {
             return false;
         }
-        var name = attr.qualifiedName();
+        var name = attr.name();
         return isPattern ? Regex.simpleMatch(name, other) : name.equals(other);
     }
 
@@ -311,8 +354,8 @@ private static Set<String> subfields(Set<String> names) {
         return names.stream().filter(name -> name.endsWith(WILDCARD) == false).map(name -> name + ".*").collect(Collectors.toSet());
     }
 
-    private PhysicalPlan logicalPlanToPhysicalPlan(LogicalPlan logicalPlan, EsqlQueryRequest request) {
-        PhysicalPlan physicalPlan = optimizedPhysicalPlan(logicalPlan);
+    private PhysicalPlan logicalPlanToPhysicalPlan(LogicalPlan optimizedPlan, EsqlQueryRequest request) {
+        PhysicalPlan physicalPlan = optimizedPhysicalPlan(optimizedPlan);
         physicalPlan = physicalPlan.transformUp(FragmentExec.class, f -> {
             QueryBuilder filter = request.filter();
             if (filter != null) {
@@ -330,20 +373,25 @@ private PhysicalPlan logicalPlanToPhysicalPlan(LogicalPlan logicalPlan, EsqlQuer
     }
 
     public LogicalPlan optimizedPlan(LogicalPlan logicalPlan) {
-        assert logicalPlan.analyzed();
+        if (logicalPlan.analyzed() == false) {
+            throw new IllegalStateException("Expected analyzed plan");
+        }
         var plan = logicalPlanOptimizer.optimize(logicalPlan);
         LOGGER.debug("Optimized logicalPlan plan:\n{}", plan);
         return plan;
     }
 
-    public PhysicalPlan physicalPlan(LogicalPlan logicalPlan) {
-        var plan = mapper.map(optimizedPlan(logicalPlan));
+    public PhysicalPlan physicalPlan(LogicalPlan optimizedPlan) {
+        if (optimizedPlan.optimized() == false) {
+            throw new IllegalStateException("Expected optimized plan");
+        }
+        var plan = mapper.map(optimizedPlan);
         LOGGER.debug("Physical plan:\n{}", plan);
         return plan;
     }
 
-    public PhysicalPlan optimizedPhysicalPlan(LogicalPlan logicalPlan) {
-        var plan = physicalPlanOptimizer.optimize(physicalPlan(logicalPlan));
+    public PhysicalPlan optimizedPhysicalPlan(LogicalPlan optimizedPlan) {
+        var plan = physicalPlanOptimizer.optimize(physicalPlan(optimizedPlan));
         LOGGER.debug("Optimized physical plan:\n{}", plan);
         return plan;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
index 8fe00b70f1ff6..954b255f6d42f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
@@ -17,16 +17,16 @@
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypeRegistry;
 import org.elasticsearch.xpack.esql.core.type.DateEsField;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
 import org.elasticsearch.xpack.esql.core.type.KeywordEsField;
 import org.elasticsearch.xpack.esql.core.type.TextEsField;
 import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
+import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -65,11 +65,9 @@ public class IndexResolver {
         .build();
 
     private final Client client;
-    private final DataTypeRegistry typeRegistry;
 
-    public IndexResolver(Client client, DataTypeRegistry typeRegistry) {
+    public IndexResolver(Client client) {
         this.client = client;
-        this.typeRegistry = typeRegistry;
     }
 
     /**
@@ -83,6 +81,7 @@ public void resolveAsMergedMapping(String indexWildcard, Set<String> fieldNames,
         );
     }
 
+    // public for testing only
     public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResponse fieldCapsResponse) {
         assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker
         if (fieldCapsResponse.getIndexResponses().isEmpty()) {
@@ -95,8 +94,9 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp
         // TODO flattened is simpler - could we get away with that?
         String[] names = fieldsCaps.keySet().toArray(new String[0]);
         Arrays.sort(names);
+        Set<String> forbiddenFields = new HashSet<>();
         Map<String, EsField> rootFields = new HashMap<>();
-        for (String name : names) {
+        name: for (String name : names) {
             Map<String, EsField> fields = rootFields;
             String fullName = name;
             boolean isAlias = false;
@@ -107,6 +107,9 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp
                     break;
                 }
                 String parent = name.substring(0, nextDot);
+                if (forbiddenFields.contains(parent)) {
+                    continue name;
+                }
                 EsField obj = fields.get(parent);
                 if (obj == null) {
                     obj = new EsField(parent, OBJECT, new HashMap<>(), false, true);
@@ -118,9 +121,16 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp
                 fields = obj.getProperties();
                 name = name.substring(nextDot + 1);
             }
+
+            List<IndexFieldCapabilities> caps = fieldsCaps.get(fullName);
+            if (allNested(caps)) {
+                forbiddenFields.add(name);
+                continue;
+            }
             // TODO we're careful to make isAlias match IndexResolver - but do we use it?
+
             EsField field = firstUnsupportedParent == null
-                ? createField(fieldCapsResponse, name, fullName, fieldsCaps.get(fullName), isAlias)
+                ? createField(fieldCapsResponse, name, fullName, caps, isAlias)
                 : new UnsupportedEsField(
                     fullName,
                     firstUnsupportedParent.getOriginalType(),
@@ -146,6 +156,15 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp
         return IndexResolution.valid(new EsIndex(indexPattern, rootFields, concreteIndices));
     }
 
+    private boolean allNested(List<IndexFieldCapabilities> caps) {
+        for (IndexFieldCapabilities cap : caps) {
+            if (false == cap.type().equalsIgnoreCase("nested")) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     private static Map<String, List<IndexFieldCapabilities>> collectFieldCaps(FieldCapabilitiesResponse fieldCapsResponse) {
         Set<String> seenHashes = new HashSet<>();
         Map<String, List<IndexFieldCapabilities>> fieldsCaps = new HashMap<>();
@@ -174,7 +193,7 @@ private EsField createField(
     ) {
         IndexFieldCapabilities first = fcs.get(0);
         List<IndexFieldCapabilities> rest = fcs.subList(1, fcs.size());
-        DataType type = typeRegistry.fromEs(first.type(), first.metricType());
+        DataType type = EsqlDataTypeRegistry.INSTANCE.fromEs(first.type(), first.metricType());
         boolean aggregatable = first.isAggregatable();
         if (rest.isEmpty() == false) {
             for (IndexFieldCapabilities fc : rest) {
@@ -183,7 +202,7 @@ private EsField createField(
                 }
             }
             for (IndexFieldCapabilities fc : rest) {
-                if (type != typeRegistry.fromEs(fc.type(), fc.metricType())) {
+                if (type != EsqlDataTypeRegistry.INSTANCE.fromEs(fc.type(), fc.metricType())) {
                     return conflictingTypes(name, fullName, fieldCapsResponse);
                 }
             }
@@ -223,7 +242,7 @@ private EsField conflictingTypes(String name, String fullName, FieldCapabilities
         for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) {
             IndexFieldCapabilities fc = ir.get().get(fullName);
             if (fc != null) {
-                DataType type = typeRegistry.fromEs(fc.type(), fc.metricType());
+                DataType type = EsqlDataTypeRegistry.INSTANCE.fromEs(fc.type(), fc.metricType());
                 if (type == UNSUPPORTED) {
                     return unsupported(name, fc);
                 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
index 23a94bde56b1e..b090708a64ad3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
@@ -46,6 +46,7 @@
 import java.time.Period;
 import java.time.ZoneId;
 import java.time.temporal.ChronoField;
+import java.time.temporal.TemporalAccessor;
 import java.time.temporal.TemporalAmount;
 import java.util.Locale;
 import java.util.Map;
@@ -57,6 +58,7 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
@@ -66,9 +68,14 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
 import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
-import static org.elasticsearch.xpack.esql.core.type.DataType.isPrimitive;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTimeOrTemporal;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isNullOrDatePeriod;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isNullOrTemporalAmount;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isNullOrTimeDuration;
 import static org.elasticsearch.xpack.esql.core.type.DataType.isString;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeDoubleToLong;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToInt;
@@ -106,18 +113,6 @@ public class EsqlDataTypeConverter {
         entry(VERSION, ToVersion::new)
     );
 
-    /**
-     * Returns true if the from type can be converted to the to type, false - otherwise
-     */
-    public static boolean canConvert(DataType from, DataType to) {
-        // Special handling for nulls and if conversion is not requires
-        if (from == to || from == NULL) {
-            return true;
-        }
-        // only primitives are supported so far
-        return isPrimitive(from) && isPrimitive(to) && converterFor(from, to) != null;
-    }
-
     public static Converter converterFor(DataType from, DataType to) {
         // TODO move EXPRESSION_TO_LONG here if there is no regression
         if (isString(from)) {
@@ -142,7 +137,7 @@ public static Converter converterFor(DataType from, DataType to) {
             if (to == DataType.BOOLEAN) {
                 return EsqlConverter.STRING_TO_BOOLEAN;
             }
-            if (EsqlDataTypes.isSpatial(to)) {
+            if (DataType.isSpatial(to)) {
                 return EsqlConverter.STRING_TO_SPATIAL;
             }
             if (to == DataType.TIME_DURATION) {
@@ -229,8 +224,63 @@ public static Object convert(Object value, DataType dataType) {
         return converter.convert(value);
     }
 
+    /**
+     * Returns the type compatible with both left and right types
+     * <p>
+     * If one of the types is null - returns another type
+     * If both types are numeric - returns type with the highest precision int &lt; long &lt; float &lt; double
+     */
     public static DataType commonType(DataType left, DataType right) {
-        return DataTypeConverter.commonType(left, right);
+        if (left == right) {
+            return left;
+        }
+        if (left == NULL) {
+            return right;
+        }
+        if (right == NULL) {
+            return left;
+        }
+        if (isDateTimeOrTemporal(left) || isDateTimeOrTemporal(right)) {
+            if ((isDateTime(left) && isNullOrTemporalAmount(right)) || (isNullOrTemporalAmount(left) && isDateTime(right))) {
+                return DATETIME;
+            }
+            if (isNullOrTimeDuration(left) && isNullOrTimeDuration(right)) {
+                return TIME_DURATION;
+            }
+            if (isNullOrDatePeriod(left) && isNullOrDatePeriod(right)) {
+                return DATE_PERIOD;
+            }
+        }
+        if (isString(left) && isString(right)) {
+            if (left == TEXT || right == TEXT) {
+                return TEXT;
+            }
+            return right;
+        }
+        if (left.isNumeric() && right.isNumeric()) {
+            int lsize = left.estimatedSize().orElseThrow();
+            int rsize = right.estimatedSize().orElseThrow();
+            // if one is int
+            if (left.isWholeNumber()) {
+                // promote the highest int
+                if (right.isWholeNumber()) {
+                    if (left == UNSIGNED_LONG || right == UNSIGNED_LONG) {
+                        return UNSIGNED_LONG;
+                    }
+                    return lsize > rsize ? left : right;
+                }
+                // promote the rational
+                return right;
+            }
+            // try the other side
+            if (right.isWholeNumber()) {
+                return left;
+            }
+            // promote the highest rational
+            return lsize > rsize ? left : right;
+        }
+        // none found
+        return null;
     }
 
     // generally supporting abbreviations from https://en.wikipedia.org/wiki/Unit_of_time
@@ -322,10 +372,24 @@ public static long dateTimeToLong(String dateTime, DateFormatter formatter) {
         return formatter == null ? dateTimeToLong(dateTime) : formatter.parseMillis(dateTime);
     }
 
+    public static long dateNanosToLong(String dateNano) {
+        return dateNanosToLong(dateNano, DateFormatter.forPattern("strict_date_optional_time_nanos"));
+    }
+
+    public static long dateNanosToLong(String dateNano, DateFormatter formatter) {
+        TemporalAccessor parsed = formatter.parse(dateNano);
+        long nanos = parsed.getLong(ChronoField.INSTANT_SECONDS) * 1_000_000_000 + parsed.getLong(ChronoField.NANO_OF_SECOND);
+        return nanos;
+    }
+
     public static String dateTimeToString(long dateTime) {
         return DEFAULT_DATE_TIME_FORMATTER.formatMillis(dateTime);
     }
 
+    public static String nanoTimeToString(long dateTime) {
+        return DateFormatter.forPattern("strict_date_optional_time_nanos").formatNanos(dateTime);
+    }
+
     public static String dateTimeToString(long dateTime, DateFormatter formatter) {
         return formatter == null ? dateTimeToString(dateTime) : formatter.formatMillis(dateTime);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java
index 4ddef25584eea..f8e8cd37dc8b2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java
@@ -9,31 +9,13 @@
 
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypeRegistry;
 
-import java.util.Collection;
+public class EsqlDataTypeRegistry {
 
-import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
-import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isDateTimeOrTemporal;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrDatePeriod;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrTemporalAmount;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrTimeDuration;
-
-public class EsqlDataTypeRegistry implements DataTypeRegistry {
-
-    public static final DataTypeRegistry INSTANCE = new EsqlDataTypeRegistry();
+    public static final EsqlDataTypeRegistry INSTANCE = new EsqlDataTypeRegistry();
 
     private EsqlDataTypeRegistry() {}
 
-    @Override
-    public Collection<DataType> dataTypes() {
-        return DataType.types();
-    }
-
-    @Override
     public DataType fromEs(String typeName, TimeSeriesParams.MetricType metricType) {
         DataType type = DataType.fromEs(typeName);
         /*
@@ -44,40 +26,4 @@ public DataType fromEs(String typeName, TimeSeriesParams.MetricType metricType)
          */
         return metricType == TimeSeriesParams.MetricType.COUNTER ? type.widenSmallNumeric().counter() : type;
     }
-
-    @Override
-    public DataType fromJava(Object value) {
-        return DataType.fromJava(value);
-    }
-
-    @Override
-    public boolean isUnsupported(DataType type) {
-        return type == DataType.UNSUPPORTED;
-    }
-
-    @Override
-    public boolean canConvert(DataType from, DataType to) {
-        return EsqlDataTypeConverter.canConvert(from, to);
-    }
-
-    @Override
-    public Object convert(Object value, DataType type) {
-        return EsqlDataTypeConverter.convert(value, type);
-    }
-
-    @Override
-    public DataType commonType(DataType left, DataType right) {
-        if (isDateTimeOrTemporal(left) || isDateTimeOrTemporal(right)) {
-            if ((isDateTime(left) && isNullOrTemporalAmount(right)) || (isNullOrTemporalAmount(left) && isDateTime(right))) {
-                return DATETIME;
-            }
-            if (isNullOrTimeDuration(left) && isNullOrTimeDuration(right)) {
-                return TIME_DURATION;
-            }
-            if (isNullOrDatePeriod(left) && isNullOrDatePeriod(right)) {
-                return DATE_PERIOD;
-            }
-        }
-        return EsqlDataTypeConverter.commonType(left, right);
-    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java
deleted file mode 100644
index aad9470ecbbb7..0000000000000
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.type;
-
-import org.elasticsearch.xpack.esql.core.type.DataType;
-
-import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE;
-import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.HALF_FLOAT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED;
-import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
-import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.PARTIAL_AGG;
-import static org.elasticsearch.xpack.esql.core.type.DataType.SCALED_FLOAT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT;
-import static org.elasticsearch.xpack.esql.core.type.DataType.SOURCE;
-import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
-import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
-import static org.elasticsearch.xpack.esql.core.type.DataType.isNull;
-
-public final class EsqlDataTypes {
-
-    private EsqlDataTypes() {}
-
-    public static boolean isPrimitive(DataType t) {
-        return t != OBJECT && t != NESTED;
-    }
-
-    public static boolean isDateTimeOrTemporal(DataType t) {
-        return DataType.isDateTime(t) || isTemporalAmount(t);
-    }
-
-    public static boolean isTemporalAmount(DataType t) {
-        return t == DataType.DATE_PERIOD || t == DataType.TIME_DURATION;
-    }
-
-    public static boolean isNullOrTemporalAmount(DataType t) {
-        return isTemporalAmount(t) || isNull(t);
-    }
-
-    public static boolean isNullOrDatePeriod(DataType t) {
-        return t == DataType.DATE_PERIOD || isNull(t);
-    }
-
-    public static boolean isNullOrTimeDuration(DataType t) {
-        return t == DataType.TIME_DURATION || isNull(t);
-    }
-
-    public static boolean isSpatial(DataType t) {
-        return t == DataType.GEO_POINT || t == DataType.CARTESIAN_POINT || t == DataType.GEO_SHAPE || t == DataType.CARTESIAN_SHAPE;
-    }
-
-    public static boolean isSpatialGeo(DataType t) {
-        return t == DataType.GEO_POINT || t == DataType.GEO_SHAPE;
-    }
-
-    public static boolean isSpatialPoint(DataType t) {
-        return t == DataType.GEO_POINT || t == DataType.CARTESIAN_POINT;
-    }
-
-    /**
-     * Supported types that can be contained in a block.
-     */
-    public static boolean isRepresentable(DataType t) {
-        return t != OBJECT
-            && t != NESTED
-            && t != UNSUPPORTED
-            && t != DATE_PERIOD
-            && t != TIME_DURATION
-            && t != BYTE
-            && t != SHORT
-            && t != FLOAT
-            && t != SCALED_FLOAT
-            && t != SOURCE
-            && t != HALF_FLOAT
-            && t != PARTIAL_AGG
-            && t.isCounter() == false;
-    }
-
-    public static boolean areCompatible(DataType left, DataType right) {
-        if (left == right) {
-            return true;
-        } else {
-            return (left == NULL || right == NULL)
-                || (DataType.isString(left) && DataType.isString(right))
-                || (left.isNumeric() && right.isNumeric());
-        }
-    }
-}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
index 6a9b7a0e0089d..f30db1bf9bba2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
@@ -52,14 +52,12 @@
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
 import org.elasticsearch.xpack.esql.analysis.PreAnalyzer;
-import org.elasticsearch.xpack.esql.core.CsvSpecReader;
-import org.elasticsearch.xpack.esql.core.SpecReader;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupService;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext;
@@ -82,7 +80,7 @@
 import org.elasticsearch.xpack.esql.planner.TestPhysicalOperationProviders;
 import org.elasticsearch.xpack.esql.plugin.EsqlFeatures;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.session.EsqlSession;
 import org.elasticsearch.xpack.esql.session.Result;
 import org.elasticsearch.xpack.esql.stats.DisabledSearchStats;
@@ -102,6 +100,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 
+import static org.elasticsearch.xpack.esql.CsvSpecReader.specParser;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.ExpectedResults;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled;
 import static org.elasticsearch.xpack.esql.CsvTestUtils.loadCsvSpecValues;
@@ -111,7 +110,6 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.cap;
-import static org.elasticsearch.xpack.esql.core.CsvSpecReader.specParser;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.greaterThan;
@@ -127,7 +125,7 @@
  * The results used in these files were manually added by running the same query on a real (debug mode) ES node. CsvTestsDataLoader loads
  * the test data helping to get the said results.
  * <p>
- * CsvTestsDataLoader creates an index using the mapping in mapping-default.json. The same mapping file is also used to create the
+ * {@link CsvTestsDataLoader} creates an index using the mapping in mapping-default.json. The same mapping file is also used to create the
  * IndexResolver that helps validate the correctness of the query and the supported field data types.
  * The created index and this class uses the data from employees.csv file as data. This class is creating one Page with Blocks in it using
  * this file and the type of blocks matches the type of the schema specified on the first line of the csv file. These being said, the
@@ -159,7 +157,7 @@ public class CsvTests extends ESTestCase {
     private final CsvSpecReader.CsvTestCase testCase;
     private final String instructions;
 
-    private final EsqlConfiguration configuration = EsqlTestUtils.configuration(
+    private final Configuration configuration = EsqlTestUtils.configuration(
         new QueryPragmas(Settings.builder().put("page_size", randomPageSize()).build())
     );
     private final EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry();
@@ -237,21 +235,29 @@ public final void test() throws Throwable {
              */
             assumeFalse("metadata fields aren't supported", testCase.requiredCapabilities.contains(cap(EsqlFeatures.METADATA_FIELDS)));
             assumeFalse("enrich can't load fields in csv tests", testCase.requiredCapabilities.contains(cap(EsqlFeatures.ENRICH_LOAD)));
+            assumeFalse(
+                "can't use match in csv tests",
+                testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_OPERATOR.capabilityName())
+            );
             assumeFalse("can't load metrics in csv tests", testCase.requiredCapabilities.contains(cap(EsqlFeatures.METRICS_SYNTAX)));
             assumeFalse(
                 "multiple indices aren't supported",
                 testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.UNION_TYPES.capabilityName())
             );
+            assumeFalse(
+                "can't use match command in csv tests",
+                testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_COMMAND.capabilityName())
+            );
 
             if (Build.current().isSnapshot()) {
                 assertThat(
-                    "nonexistent capabilities declared as required",
+                    "Capability is not included in the enabled list capabilities on a snapshot build. Spelling mistake?",
                     testCase.requiredCapabilities,
                     everyItem(in(EsqlCapabilities.CAPABILITIES))
                 );
             } else {
                 for (EsqlCapabilities.Cap c : EsqlCapabilities.Cap.values()) {
-                    if (c.snapshotOnly()) {
+                    if (false == c.isEnabled()) {
                         assumeFalse(
                             c.capabilityName() + " is not supported in non-snapshot releases",
                             testCase.requiredCapabilities.contains(c.capabilityName())
@@ -286,7 +292,8 @@ private void doTest() throws Exception {
             assertWarnings(actualResults.responseHeaders().getOrDefault("Warning", List.of()));
         } finally {
             Releasables.close(() -> Iterators.map(actualResults.pages().iterator(), p -> p::releaseBlocks));
-            assertThat(bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L));
+            // Give the breaker service some time to clear in case we got results before the rest of the driver had cleaned up
+            assertBusy(() -> assertThat(bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)));
         }
     }
 
@@ -408,10 +415,10 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception {
 
         PlainActionFuture<ActualResults> listener = new PlainActionFuture<>();
 
-        session.executeAnalyzedPlan(
+        session.executeOptimizedPlan(
             new EsqlQueryRequest(),
             runPhase(bigArrays, physicalOperationProviders),
-            analyzed,
+            session.optimizedPlan(analyzed),
             listener.delegateFailureAndWrap(
                 // Wrap so we can capture the warnings in the calling thread
                 (next, result) -> next.onResponse(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
index 8c5a5a4b3ba3b..339e7159ed87d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.ExistsQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
@@ -26,7 +27,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
@@ -36,7 +36,7 @@
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -51,7 +51,7 @@ public static void assertSerialization(PhysicalPlan plan) {
         assertSerialization(plan, EsqlTestUtils.TEST_CFG);
     }
 
-    public static void assertSerialization(PhysicalPlan plan, EsqlConfiguration configuration) {
+    public static void assertSerialization(PhysicalPlan plan, Configuration configuration) {
         var deserPlan = serializeDeserialize(
             plan,
             PlanStreamOutput::writePhysicalPlanNode,
@@ -62,7 +62,7 @@ public static void assertSerialization(PhysicalPlan plan, EsqlConfiguration conf
     }
 
     public static void assertSerialization(LogicalPlan plan) {
-        var deserPlan = serializeDeserialize(plan, PlanStreamOutput::writeLogicalPlanNode, PlanStreamInput::readLogicalPlanNode);
+        var deserPlan = serializeDeserialize(plan, PlanStreamOutput::writeNamedWriteable, in -> in.readNamedWriteable(LogicalPlan.class));
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(plan, unused -> deserPlan);
     }
 
@@ -70,7 +70,7 @@ public static void assertSerialization(Expression expression) {
         assertSerialization(expression, EsqlTestUtils.TEST_CFG);
     }
 
-    public static void assertSerialization(Expression expression, EsqlConfiguration configuration) {
+    public static void assertSerialization(Expression expression, Configuration configuration) {
         Expression deserExpression = serializeDeserialize(
             expression,
             PlanStreamOutput::writeNamedWriteable,
@@ -84,7 +84,7 @@ public static <T> T serializeDeserialize(T orig, Serializer<T> serializer, Deser
         return serializeDeserialize(orig, serializer, deserializer, EsqlTestUtils.TEST_CFG);
     }
 
-    public static <T> T serializeDeserialize(T orig, Serializer<T> serializer, Deserializer<T> deserializer, EsqlConfiguration config) {
+    public static <T> T serializeDeserialize(T orig, Serializer<T> serializer, Deserializer<T> deserializer, Configuration config) {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             PlanStreamOutput planStreamOutput = new PlanStreamOutput(out, planNameRegistry, config);
             serializer.write(planStreamOutput, orig);
@@ -118,7 +118,6 @@ public static NamedWriteableRegistry writableRegistry() {
         entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, RegexpQueryBuilder.NAME, RegexpQueryBuilder::new));
         entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, ExistsQueryBuilder.NAME, ExistsQueryBuilder::new));
         entries.add(SingleValueQuery.ENTRY);
-        entries.addAll(EsField.getNamedWriteables());
         entries.addAll(Attribute.getNamedWriteables());
         entries.add(UnsupportedAttribute.ENTRY);
         entries.addAll(NamedExpression.getNamedWriteables());
@@ -126,6 +125,8 @@ public static NamedWriteableRegistry writableRegistry() {
         entries.addAll(Expression.getNamedWriteables());
         entries.addAll(EsqlScalarFunction.getNamedWriteables());
         entries.addAll(AggregateFunction.getNamedWriteables());
+        entries.addAll(Block.getNamedWriteables());
+        entries.addAll(LogicalPlan.getNamedWriteables());
         return new NamedWriteableRegistry(entries);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
index 890a611fdea10..eef2dbbb53362 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
@@ -93,7 +93,9 @@ public void testNamedParams() throws IOException {
 
         String paramsString = """
             ,"params":[ {"n1" : "8.15.0" }, { "n2" : 0.05 }, {"n3" : -799810013 },
-             {"n4" : "127.0.0.1"}, {"n5" : "esql"}, {"n_6" : null}, {"n7_" : false}] }""";
+             {"n4" : "127.0.0.1"}, {"n5" : "esql"}, {"n_6" : null}, {"n7_" : false},
+             {"_n1" : "8.15.0" }, { "__n2" : 0.05 }, {"__3" : -799810013 },
+             {"__4n" : "127.0.0.1"}, {"_n5" : "esql"}, {"_n6" : null}, {"_n7" : false}] }""";
         List<QueryParam> params = new ArrayList<>(4);
         params.add(new QueryParam("n1", "8.15.0", DataType.KEYWORD));
         params.add(new QueryParam("n2", 0.05, DataType.DOUBLE));
@@ -102,6 +104,13 @@ public void testNamedParams() throws IOException {
         params.add(new QueryParam("n5", "esql", DataType.KEYWORD));
         params.add(new QueryParam("n_6", null, DataType.NULL));
         params.add(new QueryParam("n7_", false, DataType.BOOLEAN));
+        params.add(new QueryParam("_n1", "8.15.0", DataType.KEYWORD));
+        params.add(new QueryParam("__n2", 0.05, DataType.DOUBLE));
+        params.add(new QueryParam("__3", -799810013, DataType.INTEGER));
+        params.add(new QueryParam("__4n", "127.0.0.1", DataType.KEYWORD));
+        params.add(new QueryParam("_n5", "esql", DataType.KEYWORD));
+        params.add(new QueryParam("_n6", null, DataType.NULL));
+        params.add(new QueryParam("_n7", false, DataType.BOOLEAN));
         String json = String.format(Locale.ROOT, """
             {
                 "query": "%s",
@@ -131,7 +140,7 @@ public void testInvalidParams() throws IOException {
         QueryBuilder filter = randomQueryBuilder();
 
         String paramsString1 = """
-            "params":[ {"1" : "v1" }, {"1x" : "v1" }, {"_a" : "v1" }, {"@-#" : "v1" }, 1, 2]""";
+            "params":[ {"1" : "v1" }, {"1x" : "v1" }, {"@a" : "v1" }, {"@-#" : "v1" }, 1, 2, {"_1" : "v1" }, {"Å" : 0}, {"x " : 0}]""";
         String json1 = String.format(Locale.ROOT, """
             {
                 %s
@@ -146,16 +155,20 @@ public void testInvalidParams() throws IOException {
             e1.getCause().getMessage(),
             containsString(
                 "Failed to parse params: [2:16] [1] is not a valid parameter name, "
-                    + "a valid parameter name starts with a letter and contains letters, digits and underscores only"
+                    + "a valid parameter name starts with a letter or underscore, and contains letters, digits and underscores only"
             )
         );
         assertThat(e1.getCause().getMessage(), containsString("[2:31] [1x] is not a valid parameter name"));
-        assertThat(e1.getCause().getMessage(), containsString("[2:47] [_a] is not a valid parameter name"));
+        assertThat(e1.getCause().getMessage(), containsString("[2:47] [@a] is not a valid parameter name"));
         assertThat(e1.getCause().getMessage(), containsString("[2:63] [@-#] is not a valid parameter name"));
+        assertThat(e1.getCause().getMessage(), containsString("[2:102] [Å] is not a valid parameter name"));
+        assertThat(e1.getCause().getMessage(), containsString("[2:113] [x ] is not a valid parameter name"));
+
         assertThat(
             e1.getCause().getMessage(),
             containsString(
-                "Params cannot contain both named and unnamed parameters; got [{1:v1}, {1x:v1}, {_a:v1}, {@-#:v1}] and [{1}, {2}]"
+                "Params cannot contain both named and unnamed parameters; "
+                    + "got [{1:v1}, {1x:v1}, {@a:v1}, {@-#:v1}, {_1:v1}, {Å:0}, {x :0}] and [{1}, {2}]"
             )
         );
 
@@ -175,7 +188,7 @@ public void testInvalidParams() throws IOException {
             e2.getCause().getMessage(),
             containsString(
                 "Failed to parse params: [2:22] [1] is not a valid parameter name, "
-                    + "a valid parameter name starts with a letter and contains letters, digits and underscores only"
+                    + "a valid parameter name starts with a letter or underscore, and contains letters, digits and underscores only"
             )
         );
         assertThat(e2.getCause().getMessage(), containsString("[2:37] [1x] is not a valid parameter name"));
@@ -302,6 +315,9 @@ public void testPragmasOnlyValidOnSnapshot() throws IOException {
         request.onSnapshotBuild(false);
         assertNotNull(request.validate());
         assertThat(request.validate().getMessage(), containsString("[pragma] only allowed in snapshot builds"));
+
+        request.acceptedPragmaRisks(true);
+        assertNull(request.validate());
     }
 
     public void testTablesKeyword() throws IOException {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseProfileTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseProfileTests.java
index 782e1fb4333d8..2f3aa09868637 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseProfileTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseProfileTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.operator.AbstractPageMappingOperator;
 import org.elasticsearch.compute.operator.DriverProfile;
+import org.elasticsearch.compute.operator.DriverSleeps;
 import org.elasticsearch.compute.operator.DriverStatus;
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
@@ -51,7 +52,10 @@ private DriverProfile randomDriverProfile() {
             randomNonNegativeLong(),
             randomNonNegativeLong(),
             randomNonNegativeLong(),
-            randomList(10, this::randomOperatorStatus)
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomList(10, this::randomOperatorStatus),
+            DriverSleeps.empty()
         );
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
index cff4d274dc49c..9d4a1c21c5995 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
@@ -27,9 +27,9 @@
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.Page;
-import org.elasticsearch.compute.lucene.UnsupportedValueSource;
 import org.elasticsearch.compute.operator.AbstractPageMappingOperator;
 import org.elasticsearch.compute.operator.DriverProfile;
+import org.elasticsearch.compute.operator.DriverSleeps;
 import org.elasticsearch.compute.operator.DriverStatus;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasables;
@@ -41,6 +41,7 @@
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ParserConstructor;
 import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -63,9 +64,14 @@
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xpack.esql.action.EsqlQueryResponse.DROP_NULL_COLUMNS_OPTION;
-import static org.elasticsearch.xpack.esql.action.ResponseValueUtils.valuesToPage;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateNanosToLong;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.longToUnsignedLong;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToIP;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToVersion;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -122,7 +128,10 @@ EsqlQueryResponse randomResponseAsync(boolean columnar, EsqlQueryResponse.Profil
 
     private ColumnInfoImpl randomColumnInfo() {
         DataType type = randomValueOtherThanMany(
-            t -> false == DataType.isPrimitive(t) || t == DataType.DATE_PERIOD || t == DataType.TIME_DURATION || t == DataType.PARTIAL_AGG,
+            t -> false == DataType.isPrimitiveAndSupported(t)
+                || t == DataType.DATE_PERIOD
+                || t == DataType.TIME_DURATION
+                || t == DataType.PARTIAL_AGG,
             () -> randomFrom(DataType.types())
         ).widenSmallNumeric();
         return new ColumnInfoImpl(randomAlphaOfLength(10), type.esType());
@@ -149,9 +158,9 @@ private Page randomPage(List<ColumnInfoImpl> columns) {
                 );
                 case DATETIME -> ((LongBlock.Builder) builder).appendLong(randomInstant().toEpochMilli());
                 case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean(randomBoolean());
-                case UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendBytesRef(
-                    new BytesRef(UnsupportedValueSource.UNSUPPORTED_OUTPUT)
-                );
+                case UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendNull();
+                // TODO - add a random instant thing here?
+                case DATE_NANOS -> ((LongBlock.Builder) builder).appendLong(randomLong());
                 case VERSION -> ((BytesRefBlock.Builder) builder).appendBytesRef(new Version(randomIdentifier()).toBytesRef());
                 case GEO_POINT -> ((BytesRefBlock.Builder) builder).appendBytesRef(GEO.asWkb(GeometryTestUtils.randomPoint()));
                 case CARTESIAN_POINT -> ((BytesRefBlock.Builder) builder).appendBytesRef(CARTESIAN.asWkb(ShapeTestUtils.randomPoint()));
@@ -254,6 +263,13 @@ protected EsqlQueryResponse doParseInstance(XContentParser parser) {
         return ResponseBuilder.fromXContent(parser);
     }
 
+    /**
+     * Used to test round tripping through x-content. Unlike lots of other
+     * response objects, ESQL doesn't have production code that can parse
+     * the response because it doesn't need it. But we want to test random
+     * responses are valid. This helps with that by parsing it into a
+     * response.
+     */
     public static class ResponseBuilder {
         private static final ParseField ID = new ParseField("id");
         private static final ParseField IS_RUNNING = new ParseField("is_running");
@@ -464,10 +480,13 @@ public void testProfileXContent() {
                 new EsqlQueryResponse.Profile(
                     List.of(
                         new DriverProfile(
+                            1723489812649L,
+                            1723489819929L,
                             20021,
                             20000,
                             12,
-                            List.of(new DriverStatus.OperatorStatus("asdf", new AbstractPageMappingOperator.Status(10021, 10)))
+                            List.of(new DriverStatus.OperatorStatus("asdf", new AbstractPageMappingOperator.Status(10021, 10))),
+                            DriverSleeps.empty()
                         )
                     )
                 ),
@@ -494,6 +513,8 @@ public void testProfileXContent() {
                   "profile" : {
                     "drivers" : [
                       {
+                        "start_millis" : 1723489812649,
+                        "stop_millis" : 1723489819929,
                         "took_nanos" : 20021,
                         "cpu_nanos" : 20000,
                         "iterations" : 12,
@@ -505,7 +526,12 @@ public void testProfileXContent() {
                               "pages_processed" : 10
                             }
                           }
-                        ]
+                        ],
+                        "sleeps" : {
+                          "counts" : { },
+                          "first" : [ ],
+                          "last" : [ ]
+                        }
                       }
                     ]
                   }
@@ -625,4 +651,61 @@ static List<Object> columnValues(Iterator<Object> values) {
         values.forEachRemaining(l::add);
         return l;
     }
+
+    /**
+     * Converts a list of values to Pages so that we can parse from xcontent, so we
+     * can test round tripping. This is functionally the inverse of {@link PositionToXContent}.
+     */
+    static Page valuesToPage(BlockFactory blockFactory, List<ColumnInfoImpl> columns, List<List<Object>> values) {
+        List<DataType> dataTypes = columns.stream().map(ColumnInfoImpl::type).toList();
+        List<Block.Builder> results = dataTypes.stream()
+            .map(c -> PlannerUtils.toElementType(c).newBlockBuilder(values.size(), blockFactory))
+            .toList();
+
+        for (List<Object> row : values) {
+            for (int c = 0; c < row.size(); c++) {
+                var builder = results.get(c);
+                var value = row.get(c);
+                switch (dataTypes.get(c)) {
+                    case UNSIGNED_LONG -> ((LongBlock.Builder) builder).appendLong(longToUnsignedLong(((Number) value).longValue(), true));
+                    case LONG, COUNTER_LONG -> ((LongBlock.Builder) builder).appendLong(((Number) value).longValue());
+                    case INTEGER, COUNTER_INTEGER -> ((IntBlock.Builder) builder).appendInt(((Number) value).intValue());
+                    case DOUBLE, COUNTER_DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble(((Number) value).doubleValue());
+                    case KEYWORD, TEXT -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(value.toString()));
+                    case UNSUPPORTED -> ((BytesRefBlock.Builder) builder).appendNull();
+                    case IP -> ((BytesRefBlock.Builder) builder).appendBytesRef(stringToIP(value.toString()));
+                    case DATETIME -> {
+                        long longVal = dateTimeToLong(value.toString());
+                        ((LongBlock.Builder) builder).appendLong(longVal);
+                    }
+                    case DATE_NANOS -> {
+                        long longVal = dateNanosToLong(value.toString());
+                        ((LongBlock.Builder) builder).appendLong(longVal);
+                    }
+                    case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean(((Boolean) value));
+                    case NULL -> builder.appendNull();
+                    case VERSION -> ((BytesRefBlock.Builder) builder).appendBytesRef(stringToVersion(new BytesRef(value.toString())));
+                    case SOURCE -> {
+                        @SuppressWarnings("unchecked")
+                        Map<String, ?> o = (Map<String, ?>) value;
+                        try {
+                            try (XContentBuilder sourceBuilder = JsonXContent.contentBuilder()) {
+                                sourceBuilder.map(o);
+                                ((BytesRefBlock.Builder) builder).appendBytesRef(BytesReference.bytes(sourceBuilder).toBytesRef());
+                            }
+                        } catch (IOException e) {
+                            throw new UncheckedIOException(e);
+                        }
+                    }
+                    case GEO_POINT, GEO_SHAPE, CARTESIAN_POINT, CARTESIAN_SHAPE -> {
+                        // This just converts WKT to WKB, so does not need CRS knowledge, we could merge GEO and CARTESIAN here
+                        BytesRef wkb = stringToSpatial(value.toString());
+                        ((BytesRefBlock.Builder) builder).appendBytesRef(wkb);
+                    }
+                }
+            }
+        }
+        return new Page(results.stream().map(Block.Builder::build).toArray(Block[]::new));
+    }
+
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java
index 7c5dc73fb62af..3e43293af41e2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java
@@ -9,14 +9,14 @@
 
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -51,7 +51,7 @@ public static Analyzer analyzer(IndexResolution indexResolution, Verifier verifi
         );
     }
 
-    public static Analyzer analyzer(IndexResolution indexResolution, Verifier verifier, EsqlConfiguration config) {
+    public static Analyzer analyzer(IndexResolution indexResolution, Verifier verifier, Configuration config) {
         return new Analyzer(new AnalyzerContext(config, new EsqlFunctionRegistry(), indexResolution, defaultEnrichResolution()), verifier);
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index 73aa9ee73bfb4..e4872b24558bc 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.esql.LoadMapping;
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
@@ -29,17 +30,16 @@
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
-import org.elasticsearch.xpack.esql.core.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.TypesTests;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Max;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Min;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.parser.ParsingException;
+import org.elasticsearch.xpack.esql.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
@@ -54,7 +54,6 @@
 import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.session.IndexResolver;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -130,7 +129,7 @@ public void testIndexWithClusterResolution() {
     }
 
     public void testAttributeResolution() {
-        EsIndex idx = new EsIndex("idx", TypesTests.loadMapping("mapping-one-field.json"));
+        EsIndex idx = new EsIndex("idx", LoadMapping.loadMapping("mapping-one-field.json"));
         Analyzer analyzer = analyzer(IndexResolution.valid(idx));
 
         var plan = analyzer.analyze(
@@ -633,30 +632,6 @@ public void testRenameReuseAlias() {
             """, "_meta_field", "e", "gender", "job", "job.raw", "languages", "last_name", "long_noidx", "salary");
     }
 
-    public void testRenameUnsupportedField() {
-        assertProjectionWithMapping("""
-            from test
-            | rename unsupported as u
-            | keep int, u, float
-            """, "mapping-multi-field-variation.json", "int", "u", "float");
-    }
-
-    public void testRenameUnsupportedFieldChained() {
-        assertProjectionWithMapping("""
-            from test
-            | rename unsupported as u1, u1 as u2
-            | keep int, u2, float
-            """, "mapping-multi-field-variation.json", "int", "u2", "float");
-    }
-
-    public void testRenameUnsupportedAndResolved() {
-        assertProjectionWithMapping("""
-            from test
-            | rename unsupported as u, float as f
-            | keep int, u, f
-            """, "mapping-multi-field-variation.json", "int", "u", "f");
-    }
-
     public void testRenameUnsupportedSubFieldAndResolved() {
         assertProjectionWithMapping("""
             from test
@@ -1456,8 +1431,8 @@ public void testRegexOnInt() {
 
     public void testUnsupportedTypesWithToString() {
         // DATE_PERIOD and TIME_DURATION types have been added, but not really patched through the engine; i.e. supported.
-        final String supportedTypes =
-            "boolean or cartesian_point or cartesian_shape or datetime or geo_point or geo_shape or ip or numeric or string or version";
+        final String supportedTypes = "boolean or cartesian_point or cartesian_shape or date_nanos or datetime "
+            + "or geo_point or geo_shape or ip or numeric or string or version";
         verifyUnsupported(
             "row period = 1 year | eval to_string(period)",
             "line 1:28: argument of [to_string(period)] must be [" + supportedTypes + "], found value [period] type [date_period]"
@@ -1541,7 +1516,7 @@ public void testEnrichWrongMatchFieldType() {
             | enrich languages on x
             | keep first_name, language_name, id
             """));
-        assertThat(e.getMessage(), containsString("Unsupported type [BOOLEAN] for enrich matching field [x]; only [KEYWORD,"));
+        assertThat(e.getMessage(), containsString("Unsupported type [boolean] for enrich matching field [x]; only [keyword, "));
 
         e = expectThrows(VerificationException.class, () -> analyze("""
             FROM airports
@@ -1549,7 +1524,7 @@ public void testEnrichWrongMatchFieldType() {
             | ENRICH city_boundaries ON x
             | KEEP abbrev, airport, region
             """, "airports", "mapping-airports.json"));
-        assertThat(e.getMessage(), containsString("Unsupported type [KEYWORD] for enrich matching field [x]; only [GEO_POINT,"));
+        assertThat(e.getMessage(), containsString("Unsupported type [keyword] for enrich matching field [x]; only [geo_point, "));
     }
 
     public void testValidEnrich() {
@@ -1834,14 +1809,14 @@ public void testUnsupportedTypesInStats() {
              found value [x] type [unsigned_long]
             line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long, _source, or counter types],\
              found value [x] type [unsigned_long]
-            line 2:39: argument of [max(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\
-             found value [max(x)] type [unsigned_long]
+            line 2:39: argument of [max(x)] must be [representable except unsigned_long and spatial types],\
+             found value [x] type [unsigned_long]
             line 2:47: argument of [median(x)] must be [numeric except unsigned_long or counter types],\
              found value [x] type [unsigned_long]
             line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\
              found value [x] type [unsigned_long]
-            line 2:88: argument of [min(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\
-             found value [min(x)] type [unsigned_long]
+            line 2:88: argument of [min(x)] must be [representable except unsigned_long and spatial types],\
+             found value [x] type [unsigned_long]
             line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long],\
              found value [x] type [unsigned_long]
             line 2:115: argument of [sum(x)] must be [numeric except unsigned_long or counter types],\
@@ -1849,21 +1824,17 @@ public void testUnsupportedTypesInStats() {
 
         verifyUnsupported("""
             row x = to_version("1.2")
-            | stats  avg(x), max(x), median(x), median_absolute_deviation(x), min(x), percentile(x, 10), sum(x)
+            | stats  avg(x), median(x), median_absolute_deviation(x), percentile(x, 10), sum(x)
             """, """
-            Found 7 problems
+            Found 5 problems
             line 2:10: argument of [avg(x)] must be [numeric except unsigned_long or counter types],\
              found value [x] type [version]
-            line 2:18: argument of [max(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\
-             found value [max(x)] type [version]
-            line 2:26: argument of [median(x)] must be [numeric except unsigned_long or counter types],\
+            line 2:18: argument of [median(x)] must be [numeric except unsigned_long or counter types],\
              found value [x] type [version]
-            line 2:37: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\
+            line 2:29: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\
              found value [x] type [version]
-            line 2:67: argument of [min(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\
-             found value [min(x)] type [version]
-            line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version]
-            line 2:94: argument of [sum(x)] must be [numeric except unsigned_long or counter types], found value [x] type [version]""");
+            line 2:59: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version]
+            line 2:78: argument of [sum(x)] must be [numeric except unsigned_long or counter types], found value [x] type [version]""");
     }
 
     public void testInOnText() {
@@ -2090,6 +2061,31 @@ public void testRateRequiresCounterTypes() {
         );
     }
 
+    public void testCoalesceWithMixedNumericTypes() {
+        LogicalPlan plan = analyze("""
+            from test
+            | eval x = coalesce(salary_change, null, 0), y = coalesce(languages, null, 0), z = coalesce(languages.long, null, 0)
+            , w = coalesce(salary_change, null, 0::long)
+            | keep x, y, z, w
+            """, "mapping-default.json");
+        var limit = as(plan, Limit.class);
+        var esqlProject = as(limit.child(), EsqlProject.class);
+        List<?> projections = esqlProject.projections();
+        var projection = as(projections.get(0), ReferenceAttribute.class);
+        assertEquals(projection.name(), "x");
+        assertEquals(projection.dataType(), DataType.DOUBLE);
+        projection = as(projections.get(1), ReferenceAttribute.class);
+        assertEquals(projection.name(), "y");
+        assertEquals(projection.dataType(), DataType.INTEGER);
+        projection = as(projections.get(2), ReferenceAttribute.class);
+        assertEquals(projection.name(), "z");
+        assertEquals(projection.dataType(), DataType.LONG);
+        projection = as(projections.get(3), ReferenceAttribute.class);
+        assertEquals(projection.name(), "w");
+        assertEquals(projection.dataType(), DataType.DOUBLE);
+        assertThat(limit.limit().fold(), equalTo(1000));
+    }
+
     private void verifyUnsupported(String query, String errorMessage) {
         verifyUnsupported(query, errorMessage, "mapping-multi-field-variation.json");
     }
@@ -2126,9 +2122,11 @@ protected List<String> filteredWarnings() {
     }
 
     private static LogicalPlan analyzeWithEmptyFieldCapsResponse(String query) throws IOException {
-        List<FieldCapabilitiesIndexResponse> idxResponses = List.of(new FieldCapabilitiesIndexResponse("idx", "idx", Map.of(), true));
+        List<FieldCapabilitiesIndexResponse> idxResponses = List.of(
+            new FieldCapabilitiesIndexResponse("idx", "idx", Map.of(), true, IndexMode.STANDARD)
+        );
         FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of());
-        IndexResolution resolution = new IndexResolver(null, EsqlDataTypeRegistry.INSTANCE).mergedMappings("test*", caps);
+        IndexResolution resolution = new IndexResolver(null).mergedMappings("test*", caps);
         var analyzer = analyzer(resolution, TEST_VERIFIER, configuration(query));
         return analyze(query, analyzer);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java
index 0231dc1f4a82b..8867e7425a92e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java
@@ -11,14 +11,14 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.esql.LoadMapping;
 import org.elasticsearch.xpack.esql.core.ParsingException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.TypesTests;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.Row;
@@ -123,6 +123,6 @@ private String error(String query) {
     }
 
     private static IndexResolution loadIndexResolution(String name) {
-        return IndexResolution.valid(new EsIndex(INDEX_NAME, TypesTests.loadMapping(name)));
+        return IndexResolution.valid(new EsIndex(INDEX_NAME, LoadMapping.loadMapping(name)));
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
index 57627b92a7d47..b50b801785a9f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
@@ -10,14 +10,23 @@
 import org.elasticsearch.Build;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.VerificationException;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
+import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.parser.QueryParam;
 import org.elasticsearch.xpack.esql.parser.QueryParams;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping;
@@ -46,6 +55,189 @@ public void testIncompatibleTypesInMathOperation() {
         );
     }
 
+    public void testUnsupportedAndMultiTypedFields() {
+        final String unsupported = "unsupported";
+        final String multiTyped = "multi_typed";
+
+        EsField unsupportedField = new UnsupportedEsField(unsupported, "flattened");
+        // Use linked maps/sets to fix the order in the error message.
+        LinkedHashSet<String> ipIndices = new LinkedHashSet<>();
+        ipIndices.add("test1");
+        ipIndices.add("test2");
+        ipIndices.add("test3");
+        ipIndices.add("test4");
+        ipIndices.add("test5");
+        LinkedHashMap<String, Set<String>> typesToIndices = new LinkedHashMap<>();
+        typesToIndices.put("ip", ipIndices);
+        typesToIndices.put("keyword", Set.of("test6"));
+        EsField multiTypedField = new InvalidMappedField(multiTyped, typesToIndices);
+
+        // Also add an unsupported/multityped field under the names `int` and `double` so we can use `LOOKUP int_number_names ...` and
+        // `LOOKUP double_number_names` without renaming the fields first.
+        IndexResolution indexWithUnsupportedAndMultiTypedField = IndexResolution.valid(
+            new EsIndex(
+                "test*",
+                Map.of(unsupported, unsupportedField, multiTyped, multiTypedField, "int", unsupportedField, "double", multiTypedField)
+            )
+        );
+        Analyzer analyzer = AnalyzerTestUtils.analyzer(indexWithUnsupportedAndMultiTypedField);
+
+        assertEquals(
+            "1:22: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | dissect unsupported \"%{foo}\"", analyzer)
+        );
+        assertEquals(
+            "1:22: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+            error("from test* | dissect multi_typed \"%{foo}\"", analyzer)
+        );
+
+        assertEquals(
+            "1:19: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | grok unsupported \"%{WORD:foo}\"", analyzer)
+        );
+        assertEquals(
+            "1:19: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+            error("from test* | grok multi_typed \"%{WORD:foo}\"", analyzer)
+        );
+
+        assertEquals(
+            "1:36: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | enrich client_cidr on unsupported", analyzer)
+        );
+        assertEquals(
+            "1:36: Unsupported type [unsupported] for enrich matching field [multi_typed];"
+                + " only [keyword, text, ip, long, integer, float, double, datetime] allowed for type [range]",
+            error("from test* | enrich client_cidr on multi_typed", analyzer)
+        );
+
+        assertEquals(
+            "1:23: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | eval x = unsupported", analyzer)
+        );
+        assertEquals(
+            "1:23: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+            error("from test* | eval x = multi_typed", analyzer)
+        );
+
+        assertEquals(
+            "1:32: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | eval x = to_lower(unsupported)", analyzer)
+        );
+        assertEquals(
+            "1:32: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+            error("from test* | eval x = to_lower(multi_typed)", analyzer)
+        );
+
+        assertEquals(
+            "1:32: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | stats count(1) by unsupported", analyzer)
+        );
+        assertEquals(
+            "1:32: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+            error("from test* | stats count(1) by multi_typed", analyzer)
+        );
+        if (EsqlCapabilities.Cap.INLINESTATS.isEnabled()) {
+            assertEquals(
+                "1:38: Cannot use field [unsupported] with unsupported type [flattened]",
+                error("from test* | inlinestats count(1) by unsupported", analyzer)
+            );
+            assertEquals(
+                "1:38: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                    + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+                error("from test* | inlinestats count(1) by multi_typed", analyzer)
+            );
+        }
+
+        assertEquals(
+            "1:27: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | stats values(unsupported)", analyzer)
+        );
+        assertEquals(
+            "1:27: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+            error("from test* | stats values(multi_typed)", analyzer)
+        );
+        if (EsqlCapabilities.Cap.INLINESTATS.isEnabled()) {
+            assertEquals(
+                "1:33: Cannot use field [unsupported] with unsupported type [flattened]",
+                error("from test* | inlinestats values(unsupported)", analyzer)
+            );
+            assertEquals(
+                "1:33: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                    + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+                error("from test* | inlinestats values(multi_typed)", analyzer)
+            );
+        }
+
+        assertEquals(
+            "1:27: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | stats values(unsupported)", analyzer)
+        );
+        assertEquals(
+            "1:27: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+            error("from test* | stats values(multi_typed)", analyzer)
+        );
+
+        if (EsqlCapabilities.Cap.LOOKUP_V4.isEnabled()) {
+            // LOOKUP with unsupported type
+            assertEquals(
+                "1:41: column type mismatch, table column was [integer] and original column was [unsupported]",
+                error("from test* | lookup int_number_names on int", analyzer)
+            );
+            // LOOKUP with multi-typed field
+            assertEquals(
+                "1:44: column type mismatch, table column was [double] and original column was [unsupported]",
+                error("from test* | lookup double_number_names on double", analyzer)
+            );
+        }
+
+        assertEquals(
+            "1:24: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | mv_expand unsupported", analyzer)
+        );
+        assertEquals(
+            "1:24: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+            error("from test* | mv_expand multi_typed", analyzer)
+        );
+
+        assertEquals(
+            "1:21: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | rename unsupported as x", analyzer)
+        );
+        assertEquals(
+            "1:21: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+            error("from test* | rename multi_typed as x", analyzer)
+        );
+
+        assertEquals(
+            "1:19: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | sort unsupported asc", analyzer)
+        );
+        assertEquals(
+            "1:19: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+            error("from test* | sort multi_typed desc", analyzer)
+        );
+
+        assertEquals(
+            "1:20: Cannot use field [unsupported] with unsupported type [flattened]",
+            error("from test* | where unsupported is null", analyzer)
+        );
+        assertEquals(
+            "1:20: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types:"
+                + " [ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
+            error("from test* | where multi_typed is not null", analyzer)
+        );
+    }
+
     public void testRoundFunctionInvalidInputs() {
         assertEquals(
             "1:31: first argument of [round(b, 3)] must be [numeric], found value [b] type [keyword]",
@@ -63,10 +255,30 @@ public void testRoundFunctionInvalidInputs() {
             "1:31: second argument of [round(a, 3.5)] must be [integer], found value [3.5] type [double]",
             error("row a = 1, b = \"c\" | eval x = round(a, 3.5)")
         );
+    }
+
+    public void testImplicitCastingErrorMessages() {
         assertEquals(
             "1:23: Cannot convert string [c] to [INTEGER], error [Cannot parse number [c]]",
             error("row a = round(123.45, \"c\")")
         );
+        assertEquals(
+            "1:27: Cannot convert string [c] to [DOUBLE], error [Cannot parse number [c]]",
+            error("row a = 1 | eval x = acos(\"c\")")
+        );
+        assertEquals(
+            "1:33: Cannot convert string [c] to [DOUBLE], error [Cannot parse number [c]]\n"
+                + "line 1:38: Cannot convert string [a] to [INTEGER], error [Cannot parse number [a]]",
+            error("row a = 1 | eval x = round(acos(\"c\"),\"a\")")
+        );
+        assertEquals(
+            "1:63: Cannot convert string [x] to [INTEGER], error [Cannot parse number [x]]",
+            error("row ip4 = to_ip(\"1.2.3.4\") | eval ip4_prefix = ip_prefix(ip4, \"x\", 0)")
+        );
+        assertEquals(
+            "1:42: Cannot convert string [a] to [DOUBLE], error [Cannot parse number [a]]",
+            error("ROW a=[3, 5, 1, 6] | EVAL avg_a = MV_AVG(\"a\")")
+        );
     }
 
     public void testAggsExpressionsInStatsAggs() {
@@ -182,6 +394,66 @@ public void testGroupingInsideGrouping() {
         );
     }
 
+    public void testInvalidBucketCalls() {
+        assertThat(
+            error("from test | stats max(emp_no) by bucket(emp_no, 5, \"2000-01-01\")"),
+            containsString(
+                "function expects exactly four arguments when the first one is of type [INTEGER] and the second of type [INTEGER]"
+            )
+        );
+
+        assertThat(
+            error("from test | stats max(emp_no) by bucket(emp_no, 1 week, \"2000-01-01\")"),
+            containsString(
+                "second argument of [bucket(emp_no, 1 week, \"2000-01-01\")] must be [numeric], found value [1 week] type [date_period]"
+            )
+        );
+
+        assertThat(
+            error("from test | stats max(emp_no) by bucket(hire_date, 5.5, \"2000-01-01\")"),
+            containsString(
+                "second argument of [bucket(hire_date, 5.5, \"2000-01-01\")] must be [integral, date_period or time_duration], "
+                    + "found value [5.5] type [double]"
+            )
+        );
+
+        assertThat(
+            error("from test | stats max(emp_no) by bucket(hire_date, 5, 1 day, 1 month)"),
+            containsString(
+                "third argument of [bucket(hire_date, 5, 1 day, 1 month)] must be [datetime or string], "
+                    + "found value [1 day] type [date_period]"
+            )
+        );
+
+        assertThat(
+            error("from test | stats max(emp_no) by bucket(hire_date, 5, \"2000-01-01\", 1 month)"),
+            containsString(
+                "fourth argument of [bucket(hire_date, 5, \"2000-01-01\", 1 month)] must be [datetime or string], "
+                    + "found value [1 month] type [date_period]"
+            )
+        );
+
+        assertThat(
+            error("from test | stats max(emp_no) by bucket(hire_date, 5, \"2000-01-01\")"),
+            containsString(
+                "function expects exactly four arguments when the first one is of type [DATETIME] and the second of type [INTEGER]"
+            )
+        );
+
+        assertThat(
+            error("from test | stats max(emp_no) by bucket(emp_no, \"5\")"),
+            containsString("second argument of [bucket(emp_no, \"5\")] must be [numeric], found value [\"5\"] type [keyword]")
+        );
+
+        assertThat(
+            error("from test | stats max(emp_no) by bucket(hire_date, \"5\")"),
+            containsString(
+                "second argument of [bucket(hire_date, \"5\")] must be [integral, date_period or time_duration], "
+                    + "found value [\"5\"] type [keyword]"
+            )
+        );
+    }
+
     public void testAggsWithInvalidGrouping() {
         assertEquals(
             "1:35: column [languages] cannot be used as an aggregate once declared in the STATS BY grouping key [l = languages % 3]",
@@ -313,7 +585,7 @@ public void testMixedNumericalNonConvertibleTypesInIn() {
     public void testUnsignedLongTypeMixInComparisons() {
         List<String> types = DataType.types()
             .stream()
-            .filter(dt -> dt.isNumeric() && EsqlDataTypes.isRepresentable(dt) && dt != UNSIGNED_LONG)
+            .filter(dt -> dt.isNumeric() && DataType.isRepresentable(dt) && dt != UNSIGNED_LONG)
             .map(DataType::typeName)
             .toList();
         for (var type : types) {
@@ -351,7 +623,7 @@ public void testUnsignedLongTypeMixInComparisons() {
     public void testUnsignedLongTypeMixInArithmetics() {
         List<String> types = DataType.types()
             .stream()
-            .filter(dt -> dt.isNumeric() && EsqlDataTypes.isRepresentable(dt) && dt != UNSIGNED_LONG)
+            .filter(dt -> dt.isNumeric() && DataType.isRepresentable(dt) && dt != UNSIGNED_LONG)
             .map(DataType::typeName)
             .toList();
         for (var type : types) {
@@ -494,8 +766,8 @@ public void testAggregateOnCounter() {
             error("FROM tests | STATS min(network.bytes_in)", tsdb),
             equalTo(
                 "1:20: argument of [min(network.bytes_in)] must be"
-                    + " [boolean, datetime, ip or numeric except unsigned_long or counter types],"
-                    + " found value [min(network.bytes_in)] type [counter_long]"
+                    + " [representable except unsigned_long and spatial types],"
+                    + " found value [network.bytes_in] type [counter_long]"
             )
         );
 
@@ -503,8 +775,8 @@ public void testAggregateOnCounter() {
             error("FROM tests | STATS max(network.bytes_in)", tsdb),
             equalTo(
                 "1:20: argument of [max(network.bytes_in)] must be"
-                    + " [boolean, datetime, ip or numeric except unsigned_long or counter types],"
-                    + " found value [max(network.bytes_in)] type [counter_long]"
+                    + " [representable except unsigned_long and spatial types],"
+                    + " found value [network.bytes_in] type [counter_long]"
             )
         );
 
@@ -536,9 +808,9 @@ public void testAggsResolutionWithUnresolvedGroupings() {
         );
         assertThat(error("FROM tests | STATS " + agg_func + "(foobar) by foobar"), matchesRegex("1:\\d+: Unknown column \\[foobar]"));
         assertThat(
-            error("FROM tests | STATS " + agg_func + "(foobar) by BUCKET(languages, 10)"),
+            error("FROM tests | STATS " + agg_func + "(foobar) by BUCKET(hire_date, 10)"),
             matchesRegex(
-                "1:\\d+: function expects exactly four arguments when the first one is of type \\[INTEGER]"
+                "1:\\d+: function expects exactly four arguments when the first one is of type \\[DATETIME]"
                     + " and the second of type \\[INTEGER]\n"
                     + "line 1:\\d+: Unknown column \\[foobar]"
             )
@@ -628,6 +900,168 @@ public void testWeightedAvg() {
         );
     }
 
+    public void testMatchInsideEval() throws Exception {
+        assumeTrue("Match operator is available just for snapshots", Build.current().isSnapshot());
+
+        assertEquals("1:36: EVAL does not support MATCH expressions", error("row title = \"brown fox\" | eval x = title match \"fox\" "));
+    }
+
+    public void testMatchFilter() throws Exception {
+        assumeTrue("Match operator is available just for snapshots", Build.current().isSnapshot());
+
+        assertEquals(
+            "1:63: MATCH requires a mapped index field, found [name]",
+            error("from test | eval name = concat(first_name, last_name) | where name match \"Anna\"")
+        );
+
+        assertEquals(
+            "1:19: MATCH requires a text or keyword field, but [salary] has type [integer]",
+            error("from test | where salary match \"100\"")
+        );
+
+        assertEquals(
+            "1:19: Invalid condition using MATCH",
+            error("from test | where first_name match \"Anna\" or starts_with(first_name, \"Anne\")")
+        );
+
+        assertEquals(
+            "1:51: Invalid condition using MATCH",
+            error("from test | eval new_salary = salary + 10 | where first_name match \"Anna\" OR new_salary > 100")
+        );
+
+        assertEquals(
+            "1:45: MATCH requires a mapped index field, found [fn]",
+            error("from test | rename first_name as fn | where fn match \"Anna\"")
+        );
+    }
+
+    public void testMatchCommand() throws Exception {
+        assumeTrue("skipping because MATCH_COMMAND is not enabled", EsqlCapabilities.Cap.MATCH_COMMAND.isEnabled());
+        assertEquals("1:24: MATCH cannot be used after LIMIT", error("from test | limit 10 | match \"Anna\""));
+        assertEquals("1:13: MATCH cannot be used after SHOW", error("show info | match \"8.16.0\""));
+        assertEquals("1:17: MATCH cannot be used after ROW", error("row a= \"Anna\" | match \"Anna\""));
+        assertEquals("1:26: MATCH cannot be used after EVAL", error("from test | eval z = 2 | match \"Anna\""));
+        assertEquals("1:43: MATCH cannot be used after DISSECT", error("from test | dissect first_name \"%{foo}\" | match \"Connection\""));
+        assertEquals("1:27: MATCH cannot be used after DROP", error("from test | drop emp_no | match \"Anna\""));
+        assertEquals("1:35: MATCH cannot be used after EVAL", error("from test | eval n = emp_no * 3 | match \"Anna\""));
+        assertEquals("1:44: MATCH cannot be used after GROK", error("from test | grok last_name \"%{WORD:foo}\" | match \"Anna\""));
+        assertEquals("1:27: MATCH cannot be used after KEEP", error("from test | keep emp_no | match \"Anna\""));
+
+        // TODO Keep adding tests for all unsupported commands
+    }
+
+    public void testCoalesceWithMixedNumericTypes() {
+        assertEquals(
+            "1:22: second argument of [coalesce(languages, height)] must be [integer], found value [height] type [double]",
+            error("from test | eval x = coalesce(languages, height)")
+        );
+        assertEquals(
+            "1:22: second argument of [coalesce(languages.long, height)] must be [long], found value [height] type [double]",
+            error("from test | eval x = coalesce(languages.long, height)")
+        );
+        assertEquals(
+            "1:22: second argument of [coalesce(salary, languages.long)] must be [integer], found value [languages.long] type [long]",
+            error("from test | eval x = coalesce(salary, languages.long)")
+        );
+        assertEquals(
+            "1:22: second argument of [coalesce(languages.short, height)] must be [integer], found value [height] type [double]",
+            error("from test | eval x = coalesce(languages.short, height)")
+        );
+        assertEquals(
+            "1:22: second argument of [coalesce(languages.byte, height)] must be [integer], found value [height] type [double]",
+            error("from test | eval x = coalesce(languages.byte, height)")
+        );
+        assertEquals(
+            "1:22: second argument of [coalesce(languages, height.float)] must be [integer], found value [height.float] type [double]",
+            error("from test | eval x = coalesce(languages, height.float)")
+        );
+        assertEquals(
+            "1:22: second argument of [coalesce(languages, height.scaled_float)] must be [integer], "
+                + "found value [height.scaled_float] type [double]",
+            error("from test | eval x = coalesce(languages, height.scaled_float)")
+        );
+        assertEquals(
+            "1:22: second argument of [coalesce(languages, height.half_float)] must be [integer], "
+                + "found value [height.half_float] type [double]",
+            error("from test | eval x = coalesce(languages, height.half_float)")
+        );
+
+        assertEquals(
+            "1:22: third argument of [coalesce(null, languages, height)] must be [integer], found value [height] type [double]",
+            error("from test | eval x = coalesce(null, languages, height)")
+        );
+        assertEquals(
+            "1:22: third argument of [coalesce(null, languages.long, height)] must be [long], found value [height] type [double]",
+            error("from test | eval x = coalesce(null, languages.long, height)")
+        );
+        assertEquals(
+            "1:22: third argument of [coalesce(null, salary, languages.long)] must be [integer], "
+                + "found value [languages.long] type [long]",
+            error("from test | eval x = coalesce(null, salary, languages.long)")
+        );
+        assertEquals(
+            "1:22: third argument of [coalesce(null, languages.short, height)] must be [integer], found value [height] type [double]",
+            error("from test | eval x = coalesce(null, languages.short, height)")
+        );
+        assertEquals(
+            "1:22: third argument of [coalesce(null, languages.byte, height)] must be [integer], found value [height] type [double]",
+            error("from test | eval x = coalesce(null, languages.byte, height)")
+        );
+        assertEquals(
+            "1:22: third argument of [coalesce(null, languages, height.float)] must be [integer], "
+                + "found value [height.float] type [double]",
+            error("from test | eval x = coalesce(null, languages, height.float)")
+        );
+        assertEquals(
+            "1:22: third argument of [coalesce(null, languages, height.scaled_float)] must be [integer], "
+                + "found value [height.scaled_float] type [double]",
+            error("from test | eval x = coalesce(null, languages, height.scaled_float)")
+        );
+        assertEquals(
+            "1:22: third argument of [coalesce(null, languages, height.half_float)] must be [integer], "
+                + "found value [height.half_float] type [double]",
+            error("from test | eval x = coalesce(null, languages, height.half_float)")
+        );
+
+        // counter
+        assertEquals(
+            "1:23: second argument of [coalesce(network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]",
+            error("FROM tests | eval x = coalesce(network.bytes_in, 0)", tsdb)
+        );
+
+        assertEquals(
+            "1:23: second argument of [coalesce(network.bytes_in, to_long(0))] must be [counter_long], "
+                + "found value [to_long(0)] type [long]",
+            error("FROM tests | eval x = coalesce(network.bytes_in, to_long(0))", tsdb)
+        );
+        assertEquals(
+            "1:23: second argument of [coalesce(network.bytes_in, 0.0)] must be [counter_long], found value [0.0] type [double]",
+            error("FROM tests | eval x = coalesce(network.bytes_in, 0.0)", tsdb)
+        );
+
+        assertEquals(
+            "1:23: third argument of [coalesce(null, network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]",
+            error("FROM tests | eval x = coalesce(null, network.bytes_in, 0)", tsdb)
+        );
+
+        assertEquals(
+            "1:23: third argument of [coalesce(null, network.bytes_in, to_long(0))] must be [counter_long], "
+                + "found value [to_long(0)] type [long]",
+            error("FROM tests | eval x = coalesce(null, network.bytes_in, to_long(0))", tsdb)
+        );
+        assertEquals(
+            "1:23: third argument of [coalesce(null, network.bytes_in, 0.0)] must be [counter_long], found value [0.0] type [double]",
+            error("FROM tests | eval x = coalesce(null, network.bytes_in, 0.0)", tsdb)
+        );
+    }
+
+    public void test() {
+        assertEquals(
+            "1:23: second argument of [coalesce(network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]",
+            error("FROM tests | eval x = coalesce(network.bytes_in, 0)", tsdb)
+        );
+    }
+
     private String error(String query) {
         return error(query, defaultAnalyzer);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java
index 9f81437bd1b77..ebad8e6e13b8c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -40,7 +41,6 @@
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.session.IndexResolver;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
 import org.junit.After;
 import org.junit.Before;
 
@@ -418,7 +418,7 @@ class TestEnrichPolicyResolver extends EnrichPolicyResolver {
             super(
                 mockClusterService(policies),
                 transports.get(cluster),
-                new IndexResolver(new FieldCapsClient(threadPool, aliases, mappings), EsqlDataTypeRegistry.INSTANCE)
+                new IndexResolver(new FieldCapsClient(threadPool, aliases, mappings))
             );
             this.policies = policies;
             this.cluster = cluster;
@@ -492,7 +492,7 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
                     var f = new IndexFieldCapabilities(e.getKey(), e.getValue(), false, false, false, false, null, Map.of());
                     fieldCaps.put(e.getKey(), f);
                 }
-                var indexResponse = new FieldCapabilitiesIndexResponse(alias, null, fieldCaps, true);
+                var indexResponse = new FieldCapabilitiesIndexResponse(alias, null, fieldCaps, true, IndexMode.STANDARD);
                 response = new FieldCapabilitiesResponse(List.of(indexResponse), List.of());
             } else {
                 response = new FieldCapabilitiesResponse(List.of(), List.of());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java
index 5e50af6a0d212..596ff2af5fb5a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java
@@ -7,81 +7,25 @@
 
 package org.elasticsearch.xpack.esql.expression;
 
-import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.test.AbstractWireTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.tree.Node;
 import org.elasticsearch.xpack.esql.expression.function.ReferenceAttributeTests;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
-import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
-import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
-import org.elasticsearch.xpack.esql.session.EsqlConfigurationSerializationTests;
-import org.junit.Before;
+import org.elasticsearch.xpack.esql.plan.AbstractNodeSerializationTests;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.sameInstance;
-
-public abstract class AbstractExpressionSerializationTests<T extends Expression> extends AbstractWireTestCase<T> {
-    /**
-     * We use a single random config for all serialization because it's pretty
-     * heavy to build, especially in {@link #testConcurrentSerialization()}.
-     */
-    private EsqlConfiguration config;
-
-    public static Source randomSource() {
-        int lineNumber = between(0, EXAMPLE_QUERY.length - 1);
-        int offset = between(0, EXAMPLE_QUERY[lineNumber].length() - 2);
-        int length = between(1, EXAMPLE_QUERY[lineNumber].length() - offset - 1);
-        String text = EXAMPLE_QUERY[lineNumber].substring(offset, offset + length);
-        return new Source(lineNumber + 1, offset, text);
-    }
 
+public abstract class AbstractExpressionSerializationTests<T extends Expression> extends AbstractNodeSerializationTests<T> {
     public static Expression randomChild() {
         return ReferenceAttributeTests.randomReferenceAttribute();
     }
 
-    @Override
-    protected final T copyInstance(T instance, TransportVersion version) throws IOException {
-        return copyInstance(
-            instance,
-            getNamedWriteableRegistry(),
-            (out, v) -> new PlanStreamOutput(out, new PlanNameRegistry(), config).writeNamedWriteable(v),
-            in -> {
-                PlanStreamInput pin = new PlanStreamInput(in, new PlanNameRegistry(), in.namedWriteableRegistry(), config);
-                @SuppressWarnings("unchecked")
-                T deser = (T) pin.readNamedWriteable(Expression.class);
-                if (alwaysEmptySource()) {
-                    assertThat(deser.source(), sameInstance(Source.EMPTY));
-                } else {
-                    assertThat(deser.source(), equalTo(instance.source()));
-                }
-                return deser;
-            },
-            version
-        );
-    }
-
-    protected boolean alwaysEmptySource() {
-        return false;
-    }
-
-    public EsqlConfiguration configuration() {
-        return config;
-    }
-
     @Override
     protected final NamedWriteableRegistry getNamedWriteableRegistry() {
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(NamedExpression.getNamedWriteables());
@@ -92,23 +36,12 @@ protected final NamedWriteableRegistry getNamedWriteableRegistry() {
         entries.add(UnsupportedAttribute.ENTRY);
         entries.add(UnsupportedAttribute.NAMED_EXPRESSION_ENTRY);
         entries.add(UnsupportedAttribute.EXPRESSION_ENTRY);
-        entries.addAll(EsField.getNamedWriteables());
         entries.add(org.elasticsearch.xpack.esql.expression.Order.ENTRY);
         return new NamedWriteableRegistry(entries);
     }
 
-    private static final String[] EXAMPLE_QUERY = new String[] {
-        "I am the very model of a modern Major-Gineral,",
-        "I've information vegetable, animal, and mineral,",
-        "I know the kings of England, and I quote the fights historical",
-        "From Marathon to Waterloo, in order categorical;",
-        "I'm very well acquainted, too, with matters mathematical,",
-        "I understand equations, both the simple and quadratical,",
-        "About binomial theorem I'm teeming with a lot o' news,",
-        "With many cheerful facts about the square of the hypotenuse." };
-
-    @Before
-    public void initConfig() {
-        config = EsqlConfigurationSerializationTests.randomConfiguration(String.join("\n", EXAMPLE_QUERY), Map.of());
+    @Override
+    protected Class<? extends Node<?>> categoryClass() {
+        return Expression.class;
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java
index a6808e835bc3c..2a6791a1f5300 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.tree.SourceTests;
-import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.expression.function.ReferenceAttributeTests;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
@@ -32,31 +31,32 @@
 import static org.hamcrest.Matchers.in;
 
 public class AliasTests extends AbstractWireTestCase<Alias> {
-    @Override
-    protected Alias createTestInstance() {
+    public static Alias randomAlias() {
         Source source = SourceTests.randomSource();
         String name = randomAlphaOfLength(5);
-        String qualifier = randomBoolean() ? null : randomAlphaOfLength(3);
         // TODO better randomChild
         Expression child = ReferenceAttributeTests.randomReferenceAttribute();
         boolean synthetic = randomBoolean();
-        return new Alias(source, name, qualifier, child, new NameId(), synthetic);
+        return new Alias(source, name, child, new NameId(), synthetic);
+    }
+
+    @Override
+    protected Alias createTestInstance() {
+        return randomAlias();
     }
 
     @Override
     protected Alias mutateInstance(Alias instance) throws IOException {
         Source source = instance.source();
         String name = instance.name();
-        String qualifier = instance.qualifier();
         Expression child = instance.child();
         boolean synthetic = instance.synthetic();
-        switch (between(0, 3)) {
+        switch (between(0, 2)) {
             case 0 -> name = randomAlphaOfLength(name.length() + 1);
-            case 1 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3));
-            case 2 -> child = randomValueOtherThan(child, ReferenceAttributeTests::randomReferenceAttribute);
-            case 3 -> synthetic = false == synthetic;
+            case 1 -> child = randomValueOtherThan(child, ReferenceAttributeTests::randomReferenceAttribute);
+            case 2 -> synthetic = false == synthetic;
         }
-        return new Alias(source, name, qualifier, child, instance.id(), synthetic);
+        return new Alias(source, name, child, instance.id(), synthetic);
     }
 
     @Override
@@ -80,7 +80,6 @@ protected final NamedWriteableRegistry getNamedWriteableRegistry() {
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(NamedExpression.getNamedWriteables());
         entries.addAll(Attribute.getNamedWriteables());
         entries.add(UnsupportedAttribute.ENTRY);
-        entries.addAll(EsField.getNamedWriteables());
         entries.addAll(Expression.getNamedWriteables());
         return new NamedWriteableRegistry(entries);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/OrderSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/OrderSerializationTests.java
index 3c5a77daf8832..4b14e453201f8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/OrderSerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/OrderSerializationTests.java
@@ -13,25 +13,29 @@
 import java.io.IOException;
 
 public class OrderSerializationTests extends AbstractExpressionSerializationTests<Order> {
+    public static Order randomOrder() {
+        return new Order(randomSource(), randomChild(), randomDirection(), randomNulls());
+    }
+
     @Override
     protected Order createTestInstance() {
-        return new Order(randomSource(), randomChild(), randomDirection(), randomNulls());
+        return randomOrder();
     }
 
-    private static org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection randomDirection() {
-        return randomFrom(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.values());
+    private static Order.OrderDirection randomDirection() {
+        return randomFrom(Order.OrderDirection.values());
     }
 
-    private static org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition randomNulls() {
-        return randomFrom(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.values());
+    private static Order.NullsPosition randomNulls() {
+        return randomFrom(Order.NullsPosition.values());
     }
 
     @Override
     protected Order mutateInstance(Order instance) throws IOException {
         Source source = instance.source();
         Expression child = instance.child();
-        org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection direction = instance.direction();
-        org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition nulls = instance.nullsPosition();
+        Order.OrderDirection direction = instance.direction();
+        Order.NullsPosition nulls = instance.nullsPosition();
         switch (between(0, 2)) {
             case 0 -> child = randomValueOtherThan(child, AbstractExpressionSerializationTests::randomChild);
             case 1 -> direction = randomValueOtherThan(direction, OrderSerializationTests::randomDirection);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java
index 25ff4f9c2122d..f3c87e0e9d1d7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java
@@ -57,6 +57,25 @@ public abstract class AbstractAggregationTestCase extends AbstractFunctionTestCa
      *     Use if possible, as this method may get updated with new checks in the future.
      * </p>
      */
+    protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecks(
+        List<TestCaseSupplier> suppliers,
+        boolean entirelyNullPreservesType,
+        PositionalErrorMessageSupplier positionalErrorMessageSupplier
+    ) {
+        return parameterSuppliersFromTypedData(
+            errorsForCasesWithoutExamples(
+                withNoRowsExpectingNull(anyNullIsNull(entirelyNullPreservesType, randomizeBytesRefsOffset(suppliers))),
+                positionalErrorMessageSupplier
+            )
+        );
+    }
+
+    // TODO: Remove and migrate everything to the method with all the parameters
+    /**
+     * @deprecated Use {@link #parameterSuppliersFromTypedDataWithDefaultChecks(List, boolean, PositionalErrorMessageSupplier)} instead.
+     * This method doesn't add all the default checks.
+     */
+    @Deprecated
     protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecks(List<TestCaseSupplier> suppliers) {
         return parameterSuppliersFromTypedData(withNoRowsExpectingNull(randomizeBytesRefsOffset(suppliers)));
     }
@@ -119,24 +138,9 @@ public void testFold() {
         Expression expression = buildLiteralExpression(testCase);
 
         resolveExpression(expression, aggregatorFunctionSupplier -> {
-            // An aggregation cannot be folded
-        }, evaluableExpression -> {
-            assertTrue(evaluableExpression.foldable());
-            if (testCase.foldingExceptionClass() == null) {
-                Object result = evaluableExpression.fold();
-                // Decode unsigned longs into BigIntegers
-                if (testCase.expectedType() == DataType.UNSIGNED_LONG && result != null) {
-                    result = NumericUtils.unsignedLongAsBigInteger((Long) result);
-                }
-                assertThat(result, testCase.getMatcher());
-                if (testCase.getExpectedWarnings() != null) {
-                    assertWarnings(testCase.getExpectedWarnings());
-                }
-            } else {
-                Throwable t = expectThrows(testCase.foldingExceptionClass(), evaluableExpression::fold);
-                assertThat(t.getMessage(), equalTo(testCase.foldingExceptionMessage()));
-            }
-        });
+            // An aggregation cannot be folded.
+            // It's not an error either as not all aggregations are foldable.
+        }, this::evaluate);
     }
 
     private void aggregateSingleMode(Expression expression) {
@@ -263,13 +267,19 @@ private void aggregateWithIntermediates(Expression expression) {
     }
 
     private void evaluate(Expression evaluableExpression) {
-        Object result;
-        try (var evaluator = evaluator(evaluableExpression).get(driverContext())) {
-            try (Block block = evaluator.eval(row(testCase.getDataValues()))) {
-                result = toJavaObjectUnsignedLongAware(block, 0);
-            }
+        assertTrue(evaluableExpression.foldable());
+
+        if (testCase.foldingExceptionClass() != null) {
+            Throwable t = expectThrows(testCase.foldingExceptionClass(), evaluableExpression::fold);
+            assertThat(t.getMessage(), equalTo(testCase.foldingExceptionMessage()));
+            return;
         }
 
+        Object result = evaluableExpression.fold();
+        // Decode unsigned longs into BigIntegers
+        if (testCase.expectedType() == DataType.UNSIGNED_LONG && result != null) {
+            result = NumericUtils.unsignedLongAsBigInteger((Long) result);
+        }
         assertThat(result, not(equalTo(Double.NaN)));
         assert testCase.getMatcher().matches(Double.POSITIVE_INFINITY) == false;
         assertThat(result, not(equalTo(Double.POSITIVE_INFINITY)));
@@ -291,14 +301,15 @@ private void resolveExpression(Expression expression, Consumer<Expression> onAgg
         }
         expression = resolveSurrogates(expression);
 
+        // As expressions may be composed of multiple functions, we need to fold nulls bottom-up
+        expression = expression.transformUp(e -> new FoldNull().rule(e));
+        assertThat(expression.dataType(), equalTo(testCase.expectedType()));
+
         Expression.TypeResolution resolution = expression.typeResolved();
         if (resolution.unresolved()) {
             throw new AssertionError("expected resolved " + resolution.message());
         }
 
-        expression = new FoldNull().rule(expression);
-        assertThat(expression.dataType(), equalTo(testCase.expectedType()));
-
         assumeTrue(
             "Surrogate expression with non-trivial children cannot be evaluated",
             expression.children()
@@ -435,16 +446,23 @@ private IntBlock makeGroupsVector(int groupStart, int groupEnd, int rowCount) {
      */
     private void processPageGrouping(GroupingAggregator aggregator, Page inputPage, int groupCount) {
         var groupSliceSize = 1;
+        var allValuesNull = IntStream.range(0, inputPage.getBlockCount())
+            .<Block>mapToObj(inputPage::getBlock)
+            .anyMatch(Block::areAllValuesNull);
         // Add data to chunks of groups
         for (int currentGroupOffset = 0; currentGroupOffset < groupCount;) {
-            var seenGroupIds = new SeenGroupIds.Range(0, currentGroupOffset + groupSliceSize);
+            int groupSliceRemainingSize = Math.min(groupSliceSize, groupCount - currentGroupOffset);
+            var seenGroupIds = new SeenGroupIds.Range(0, allValuesNull ? 0 : currentGroupOffset + groupSliceRemainingSize);
             var addInput = aggregator.prepareProcessPage(seenGroupIds, inputPage);
 
             var positionCount = inputPage.getPositionCount();
             var dataSliceSize = 1;
             // Divide data in chunks
             for (int currentDataOffset = 0; currentDataOffset < positionCount;) {
-                try (var groups = makeGroupsVector(currentGroupOffset, currentGroupOffset + groupSliceSize, dataSliceSize)) {
+                int dataSliceRemainingSize = Math.min(dataSliceSize, positionCount - currentDataOffset);
+                try (
+                    var groups = makeGroupsVector(currentGroupOffset, currentGroupOffset + groupSliceRemainingSize, dataSliceRemainingSize)
+                ) {
                     addInput.add(currentDataOffset, groups);
                 }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAttributeTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAttributeTestCase.java
index 17dcab2048eb1..76b813f08d818 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAttributeTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAttributeTestCase.java
@@ -8,28 +8,34 @@
 package org.elasticsearch.xpack.esql.expression.function;
 
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
-import org.elasticsearch.xpack.esql.session.EsqlConfigurationSerializationTests;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 
 import static org.hamcrest.Matchers.sameInstance;
 
 public abstract class AbstractAttributeTestCase<T extends Attribute> extends AbstractWireSerializingTestCase<
     AbstractAttributeTestCase.ExtraAttribute> {
+
+    /**
+     * We use a single random config for all serialization because it's pretty
+     * heavy to build, especially in {@link #testConcurrentSerialization()}.
+     */
+    private Configuration config;
+
     protected abstract T create();
 
     protected abstract T mutate(T instance);
@@ -49,13 +55,16 @@ protected final ExtraAttribute mutateInstance(ExtraAttribute instance) {
     protected final NamedWriteableRegistry getNamedWriteableRegistry() {
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(Attribute.getNamedWriteables());
         entries.add(UnsupportedAttribute.ENTRY);
-        entries.addAll(EsField.getNamedWriteables());
         return new NamedWriteableRegistry(entries);
     }
 
     @Override
     protected final Writeable.Reader<ExtraAttribute> instanceReader() {
-        return ExtraAttribute::new;
+        return in -> {
+            PlanStreamInput pin = new PlanStreamInput(in, PlanNameRegistry.INSTANCE, in.namedWriteableRegistry(), config);
+            pin.setTransportVersion(in.getTransportVersion());
+            return new ExtraAttribute(pin);
+        };
     }
 
     /**
@@ -69,20 +78,13 @@ public static class ExtraAttribute implements Writeable {
             assertThat(a.source(), sameInstance(Source.EMPTY));
         }
 
-        ExtraAttribute(StreamInput in) throws IOException {
-            PlanStreamInput ps = new PlanStreamInput(
-                in,
-                PlanNameRegistry.INSTANCE,
-                in.namedWriteableRegistry(),
-                EsqlConfigurationSerializationTests.randomConfiguration("", Map.of())
-            );
-            ps.setTransportVersion(in.getTransportVersion());
-            a = ps.readNamedWriteable(Attribute.class);
+        ExtraAttribute(PlanStreamInput in) throws IOException {
+            a = in.readNamedWriteable(Attribute.class);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeNamedWriteable(a);
+            new PlanStreamOutput(out, new PlanNameRegistry(), EsqlTestUtils.TEST_CFG).writeNamedWriteable(a);
         }
 
         @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
index e1d48a58aaed4..c79f5ab8d086b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
@@ -10,12 +10,9 @@
 import com.carrotsearch.randomizedtesting.ClassModel;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.apache.lucene.document.InetAddressPoint;
-import org.apache.lucene.sandbox.document.HalfFloatPoint;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
@@ -27,8 +24,6 @@
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.core.PathUtils;
-import org.elasticsearch.geo.GeometryTestUtils;
-import org.elasticsearch.geo.ShapeTestUtils;
 import org.elasticsearch.indices.CrankyCircuitBreakerService;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
@@ -40,28 +35,48 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
+import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
+import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
+import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest;
+import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.esql.optimizer.FoldNull;
 import org.elasticsearch.xpack.esql.parser.ExpressionBuilder;
 import org.elasticsearch.xpack.esql.planner.Layout;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
-import org.elasticsearch.xpack.versionfield.Version;
+import org.elasticsearch.xpack.esql.session.Configuration;
+import org.hamcrest.Matcher;
 import org.junit.After;
 import org.junit.AfterClass;
 
 import java.io.IOException;
-import java.io.UncheckedIOException;
+import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.math.BigInteger;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.time.Duration;
-import java.time.Period;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -73,62 +88,47 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
-import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
 
+import static java.util.Map.entry;
 import static org.elasticsearch.compute.data.BlockUtils.toJavaObject;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization;
-import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
-import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  * Base class for function tests.
  */
 public abstract class AbstractFunctionTestCase extends ESTestCase {
     /**
-     * Generate a random value of the appropriate type to fit into blocks of {@code e}.
+     * Operators are unregistered functions.
      */
-    public static Literal randomLiteral(DataType type) {
-        return new Literal(Source.EMPTY, switch (type) {
-            case BOOLEAN -> randomBoolean();
-            case BYTE -> randomByte();
-            case SHORT -> randomShort();
-            case INTEGER, COUNTER_INTEGER -> randomInt();
-            case UNSIGNED_LONG, LONG, COUNTER_LONG -> randomLong();
-            case DATE_PERIOD -> Period.of(randomIntBetween(-1000, 1000), randomIntBetween(-13, 13), randomIntBetween(-32, 32));
-            case DATETIME -> randomMillisUpToYear9999();
-            case DOUBLE, SCALED_FLOAT, COUNTER_DOUBLE -> randomDouble();
-            case FLOAT -> randomFloat();
-            case HALF_FLOAT -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat()));
-            case KEYWORD -> new BytesRef(randomAlphaOfLength(5));
-            case IP -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())));
-            case TIME_DURATION -> Duration.ofMillis(randomLongBetween(-604800000L, 604800000L)); // plus/minus 7 days
-            case TEXT -> new BytesRef(randomAlphaOfLength(50));
-            case VERSION -> randomVersion().toBytesRef();
-            case GEO_POINT -> GEO.asWkb(GeometryTestUtils.randomPoint());
-            case CARTESIAN_POINT -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint());
-            case GEO_SHAPE -> GEO.asWkb(GeometryTestUtils.randomGeometry(randomBoolean()));
-            case CARTESIAN_SHAPE -> CARTESIAN.asWkb(ShapeTestUtils.randomGeometry(randomBoolean()));
-            case NULL -> null;
-            case SOURCE -> {
-                try {
-                    yield BytesReference.bytes(
-                        JsonXContent.contentBuilder().startObject().field(randomAlphaOfLength(3), randomAlphaOfLength(10)).endObject()
-                    ).toBytesRef();
-                } catch (IOException e) {
-                    throw new UncheckedIOException(e);
-                }
-            }
-            case UNSUPPORTED, OBJECT, NESTED, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException(
-                "can't make random values for [" + type.typeName() + "]"
-            );
-        }, type);
-    }
+    private static final Map<String, Class<?>> OPERATORS = Map.ofEntries(
+        entry("in", In.class),
+        entry("like", WildcardLike.class),
+        entry("rlike", RLike.class),
+        entry("equals", Equals.class),
+        entry("not_equals", NotEquals.class),
+        entry("greater_than", GreaterThan.class),
+        entry("greater_than_or_equal", GreaterThanOrEqual.class),
+        entry("less_than", LessThan.class),
+        entry("less_than_or_equal", LessThanOrEqual.class),
+        entry("add", Add.class),
+        entry("sub", Sub.class),
+        entry("mul", Mul.class),
+        entry("div", Div.class),
+        entry("mod", Mod.class),
+        entry("neg", Neg.class),
+        entry("is_null", IsNull.class),
+        entry("is_not_null", IsNotNull.class)
+    );
 
     protected TestCaseSupplier.TestCase testCase;
 
@@ -147,6 +147,326 @@ protected static Iterable<Object[]> parameterSuppliersFromTypedData(List<TestCas
         return parameters;
     }
 
+    /**
+     * Adds cases with {@code null} and asserts that the result is {@code null}.
+     * <p>
+     * Note: This won't add more than a single null to any existing test case,
+     * just to keep the number of test cases from exploding totally.
+     * </p>
+     *
+     * @param entirelyNullPreservesType should a test case that only contains parameters
+     *                                  with the {@code null} type keep it's expected type?
+     *                                  This is <strong>mostly</strong> going to be {@code true}
+     *                                  except for functions that base their type entirely
+     *                                  on input types like {@link Greatest} or {@link Coalesce}.
+     */
+    protected static List<TestCaseSupplier> anyNullIsNull(boolean entirelyNullPreservesType, List<TestCaseSupplier> testCaseSuppliers) {
+        return anyNullIsNull(
+            testCaseSuppliers,
+            (nullPosition, nullValueDataType, original) -> entirelyNullPreservesType == false
+                && nullValueDataType == DataType.NULL
+                && original.getData().size() == 1 ? DataType.NULL : original.expectedType(),
+            (nullPosition, nullData, original) -> original
+        );
+    }
+
+    public interface ExpectedType {
+        DataType expectedType(int nullPosition, DataType nullValueDataType, TestCaseSupplier.TestCase original);
+    }
+
+    public interface ExpectedEvaluatorToString {
+        Matcher<String> evaluatorToString(int nullPosition, TestCaseSupplier.TypedData nullData, Matcher<String> original);
+    }
+
+    protected static List<TestCaseSupplier> anyNullIsNull(
+        List<TestCaseSupplier> testCaseSuppliers,
+        ExpectedType expectedType,
+        ExpectedEvaluatorToString evaluatorToString
+    ) {
+        typesRequired(testCaseSuppliers);
+        List<TestCaseSupplier> suppliers = new ArrayList<>(testCaseSuppliers.size());
+        suppliers.addAll(testCaseSuppliers);
+
+        /*
+         * For each original test case, add as many copies as there were
+         * arguments, replacing one of the arguments with null and keeping
+         * the others.
+         *
+         * Also, if this was the first time we saw the signature we copy it
+         * *again*, replacing the argument with null, but annotating the
+         * argument's type as `null` explicitly.
+         */
+        Set<List<DataType>> uniqueSignatures = new HashSet<>();
+        for (TestCaseSupplier original : testCaseSuppliers) {
+            boolean firstTimeSeenSignature = uniqueSignatures.add(original.types());
+            for (int nullPosition = 0; nullPosition < original.types().size(); nullPosition++) {
+                int finalNullPosition = nullPosition;
+                suppliers.add(new TestCaseSupplier(original.name() + " null in " + nullPosition, original.types(), () -> {
+                    TestCaseSupplier.TestCase oc = original.get();
+                    List<TestCaseSupplier.TypedData> data = IntStream.range(0, oc.getData().size()).mapToObj(i -> {
+                        TestCaseSupplier.TypedData od = oc.getData().get(i);
+                        if (i != finalNullPosition) {
+                            return od;
+                        }
+                        return od.withData(od.isMultiRow() ? Collections.singletonList(null) : null);
+                    }).toList();
+                    TestCaseSupplier.TypedData nulledData = oc.getData().get(finalNullPosition);
+                    return new TestCaseSupplier.TestCase(
+                        data,
+                        evaluatorToString.evaluatorToString(finalNullPosition, nulledData, oc.evaluatorToString()),
+                        expectedType.expectedType(finalNullPosition, nulledData.type(), oc),
+                        nullValue(),
+                        null,
+                        oc.getExpectedTypeError(),
+                        null,
+                        null
+                    );
+                }));
+
+                if (firstTimeSeenSignature) {
+                    List<DataType> typesWithNull = IntStream.range(0, original.types().size())
+                        .mapToObj(i -> i == finalNullPosition ? DataType.NULL : original.types().get(i))
+                        .toList();
+                    boolean newSignature = uniqueSignatures.add(typesWithNull);
+                    if (newSignature) {
+                        suppliers.add(new TestCaseSupplier(typesWithNull, () -> {
+                            TestCaseSupplier.TestCase oc = original.get();
+                            List<TestCaseSupplier.TypedData> data = IntStream.range(0, oc.getData().size())
+                                .mapToObj(
+                                    i -> i == finalNullPosition
+                                        ? (oc.getData().get(i).isMultiRow()
+                                            ? TestCaseSupplier.TypedData.MULTI_ROW_NULL
+                                            : TestCaseSupplier.TypedData.NULL)
+                                        : oc.getData().get(i)
+                                )
+                                .toList();
+                            return new TestCaseSupplier.TestCase(
+                                data,
+                                equalTo("LiteralsEvaluator[lit=null]"),
+                                expectedType.expectedType(finalNullPosition, DataType.NULL, oc),
+                                nullValue(),
+                                null,
+                                oc.getExpectedTypeError(),
+                                null,
+                                null
+                            );
+                        }));
+                    }
+                }
+            }
+        }
+
+        return suppliers;
+    }
+
+    @FunctionalInterface
+    protected interface PositionalErrorMessageSupplier {
+        /**
+         * This interface defines functions to supply error messages for incorrect types in specific positions. Functions which have
+         * the same type requirements for all positions can simplify this with a lambda returning a string constant.
+         *
+         * @param validForPosition - the set of {@link DataType}s that the test infrastructure believes to be allowable in the
+         *                         given position.
+         * @param position - the zero-index position in the list of parameters the function has detected the bad argument to be.
+         * @return The string describing the acceptable parameters for that position.  Note that this function should not return
+         *         the full error string; that will be constructed by the test.  Just return the type string for that position.
+         */
+        String apply(Set<DataType> validForPosition, int position);
+    }
+
+    /**
+     * Adds test cases containing unsupported parameter types that assert
+     * that they throw type errors.
+     */
+    protected static List<TestCaseSupplier> errorsForCasesWithoutExamples(
+        List<TestCaseSupplier> testCaseSuppliers,
+        PositionalErrorMessageSupplier positionalErrorMessageSupplier
+    ) {
+        return errorsForCasesWithoutExamples(testCaseSuppliers, (i, v, t) -> typeErrorMessage(i, v, t, positionalErrorMessageSupplier));
+    }
+
+    /**
+     * Build the expected error message for an invalid type signature.
+     */
+    protected static String typeErrorMessage(
+        boolean includeOrdinal,
+        List<Set<DataType>> validPerPosition,
+        List<DataType> types,
+        PositionalErrorMessageSupplier expectedTypeSupplier
+    ) {
+        int badArgPosition = -1;
+        for (int i = 0; i < types.size(); i++) {
+            if (validPerPosition.get(i).contains(types.get(i)) == false) {
+                badArgPosition = i;
+                break;
+            }
+        }
+        if (badArgPosition == -1) {
+            throw new IllegalStateException(
+                "Can't generate error message for these types, you probably need a custom error message function"
+            );
+        }
+        String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : "";
+        String expectedTypeString = expectedTypeSupplier.apply(validPerPosition.get(badArgPosition), badArgPosition);
+        String name = types.get(badArgPosition).typeName();
+        return ordinal + "argument of [] must be [" + expectedTypeString + "], found value [" + name + "] type [" + name + "]";
+    }
+
+    @FunctionalInterface
+    protected interface TypeErrorMessageSupplier {
+        String apply(boolean includeOrdinal, List<Set<DataType>> validPerPosition, List<DataType> types);
+    }
+
+    protected static List<TestCaseSupplier> errorsForCasesWithoutExamples(
+        List<TestCaseSupplier> testCaseSuppliers,
+        TypeErrorMessageSupplier typeErrorMessageSupplier
+    ) {
+        typesRequired(testCaseSuppliers);
+        List<TestCaseSupplier> suppliers = new ArrayList<>(testCaseSuppliers.size());
+        suppliers.addAll(testCaseSuppliers);
+
+        Set<List<DataType>> valid = testCaseSuppliers.stream().map(TestCaseSupplier::types).collect(Collectors.toSet());
+        List<Set<DataType>> validPerPosition = validPerPosition(valid);
+
+        testCaseSuppliers.stream()
+            .map(s -> s.types().size())
+            .collect(Collectors.toSet())
+            .stream()
+            .flatMap(count -> allPermutations(count))
+            .filter(types -> valid.contains(types) == false)
+            /*
+             * Skip any cases with more than one null. Our tests don't generate
+             * the full combinatorial explosions of all nulls - just a single null.
+             * Hopefully <null>, <null> cases will function the same as <null>, <valid>
+             * cases.
+             */.filter(types -> types.stream().filter(t -> t == DataType.NULL).count() <= 1)
+            .map(types -> typeErrorSupplier(validPerPosition.size() != 1, validPerPosition, types, typeErrorMessageSupplier))
+            .forEach(suppliers::add);
+        return suppliers;
+    }
+
+    private static List<DataType> append(List<DataType> orig, DataType extra) {
+        List<DataType> longer = new ArrayList<>(orig.size() + 1);
+        longer.addAll(orig);
+        longer.add(extra);
+        return longer;
+    }
+
+    protected static Stream<DataType> representable() {
+        return DataType.types().stream().filter(DataType::isRepresentable);
+    }
+
+    protected static TestCaseSupplier typeErrorSupplier(
+        boolean includeOrdinal,
+        List<Set<DataType>> validPerPosition,
+        List<DataType> types,
+        PositionalErrorMessageSupplier errorMessageSupplier
+    ) {
+        return typeErrorSupplier(includeOrdinal, validPerPosition, types, (o, v, t) -> typeErrorMessage(o, v, t, errorMessageSupplier));
+    }
+
+    /**
+     * Build a test case that asserts that the combination of parameter types is an error.
+     */
+    protected static TestCaseSupplier typeErrorSupplier(
+        boolean includeOrdinal,
+        List<Set<DataType>> validPerPosition,
+        List<DataType> types,
+        TypeErrorMessageSupplier errorMessageSupplier
+    ) {
+        return new TestCaseSupplier(
+            "type error for " + TestCaseSupplier.nameFromTypes(types),
+            types,
+            () -> TestCaseSupplier.TestCase.typeError(
+                types.stream().map(type -> new TestCaseSupplier.TypedData(randomLiteral(type).value(), type, type.typeName())).toList(),
+                errorMessageSupplier.apply(includeOrdinal, validPerPosition, types)
+            )
+        );
+    }
+
+    private static List<Set<DataType>> validPerPosition(Set<List<DataType>> valid) {
+        int max = valid.stream().mapToInt(List::size).max().getAsInt();
+        List<Set<DataType>> result = new ArrayList<>(max);
+        for (int i = 0; i < max; i++) {
+            result.add(new HashSet<>());
+        }
+        for (List<DataType> signature : valid) {
+            for (int i = 0; i < signature.size(); i++) {
+                result.get(i).add(signature.get(i));
+            }
+        }
+        return result;
+    }
+
+    protected static Stream<List<DataType>> allPermutations(int argumentCount) {
+        if (argumentCount == 0) {
+            return Stream.of(List.of());
+        }
+        if (argumentCount > 3) {
+            throw new IllegalArgumentException("would generate too many combinations");
+        }
+        Stream<List<DataType>> stream = validFunctionParameters().map(List::of);
+        for (int i = 1; i < argumentCount; i++) {
+            stream = stream.flatMap(types -> validFunctionParameters().map(t -> append(types, t)));
+        }
+        return stream;
+    }
+
+    /**
+     * The types that are valid in function parameters. This is used by the
+     * function tests to enumerate all possible parameters to test error messages
+     * for invalid combinations.
+     */
+    public static Stream<DataType> validFunctionParameters() {
+        return Arrays.stream(DataType.values()).filter(t -> {
+            if (t == DataType.UNSUPPORTED) {
+                // By definition, functions never support UNSUPPORTED
+                return false;
+            }
+            if (t == DataType.DOC_DATA_TYPE || t == DataType.PARTIAL_AGG) {
+                /*
+                 * Doc and partial_agg are special and functions aren't
+                 * defined to take these. They'll use them implicitly if needed.
+                 */
+                return false;
+            }
+            if (t == DataType.OBJECT) {
+                // Object and nested fields aren't supported by any functions yet
+                return false;
+            }
+            if (t == DataType.SOURCE || t == DataType.TSID_DATA_TYPE) {
+                // No functions take source or tsid fields yet. We'll make some eventually and remove this.
+                return false;
+            }
+            if (t == DataType.DATE_PERIOD || t == DataType.TIME_DURATION) {
+                // We don't test that functions don't take date_period or time_duration. We should.
+                return false;
+            }
+            if (DataType.UNDER_CONSTRUCTION.containsKey(t)) {
+                /*
+                 * Types under construction aren't checked because we're actively
+                 * adding support for them to functions. That's *why* they are
+                 * under construction.
+                 */
+                return false;
+            }
+            if (t.isCounter()) {
+                /*
+                 * For now, we're assuming no functions take counters
+                 * as parameters. That's not true - some do. But we'll
+                 * need to update the tests to handle that.
+                 */
+                return false;
+            }
+            if (t.widenSmallNumeric() != t) {
+                // Small numeric types are widened long before they arrive at functions.
+                return false;
+            }
+
+            return true;
+        }).sorted();
+    }
+
     /**
      * Build an {@link Attribute} that loads a field.
      */
@@ -255,7 +575,12 @@ protected final List<Page> rows(List<TestCaseSupplier.TypedData> multirowFields)
                 ) {
                     var multiRowData = field.multiRowData();
                     for (int row = initialRow; row < initialRow + pageSize; row++) {
-                        wrapper.accept(multiRowData.get(row));
+                        var data = multiRowData.get(row);
+                        if (data instanceof BigInteger bigIntegerData) {
+                            wrapper.accept(NumericUtils.asLongUnsigned(bigIntegerData));
+                        } else {
+                            wrapper.accept(data);
+                        }
                     }
 
                     blocks[i] = wrapper.builder().build();
@@ -294,11 +619,6 @@ protected Object toJavaObjectUnsignedLongAware(Block block, int position) {
         return result;
     }
 
-    protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlock) {
-        // TODO remove me in favor of cases containing null
-        assertTrue("argument " + nullBlock + " is null", value.isNull(0));
-    }
-
     /**
      * Modifies suppliers to generate BytesRefs with random offsets.
      */
@@ -360,6 +680,10 @@ public void testSerializationOfSimple() {
         assertSerialization(buildFieldExpression(testCase));
     }
 
+    /**
+     * This test is meant to validate that the params annotations for the function being tested align with the supported types the
+     * test framework has detected.
+     */
     @AfterClass
     public static void testFunctionInfo() {
         Logger log = LogManager.getLogger(getTestClass());
@@ -387,25 +711,33 @@ public static void testFunctionInfo() {
         for (int i = 0; i < args.size(); i++) {
             typesFromSignature.add(new HashSet<>());
         }
-        Function<DataType, String> typeName = dt -> dt.esType() != null ? dt.esType() : dt.typeName();
         for (Map.Entry<List<DataType>, DataType> entry : signatures().entrySet()) {
             List<DataType> types = entry.getKey();
             for (int i = 0; i < args.size() && i < types.size(); i++) {
-                typesFromSignature.get(i).add(typeName.apply(types.get(i)));
+                typesFromSignature.get(i).add(types.get(i).esNameIfPossible());
             }
-            returnFromSignature.add(typeName.apply(entry.getValue()));
+            returnFromSignature.add(entry.getValue().esNameIfPossible());
         }
 
         for (int i = 0; i < args.size(); i++) {
             EsqlFunctionRegistry.ArgSignature arg = args.get(i);
-            Set<String> annotationTypes = Arrays.stream(arg.type()).collect(Collectors.toCollection(TreeSet::new));
-            Set<String> signatureTypes = typesFromSignature.get(i);
+            Set<String> annotationTypes = Arrays.stream(arg.type())
+                .filter(DataType.UNDER_CONSTRUCTION::containsKey)
+                .collect(Collectors.toCollection(TreeSet::new));
+            Set<String> signatureTypes = typesFromSignature.get(i)
+                .stream()
+                .filter(DataType.UNDER_CONSTRUCTION::containsKey)
+                .collect(Collectors.toCollection(TreeSet::new));
             if (signatureTypes.isEmpty()) {
                 log.info("{}: skipping", arg.name());
                 continue;
             }
             log.info("{}: tested {} vs annotated {}", arg.name(), signatureTypes, annotationTypes);
-            assertEquals(signatureTypes, annotationTypes);
+            assertEquals(
+                "Missmatch between actual and declared parameter types. You probably need to update your @params annotations.",
+                signatureTypes,
+                annotationTypes
+            );
         }
 
         Set<String> returnTypes = Arrays.stream(description.returnType()).collect(Collectors.toCollection(TreeSet::new));
@@ -489,20 +821,8 @@ public static void renderDocs() throws IOException {
             return;
         }
         String name = functionName();
-        if (binaryOperator(name) != null) {
-            renderTypes(List.of("lhs", "rhs"));
-            return;
-        }
-        if (unaryOperator(name) != null) {
-            renderTypes(List.of("v"));
-            return;
-        }
-        if (name.equalsIgnoreCase("rlike")) {
-            renderTypes(List.of("str", "pattern", "caseInsensitive"));
-            return;
-        }
-        if (name.equalsIgnoreCase("like")) {
-            renderTypes(List.of("str", "pattern"));
+        if (binaryOperator(name) != null || unaryOperator(name) != null || likeOrInOperator(name)) {
+            renderDocsForOperators(name);
             return;
         }
         FunctionDefinition definition = definition(name);
@@ -514,7 +834,7 @@ public static void renderDocs() throws IOException {
             renderDescription(description.description(), info.detailedDescription(), info.note());
             boolean hasExamples = renderExamples(info);
             boolean hasAppendix = renderAppendix(info.appendix());
-            renderFullLayout(name, hasExamples, hasAppendix);
+            renderFullLayout(name, info.preview(), hasExamples, hasAppendix);
             renderKibanaInlineDocs(name, info);
             List<EsqlFunctionRegistry.ArgSignature> args = description.args();
             if (name.equals("case")) {
@@ -540,6 +860,11 @@ public static void renderDocs() throws IOException {
     private static final String DOCS_WARNING =
         "// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.\n\n";
 
+    private static final String PREVIEW_CALLOUT =
+        "\npreview::[\"Do not use `VALUES` on production environments. This functionality is in technical preview and "
+            + "may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview "
+            + "are not subject to the support SLA of official GA features.\"]\n";
+
     private static void renderTypes(List<String> argNames) throws IOException {
         StringBuilder header = new StringBuilder();
         for (String arg : argNames) {
@@ -549,20 +874,23 @@ private static void renderTypes(List<String> argNames) throws IOException {
 
         List<String> table = new ArrayList<>();
         for (Map.Entry<List<DataType>, DataType> sig : signatures().entrySet()) { // TODO flip to using sortedSignatures
+            if (shouldHideSignature(sig.getKey(), sig.getValue())) {
+                continue;
+            }
             if (sig.getKey().size() > argNames.size()) { // skip variadic [test] cases (but not those with optional parameters)
                 continue;
             }
             StringBuilder b = new StringBuilder();
             for (DataType arg : sig.getKey()) {
-                b.append(arg.typeName()).append(" | ");
+                b.append(arg.esNameIfPossible()).append(" | ");
             }
             b.append("| ".repeat(argNames.size() - sig.getKey().size()));
-            b.append(sig.getValue().typeName());
+            b.append(sig.getValue().esNameIfPossible());
             table.add(b.toString());
         }
         Collections.sort(table);
         if (table.isEmpty()) {
-            table.add(signatures.values().iterator().next().typeName());
+            table.add(signatures.values().iterator().next().esNameIfPossible());
         }
 
         String rendered = DOCS_WARNING + """
@@ -655,12 +983,12 @@ private static boolean renderAppendix(String appendix) throws IOException {
         return true;
     }
 
-    private static void renderFullLayout(String name, boolean hasExamples, boolean hasAppendix) throws IOException {
+    private static void renderFullLayout(String name, boolean preview, boolean hasExamples, boolean hasAppendix) throws IOException {
         String rendered = DOCS_WARNING + """
             [discrete]
             [[esql-$NAME$]]
             === `$UPPER_NAME$`
-
+            $PREVIEW_CALLOUT$
             *Syntax*
 
             [.text-center]
@@ -669,7 +997,9 @@ private static void renderFullLayout(String name, boolean hasExamples, boolean h
             include::../parameters/$NAME$.asciidoc[]
             include::../description/$NAME$.asciidoc[]
             include::../types/$NAME$.asciidoc[]
-            """.replace("$NAME$", name).replace("$UPPER_NAME$", name.toUpperCase(Locale.ROOT));
+            """.replace("$NAME$", name)
+            .replace("$UPPER_NAME$", name.toUpperCase(Locale.ROOT))
+            .replace("$PREVIEW_CALLOUT$", preview ? PREVIEW_CALLOUT : "");
         if (hasExamples) {
             rendered += "include::../examples/" + name + ".asciidoc[]\n";
         }
@@ -680,6 +1010,41 @@ private static void renderFullLayout(String name, boolean hasExamples, boolean h
         writeToTempDir("layout", rendered, "asciidoc");
     }
 
+    private static Constructor<?> constructorWithFunctionInfo(Class<?> clazz) {
+        for (Constructor<?> ctor : clazz.getConstructors()) {
+            FunctionInfo functionInfo = ctor.getAnnotation(FunctionInfo.class);
+            if (functionInfo != null) {
+                return ctor;
+            }
+        }
+        return null;
+    }
+
+    private static void renderDocsForOperators(String name) throws IOException {
+        Constructor<?> ctor = constructorWithFunctionInfo(OPERATORS.get(name));
+        assert ctor != null;
+        FunctionInfo functionInfo = ctor.getAnnotation(FunctionInfo.class);
+        assert functionInfo != null;
+        renderKibanaInlineDocs(name, functionInfo);
+
+        var params = ctor.getParameters();
+
+        List<EsqlFunctionRegistry.ArgSignature> args = new ArrayList<>(params.length);
+        for (int i = 1; i < params.length; i++) { // skipping 1st argument, the source
+            if (Configuration.class.isAssignableFrom(params[i].getType()) == false) {
+                Param paramInfo = params[i].getAnnotation(Param.class);
+                String paramName = paramInfo == null ? params[i].getName() : paramInfo.name();
+                String[] type = paramInfo == null ? new String[] { "?" } : paramInfo.type();
+                String desc = paramInfo == null ? "" : paramInfo.description().replace('\n', ' ');
+                boolean optional = paramInfo == null ? false : paramInfo.optional();
+                DataType targetDataType = EsqlFunctionRegistry.getTargetType(type);
+                args.add(new EsqlFunctionRegistry.ArgSignature(paramName, type, desc, optional, targetDataType));
+            }
+        }
+        renderKibanaFunctionDefinition(name, functionInfo, args, likeOrInOperator(name));
+        renderTypes(args.stream().map(EsqlFunctionRegistry.ArgSignature::name).toList());
+    }
+
     private static void renderKibanaInlineDocs(String name, FunctionInfo info) throws IOException {
         StringBuilder builder = new StringBuilder();
         builder.append("""
@@ -717,7 +1082,7 @@ private static void renderKibanaFunctionDefinition(
             "comment",
             "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it."
         );
-        builder.field("type", isAggregation() ? "agg" : "eval");
+        builder.field("type", isAggregation() ? "agg" : OPERATORS.get(name) != null ? "operator" : "eval");
         builder.field("name", name);
         builder.field("description", removeAsciidocLinks(info.description()));
         if (Strings.isNullOrEmpty(info.note()) == false) {
@@ -731,7 +1096,7 @@ private static void renderKibanaFunctionDefinition(
             builder.startArray("params");
             builder.endArray();
             // There should only be one return type so just use that as the example
-            builder.field("returnType", signatures().values().iterator().next().typeName());
+            builder.field("returnType", signatures().values().iterator().next().esNameIfPossible());
             builder.endObject();
         } else {
             int minArgCount = (int) args.stream().filter(a -> false == a.optional()).count();
@@ -743,20 +1108,23 @@ private static void renderKibanaFunctionDefinition(
                 if (sig.getKey().size() < minArgCount) {
                     throw new IllegalArgumentException("signature " + sig.getKey() + " is missing non-optional arg for " + args);
                 }
+                if (shouldHideSignature(sig.getKey(), sig.getValue())) {
+                    continue;
+                }
                 builder.startObject();
                 builder.startArray("params");
                 for (int i = 0; i < sig.getKey().size(); i++) {
                     EsqlFunctionRegistry.ArgSignature arg = args.get(i);
                     builder.startObject();
                     builder.field("name", arg.name());
-                    builder.field("type", sig.getKey().get(i).typeName());
+                    builder.field("type", sig.getKey().get(i).esNameIfPossible());
                     builder.field("optional", arg.optional());
                     builder.field("description", arg.description());
                     builder.endObject();
                 }
                 builder.endArray();
                 builder.field("variadic", variadic);
-                builder.field("returnType", sig.getValue().typeName());
+                builder.field("returnType", sig.getValue().esNameIfPossible());
                 builder.endObject();
             }
         }
@@ -792,12 +1160,12 @@ public int compare(Map.Entry<List<DataType>, DataType> lhs, Map.Entry<List<DataT
                     if (rhs.getKey().size() <= i) {
                         return 1;
                     }
-                    int c = lhs.getKey().get(i).typeName().compareTo(rhs.getKey().get(i).typeName());
+                    int c = lhs.getKey().get(i).esNameIfPossible().compareTo(rhs.getKey().get(i).esNameIfPossible());
                     if (c != 0) {
                         return c;
                     }
                 }
-                return lhs.getValue().typeName().compareTo(rhs.getValue().typeName());
+                return lhs.getValue().esNameIfPossible().compareTo(rhs.getValue().esNameIfPossible());
             }
         });
         return sortedSignatures;
@@ -854,6 +1222,13 @@ private static String unaryOperator(String name) {
         };
     }
 
+    /**
+     * If this tests is for a like or rlike operator return true, otherwise return {@code null}.
+     */
+    private static boolean likeOrInOperator(String name) {
+        return name.equalsIgnoreCase("rlike") || name.equalsIgnoreCase("like") || name.equalsIgnoreCase("in");
+    }
+
     /**
      * Write some text to a tempdir so we can copy it to the docs later.
      * <p>
@@ -894,16 +1269,6 @@ public void allMemoryReleased() {
         }
     }
 
-    static Version randomVersion() {
-        // TODO degenerate versions and stuff
-        return switch (between(0, 2)) {
-            case 0 -> new Version(Integer.toString(between(0, 100)));
-            case 1 -> new Version(between(0, 100) + "." + between(0, 100));
-            case 2 -> new Version(between(0, 100) + "." + between(0, 100) + "." + between(0, 100));
-            default -> throw new IllegalArgumentException();
-        };
-    }
-
     /**
      * All string types (keyword, text, match_only_text, etc).
      */
@@ -911,6 +1276,17 @@ protected static DataType[] strings() {
         return DataType.types().stream().filter(DataType::isString).toArray(DataType[]::new);
     }
 
+    /**
+     * Validate that we know the types for all the test cases already created
+     * @param suppliers - list of suppliers before adding in the illegal type combinations
+     */
+    protected static void typesRequired(List<TestCaseSupplier> suppliers) {
+        String bad = suppliers.stream().filter(s -> s.types() == null).map(s -> s.name()).collect(Collectors.joining("\n"));
+        if (bad.equals("") == false) {
+            throw new IllegalArgumentException("types required but not found for these tests:\n" + bad);
+        }
+    }
+
     /**
      * Returns true if the current test case is for an aggregation function.
      * <p>
@@ -920,4 +1296,17 @@ protected static DataType[] strings() {
     private static boolean isAggregation() {
         return AbstractAggregationTestCase.class.isAssignableFrom(getTestClass());
     }
+
+    /**
+     * Should this particular signature be hidden from the docs even though we test it?
+     */
+    private static boolean shouldHideSignature(List<DataType> argTypes, DataType returnType) {
+        for (DataType dt : DataType.UNDER_CONSTRUCTION.keySet()) {
+            if (returnType == dt) {
+                return true;
+            }
+            return argTypes.contains(dt);
+        }
+        return false;
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
index 1aa90d367099a..3ef2a7f821457 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
-import org.elasticsearch.compute.data.BlockUtils;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.data.Vector;
@@ -20,39 +19,29 @@
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.indices.CrankyCircuitBreakerService;
-import org.elasticsearch.xpack.esql.TestBlockFactory;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
-import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest;
-import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.AbstractMultivalueFunctionTestCase;
-import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
 import org.elasticsearch.xpack.esql.optimizer.FoldNull;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.hamcrest.Matcher;
 
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.compute.data.BlockUtils.toJavaObject;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatial;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
@@ -74,10 +63,38 @@ public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTes
      */
     protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecks(
         boolean entirelyNullPreservesType,
-        List<TestCaseSupplier> suppliers
+        List<TestCaseSupplier> suppliers,
+        PositionalErrorMessageSupplier positionalErrorMessageSupplier
     ) {
         return parameterSuppliersFromTypedData(
-            errorsForCasesWithoutExamples(anyNullIsNull(entirelyNullPreservesType, randomizeBytesRefsOffset(suppliers)))
+            errorsForCasesWithoutExamples(
+                anyNullIsNull(entirelyNullPreservesType, randomizeBytesRefsOffset(suppliers)),
+                positionalErrorMessageSupplier
+            )
+        );
+    }
+
+    /**
+     * Converts a list of test cases into a list of parameter suppliers.
+     * Also, adds a default set of extra test cases.
+     * <p>
+     *     Use if possible, as this method may get updated with new checks in the future.
+     * </p>
+     *
+     * @param nullsExpectedType See {@link #anyNullIsNull(List, ExpectedType, ExpectedEvaluatorToString)}
+     * @param evaluatorToString See {@link #anyNullIsNull(List, ExpectedType, ExpectedEvaluatorToString)}
+     */
+    protected static Iterable<Object[]> parameterSuppliersFromTypedDataWithDefaultChecks(
+        ExpectedType nullsExpectedType,
+        ExpectedEvaluatorToString evaluatorToString,
+        List<TestCaseSupplier> suppliers,
+        PositionalErrorMessageSupplier positionalErrorMessageSupplier
+    ) {
+        return parameterSuppliersFromTypedData(
+            errorsForCasesWithoutExamples(
+                anyNullIsNull(randomizeBytesRefsOffset(suppliers), nullsExpectedType, evaluatorToString),
+                positionalErrorMessageSupplier
+            )
         );
     }
 
@@ -89,7 +106,7 @@ public final void testEvaluate() {
             assertTypeResolutionFailure(expression);
             return;
         }
-        assumeTrue("Expected type must be representable to build an evaluator", EsqlDataTypes.isRepresentable(testCase.expectedType()));
+        assumeTrue("Expected type must be representable to build an evaluator", DataType.isRepresentable(testCase.expectedType()));
         logger.info(
             "Test Values: " + testCase.getData().stream().map(TestCaseSupplier.TypedData::toString).collect(Collectors.joining(","))
         );
@@ -104,6 +121,7 @@ public final void testEvaluate() {
         Object result;
         try (ExpressionEvaluator evaluator = evaluator(expression).get(driverContext())) {
             try (Block block = evaluator.eval(row(testCase.getDataValues()))) {
+                assertThat(block.getPositionCount(), is(1));
                 result = toJavaObjectUnsignedLongAware(block, 0);
             }
         }
@@ -190,7 +208,7 @@ private void testEvaluateBlock(BlockFactory inputBlockFactory, DriverContext con
             return;
         }
         assumeTrue("Can't build evaluator", testCase.canBuildEvaluator());
-        assumeTrue("Expected type must be representable to build an evaluator", EsqlDataTypes.isRepresentable(testCase.expectedType()));
+        assumeTrue("Expected type must be representable to build an evaluator", DataType.isRepresentable(testCase.expectedType()));
         int positions = between(1, 1024);
         List<TestCaseSupplier.TypedData> data = testCase.getData();
         Page onePositionPage = row(testCase.getDataValues());
@@ -224,6 +242,7 @@ private void testEvaluateBlock(BlockFactory inputBlockFactory, DriverContext con
                 ExpressionEvaluator eval = evaluator(expression).get(context);
                 Block block = eval.eval(new Page(positions, manyPositionsBlocks))
             ) {
+                assertThat(block.getPositionCount(), is(positions));
                 for (int p = 0; p < positions; p++) {
                     if (nullPositions.contains(p)) {
                         assertThat(toJavaObject(block, p), allNullsMatcher());
@@ -245,47 +264,6 @@ private void testEvaluateBlock(BlockFactory inputBlockFactory, DriverContext con
         }
     }
 
-    public void testSimpleWithNulls() { // TODO replace this with nulls inserted into the test case like anyNullIsNull
-        Expression expression = buildFieldExpression(testCase);
-        if (testCase.getExpectedTypeError() != null) {
-            assertTypeResolutionFailure(expression);
-            return;
-        }
-        assumeTrue("Can't build evaluator", testCase.canBuildEvaluator());
-        List<Object> simpleData = testCase.getDataValues();
-        try (EvalOperator.ExpressionEvaluator eval = evaluator(expression).get(driverContext())) {
-            BlockFactory blockFactory = TestBlockFactory.getNonBreakingInstance();
-            Block[] orig = BlockUtils.fromListRow(blockFactory, simpleData);
-            for (int i = 0; i < orig.length; i++) {
-                List<Object> data = new ArrayList<>();
-                Block[] blocks = new Block[orig.length];
-                for (int b = 0; b < blocks.length; b++) {
-                    if (b == i) {
-                        blocks[b] = orig[b].elementType().newBlockBuilder(1, blockFactory).appendNull().build();
-                        data.add(null);
-                    } else {
-                        blocks[b] = orig[b];
-                        data.add(simpleData.get(b));
-                    }
-                }
-                try (Block block = eval.eval(new Page(blocks))) {
-                    assertSimpleWithNulls(data, block, i);
-                }
-            }
-
-            // Note: the null-in-fast-null-out handling prevents any exception from being thrown, so the warnings provided in some test
-            // cases won't actually be registered. This isn't an issue for unary functions, but could be an issue for n-ary ones, if
-            // function processing of the first parameter(s) could raise an exception/warning. (But hasn't been the case so far.)
-            // N-ary non-MV functions dealing with one multivalue (before hitting the null parameter injected above) will now trigger
-            // a warning ("SV-function encountered a MV") that thus needs to be checked.
-            if (this instanceof AbstractMultivalueFunctionTestCase == false
-                && simpleData.stream().anyMatch(List.class::isInstance)
-                && testCase.getExpectedWarnings() != null) {
-                assertWarnings(testCase.getExpectedWarnings());
-            }
-        }
-    }
-
     public final void testEvaluateInManyThreads() throws ExecutionException, InterruptedException {
         Expression expression = buildFieldExpression(testCase);
         if (testCase.getExpectedTypeError() != null) {
@@ -293,7 +271,7 @@ public final void testEvaluateInManyThreads() throws ExecutionException, Interru
             return;
         }
         assumeTrue("Can't build evaluator", testCase.canBuildEvaluator());
-        assumeTrue("Expected type must be representable to build an evaluator", EsqlDataTypes.isRepresentable(testCase.expectedType()));
+        assumeTrue("Expected type must be representable to build an evaluator", DataType.isRepresentable(testCase.expectedType()));
         int count = 10_000;
         int threads = 5;
         var evalSupplier = evaluator(expression);
@@ -308,6 +286,7 @@ public final void testEvaluateInManyThreads() throws ExecutionException, Interru
                     try (EvalOperator.ExpressionEvaluator eval = evalSupplier.get(driverContext())) {
                         for (int c = 0; c < count; c++) {
                             try (Block block = eval.eval(page)) {
+                                assertThat(block.getPositionCount(), is(1));
                                 assertThat(toJavaObjectUnsignedLongAware(block, 0), testCase.getMatcher());
                             }
                         }
@@ -352,7 +331,7 @@ public final void testFold() {
             assertTypeResolutionFailure(expression);
             return;
         }
-        assertFalse(expression.typeResolved().unresolved());
+        assertFalse("expected resolved", expression.typeResolved().unresolved());
         Expression nullOptimized = new FoldNull().rule(expression);
         assertThat(nullOptimized.dataType(), equalTo(testCase.expectedType()));
         assertTrue(nullOptimized.foldable());
@@ -372,153 +351,14 @@ public final void testFold() {
         }
     }
 
-    /**
-     * Adds cases with {@code null} and asserts that the result is {@code null}.
-     * <p>
-     * Note: This won't add more than a single null to any existing test case,
-     * just to keep the number of test cases from exploding totally.
-     * </p>
-     *
-     * @param entirelyNullPreservesType should a test case that only contains parameters
-     *                                  with the {@code null} type keep it's expected type?
-     *                                  This is <strong>mostly</strong> going to be {@code true}
-     *                                  except for functions that base their type entirely
-     *                                  on input types like {@link Greatest} or {@link Coalesce}.
-     */
-    protected static List<TestCaseSupplier> anyNullIsNull(boolean entirelyNullPreservesType, List<TestCaseSupplier> testCaseSuppliers) {
-        return anyNullIsNull(
-            testCaseSuppliers,
-            (nullPosition, nullValueDataType, original) -> entirelyNullPreservesType == false
-                && nullValueDataType == DataType.NULL
-                && original.getData().size() == 1 ? DataType.NULL : original.expectedType(),
-            (nullPosition, nullData, original) -> original
-        );
-    }
-
-    public interface ExpectedType {
-        DataType expectedType(int nullPosition, DataType nullValueDataType, TestCaseSupplier.TestCase original);
-    }
-
-    public interface ExpectedEvaluatorToString {
-        Matcher<String> evaluatorToString(int nullPosition, TestCaseSupplier.TypedData nullData, Matcher<String> original);
-    }
-
-    protected static List<TestCaseSupplier> anyNullIsNull(
-        List<TestCaseSupplier> testCaseSuppliers,
-        ExpectedType expectedType,
-        ExpectedEvaluatorToString evaluatorToString
-    ) {
-        typesRequired(testCaseSuppliers);
-        List<TestCaseSupplier> suppliers = new ArrayList<>(testCaseSuppliers.size());
-        suppliers.addAll(testCaseSuppliers);
-
-        /*
-         * For each original test case, add as many copies as there were
-         * arguments, replacing one of the arguments with null and keeping
-         * the others.
-         *
-         * Also, if this was the first time we saw the signature we copy it
-         * *again*, replacing the argument with null, but annotating the
-         * argument's type as `null` explicitly.
-         */
-        Set<List<DataType>> uniqueSignatures = new HashSet<>();
-        for (TestCaseSupplier original : testCaseSuppliers) {
-            boolean firstTimeSeenSignature = uniqueSignatures.add(original.types());
-            for (int nullPosition = 0; nullPosition < original.types().size(); nullPosition++) {
-                int finalNullPosition = nullPosition;
-                suppliers.add(new TestCaseSupplier(original.name() + " null in " + nullPosition, original.types(), () -> {
-                    TestCaseSupplier.TestCase oc = original.get();
-                    List<TestCaseSupplier.TypedData> data = IntStream.range(0, oc.getData().size()).mapToObj(i -> {
-                        TestCaseSupplier.TypedData od = oc.getData().get(i);
-                        return i == finalNullPosition ? od.withData(null) : od;
-                    }).toList();
-                    TestCaseSupplier.TypedData nulledData = oc.getData().get(finalNullPosition);
-                    return new TestCaseSupplier.TestCase(
-                        data,
-                        evaluatorToString.evaluatorToString(finalNullPosition, nulledData, oc.evaluatorToString()),
-                        expectedType.expectedType(finalNullPosition, nulledData.type(), oc),
-                        nullValue(),
-                        null,
-                        oc.getExpectedTypeError(),
-                        null,
-                        null
-                    );
-                }));
-
-                if (firstTimeSeenSignature) {
-                    List<DataType> typesWithNull = IntStream.range(0, original.types().size())
-                        .mapToObj(i -> i == finalNullPosition ? DataType.NULL : original.types().get(i))
-                        .toList();
-                    boolean newSignature = uniqueSignatures.add(typesWithNull);
-                    if (newSignature) {
-                        suppliers.add(new TestCaseSupplier(typesWithNull, () -> {
-                            TestCaseSupplier.TestCase oc = original.get();
-                            List<TestCaseSupplier.TypedData> data = IntStream.range(0, oc.getData().size())
-                                .mapToObj(i -> i == finalNullPosition ? TestCaseSupplier.TypedData.NULL : oc.getData().get(i))
-                                .toList();
-                            return new TestCaseSupplier.TestCase(
-                                data,
-                                equalTo("LiteralsEvaluator[lit=null]"),
-                                expectedType.expectedType(finalNullPosition, DataType.NULL, oc),
-                                nullValue(),
-                                null,
-                                oc.getExpectedTypeError(),
-                                null,
-                                null
-                            );
-                        }));
-                    }
-                }
-            }
-        }
-
-        return suppliers;
-
-    }
-
-    /**
-     * Adds test cases containing unsupported parameter types that assert
-     * that they throw type errors.
-     */
-    protected static List<TestCaseSupplier> errorsForCasesWithoutExamples(List<TestCaseSupplier> testCaseSuppliers) {
-        return errorsForCasesWithoutExamples(testCaseSuppliers, AbstractScalarFunctionTestCase::typeErrorMessage);
-    }
-
-    protected static List<TestCaseSupplier> errorsForCasesWithoutExamples(
-        List<TestCaseSupplier> testCaseSuppliers,
-        TypeErrorMessageSupplier typeErrorMessageSupplier
-    ) {
-        typesRequired(testCaseSuppliers);
-        List<TestCaseSupplier> suppliers = new ArrayList<>(testCaseSuppliers.size());
-        suppliers.addAll(testCaseSuppliers);
-
-        Set<List<DataType>> valid = testCaseSuppliers.stream().map(TestCaseSupplier::types).collect(Collectors.toSet());
-        List<Set<DataType>> validPerPosition = validPerPosition(valid);
-
-        testCaseSuppliers.stream()
-            .map(s -> s.types().size())
-            .collect(Collectors.toSet())
-            .stream()
-            .flatMap(count -> allPermutations(count))
-            .filter(types -> valid.contains(types) == false)
-            /*
-             * Skip any cases with more than one null. Our tests don't generate
-             * the full combinatorial explosions of all nulls - just a single null.
-             * Hopefully <null>, <null> cases will function the same as <null>, <valid>
-             * cases.
-             */.filter(types -> types.stream().filter(t -> t == DataType.NULL).count() <= 1)
-            .map(types -> typeErrorSupplier(validPerPosition.size() != 1, validPerPosition, types, typeErrorMessageSupplier))
-            .forEach(suppliers::add);
-        return suppliers;
-    }
-
     public static String errorMessageStringForBinaryOperators(
         boolean includeOrdinal,
         List<Set<DataType>> validPerPosition,
-        List<DataType> types
+        List<DataType> types,
+        PositionalErrorMessageSupplier positionalErrorMessageSupplier
     ) {
         try {
-            return typeErrorMessage(includeOrdinal, validPerPosition, types);
+            return typeErrorMessage(includeOrdinal, validPerPosition, types, positionalErrorMessageSupplier);
         } catch (IllegalStateException e) {
             // This means all the positional args were okay, so the expected error is from the combination
             if (types.get(0).equals(DataType.UNSIGNED_LONG)) {
@@ -567,318 +407,28 @@ protected static List<TestCaseSupplier> failureForCasesWithoutExamples(List<Test
     }
 
     /**
-     * Validate that we know the types for all the test cases already created
-     * @param suppliers - list of suppliers before adding in the illegal type combinations
-     */
-    private static void typesRequired(List<TestCaseSupplier> suppliers) {
-        String bad = suppliers.stream().filter(s -> s.types() == null).map(s -> s.name()).collect(Collectors.joining("\n"));
-        if (bad.equals("") == false) {
-            throw new IllegalArgumentException("types required but not found for these tests:\n" + bad);
-        }
-    }
-
-    private static List<Set<DataType>> validPerPosition(Set<List<DataType>> valid) {
-        int max = valid.stream().mapToInt(List::size).max().getAsInt();
-        List<Set<DataType>> result = new ArrayList<>(max);
-        for (int i = 0; i < max; i++) {
-            result.add(new HashSet<>());
-        }
-        for (List<DataType> signature : valid) {
-            for (int i = 0; i < signature.size(); i++) {
-                result.get(i).add(signature.get(i));
-            }
-        }
-        return result;
-    }
-
-    private static Stream<List<DataType>> allPermutations(int argumentCount) {
-        if (argumentCount == 0) {
-            return Stream.of(List.of());
-        }
-        if (argumentCount > 3) {
-            throw new IllegalArgumentException("would generate too many combinations");
-        }
-        Stream<List<DataType>> stream = representable().map(t -> List.of(t));
-        for (int i = 1; i < argumentCount; i++) {
-            stream = stream.flatMap(types -> representable().map(t -> append(types, t)));
-        }
-        return stream;
-    }
-
-    private static List<DataType> append(List<DataType> orig, DataType extra) {
-        List<DataType> longer = new ArrayList<>(orig.size() + 1);
-        longer.addAll(orig);
-        longer.add(extra);
-        return longer;
-    }
-
-    @FunctionalInterface
-    protected interface TypeErrorMessageSupplier {
-        String apply(boolean includeOrdinal, List<Set<DataType>> validPerPosition, List<DataType> types);
-    }
-
-    protected static TestCaseSupplier typeErrorSupplier(
-        boolean includeOrdinal,
-        List<Set<DataType>> validPerPosition,
-        List<DataType> types
-    ) {
-        return typeErrorSupplier(includeOrdinal, validPerPosition, types, AbstractScalarFunctionTestCase::typeErrorMessage);
-    }
-
-    /**
-     * Build a test case that asserts that the combination of parameter types is an error.
+     * Build a test case checking for arithmetic overflow.
      */
-    protected static TestCaseSupplier typeErrorSupplier(
-        boolean includeOrdinal,
-        List<Set<DataType>> validPerPosition,
-        List<DataType> types,
-        TypeErrorMessageSupplier errorMessageSupplier
+    protected static TestCaseSupplier arithmeticExceptionOverflowCase(
+        DataType dataType,
+        Supplier<Object> lhsSupplier,
+        Supplier<Object> rhsSupplier,
+        String evaluator
     ) {
+        String typeNameOverflow = dataType.typeName().toLowerCase(Locale.ROOT) + " overflow";
         return new TestCaseSupplier(
-            "type error for " + TestCaseSupplier.nameFromTypes(types),
-            types,
-            () -> TestCaseSupplier.TestCase.typeError(
-                types.stream().map(type -> new TestCaseSupplier.TypedData(randomLiteral(type).value(), type, type.typeName())).toList(),
-                errorMessageSupplier.apply(includeOrdinal, validPerPosition, types)
-            )
+            "<" + typeNameOverflow + ">",
+            List.of(dataType),
+            () -> new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(lhsSupplier.get(), dataType, "lhs"),
+                    new TestCaseSupplier.TypedData(rhsSupplier.get(), dataType, "rhs")
+                ),
+                evaluator + "[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
+                dataType,
+                is(nullValue())
+            ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
+                .withWarning("Line -1:-1: java.lang.ArithmeticException: " + typeNameOverflow)
         );
     }
-
-    /**
-     * Build the expected error message for an invalid type signature.
-     */
-    protected static String typeErrorMessage(boolean includeOrdinal, List<Set<DataType>> validPerPosition, List<DataType> types) {
-        int badArgPosition = -1;
-        for (int i = 0; i < types.size(); i++) {
-            if (validPerPosition.get(i).contains(types.get(i)) == false) {
-                badArgPosition = i;
-                break;
-            }
-        }
-        if (badArgPosition == -1) {
-            throw new IllegalStateException(
-                "Can't generate error message for these types, you probably need a custom error message function"
-            );
-        }
-        String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : "";
-        String expectedType = expectedType(validPerPosition.get(badArgPosition));
-        String name = types.get(badArgPosition).typeName();
-        return ordinal + "argument of [] must be [" + expectedType + "], found value [" + name + "] type [" + name + "]";
-    }
-
-    private static final Map<Set<DataType>, String> NAMED_EXPECTED_TYPES = Map.ofEntries(
-        Map.entry(
-            Set.of(DataType.DATE_PERIOD, DataType.DOUBLE, DataType.INTEGER, DataType.LONG, DataType.TIME_DURATION, DataType.NULL),
-            "numeric, date_period or time_duration"
-        ),
-        Map.entry(Set.of(DataType.DATETIME, DataType.NULL), "datetime"),
-        Map.entry(Set.of(DataType.DOUBLE, DataType.NULL), "double"),
-        Map.entry(Set.of(DataType.INTEGER, DataType.NULL), "integer"),
-        Map.entry(Set.of(DataType.IP, DataType.NULL), "ip"),
-        Map.entry(Set.of(DataType.LONG, DataType.INTEGER, DataType.UNSIGNED_LONG, DataType.DOUBLE, DataType.NULL), "numeric"),
-        Map.entry(Set.of(DataType.LONG, DataType.INTEGER, DataType.UNSIGNED_LONG, DataType.DOUBLE), "numeric"),
-        Map.entry(Set.of(DataType.KEYWORD, DataType.TEXT, DataType.VERSION, DataType.NULL), "string or version"),
-        Map.entry(Set.of(DataType.KEYWORD, DataType.TEXT, DataType.NULL), "string"),
-        Map.entry(Set.of(DataType.IP, DataType.KEYWORD, DataType.TEXT, DataType.NULL), "ip or string"),
-        Map.entry(Set.copyOf(Arrays.asList(representableTypes())), "representable"),
-        Map.entry(Set.copyOf(Arrays.asList(representableNonSpatialTypes())), "representableNonSpatial"),
-        Map.entry(
-            Set.of(
-                DataType.BOOLEAN,
-                DataType.DOUBLE,
-                DataType.INTEGER,
-                DataType.KEYWORD,
-                DataType.LONG,
-                DataType.TEXT,
-                DataType.UNSIGNED_LONG,
-                DataType.NULL
-            ),
-            "boolean or numeric or string"
-        ),
-        Map.entry(
-            Set.of(
-                DataType.DATETIME,
-                DataType.DOUBLE,
-                DataType.INTEGER,
-                DataType.KEYWORD,
-                DataType.LONG,
-                DataType.TEXT,
-                DataType.UNSIGNED_LONG,
-                DataType.NULL
-            ),
-            "datetime or numeric or string"
-        ),
-        // What Add accepts
-        Map.entry(
-            Set.of(
-                DataType.DATE_PERIOD,
-                DataType.DATETIME,
-                DataType.DOUBLE,
-                DataType.INTEGER,
-                DataType.LONG,
-                DataType.NULL,
-                DataType.TIME_DURATION,
-                DataType.UNSIGNED_LONG
-            ),
-            "datetime or numeric"
-        ),
-        Map.entry(
-            Set.of(
-                DataType.BOOLEAN,
-                DataType.DATETIME,
-                DataType.DOUBLE,
-                DataType.INTEGER,
-                DataType.KEYWORD,
-                DataType.LONG,
-                DataType.TEXT,
-                DataType.UNSIGNED_LONG,
-                DataType.NULL
-            ),
-            "boolean or datetime or numeric or string"
-        ),
-        // to_int
-        Map.entry(
-            Set.of(
-                DataType.BOOLEAN,
-                DataType.COUNTER_INTEGER,
-                DataType.DATETIME,
-                DataType.DOUBLE,
-                DataType.INTEGER,
-                DataType.KEYWORD,
-                DataType.LONG,
-                DataType.TEXT,
-                DataType.UNSIGNED_LONG,
-                DataType.NULL
-            ),
-            "boolean or counter_integer or datetime or numeric or string"
-        ),
-        // to_long
-        Map.entry(
-            Set.of(
-                DataType.BOOLEAN,
-                DataType.COUNTER_INTEGER,
-                DataType.COUNTER_LONG,
-                DataType.DATETIME,
-                DataType.DOUBLE,
-                DataType.INTEGER,
-                DataType.KEYWORD,
-                DataType.LONG,
-                DataType.TEXT,
-                DataType.UNSIGNED_LONG,
-                DataType.NULL
-            ),
-            "boolean or counter_integer or counter_long or datetime or numeric or string"
-        ),
-        // to_double
-        Map.entry(
-            Set.of(
-                DataType.BOOLEAN,
-                DataType.COUNTER_DOUBLE,
-                DataType.COUNTER_INTEGER,
-                DataType.COUNTER_LONG,
-                DataType.DATETIME,
-                DataType.DOUBLE,
-                DataType.INTEGER,
-                DataType.KEYWORD,
-                DataType.LONG,
-                DataType.TEXT,
-                DataType.UNSIGNED_LONG,
-                DataType.NULL
-            ),
-            "boolean or counter_double or counter_integer or counter_long or datetime or numeric or string"
-        ),
-        Map.entry(
-            Set.of(
-                DataType.BOOLEAN,
-                DataType.CARTESIAN_POINT,
-                DataType.DATETIME,
-                DataType.DOUBLE,
-                DataType.GEO_POINT,
-                DataType.INTEGER,
-                DataType.KEYWORD,
-                DataType.LONG,
-                DataType.TEXT,
-                DataType.UNSIGNED_LONG,
-                DataType.NULL
-            ),
-            "boolean or cartesian_point or datetime or geo_point or numeric or string"
-        ),
-        Map.entry(
-            Set.of(
-                DataType.DATETIME,
-                DataType.DOUBLE,
-                DataType.INTEGER,
-                DataType.IP,
-                DataType.KEYWORD,
-                DataType.LONG,
-                DataType.TEXT,
-                DataType.UNSIGNED_LONG,
-                DataType.VERSION,
-                DataType.NULL
-            ),
-            "datetime, double, integer, ip, keyword, long, text, unsigned_long or version"
-        ),
-        Map.entry(
-            Set.of(
-                DataType.BOOLEAN,
-                DataType.DATETIME,
-                DataType.DOUBLE,
-                DataType.GEO_POINT,
-                DataType.GEO_SHAPE,
-                DataType.INTEGER,
-                DataType.IP,
-                DataType.KEYWORD,
-                DataType.LONG,
-                DataType.TEXT,
-                DataType.UNSIGNED_LONG,
-                DataType.VERSION,
-                DataType.NULL
-            ),
-            "cartesian_point or datetime or geo_point or numeric or string"
-        ),
-        Map.entry(Set.of(DataType.GEO_POINT, DataType.KEYWORD, DataType.TEXT, DataType.NULL), "geo_point or string"),
-        Map.entry(Set.of(DataType.CARTESIAN_POINT, DataType.KEYWORD, DataType.TEXT, DataType.NULL), "cartesian_point or string"),
-        Map.entry(
-            Set.of(DataType.GEO_POINT, DataType.GEO_SHAPE, DataType.KEYWORD, DataType.TEXT, DataType.NULL),
-            "geo_point or geo_shape or string"
-        ),
-        Map.entry(
-            Set.of(DataType.CARTESIAN_POINT, DataType.CARTESIAN_SHAPE, DataType.KEYWORD, DataType.TEXT, DataType.NULL),
-            "cartesian_point or cartesian_shape or string"
-        ),
-        Map.entry(Set.of(DataType.GEO_POINT, DataType.CARTESIAN_POINT, DataType.NULL), "geo_point or cartesian_point"),
-        Map.entry(Set.of(DataType.DATE_PERIOD, DataType.TIME_DURATION, DataType.NULL), "dateperiod or timeduration")
-    );
-
-    // TODO: generate this message dynamically, a la AbstractConvertFunction#supportedTypesNames()?
-    private static String expectedType(Set<DataType> validTypes) {
-        String named = NAMED_EXPECTED_TYPES.get(validTypes);
-        if (named == null) {
-            /*
-             * Note for anyone who's test lands here - it's likely that you
-             * don't have a test case covering explicit `null` arguments in
-             * this position. Generally you can get that with anyNullIsNull.
-             */
-            throw new UnsupportedOperationException(
-                "can't guess expected types for " + validTypes.stream().sorted(Comparator.comparing(t -> t.typeName())).toList()
-            );
-        }
-        return named;
-    }
-
-    protected static Stream<DataType> representable() {
-        return DataType.types().stream().filter(EsqlDataTypes::isRepresentable);
-    }
-
-    protected static DataType[] representableTypes() {
-        return representable().toArray(DataType[]::new);
-    }
-
-    protected static Stream<DataType> representableNonSpatial() {
-        return representable().filter(t -> isSpatial(t) == false);
-    }
-
-    protected static DataType[] representableNonSpatialTypes() {
-        return representableNonSpatial().toArray(DataType[]::new);
-    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java
index 94549f6dfbdec..7f19419b21816 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java
@@ -12,15 +12,14 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.ParsingException;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
-import org.elasticsearch.xpack.esql.core.TestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.tree.SourceTests;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -28,7 +27,7 @@
 import java.util.function.Function;
 
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomConfiguration;
+import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
 import static org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry.def;
 import static org.elasticsearch.xpack.esql.expression.function.FunctionResolutionStrategy.DEFAULT;
 import static org.hamcrest.Matchers.endsWith;
@@ -41,7 +40,7 @@ public void testNoArgFunction() {
         UnresolvedFunction ur = uf(DEFAULT);
         EsqlFunctionRegistry r = new EsqlFunctionRegistry(def(DummyFunction.class, DummyFunction::new, "dummyFunction"));
         FunctionDefinition def = r.resolveFunction(ur.name());
-        assertEquals(ur.source(), ur.buildResolved(TestUtils.randomConfiguration(), def).source());
+        assertEquals(ur.source(), ur.buildResolved(randomConfiguration(), def).source());
     }
 
     public void testBinaryFunction() {
@@ -52,24 +51,21 @@ public void testBinaryFunction() {
             return new DummyFunction(l);
         }, "dummyFunction"));
         FunctionDefinition def = r.resolveFunction(ur.name());
-        assertEquals(ur.source(), ur.buildResolved(TestUtils.randomConfiguration(), def).source());
+        assertEquals(ur.source(), ur.buildResolved(randomConfiguration(), def).source());
 
         // No children aren't supported
-        ParsingException e = expectThrows(ParsingException.class, () -> uf(DEFAULT).buildResolved(TestUtils.randomConfiguration(), def));
+        ParsingException e = expectThrows(ParsingException.class, () -> uf(DEFAULT).buildResolved(randomConfiguration(), def));
         assertThat(e.getMessage(), endsWith("expects exactly two arguments"));
 
         // One child isn't supported
-        e = expectThrows(
-            ParsingException.class,
-            () -> uf(DEFAULT, mock(Expression.class)).buildResolved(TestUtils.randomConfiguration(), def)
-        );
+        e = expectThrows(ParsingException.class, () -> uf(DEFAULT, mock(Expression.class)).buildResolved(randomConfiguration(), def));
         assertThat(e.getMessage(), endsWith("expects exactly two arguments"));
 
         // Many children aren't supported
         e = expectThrows(
             ParsingException.class,
             () -> uf(DEFAULT, mock(Expression.class), mock(Expression.class), mock(Expression.class)).buildResolved(
-                TestUtils.randomConfiguration(),
+                randomConfiguration(),
                 def
             )
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
index 7ff87a682a789..8090a20ddc836 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
@@ -11,26 +11,27 @@
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.AbstractEsFieldTypeTests;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.type.AbstractEsFieldTypeTests;
 
 public class FieldAttributeTests extends AbstractAttributeTestCase<FieldAttribute> {
-    static FieldAttribute createFieldAttribute(int maxDepth) {
+    public static FieldAttribute createFieldAttribute(int maxDepth, boolean onlyRepresentable) {
         Source source = Source.EMPTY;
-        FieldAttribute parent = maxDepth == 0 || randomBoolean() ? null : createFieldAttribute(maxDepth - 1);
+        FieldAttribute parent = maxDepth == 0 || randomBoolean() ? null : createFieldAttribute(maxDepth - 1, onlyRepresentable);
         String name = randomAlphaOfLength(5);
-        DataType type = randomFrom(DataType.types());
+        DataType type = onlyRepresentable
+            ? randomValueOtherThanMany(t -> false == DataType.isRepresentable(t), () -> randomFrom(DataType.types()))
+            : randomFrom(DataType.types());
         EsField field = AbstractEsFieldTypeTests.randomAnyEsField(maxDepth);
-        String qualifier = randomBoolean() ? null : randomAlphaOfLength(3);
         Nullability nullability = randomFrom(Nullability.values());
         boolean synthetic = randomBoolean();
-        return new FieldAttribute(source, parent, name, type, field, qualifier, nullability, new NameId(), synthetic);
+        return new FieldAttribute(source, parent, name, type, field, nullability, new NameId(), synthetic);
     }
 
     @Override
     protected FieldAttribute create() {
-        return createFieldAttribute(3);
+        return createFieldAttribute(3, false);
     }
 
     @Override
@@ -40,18 +41,16 @@ protected FieldAttribute mutate(FieldAttribute instance) {
         String name = instance.name();
         DataType type = instance.dataType();
         EsField field = instance.field();
-        String qualifier = instance.qualifier();
         Nullability nullability = instance.nullable();
         boolean synthetic = instance.synthetic();
-        switch (between(0, 6)) {
-            case 0 -> parent = randomValueOtherThan(parent, () -> randomBoolean() ? null : createFieldAttribute(2));
+        switch (between(0, 5)) {
+            case 0 -> parent = randomValueOtherThan(parent, () -> randomBoolean() ? null : createFieldAttribute(2, false));
             case 1 -> name = randomAlphaOfLength(name.length() + 1);
             case 2 -> type = randomValueOtherThan(type, () -> randomFrom(DataType.types()));
             case 3 -> field = randomValueOtherThan(field, () -> AbstractEsFieldTypeTests.randomAnyEsField(3));
-            case 4 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3));
-            case 5 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
-            case 6 -> synthetic = false == synthetic;
+            case 4 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
+            case 5 -> synthetic = false == synthetic;
         }
-        return new FieldAttribute(source, parent, name, type, field, qualifier, nullability, new NameId(), synthetic);
+        return new FieldAttribute(source, parent, name, type, field, nullability, new NameId(), synthetic);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FunctionName.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FunctionName.java
index 9807cb5365e54..ac32b50d1045a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FunctionName.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FunctionName.java
@@ -13,7 +13,7 @@
 import java.lang.annotation.Target;
 
 /**
- * Tests that extend {@link AbstractScalarFunctionTestCase} can use this annotation to specify the name of the function
+ * Tests that extend {@link AbstractFunctionTestCase} can use this annotation to specify the name of the function
  * to use when generating documentation files while running tests.
  * If this is not used, the name will be deduced from the test class name, by removing the "Test" suffix, and converting
  * the class name to snake case. This annotation can be used to override that behavior, for cases where the deduced name
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java
index 573af9c17bb1d..dded8bbb478c1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java
@@ -16,6 +16,10 @@
 public class MetadataAttributeTests extends AbstractAttributeTestCase<MetadataAttribute> {
     @Override
     protected MetadataAttribute create() {
+        return randomMetadataAttribute();
+    }
+
+    public static MetadataAttribute randomMetadataAttribute() {
         Source source = Source.EMPTY;
         String name = randomAlphaOfLength(5);
         DataType type = randomFrom(DataType.types());
@@ -23,7 +27,7 @@ protected MetadataAttribute create() {
         Nullability nullability = randomFrom(Nullability.values());
         boolean synthetic = randomBoolean();
         boolean searchable = randomBoolean();
-        return new MetadataAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic, searchable);
+        return new MetadataAttribute(source, name, type, nullability, new NameId(), synthetic, searchable);
     }
 
     @Override
@@ -31,18 +35,16 @@ protected MetadataAttribute mutate(MetadataAttribute instance) {
         Source source = instance.source();
         String name = instance.name();
         DataType type = instance.dataType();
-        String qualifier = instance.qualifier();
         Nullability nullability = instance.nullable();
         boolean synthetic = instance.synthetic();
         boolean searchable = instance.searchable();
-        switch (between(0, 5)) {
+        switch (between(0, 4)) {
             case 0 -> name = randomAlphaOfLength(name.length() + 1);
             case 1 -> type = randomValueOtherThan(type, () -> randomFrom(DataType.types()));
-            case 2 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3));
-            case 3 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
-            case 4 -> synthetic = false == synthetic;
-            case 5 -> searchable = false == searchable;
+            case 2 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
+            case 3 -> synthetic = false == synthetic;
+            case 4 -> searchable = false == searchable;
         }
-        return new MetadataAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic, searchable);
+        return new MetadataAttribute(source, name, type, nullability, new NameId(), synthetic, searchable);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java
index dd73e64fbd8da..e740533462746 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java
@@ -10,14 +10,20 @@
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geo.ShapeTestUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.versionfield.Version;
 
+import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.List;
 
 import static org.elasticsearch.test.ESTestCase.randomBoolean;
 import static org.elasticsearch.test.ESTestCase.randomList;
+import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
+import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
 import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.TypedDataSupplier;
 
 /**
@@ -145,6 +151,55 @@ public static List<TypedDataSupplier> longCases(int minRows, int maxRows, long m
         return cases;
     }
 
+    public static List<TypedDataSupplier> ulongCases(int minRows, int maxRows, BigInteger min, BigInteger max, boolean includeZero) {
+        List<TypedDataSupplier> cases = new ArrayList<>();
+
+        // Zero
+        if (BigInteger.ZERO.compareTo(max) <= 0 && BigInteger.ZERO.compareTo(min) >= 0 && includeZero) {
+            cases.add(
+                new TypedDataSupplier(
+                    "<0 unsigned longs>",
+                    () -> randomList(minRows, maxRows, () -> BigInteger.ZERO),
+                    DataType.UNSIGNED_LONG,
+                    false,
+                    true
+                )
+            );
+        }
+
+        // Small values, less than Long.MAX_VALUE
+        BigInteger lower1 = min.max(BigInteger.ONE);
+        BigInteger upper1 = max.min(BigInteger.valueOf(Long.MAX_VALUE));
+        if (lower1.compareTo(upper1) < 0) {
+            cases.add(
+                new TypedDataSupplier(
+                    "<small unsigned longs>",
+                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomUnsignedLongBetween(lower1, upper1)),
+                    DataType.UNSIGNED_LONG,
+                    false,
+                    true
+                )
+            );
+        }
+
+        // Big values, greater than Long.MAX_VALUE
+        BigInteger lower2 = min.max(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE));
+        BigInteger upper2 = max.min(ESTestCase.UNSIGNED_LONG_MAX);
+        if (lower2.compareTo(upper2) < 0) {
+            cases.add(
+                new TypedDataSupplier(
+                    "<big unsigned longs>",
+                    () -> randomList(minRows, maxRows, () -> ESTestCase.randomUnsignedLongBetween(lower2, upper2)),
+                    DataType.UNSIGNED_LONG,
+                    false,
+                    true
+                )
+            );
+        }
+
+        return cases;
+    }
+
     public static List<TypedDataSupplier> doubleCases(int minRows, int maxRows, double min, double max, boolean includeZero) {
         List<TypedDataSupplier> cases = new ArrayList<>();
 
@@ -321,4 +376,154 @@ public static List<TypedDataSupplier> ipCases(int minRows, int maxRows) {
             )
         );
     }
+
+    public static List<TypedDataSupplier> versionCases(int minRows, int maxRows) {
+        return List.of(
+            new TypedDataSupplier(
+                "<major versions>",
+                () -> randomList(minRows, maxRows, () -> new Version(Integer.toString(ESTestCase.between(0, 100))).toBytesRef()),
+                DataType.VERSION,
+                false,
+                true
+            ),
+            new TypedDataSupplier(
+                "<major.minor versions>",
+                () -> randomList(
+                    minRows,
+                    maxRows,
+                    () -> new Version(ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100)).toBytesRef()
+                ),
+                DataType.VERSION,
+                false,
+                true
+            ),
+            new TypedDataSupplier(
+                "<major.minor.patch versions>",
+                () -> randomList(
+                    minRows,
+                    maxRows,
+                    () -> new Version(ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100))
+                        .toBytesRef()
+                ),
+                DataType.VERSION,
+                false,
+                true
+            )
+        );
+    }
+
+    public static List<TypedDataSupplier> geoPointCases(int minRows, int maxRows, boolean withAltitude) {
+        List<TypedDataSupplier> cases = new ArrayList<>();
+
+        cases.add(
+            new TypedDataSupplier(
+                "<no alt geo_points>",
+                () -> randomList(minRows, maxRows, () -> GEO.asWkb(GeometryTestUtils.randomPoint(false))),
+                DataType.GEO_POINT,
+                false,
+                true
+            )
+        );
+
+        if (withAltitude) {
+            cases.add(
+                new TypedDataSupplier(
+                    "<with alt geo_points>",
+                    () -> randomList(minRows, maxRows, () -> GEO.asWkb(GeometryTestUtils.randomPoint(false))),
+                    DataType.GEO_POINT,
+                    false,
+                    true
+                )
+            );
+        }
+
+        return cases;
+    }
+
+    public static List<TypedDataSupplier> cartesianPointCases(int minRows, int maxRows, boolean withAltitude) {
+        List<TypedDataSupplier> cases = new ArrayList<>();
+
+        cases.add(
+            new TypedDataSupplier(
+                "<no alt cartesian_points>",
+                () -> randomList(minRows, maxRows, () -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint(false))),
+                DataType.CARTESIAN_POINT,
+                false,
+                true
+            )
+        );
+
+        if (withAltitude) {
+            cases.add(
+                new TypedDataSupplier(
+                    "<with alt cartesian_points>",
+                    () -> randomList(minRows, maxRows, () -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint(true))),
+                    DataType.CARTESIAN_POINT,
+                    false,
+                    true
+                )
+            );
+        }
+
+        return cases;
+    }
+
+    public static List<TypedDataSupplier> stringCases(int minRows, int maxRows, DataType type) {
+        List<TypedDataSupplier> cases = new ArrayList<>();
+
+        cases.addAll(
+            List.of(
+                new TypedDataSupplier(
+                    "<empty " + type + "s>",
+                    () -> randomList(minRows, maxRows, () -> new BytesRef("")),
+                    type,
+                    false,
+                    true
+                ),
+                new TypedDataSupplier(
+                    "<short alpha " + type + "s>",
+                    () -> randomList(minRows, maxRows, () -> new BytesRef(ESTestCase.randomAlphaOfLengthBetween(1, 30))),
+                    type,
+                    false,
+                    true
+                ),
+                new TypedDataSupplier(
+                    "<short unicode " + type + "s>",
+                    () -> randomList(minRows, maxRows, () -> new BytesRef(ESTestCase.randomRealisticUnicodeOfLengthBetween(1, 30))),
+                    type,
+                    false,
+                    true
+                )
+            )
+        );
+
+        if (minRows <= 100) {
+            var longStringsMaxRows = Math.min(maxRows, 100);
+
+            cases.addAll(
+                List.of(
+                    new TypedDataSupplier(
+                        "<long alpha " + type + "s>",
+                        () -> randomList(minRows, longStringsMaxRows, () -> new BytesRef(ESTestCase.randomAlphaOfLengthBetween(300, 1000))),
+                        type,
+                        false,
+                        true
+                    ),
+                    new TypedDataSupplier(
+                        "<long unicode " + type + "s>",
+                        () -> randomList(
+                            minRows,
+                            longStringsMaxRows,
+                            () -> new BytesRef(ESTestCase.randomRealisticUnicodeOfLengthBetween(300, 1000))
+                        ),
+                        type,
+                        false,
+                        true
+                    )
+                )
+            );
+        }
+
+        return cases;
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultivalueTestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultivalueTestCaseSupplier.java
new file mode 100644
index 0000000000000..01c73e9ef0482
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultivalueTestCaseSupplier.java
@@ -0,0 +1,325 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+
+import static org.elasticsearch.test.ESTestCase.randomBoolean;
+import static org.elasticsearch.test.ESTestCase.randomList;
+import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.TypedDataSupplier;
+
+/**
+ * Extension of {@link TestCaseSupplier} that provided multivalue test cases.
+ */
+public final class MultivalueTestCaseSupplier {
+
+    private static final int MIN_VALUES = 1;
+    private static final int MAX_VALUES = 1000;
+
+    private MultivalueTestCaseSupplier() {}
+
+    public static List<TypedDataSupplier> intCases(int min, int max, boolean includeZero) {
+        List<TypedDataSupplier> cases = new ArrayList<>();
+
+        for (Block.MvOrdering ordering : Block.MvOrdering.values()) {
+            if (0 <= max && 0 >= min && includeZero) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<0 mv " + ordering + " ints>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> 0), ordering),
+                        DataType.INTEGER
+                    )
+                );
+            }
+
+            if (max != 0) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<" + max + " mv " + ordering + " ints>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> max), ordering),
+                        DataType.INTEGER
+                    )
+                );
+            }
+
+            if (min != 0 && min != max) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<" + min + " mv " + ordering + " ints>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> min), ordering),
+                        DataType.INTEGER
+                    )
+                );
+            }
+
+            int lower = Math.max(min, 1);
+            int upper = Math.min(max, Integer.MAX_VALUE);
+            if (lower < upper) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<positive mv " + ordering + " ints>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomIntBetween(lower, upper)), ordering),
+                        DataType.INTEGER
+                    )
+                );
+            }
+
+            int lower1 = Math.max(min, Integer.MIN_VALUE);
+            int upper1 = Math.min(max, -1);
+            if (lower1 < upper1) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<negative mv " + ordering + " ints>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomIntBetween(lower1, upper1)), ordering),
+                        DataType.INTEGER
+                    )
+                );
+            }
+
+            if (min < 0 && max > 0) {
+                cases.add(
+                    new TypedDataSupplier("<random mv " + ordering + " ints>", () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> {
+                        if (includeZero) {
+                            return ESTestCase.randomIntBetween(min, max);
+                        }
+                        return randomBoolean() ? ESTestCase.randomIntBetween(min, -1) : ESTestCase.randomIntBetween(1, max);
+                    }), ordering), DataType.INTEGER)
+                );
+            }
+        }
+
+        return cases;
+    }
+
+    public static List<TypedDataSupplier> longCases(long min, long max, boolean includeZero) {
+        List<TypedDataSupplier> cases = new ArrayList<>();
+
+        for (Block.MvOrdering ordering : Block.MvOrdering.values()) {
+            if (0 <= max && 0 >= min && includeZero) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<0 mv " + ordering + " longs>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> 0L), ordering),
+                        DataType.LONG
+                    )
+                );
+            }
+
+            if (max != 0) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<" + max + " mv " + ordering + " longs>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> max), ordering),
+                        DataType.LONG
+                    )
+                );
+            }
+
+            if (min != 0 && min != max) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<" + min + " mv " + ordering + " longs>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> min), ordering),
+                        DataType.LONG
+                    )
+                );
+            }
+
+            long lower = Math.max(min, 1);
+            long upper = Math.min(max, Long.MAX_VALUE);
+            if (lower < upper) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<positive mv " + ordering + " longs>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomLongBetween(lower, upper)), ordering),
+                        DataType.LONG
+                    )
+                );
+            }
+
+            long lower1 = Math.max(min, Long.MIN_VALUE);
+            long upper1 = Math.min(max, -1);
+            if (lower1 < upper1) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<negative mv " + ordering + " longs>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomLongBetween(lower1, upper1)), ordering),
+                        DataType.LONG
+                    )
+                );
+            }
+
+            if (min < 0 && max > 0) {
+                cases.add(
+                    new TypedDataSupplier("<random mv " + ordering + " longs>", () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> {
+                        if (includeZero) {
+                            return ESTestCase.randomLongBetween(min, max);
+                        }
+                        return randomBoolean() ? ESTestCase.randomLongBetween(min, -1) : ESTestCase.randomLongBetween(1, max);
+                    }), ordering), DataType.LONG)
+                );
+            }
+        }
+
+        return cases;
+    }
+
+    public static List<TypedDataSupplier> doubleCases(double min, double max, boolean includeZero) {
+        List<TypedDataSupplier> cases = new ArrayList<>();
+
+        for (Block.MvOrdering ordering : Block.MvOrdering.values()) {
+            if (0d <= max && 0d >= min && includeZero) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<0 mv " + ordering + " doubles>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> 0d), ordering),
+                        DataType.DOUBLE
+                    )
+                );
+                cases.add(
+                    new TypedDataSupplier(
+                        "<-0 mv " + ordering + " doubles>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> -0d), ordering),
+                        DataType.DOUBLE
+                    )
+                );
+            }
+
+            if (max != 0d) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<" + max + " mv " + ordering + " doubles>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> max), ordering),
+                        DataType.DOUBLE
+                    )
+                );
+            }
+
+            if (min != 0d && min != max) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<" + min + " mv " + ordering + " doubles>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> min), ordering),
+                        DataType.DOUBLE
+                    )
+                );
+            }
+
+            double lower1 = Math.max(min, 0d);
+            double upper1 = Math.min(max, 1d);
+            if (lower1 < upper1) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<small positive mv " + ordering + " doubles>",
+                        () -> putInOrder(
+                            randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomDoubleBetween(lower1, upper1, true)),
+                            ordering
+                        ),
+                        DataType.DOUBLE
+                    )
+                );
+            }
+
+            double lower2 = Math.max(min, -1d);
+            double upper2 = Math.min(max, 0d);
+            if (lower2 < upper2) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<small negative mv " + ordering + " doubles>",
+                        () -> putInOrder(
+                            randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomDoubleBetween(lower2, upper2, true)),
+                            ordering
+                        ),
+                        DataType.DOUBLE
+                    )
+                );
+            }
+
+            double lower3 = Math.max(min, 1d);
+            double upper3 = Math.min(max, Double.MAX_VALUE);
+            if (lower3 < upper3) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<big positive mv " + ordering + " doubles>",
+                        () -> putInOrder(
+                            randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomDoubleBetween(lower3, upper3, true)),
+                            ordering
+                        ),
+                        DataType.DOUBLE
+                    )
+                );
+            }
+
+            double lower4 = Math.max(min, -Double.MAX_VALUE);
+            double upper4 = Math.min(max, -1d);
+            if (lower4 < upper4) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<big negative mv " + ordering + " doubles>",
+                        () -> putInOrder(
+                            randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomDoubleBetween(lower4, upper4, true)),
+                            ordering
+                        ),
+                        DataType.DOUBLE
+                    )
+                );
+            }
+
+            if (min < 0 && max > 0) {
+                cases.add(
+                    new TypedDataSupplier(
+                        "<random mv " + ordering + " doubles>",
+                        () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> {
+                            if (includeZero) {
+                                return ESTestCase.randomDoubleBetween(min, max, true);
+                            }
+                            return randomBoolean()
+                                ? ESTestCase.randomDoubleBetween(min, -1, true)
+                                : ESTestCase.randomDoubleBetween(1, max, true);
+                        }), ordering),
+                        DataType.DOUBLE
+                    )
+                );
+            }
+        }
+
+        return cases;
+    }
+
+    private static <T extends Comparable<T>> List<T> putInOrder(List<T> mvData, Block.MvOrdering ordering) {
+        switch (ordering) {
+            case UNORDERED -> {
+            }
+            case DEDUPLICATED_UNORDERD -> {
+                var dedup = new LinkedHashSet<>(mvData);
+                mvData.clear();
+                mvData.addAll(dedup);
+            }
+            case DEDUPLICATED_AND_SORTED_ASCENDING -> {
+                var dedup = new HashSet<>(mvData);
+                mvData.clear();
+                mvData.addAll(dedup);
+                Collections.sort(mvData);
+            }
+            case SORTED_ASCENDING -> {
+                Collections.sort(mvData);
+            }
+            default -> throw new UnsupportedOperationException("unsupported ordering [" + ordering + "]");
+        }
+
+        return mvData;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
index 31d1018bacc91..493cecffe8b3f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
@@ -18,10 +18,9 @@ public static ReferenceAttribute randomReferenceAttribute() {
         Source source = Source.EMPTY;
         String name = randomAlphaOfLength(5);
         DataType type = randomFrom(DataType.types());
-        String qualifier = randomBoolean() ? null : randomAlphaOfLength(3);
         Nullability nullability = randomFrom(Nullability.values());
         boolean synthetic = randomBoolean();
-        return new ReferenceAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic);
+        return new ReferenceAttribute(source, name, type, nullability, new NameId(), synthetic);
     }
 
     @Override
@@ -34,16 +33,14 @@ protected ReferenceAttribute mutate(ReferenceAttribute instance) {
         Source source = instance.source();
         String name = instance.name();
         DataType type = instance.dataType();
-        String qualifier = instance.qualifier();
         Nullability nullability = instance.nullable();
         boolean synthetic = instance.synthetic();
-        switch (between(0, 4)) {
+        switch (between(0, 3)) {
             case 0 -> name = randomAlphaOfLength(name.length() + 1);
             case 1 -> type = randomValueOtherThan(type, () -> randomFrom(DataType.types()));
-            case 2 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3));
-            case 3 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
-            case 4 -> synthetic = false == synthetic;
+            case 2 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
+            case 3 -> synthetic = false == synthetic;
         }
-        return new ReferenceAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic);
+        return new ReferenceAttribute(source, name, type, nullability, new NameId(), synthetic);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
index 3ecdf83fe36c9..a1caa784c9787 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
@@ -10,6 +10,7 @@
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geo.ShapeTestUtils;
 import org.elasticsearch.logging.LogManager;
@@ -21,7 +22,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.elasticsearch.xpack.versionfield.Version;
 import org.hamcrest.Matcher;
 
@@ -31,6 +31,7 @@
 import java.time.Period;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.function.BiFunction;
 import java.util.function.BinaryOperator;
@@ -54,15 +55,6 @@ public record TestCaseSupplier(String name, List<DataType> types, Supplier<TestC
         Supplier<TestCaseSupplier.TestCase> {
 
     private static final Logger logger = LogManager.getLogger(TestCaseSupplier.class);
-    /**
-     * Build a test case without types.
-     *
-     * @deprecated Supply types
-     */
-    @Deprecated
-    public TestCaseSupplier(String name, Supplier<TestCase> supplier) {
-        this(name, null, supplier);
-    }
 
     /**
      * Build a test case named after the types it takes.
@@ -634,6 +626,26 @@ public static void forUnaryDatetime(
         );
     }
 
+    /**
+     * Generate positive test cases for a unary function operating on an {@link DataType#DATE_NANOS}.
+     */
+    public static void forUnaryDateNanos(
+        List<TestCaseSupplier> suppliers,
+        String expectedEvaluatorToString,
+        DataType expectedType,
+        Function<Instant, Object> expectedValue,
+        List<String> warnings
+    ) {
+        unaryNumeric(
+            suppliers,
+            expectedEvaluatorToString,
+            dateNanosCases(),
+            expectedType,
+            n -> expectedValue.apply(DateUtils.toInstant((long) n)),
+            warnings
+        );
+    }
+
     /**
      * Generate positive test cases for a unary function operating on an {@link DataType#GEO_POINT}.
      */
@@ -881,6 +893,12 @@ public static List<TypedDataSupplier> longCases(long min, long max, boolean incl
         return cases;
     }
 
+    /**
+     * Generate cases for {@link DataType#UNSIGNED_LONG}.
+     * <p>
+     *     For multi-row parameters, see {@link MultiRowTestCaseSupplier#ulongCases}.
+     * </p>
+     */
     public static List<TypedDataSupplier> ulongCases(BigInteger min, BigInteger max, boolean includeZero) {
         List<TypedDataSupplier> cases = new ArrayList<>();
 
@@ -1033,6 +1051,27 @@ public static List<TypedDataSupplier> dateCases() {
         );
     }
 
+    /**
+     * Generate cases for {@link DataType#DATE_NANOS}.
+     *
+     */
+    public static List<TypedDataSupplier> dateNanosCases() {
+        return List.of(
+            new TypedDataSupplier("<1970-01-01T00:00:00.000000000Z>", () -> 0L, DataType.DATE_NANOS),
+            new TypedDataSupplier("<date nanos>", () -> ESTestCase.randomLongBetween(0, 10 * (long) 10e11), DataType.DATE_NANOS),
+            new TypedDataSupplier(
+                "<far future date nanos>",
+                () -> ESTestCase.randomLongBetween(10 * (long) 10e11, Long.MAX_VALUE),
+                DataType.DATE_NANOS
+            ),
+            new TypedDataSupplier(
+                "<nanos near the end of time>",
+                () -> ESTestCase.randomLongBetween(Long.MAX_VALUE / 100 * 99, Long.MAX_VALUE),
+                DataType.DATE_NANOS
+            )
+        );
+    }
+
     public static List<TypedDataSupplier> datePeriodCases() {
         return List.of(
             new TypedDataSupplier("<zero date period>", () -> Period.ZERO, DataType.DATE_PERIOD, true),
@@ -1077,12 +1116,24 @@ public static List<TypedDataSupplier> cartesianShapeCases() {
         return cartesianShapeCases(ESTestCase::randomBoolean);
     }
 
+    /**
+     * Generate cases for {@link DataType#GEO_POINT}.
+     * <p>
+     *     For multi-row parameters, see {@link MultiRowTestCaseSupplier#geoPointCases}.
+     * </p>
+     */
     public static List<TypedDataSupplier> geoPointCases(Supplier<Boolean> hasAlt) {
         return List.of(
             new TypedDataSupplier("<geo_point>", () -> GEO.asWkb(GeometryTestUtils.randomPoint(hasAlt.get())), DataType.GEO_POINT)
         );
     }
 
+    /**
+     * Generate cases for {@link DataType#CARTESIAN_POINT}.
+     * <p>
+     *     For multi-row parameters, see {@link MultiRowTestCaseSupplier#cartesianPointCases}.
+     * </p>
+     */
     public static List<TypedDataSupplier> cartesianPointCases(Supplier<Boolean> hasAlt) {
         return List.of(
             new TypedDataSupplier(
@@ -1131,6 +1182,12 @@ public static List<TypedDataSupplier> ipCases() {
         );
     }
 
+    /**
+     * Generate cases for String DataTypes.
+     * <p>
+     *     For multi-row parameters, see {@link MultiRowTestCaseSupplier#stringCases}.
+     * </p>
+     */
     public static List<TypedDataSupplier> stringCases(DataType type) {
         List<TypedDataSupplier> result = new ArrayList<>();
         result.add(new TypedDataSupplier("<empty " + type + ">", () -> new BytesRef(""), type));
@@ -1159,6 +1216,9 @@ public static List<TypedDataSupplier> stringCases(DataType type) {
 
     /**
      * Supplier test case data for {@link Version} fields.
+     * <p>
+     *     For multi-row parameters, see {@link MultiRowTestCaseSupplier#versionCases}.
+     * </p>
      */
     public static List<TypedDataSupplier> versionCases(String prefix) {
         return List.of(
@@ -1229,7 +1289,7 @@ private static String castToUnsignedLongEvaluator(String original, DataType curr
         throw new UnsupportedOperationException();
     }
 
-    private static String castToDoubleEvaluator(String original, DataType current) {
+    public static String castToDoubleEvaluator(String original, DataType current) {
         if (current == DataType.DOUBLE) {
             return original;
         }
@@ -1279,11 +1339,11 @@ public static class TestCase {
         private final Class<? extends Throwable> foldingExceptionClass;
         private final String foldingExceptionMessage;
 
-        public TestCase(List<TypedData> data, String evaluatorToString, DataType expectedType, Matcher<Object> matcher) {
+        public TestCase(List<TypedData> data, String evaluatorToString, DataType expectedType, Matcher<?> matcher) {
             this(data, equalTo(evaluatorToString), expectedType, matcher);
         }
 
-        public TestCase(List<TypedData> data, Matcher<String> evaluatorToString, DataType expectedType, Matcher<Object> matcher) {
+        public TestCase(List<TypedData> data, Matcher<String> evaluatorToString, DataType expectedType, Matcher<?> matcher) {
             this(data, evaluatorToString, expectedType, matcher, null, null, null, null);
         }
 
@@ -1295,7 +1355,7 @@ public static TestCase typeError(List<TypedData> data, String expectedTypeError)
             List<TypedData> data,
             Matcher<String> evaluatorToString,
             DataType expectedType,
-            Matcher<Object> matcher,
+            Matcher<?> matcher,
             String[] expectedWarnings,
             String expectedTypeError,
             Class<? extends Throwable> foldingExceptionClass,
@@ -1305,10 +1365,12 @@ public static TestCase typeError(List<TypedData> data, String expectedTypeError)
             this.data = data;
             this.evaluatorToString = evaluatorToString;
             this.expectedType = expectedType;
-            this.matcher = matcher;
+            @SuppressWarnings("unchecked")
+            Matcher<Object> downcast = (Matcher<Object>) matcher;
+            this.matcher = downcast;
             this.expectedWarnings = expectedWarnings;
             this.expectedTypeError = expectedTypeError;
-            this.canBuildEvaluator = data.stream().allMatch(d -> d.forceLiteral || EsqlDataTypes.isRepresentable(d.type));
+            this.canBuildEvaluator = data.stream().allMatch(d -> d.forceLiteral || DataType.isRepresentable(d.type));
             this.foldingExceptionClass = foldingExceptionClass;
             this.foldingExceptionMessage = foldingExceptionMessage;
         }
@@ -1427,6 +1489,7 @@ public TypedData get() {
      */
     public static class TypedData {
         public static final TypedData NULL = new TypedData(null, DataType.NULL, "<null>");
+        public static final TypedData MULTI_ROW_NULL = TypedData.multiRow(Collections.singletonList(null), DataType.NULL, "<null>");
 
         private final Object data;
         private final DataType type;
@@ -1555,7 +1618,7 @@ public Literal asLiteral() {
                     throw new IllegalStateException("Multirow values require exactly 1 element to be a literal, got " + values.size());
                 }
 
-                return new Literal(Source.synthetic(name), values, type);
+                return new Literal(Source.synthetic(name), values.get(0), type);
             }
             return new Literal(Source.synthetic(name), data, type);
         }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttributeTests.java
index e195f31664774..8e5c098c429db 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttributeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttributeTests.java
@@ -10,11 +10,15 @@
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
-import org.elasticsearch.xpack.esql.core.type.UnsupportedEsFieldTests;
+import org.elasticsearch.xpack.esql.type.UnsupportedEsFieldTests;
 
 public class UnsupportedAttributeTests extends AbstractAttributeTestCase<UnsupportedAttribute> {
     @Override
     protected UnsupportedAttribute create() {
+        return randomUnsupportedAttribute();
+    }
+
+    public static UnsupportedAttribute randomUnsupportedAttribute() {
         String name = randomAlphaOfLength(5);
         UnsupportedEsField field = UnsupportedEsFieldTests.randomUnsupportedEsField(4);
         String customMessage = randomBoolean() ? null : randomAlphaOfLength(9);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AvgTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AvgTests.java
index f456bd409059a..80737dac1aa58 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AvgTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AvgTests.java
@@ -53,7 +53,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers, true, (v, p) -> "numeric except unsigned_long or counter types");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java
new file mode 100644
index 0000000000000..5e23083d7c810
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.aggregate;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.lucene.internal.hppc.BitMixer;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.hash.MurmurHash3;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
+import org.elasticsearch.search.aggregations.metrics.HyperLogLogPlusPlus;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase;
+import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CountDistinctTests extends AbstractAggregationTestCase {
+    public CountDistinctTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        var suppliers = new ArrayList<TestCaseSupplier>();
+
+        var precisionSuppliers = Stream.of(
+            TestCaseSupplier.intCases(0, 100_000, true),
+            TestCaseSupplier.longCases(0L, 100_000L, true),
+            TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(100_000L), true)
+        ).flatMap(List::stream).toList();
+
+        Stream.of(
+            MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.dateCases(1, 1000),
+            MultiRowTestCaseSupplier.booleanCases(1, 1000),
+            MultiRowTestCaseSupplier.ipCases(1, 1000),
+            MultiRowTestCaseSupplier.versionCases(1, 1000),
+            MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD),
+            MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT)
+        ).flatMap(List::stream).forEach(fieldCaseSupplier -> {
+            // With precision
+            for (var precisionCaseSupplier : precisionSuppliers) {
+                suppliers.add(makeSupplier(fieldCaseSupplier, precisionCaseSupplier));
+            }
+
+            // Without precision
+            suppliers.add(makeSupplier(fieldCaseSupplier));
+        });
+
+        // No rows
+        for (var dataType : List.of(
+            DataType.INTEGER,
+            DataType.LONG,
+            DataType.DOUBLE,
+            DataType.DATETIME,
+            DataType.BOOLEAN,
+            DataType.IP,
+            DataType.VERSION,
+            DataType.KEYWORD,
+            DataType.TEXT
+        )) {
+            var emptyFieldSupplier = new TestCaseSupplier.TypedDataSupplier("No rows (" + dataType + ")", List::of, dataType, false, true);
+
+            // With precision
+            for (var precisionCaseSupplier : precisionSuppliers) {
+                suppliers.add(makeSupplier(emptyFieldSupplier, precisionCaseSupplier));
+            }
+
+            // Without precision
+            suppliers.add(makeSupplier(emptyFieldSupplier));
+        }
+
+        // "No rows" expects 0 here instead of null
+        // return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
+        return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers));
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new CountDistinct(source, args.get(0), args.size() > 1 ? args.get(1) : null);
+    }
+
+    private static TestCaseSupplier makeSupplier(
+        TestCaseSupplier.TypedDataSupplier fieldSupplier,
+        TestCaseSupplier.TypedDataSupplier precisionSupplier
+    ) {
+        return new TestCaseSupplier(fieldSupplier.name(), List.of(fieldSupplier.type(), precisionSupplier.type()), () -> {
+            var fieldTypedData = fieldSupplier.get();
+            var precisionTypedData = precisionSupplier.get().forceLiteral();
+            var values = fieldTypedData.multiRowData();
+            var precision = ((Number) precisionTypedData.data()).intValue();
+
+            long result;
+
+            if (fieldTypedData.type() == DataType.BOOLEAN) {
+                result = values.stream().distinct().count();
+            } else {
+                result = calculateExpectedResult(values, precision);
+            }
+
+            return new TestCaseSupplier.TestCase(
+                List.of(fieldTypedData, precisionTypedData),
+                "CountDistinct[field=Attribute[channel=0],precision=Attribute[channel=1]]",
+                DataType.LONG,
+                equalTo(result)
+            );
+        });
+    }
+
+    private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) {
+        return new TestCaseSupplier(fieldSupplier.name() + ", no precision", List.of(fieldSupplier.type()), () -> {
+            var fieldTypedData = fieldSupplier.get();
+            var values = fieldTypedData.multiRowData();
+
+            long result;
+
+            if (fieldTypedData.type() == DataType.BOOLEAN) {
+                result = values.stream().distinct().count();
+            } else {
+                result = calculateExpectedResult(values, 3000);
+            }
+
+            return new TestCaseSupplier.TestCase(
+                List.of(fieldTypedData),
+                "CountDistinct[field=Attribute[channel=0]]",
+                DataType.LONG,
+                equalTo(result)
+            );
+        });
+    }
+
+    private static long calculateExpectedResult(List<Object> values, int precision) {
+        // Can't use driverContext().bigArrays() from a static context
+        var bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofMb(256)).withCircuitBreaking();
+        try (var hll = new HyperLogLogPlusPlus(HyperLogLogPlusPlus.precisionFromThreshold(precision), bigArrays, 1)) {
+            var hash = new MurmurHash3.Hash128();
+            for (var value : values) {
+                if (value instanceof Integer casted) {
+                    hll.collect(0, BitMixer.mix64(casted));
+                } else if (value instanceof Long casted) {
+                    hll.collect(0, BitMixer.mix64(casted));
+                } else if (value instanceof Double casted) {
+                    hll.collect(0, BitMixer.mix64(Double.doubleToLongBits(casted)));
+                } else if (value instanceof BytesRef casted) {
+                    MurmurHash3.hash128(casted.bytes, casted.offset, casted.length, 0, hash);
+                    hll.collect(0, BitMixer.mix64(hash.h1));
+                } else {
+                    throw new IllegalArgumentException("Unsupported data type: " + value.getClass());
+                }
+            }
+
+            return hll.cardinality(0);
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountTests.java
new file mode 100644
index 0000000000000..979048534edbf
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountTests.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.aggregate;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase;
+import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CountTests extends AbstractAggregationTestCase {
+    public CountTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        var suppliers = new ArrayList<TestCaseSupplier>();
+
+        Stream.of(
+            MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.ulongCases(1, 1000, BigInteger.ZERO, UNSIGNED_LONG_MAX, true),
+            MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.dateCases(1, 1000),
+            MultiRowTestCaseSupplier.booleanCases(1, 1000),
+            MultiRowTestCaseSupplier.ipCases(1, 1000),
+            MultiRowTestCaseSupplier.versionCases(1, 1000),
+            MultiRowTestCaseSupplier.geoPointCases(1, 1000, true),
+            MultiRowTestCaseSupplier.cartesianPointCases(1, 1000, true),
+            MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD),
+            MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT)
+        ).flatMap(List::stream).map(CountTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers));
+
+        // No rows
+        for (var dataType : List.of(
+            DataType.INTEGER,
+            DataType.LONG,
+            DataType.DOUBLE,
+            DataType.DATETIME,
+            DataType.BOOLEAN,
+            DataType.IP,
+            DataType.VERSION,
+            DataType.KEYWORD,
+            DataType.TEXT,
+            DataType.GEO_POINT,
+            DataType.CARTESIAN_POINT,
+            DataType.UNSIGNED_LONG
+        )) {
+            suppliers.add(
+                new TestCaseSupplier(
+                    "No rows (" + dataType + ")",
+                    List.of(dataType),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(), dataType, "field")),
+                        "Count[field=Attribute[channel=0]]",
+                        DataType.LONG,
+                        equalTo(0L)
+                    )
+                )
+            );
+        }
+
+        // "No rows" expects 0 here instead of null
+        // return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
+        return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers));
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Count(source, args.get(0));
+    }
+
+    private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) {
+        return new TestCaseSupplier(fieldSupplier.name(), List.of(fieldSupplier.type()), () -> {
+            var fieldTypedData = fieldSupplier.get();
+            var rowCount = fieldTypedData.multiRowData().size();
+
+            return new TestCaseSupplier.TestCase(
+                List.of(fieldTypedData),
+                "Count[field=Attribute[channel=0]]",
+                DataType.LONG,
+                equalTo((long) rowCount)
+            );
+        });
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java
index 1d489e0146ad3..ce2bf7e262ae9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase;
 import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.elasticsearch.xpack.versionfield.Version;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -44,78 +45,14 @@ public static Iterable<Object[]> parameters() {
             MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true),
             MultiRowTestCaseSupplier.dateCases(1, 1000),
             MultiRowTestCaseSupplier.booleanCases(1, 1000),
-            MultiRowTestCaseSupplier.ipCases(1, 1000)
+            MultiRowTestCaseSupplier.ipCases(1, 1000),
+            MultiRowTestCaseSupplier.versionCases(1, 1000),
+            MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD),
+            MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT)
         ).flatMap(List::stream).map(MaxTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers));
 
         suppliers.addAll(
             List.of(
-                // Surrogates
-                new TestCaseSupplier(
-                    List.of(DataType.INTEGER),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(5, 8, -2, 0, 200), DataType.INTEGER, "field")),
-                        "Max[field=Attribute[channel=0]]",
-                        DataType.INTEGER,
-                        equalTo(200)
-                    )
-                ),
-                new TestCaseSupplier(
-                    List.of(DataType.LONG),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(5L, 8L, -2L, 0L, 200L), DataType.LONG, "field")),
-                        "Max[field=Attribute[channel=0]]",
-                        DataType.LONG,
-                        equalTo(200L)
-                    )
-                ),
-                new TestCaseSupplier(
-                    List.of(DataType.DOUBLE),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(5., 8., -2., 0., 200.), DataType.DOUBLE, "field")),
-                        "Max[field=Attribute[channel=0]]",
-                        DataType.DOUBLE,
-                        equalTo(200.)
-                    )
-                ),
-                new TestCaseSupplier(
-                    List.of(DataType.DATETIME),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(5L, 8L, 2L, 0L, 200L), DataType.DATETIME, "field")),
-                        "Max[field=Attribute[channel=0]]",
-                        DataType.DATETIME,
-                        equalTo(200L)
-                    )
-                ),
-                new TestCaseSupplier(
-                    List.of(DataType.BOOLEAN),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(true, false, false, true), DataType.BOOLEAN, "field")),
-                        "Max[field=Attribute[channel=0]]",
-                        DataType.BOOLEAN,
-                        equalTo(true)
-                    )
-                ),
-                new TestCaseSupplier(
-                    List.of(DataType.IP),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(
-                            TestCaseSupplier.TypedData.multiRow(
-                                List.of(
-                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))),
-                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))),
-                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::"))),
-                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("ffff::")))
-                                ),
-                                DataType.IP,
-                                "field"
-                            )
-                        ),
-                        "Max[field=Attribute[channel=0]]",
-                        DataType.IP,
-                        equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("ffff::"))))
-                    )
-                ),
-
                 // Folding
                 new TestCaseSupplier(
                     List.of(DataType.INTEGER),
@@ -176,11 +113,45 @@ public static Iterable<Object[]> parameters() {
                         DataType.IP,
                         equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))))
                     )
-                )
+                ),
+                new TestCaseSupplier(List.of(DataType.KEYWORD), () -> {
+                    var value = new BytesRef(randomAlphaOfLengthBetween(0, 50));
+                    return new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.KEYWORD, "field")),
+                        "Max[field=Attribute[channel=0]]",
+                        DataType.KEYWORD,
+                        equalTo(value)
+                    );
+                }),
+                new TestCaseSupplier(List.of(DataType.TEXT), () -> {
+                    var value = new BytesRef(randomAlphaOfLengthBetween(0, 50));
+                    return new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.TEXT, "field")),
+                        "Max[field=Attribute[channel=0]]",
+                        DataType.TEXT,
+                        equalTo(value)
+                    );
+                }),
+                new TestCaseSupplier(List.of(DataType.VERSION), () -> {
+                    var value = randomBoolean()
+                        ? new Version(randomAlphaOfLengthBetween(1, 10)).toBytesRef()
+                        : new Version(randomIntBetween(0, 100) + "." + randomIntBetween(0, 100) + "." + randomIntBetween(0, 100))
+                            .toBytesRef();
+                    return new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.VERSION, "field")),
+                        "Max[field=Attribute[channel=0]]",
+                        DataType.VERSION,
+                        equalTo(value)
+                    );
+                })
             )
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
+            suppliers,
+            false,
+            (v, p) -> "representable except unsigned_long and spatial types"
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviationTests.java
new file mode 100644
index 0000000000000..2ba3345c95bdc
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianAbsoluteDeviationTests.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.aggregate;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.search.aggregations.metrics.InternalMedianAbsoluteDeviation;
+import org.elasticsearch.search.aggregations.metrics.TDigestState;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase;
+import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class MedianAbsoluteDeviationTests extends AbstractAggregationTestCase {
+    public MedianAbsoluteDeviationTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        var suppliers = Stream.of(
+            MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true)
+        ).flatMap(List::stream).map(MedianAbsoluteDeviationTests::makeSupplier).toList();
+
+        return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new MedianAbsoluteDeviation(source, args.get(0));
+    }
+
+    private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) {
+        return new TestCaseSupplier(List.of(fieldSupplier.type()), () -> {
+            var fieldTypedData = fieldSupplier.get();
+
+            var digest = TDigestState.create(1000);
+
+            for (var value : fieldTypedData.multiRowData()) {
+                digest.add(((Number) value).doubleValue());
+            }
+
+            var expected = digest.size() == 0 ? null : InternalMedianAbsoluteDeviation.computeMedianAbsoluteDeviation(digest);
+
+            return new TestCaseSupplier.TestCase(
+                List.of(fieldTypedData),
+                "MedianAbsoluteDeviation[number=Attribute[channel=0]]",
+                DataType.DOUBLE,
+                equalTo(expected)
+            );
+        });
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianTests.java
new file mode 100644
index 0000000000000..b94045a9c732e
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MedianTests.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.aggregate;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.search.aggregations.metrics.TDigestState;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase;
+import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class MedianTests extends AbstractAggregationTestCase {
+    public MedianTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        var suppliers = Stream.of(
+            MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true)
+        ).flatMap(List::stream).map(MedianTests::makeSupplier).collect(Collectors.toCollection(ArrayList::new));
+
+        suppliers.addAll(
+            List.of(
+                // Folding
+                new TestCaseSupplier(
+                    List.of(DataType.INTEGER),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(200), DataType.INTEGER, "number")),
+                        "Median[field=Attribute[channel=0]]",
+                        DataType.DOUBLE,
+                        equalTo(200.)
+                    )
+                ),
+                new TestCaseSupplier(
+                    List.of(DataType.LONG),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(200L), DataType.LONG, "number")),
+                        "Median[field=Attribute[channel=0]]",
+                        DataType.DOUBLE,
+                        equalTo(200.)
+                    )
+                ),
+                new TestCaseSupplier(
+                    List.of(DataType.DOUBLE),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(200.), DataType.DOUBLE, "number")),
+                        "Median[field=Attribute[channel=0]]",
+                        DataType.DOUBLE,
+                        equalTo(200.)
+                    )
+                )
+            )
+        );
+
+        return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Median(source, args.get(0));
+    }
+
+    private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) {
+        return new TestCaseSupplier(List.of(fieldSupplier.type()), () -> {
+            var fieldTypedData = fieldSupplier.get();
+
+            var digest = TDigestState.create(1000);
+
+            for (var value : fieldTypedData.multiRowData()) {
+                digest.add(((Number) value).doubleValue());
+            }
+
+            var expected = digest.size() == 0 ? null : digest.quantile(0.5);
+
+            return new TestCaseSupplier.TestCase(
+                List.of(fieldTypedData),
+                "Median[number=Attribute[channel=0]]",
+                DataType.DOUBLE,
+                equalTo(expected)
+            );
+        });
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java
index b5fb5b2c1c414..7250072cd2003 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase;
 import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.elasticsearch.xpack.versionfield.Version;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -44,78 +45,14 @@ public static Iterable<Object[]> parameters() {
             MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true),
             MultiRowTestCaseSupplier.dateCases(1, 1000),
             MultiRowTestCaseSupplier.booleanCases(1, 1000),
-            MultiRowTestCaseSupplier.ipCases(1, 1000)
+            MultiRowTestCaseSupplier.ipCases(1, 1000),
+            MultiRowTestCaseSupplier.versionCases(1, 1000),
+            MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD),
+            MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT)
         ).flatMap(List::stream).map(MinTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers));
 
         suppliers.addAll(
             List.of(
-                // Surrogates
-                new TestCaseSupplier(
-                    List.of(DataType.INTEGER),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(5, 8, -2, 0, 200), DataType.INTEGER, "field")),
-                        "Min[field=Attribute[channel=0]]",
-                        DataType.INTEGER,
-                        equalTo(-2)
-                    )
-                ),
-                new TestCaseSupplier(
-                    List.of(DataType.LONG),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(5L, 8L, -2L, 0L, 200L), DataType.LONG, "field")),
-                        "Min[field=Attribute[channel=0]]",
-                        DataType.LONG,
-                        equalTo(-2L)
-                    )
-                ),
-                new TestCaseSupplier(
-                    List.of(DataType.DOUBLE),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(5., 8., -2., 0., 200.), DataType.DOUBLE, "field")),
-                        "Min[field=Attribute[channel=0]]",
-                        DataType.DOUBLE,
-                        equalTo(-2.)
-                    )
-                ),
-                new TestCaseSupplier(
-                    List.of(DataType.DATETIME),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(5L, 8L, 2L, 0L, 200L), DataType.DATETIME, "field")),
-                        "Min[field=Attribute[channel=0]]",
-                        DataType.DATETIME,
-                        equalTo(0L)
-                    )
-                ),
-                new TestCaseSupplier(
-                    List.of(DataType.BOOLEAN),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(true, false, false, true), DataType.BOOLEAN, "field")),
-                        "Min[field=Attribute[channel=0]]",
-                        DataType.BOOLEAN,
-                        equalTo(false)
-                    )
-                ),
-                new TestCaseSupplier(
-                    List.of(DataType.IP),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(
-                            TestCaseSupplier.TypedData.multiRow(
-                                List.of(
-                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))),
-                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))),
-                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::"))),
-                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("ffff::")))
-                                ),
-                                DataType.IP,
-                                "field"
-                            )
-                        ),
-                        "Min[field=Attribute[channel=0]]",
-                        DataType.IP,
-                        equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::"))))
-                    )
-                ),
-
                 // Folding
                 new TestCaseSupplier(
                     List.of(DataType.INTEGER),
@@ -176,11 +113,45 @@ public static Iterable<Object[]> parameters() {
                         DataType.IP,
                         equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))))
                     )
-                )
+                ),
+                new TestCaseSupplier(List.of(DataType.KEYWORD), () -> {
+                    var value = new BytesRef(randomAlphaOfLengthBetween(0, 50));
+                    return new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.KEYWORD, "field")),
+                        "Min[field=Attribute[channel=0]]",
+                        DataType.KEYWORD,
+                        equalTo(value)
+                    );
+                }),
+                new TestCaseSupplier(List.of(DataType.TEXT), () -> {
+                    var value = new BytesRef(randomAlphaOfLengthBetween(0, 50));
+                    return new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.TEXT, "field")),
+                        "Min[field=Attribute[channel=0]]",
+                        DataType.TEXT,
+                        equalTo(value)
+                    );
+                }),
+                new TestCaseSupplier(List.of(DataType.VERSION), () -> {
+                    var value = randomBoolean()
+                        ? new Version(randomAlphaOfLengthBetween(1, 10)).toBytesRef()
+                        : new Version(randomIntBetween(0, 100) + "." + randomIntBetween(0, 100) + "." + randomIntBetween(0, 100))
+                            .toBytesRef();
+                    return new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.VERSION, "field")),
+                        "Min[field=Attribute[channel=0]]",
+                        DataType.VERSION,
+                        equalTo(value)
+                    );
+                })
             )
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
+            suppliers,
+            false,
+            (v, p) -> "representable except unsigned_long and spatial types"
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java
index 5271431bd43b8..be11515876966 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java
@@ -52,7 +52,7 @@ public static Iterable<Object[]> parameters() {
             }
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers, false, (v, p) -> "numeric except unsigned_long");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java
new file mode 100644
index 0000000000000..b79252c694084
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroidTests.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.aggregate;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.utils.GeometryValidator;
+import org.elasticsearch.geometry.utils.WellKnownBinary;
+import org.elasticsearch.search.aggregations.metrics.CompensatedSum;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase;
+import org.elasticsearch.xpack.esql.expression.function.FunctionName;
+import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.nio.ByteOrder;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+@FunctionName("st_centroid_agg")
+public class SpatialCentroidTests extends AbstractAggregationTestCase {
+    public SpatialCentroidTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        var suppliers = Stream.of(
+            MultiRowTestCaseSupplier.geoPointCases(1, 1000, true),
+            MultiRowTestCaseSupplier.cartesianPointCases(1, 1000, true)
+        ).flatMap(List::stream).map(SpatialCentroidTests::makeSupplier).toList();
+
+        // The withNoRowsExpectingNull() cases don't work here, as this aggregator doesn't return nulls.
+        // return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
+        return parameterSuppliersFromTypedData(randomizeBytesRefsOffset(suppliers));
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new SpatialCentroid(source, args.get(0));
+    }
+
+    private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) {
+        if (fieldSupplier.type() != DataType.CARTESIAN_POINT && fieldSupplier.type() != DataType.GEO_POINT) {
+            throw new IllegalStateException("Unexpected type: " + fieldSupplier.type());
+        }
+
+        return new TestCaseSupplier(List.of(fieldSupplier.type()), () -> {
+            var fieldTypedData = fieldSupplier.get();
+            var values = fieldTypedData.multiRowData();
+
+            var xSum = new CompensatedSum(0, 0);
+            var ySum = new CompensatedSum(0, 0);
+            long count = 0;
+
+            for (var value : values) {
+                var wkb = (BytesRef) value;
+                var point = (Point) WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, wkb.bytes, wkb.offset, wkb.length);
+                xSum.add(point.getX());
+                ySum.add(point.getY());
+                count++;
+            }
+
+            var expected = new BytesRef(
+                WellKnownBinary.toWKB(new Point(xSum.value() / count, ySum.value() / count), ByteOrder.LITTLE_ENDIAN)
+            );
+
+            return new TestCaseSupplier.TestCase(
+                List.of(fieldTypedData),
+                "SpatialCentroid[field=Attribute[channel=0]]",
+                fieldTypedData.type(),
+                equalTo(expected)
+            );
+        });
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SumTests.java
new file mode 100644
index 0000000000000..4f14dafc8b30d
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SumTests.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.aggregate;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase;
+import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.hamcrest.Matchers.equalTo;
+
+public class SumTests extends AbstractAggregationTestCase {
+    public SumTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        var suppliers = new ArrayList<TestCaseSupplier>();
+
+        Stream.of(MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true)
+        // Longs currently throw on overflow.
+        // Restore after https://github.com/elastic/elasticsearch/issues/110437
+        // MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true),
+        // Doubles currently return +/-Infinity on overflow.
+        // Restore after https://github.com/elastic/elasticsearch/issues/111026
+        // MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true)
+        ).flatMap(List::stream).map(SumTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers));
+
+        suppliers.addAll(
+            List.of(
+                // Folding
+                new TestCaseSupplier(
+                    List.of(DataType.INTEGER),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(200), DataType.INTEGER, "field")),
+                        "Sum[field=Attribute[channel=0]]",
+                        DataType.LONG,
+                        equalTo(200L)
+                    )
+                ),
+                new TestCaseSupplier(
+                    List.of(DataType.LONG),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(200L), DataType.LONG, "field")),
+                        "Sum[field=Attribute[channel=0]]",
+                        DataType.LONG,
+                        equalTo(200L)
+                    )
+                ),
+                new TestCaseSupplier(
+                    List.of(DataType.DOUBLE),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(200.), DataType.DOUBLE, "field")),
+                        "Sum[field=Attribute[channel=0]]",
+                        DataType.DOUBLE,
+                        equalTo(200.)
+                    )
+                )
+            )
+        );
+
+        return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Sum(source, args.get(0));
+    }
+
+    private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) {
+        return new TestCaseSupplier(List.of(fieldSupplier.type()), () -> {
+            var fieldTypedData = fieldSupplier.get();
+
+            Object expected;
+
+            try {
+                expected = switch (fieldTypedData.type().widenSmallNumeric()) {
+                    case INTEGER -> fieldTypedData.multiRowData()
+                        .stream()
+                        .map(v -> (Integer) v)
+                        .collect(Collectors.summarizingInt(Integer::intValue))
+                        .getSum();
+                    case LONG -> fieldTypedData.multiRowData()
+                        .stream()
+                        .map(v -> (Long) v)
+                        .collect(Collectors.summarizingLong(Long::longValue))
+                        .getSum();
+                    case DOUBLE -> {
+                        var value = fieldTypedData.multiRowData()
+                            .stream()
+                            .map(v -> (Double) v)
+                            .collect(Collectors.summarizingDouble(Double::doubleValue))
+                            .getSum();
+
+                        if (Double.isInfinite(value) || Double.isNaN(value)) {
+                            yield null;
+                        }
+
+                        yield value;
+                    }
+                    default -> throw new IllegalStateException("Unexpected value: " + fieldTypedData.type());
+                };
+            } catch (Exception e) {
+                expected = null;
+            }
+
+            var dataType = fieldTypedData.type().isWholeNumber() == false || fieldTypedData.type() == UNSIGNED_LONG
+                ? DataType.DOUBLE
+                : DataType.LONG;
+
+            return new TestCaseSupplier.TestCase(List.of(fieldTypedData), "Sum[field=Attribute[channel=0]]", dataType, equalTo(expected));
+        });
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java
index ffb9e0dcd584e..f64d6a200a031 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopTests.java
@@ -10,7 +10,9 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -43,7 +45,8 @@ public static Iterable<Object[]> parameters() {
                     MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true),
                     MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true),
                     MultiRowTestCaseSupplier.dateCases(1, 1000),
-                    MultiRowTestCaseSupplier.booleanCases(1, 1000)
+                    MultiRowTestCaseSupplier.booleanCases(1, 1000),
+                    MultiRowTestCaseSupplier.ipCases(1, 1000)
                 )
                     .flatMap(List::stream)
                     .map(fieldCaseSupplier -> TopTests.makeSupplier(fieldCaseSupplier, limitCaseSupplier, order))
@@ -119,6 +122,28 @@ public static Iterable<Object[]> parameters() {
                         equalTo(200L)
                     )
                 ),
+                new TestCaseSupplier(
+                    List.of(DataType.IP),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(
+                            TestCaseSupplier.TypedData.multiRow(
+                                List.of(
+                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))),
+                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))),
+                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::"))),
+                                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("ffff::")))
+                                ),
+                                DataType.IP,
+                                "field"
+                            ),
+                            new TestCaseSupplier.TypedData(1, DataType.INTEGER, "limit").forceLiteral(),
+                            new TestCaseSupplier.TypedData(new BytesRef("desc"), DataType.KEYWORD, "order").forceLiteral()
+                        ),
+                        "Top[field=Attribute[channel=0], limit=Attribute[channel=1], order=Attribute[channel=2]]",
+                        DataType.IP,
+                        equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("ffff::"))))
+                    )
+                ),
 
                 // Folding
                 new TestCaseSupplier(
@@ -186,6 +211,23 @@ public static Iterable<Object[]> parameters() {
                         equalTo(200L)
                     )
                 ),
+                new TestCaseSupplier(
+                    List.of(DataType.IP),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(
+                            TestCaseSupplier.TypedData.multiRow(
+                                List.of(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))),
+                                DataType.IP,
+                                "field"
+                            ),
+                            new TestCaseSupplier.TypedData(1, DataType.INTEGER, "limit").forceLiteral(),
+                            new TestCaseSupplier.TypedData(new BytesRef("desc"), DataType.KEYWORD, "order").forceLiteral()
+                        ),
+                        "Top[field=Attribute[channel=0], limit=Attribute[channel=1], order=Attribute[channel=2]]",
+                        DataType.IP,
+                        equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))))
+                    )
+                ),
 
                 // Resolution errors
                 new TestCaseSupplier(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java
new file mode 100644
index 0000000000000..55320543d0ec3
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.aggregate;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase;
+import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class ValuesTests extends AbstractAggregationTestCase {
+    public ValuesTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        var suppliers = new ArrayList<TestCaseSupplier>();
+
+        Stream.of(
+            MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.dateCases(1, 1000),
+            MultiRowTestCaseSupplier.booleanCases(1, 1000),
+            MultiRowTestCaseSupplier.ipCases(1, 1000),
+            MultiRowTestCaseSupplier.versionCases(1, 1000),
+            // Lower values for strings, as they take more space and may trigger the circuit breaker
+            MultiRowTestCaseSupplier.stringCases(1, 20, DataType.KEYWORD),
+            MultiRowTestCaseSupplier.stringCases(1, 20, DataType.TEXT)
+        ).flatMap(List::stream).map(ValuesTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers));
+
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
+            suppliers,
+            false,
+            (v, p) -> "any type except unsigned_long and spatial types"
+        );
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Values(source, args.get(0));
+    }
+
+    @SuppressWarnings("unchecked")
+    private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) {
+        return new TestCaseSupplier(fieldSupplier.name(), List.of(fieldSupplier.type()), () -> {
+            var fieldTypedData = fieldSupplier.get();
+
+            var expected = fieldTypedData.multiRowData()
+                .stream()
+                .map(v -> (Comparable<? super Comparable<?>>) v)
+                .collect(Collectors.toSet());
+
+            return new TestCaseSupplier.TestCase(
+                List.of(fieldTypedData),
+                "Values[field=Attribute[channel=0]]",
+                fieldSupplier.type(),
+                expected.isEmpty() ? nullValue() : valuesInAnyOrder(expected)
+            );
+        });
+    }
+
+    private static <T, C extends T> Matcher<Object> valuesInAnyOrder(Collection<T> data) {
+        if (data == null) {
+            return nullValue();
+        }
+        if (data.size() == 1) {
+            return equalTo(data.iterator().next());
+        }
+        var matcher = containsInAnyOrder(data.toArray());
+        // New Matcher, as `containsInAnyOrder` returns Matcher<Iterable<...>> instead of Matcher<Object>
+        return new BaseMatcher<>() {
+            @Override
+            public void describeTo(Description description) {
+                matcher.describeTo(description);
+            }
+
+            @Override
+            public boolean matches(Object item) {
+                if (item instanceof Iterable<?> == false) {
+                    return false;
+                }
+
+                var castedItem = (Iterable<?>) item;
+
+                return matcher.matches(castedItem);
+            }
+        };
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvgTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvgTests.java
new file mode 100644
index 0000000000000..2c2ffc97f268c
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvgTests.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.aggregate;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase;
+import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class WeightedAvgTests extends AbstractAggregationTestCase {
+    public WeightedAvgTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        var suppliers = new ArrayList<TestCaseSupplier>();
+
+        var numberCases = Stream.of(
+            MultiRowTestCaseSupplier.intCases(1000, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.longCases(1000, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true),
+            MultiRowTestCaseSupplier.doubleCases(1000, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true)
+        ).flatMap(List::stream).toList();
+
+        // Most of those cases, if not all, will be ignored, as they use complex surrogates.
+        // Kept here to correctly generate the function types docs, and in case the tests are later improved to support them.
+        for (var number : numberCases) {
+            for (var weight : numberCases) {
+                suppliers.add(makeSupplier(number, weight));
+            }
+        }
+
+        suppliers.addAll(
+            List.of(
+                // Folding
+                new TestCaseSupplier(
+                    List.of(DataType.INTEGER, DataType.INTEGER),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(
+                            TestCaseSupplier.TypedData.multiRow(List.of(5), DataType.INTEGER, "number"),
+                            TestCaseSupplier.TypedData.multiRow(List.of(100), DataType.INTEGER, "weight")
+                        ),
+                        "WeightedAvg[number=Attribute[channel=0],weight=Attribute[channel=1]]",
+                        DataType.DOUBLE,
+                        equalTo(5.)
+                    )
+                ),
+                new TestCaseSupplier(
+                    List.of(DataType.LONG, DataType.INTEGER),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(
+                            TestCaseSupplier.TypedData.multiRow(List.of(5L), DataType.LONG, "number"),
+                            TestCaseSupplier.TypedData.multiRow(List.of(100), DataType.INTEGER, "weight")
+                        ),
+                        "WeightedAvg[number=Attribute[channel=0],weight=Attribute[channel=1]]",
+                        DataType.DOUBLE,
+                        equalTo(5.)
+                    )
+                ),
+                new TestCaseSupplier(
+                    List.of(DataType.DOUBLE, DataType.INTEGER),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(
+                            TestCaseSupplier.TypedData.multiRow(List.of(5.), DataType.DOUBLE, "number"),
+                            TestCaseSupplier.TypedData.multiRow(List.of(100), DataType.INTEGER, "weight")
+                        ),
+                        "WeightedAvg[number=Attribute[channel=0],weight=Attribute[channel=1]]",
+                        DataType.DOUBLE,
+                        equalTo(5.)
+                    )
+                )
+            )
+        );
+
+        return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers);
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new WeightedAvg(source, args.get(0), args.get(1));
+    }
+
+    private static TestCaseSupplier makeSupplier(
+        TestCaseSupplier.TypedDataSupplier fieldSupplier,
+        TestCaseSupplier.TypedDataSupplier weightSupplier
+    ) {
+        return new TestCaseSupplier(List.of(fieldSupplier.type(), weightSupplier.type()), () -> {
+            var fieldTypedData = fieldSupplier.get();
+            var weightTypedData = weightSupplier.get();
+
+            var fieldValues = fieldTypedData.multiRowData();
+            var weightValues = weightTypedData.multiRowData();
+
+            if (fieldValues.size() != weightValues.size()) {
+                throw new IllegalArgumentException("Field and weight values must have the same size");
+            }
+
+            var weightedSum = IntStream.range(0, fieldValues.size())
+                .mapToDouble(i -> ((Number) fieldValues.get(i)).doubleValue() * ((Number) weightValues.get(i)).doubleValue())
+                .sum();
+            var totalWeights = weightValues.stream().mapToDouble(v -> ((Number) v).doubleValue()).sum();
+
+            var expected = totalWeights == 0 ? null : weightedSum / totalWeights;
+
+            return new TestCaseSupplier.TestCase(
+                List.of(fieldTypedData, weightTypedData),
+                "WeightedAvg[number=Attribute[channel=0],weight=Attribute[channel=1]]",
+                DataType.DOUBLE,
+                equalTo(expected)
+            );
+        });
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java
index 9100e71de76df..a26504b8ced9a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java
@@ -73,7 +73,7 @@ public static Iterable<Object[]> parameters() {
     }
 
     // TODO once we cast above the functions we can drop these
-    private static final DataType[] DATE_BOUNDS_TYPE = new DataType[] { DataType.DATETIME };
+    private static final DataType[] DATE_BOUNDS_TYPE = new DataType[] { DataType.DATETIME, DataType.KEYWORD, DataType.TEXT };
 
     private static void dateCases(List<TestCaseSupplier> suppliers, String name, LongSupplier date) {
         for (DataType fromType : DATE_BOUNDS_TYPE) {
@@ -89,7 +89,21 @@ private static void dateCases(List<TestCaseSupplier> suppliers, String name, Lon
                         args,
                         "DateTruncEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding[DAY_OF_MONTH in Z][fixed to midnight]]",
                         DataType.DATETIME,
-                        dateResultsMatcher(args)
+                        resultsMatcher(args)
+                    );
+                }));
+                // same as above, but a low bucket count and datetime bounds that match it (at hour span)
+                suppliers.add(new TestCaseSupplier(name, List.of(DataType.DATETIME, DataType.INTEGER, fromType, toType), () -> {
+                    List<TestCaseSupplier.TypedData> args = new ArrayList<>();
+                    args.add(new TestCaseSupplier.TypedData(date.getAsLong(), DataType.DATETIME, "field"));
+                    args.add(new TestCaseSupplier.TypedData(4, DataType.INTEGER, "buckets").forceLiteral());
+                    args.add(dateBound("from", fromType, "2023-02-17T09:00:00Z"));
+                    args.add(dateBound("to", toType, "2023-02-17T12:00:00Z"));
+                    return new TestCaseSupplier.TestCase(
+                        args,
+                        "DateTruncEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding[3600000 in Z][fixed]]",
+                        DataType.DATETIME,
+                        equalTo(Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).build().prepareForUnknown().round(date.getAsLong()))
                     );
                 }));
             }
@@ -122,7 +136,7 @@ private static void dateCasesWithSpan(
                 args,
                 "DateTruncEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding" + spanStr + "]",
                 DataType.DATETIME,
-                dateResultsMatcher(args)
+                resultsMatcher(args)
             );
         }));
     }
@@ -153,7 +167,7 @@ private static void numberCases(List<TestCaseSupplier> suppliers, String name, D
                             + ", "
                             + "rhs=LiteralsEvaluator[lit=50.0]]], rhs=LiteralsEvaluator[lit=50.0]]",
                         DataType.DOUBLE,
-                        dateResultsMatcher(args)
+                        resultsMatcher(args)
                     );
                 }));
             }
@@ -173,26 +187,29 @@ private static TestCaseSupplier.TypedData numericBound(String name, DataType typ
     }
 
     private static void numberCasesWithSpan(List<TestCaseSupplier> suppliers, String name, DataType numberType, Supplier<Number> number) {
-        suppliers.add(new TestCaseSupplier(name, List.of(numberType, DataType.DOUBLE), () -> {
-            List<TestCaseSupplier.TypedData> args = new ArrayList<>();
-            args.add(new TestCaseSupplier.TypedData(number.get(), "field"));
-            args.add(new TestCaseSupplier.TypedData(50., DataType.DOUBLE, "span").forceLiteral());
-            String attr = "Attribute[channel=0]";
-            if (numberType == DataType.INTEGER) {
-                attr = "CastIntToDoubleEvaluator[v=" + attr + "]";
-            } else if (numberType == DataType.LONG) {
-                attr = "CastLongToDoubleEvaluator[v=" + attr + "]";
-            }
-            return new TestCaseSupplier.TestCase(
-                args,
-                "MulDoublesEvaluator[lhs=FloorDoubleEvaluator[val=DivDoublesEvaluator[lhs="
-                    + attr
-                    + ", "
-                    + "rhs=LiteralsEvaluator[lit=50.0]]], rhs=LiteralsEvaluator[lit=50.0]]",
-                DataType.DOUBLE,
-                dateResultsMatcher(args)
-            );
-        }));
+        for (Number span : List.of(50, 50L, 50d)) {
+            DataType spanType = DataType.fromJava(span);
+            suppliers.add(new TestCaseSupplier(name, List.of(numberType, spanType), () -> {
+                List<TestCaseSupplier.TypedData> args = new ArrayList<>();
+                args.add(new TestCaseSupplier.TypedData(number.get(), "field"));
+                args.add(new TestCaseSupplier.TypedData(span, spanType, "span").forceLiteral());
+                String attr = "Attribute[channel=0]";
+                if (numberType == DataType.INTEGER) {
+                    attr = "CastIntToDoubleEvaluator[v=" + attr + "]";
+                } else if (numberType == DataType.LONG) {
+                    attr = "CastLongToDoubleEvaluator[v=" + attr + "]";
+                }
+                return new TestCaseSupplier.TestCase(
+                    args,
+                    "MulDoublesEvaluator[lhs=FloorDoubleEvaluator[val=DivDoublesEvaluator[lhs="
+                        + attr
+                        + ", "
+                        + "rhs=LiteralsEvaluator[lit=50.0]]], rhs=LiteralsEvaluator[lit=50.0]]",
+                    DataType.DOUBLE,
+                    resultsMatcher(args)
+                );
+            }));
+        }
 
     }
 
@@ -200,7 +217,7 @@ private static TestCaseSupplier.TypedData keywordDateLiteral(String name, DataTy
         return new TestCaseSupplier.TypedData(date, type, name).forceLiteral();
     }
 
-    private static Matcher<Object> dateResultsMatcher(List<TestCaseSupplier.TypedData> typedData) {
+    private static Matcher<Object> resultsMatcher(List<TestCaseSupplier.TypedData> typedData) {
         if (typedData.get(0).type() == DataType.DATETIME) {
             long millis = ((Number) typedData.get(0).data()).longValue();
             return equalTo(Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build().prepareForUnknown().round(millis));
@@ -218,9 +235,4 @@ protected Expression build(Source source, List<Expression> args) {
         }
         return new Bucket(source, args.get(0), args.get(1), from, to);
     }
-
-    @Override
-    public void testSimpleWithNulls() {
-        assumeFalse("we test nulls in parameters", true);
-    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java
index 760c57f6570bb..ef4fa6d51a888 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.List;
 import java.util.Map;
@@ -23,16 +23,16 @@
 import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization;
 
 public abstract class AbstractConfigurationFunctionTestCase extends AbstractScalarFunctionTestCase {
-    protected abstract Expression buildWithConfiguration(Source source, List<Expression> args, EsqlConfiguration configuration);
+    protected abstract Expression buildWithConfiguration(Source source, List<Expression> args, Configuration configuration);
 
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return buildWithConfiguration(source, args, EsqlTestUtils.TEST_CFG);
     }
 
-    static EsqlConfiguration randomConfiguration() {
+    static Configuration randomConfiguration() {
         // TODO: Randomize the query and maybe the pragmas.
-        return new EsqlConfiguration(
+        return new Configuration(
             randomZone(),
             randomLocale(random()),
             randomBoolean() ? null : randomAlphaOfLength(randomInt(64)),
@@ -47,12 +47,12 @@ static EsqlConfiguration randomConfiguration() {
     }
 
     public void testSerializationWithConfiguration() {
-        EsqlConfiguration config = randomConfiguration();
+        Configuration config = randomConfiguration();
         Expression expr = buildWithConfiguration(testCase.getSource(), testCase.getDataAsFields(), config);
 
         assertSerialization(expr, config);
 
-        EsqlConfiguration differentConfig;
+        Configuration differentConfig;
         do {
             differentConfig = randomConfiguration();
         } while (config.equals(differentConfig));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
index 0a03af206b846..f19ac0e08a419 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
@@ -30,6 +30,7 @@
 import java.util.stream.Stream;
 
 import static org.elasticsearch.compute.data.BlockUtils.toJavaObject;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -44,6 +45,7 @@ public CaseTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseS
      */
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
+        // TODO this needs lots of stuff flipped to parameters
         return parameterSuppliersFromTypedData(
             List.of(new TestCaseSupplier("keyword", List.of(DataType.BOOLEAN, DataType.KEYWORD, DataType.KEYWORD), () -> {
                 List<TestCaseSupplier.TypedData> typedData = List.of(
@@ -203,35 +205,6 @@ public static Iterable<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlock) {
-        if (nullBlock == 0) {
-            if (data.size() == 2) {
-                assertThat(value.isNull(0), equalTo(true));
-            } else if (data.size() > 2) {
-                assertThat(toJavaObject(value, 0), equalTo(data.get(2)));
-            }
-            return;
-        }
-        if (((Boolean) data.get(0)).booleanValue()) {
-            if (nullBlock == 1) {
-                super.assertSimpleWithNulls(data, value, nullBlock);
-            } else {
-                assertThat(toJavaObject(value, 0), equalTo(data.get(1)));
-            }
-            return;
-        }
-        if (nullBlock == 2) {
-            super.assertSimpleWithNulls(data, value, nullBlock);
-        } else {
-            if (data.size() > 2) {
-                assertThat(toJavaObject(value, 0), equalTo(data.get(2)));
-            } else {
-                super.assertSimpleWithNulls(data, value, nullBlock);
-            }
-        }
-    }
-
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new Case(Source.EMPTY, args.get(0), args.subList(1, args.size()));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java
index e08da9850b555..f472e5ef5efd9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java
@@ -55,7 +55,7 @@ public static Iterable<Object[]> parameters() {
             );
         }));
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java
index 88ca7d0452b3e..06b5af8d7067b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java
@@ -24,6 +24,7 @@
 import java.util.List;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.hamcrest.Matchers.equalTo;
 
 @FunctionName("to_base64")
@@ -55,7 +56,7 @@ public static Iterable<Object[]> parameters() {
             );
         }));
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java
index c5b9b2501aeae..73837c23ff640 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java
@@ -80,7 +80,7 @@ public static Iterable<Object[]> parameters() {
             emptyList()
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "boolean or numeric or string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
index a59e7b0085e4c..9548b0476f3c2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
@@ -72,7 +72,7 @@ public static Iterable<Object[]> parameters() {
             );
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "cartesian_point or string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
index 973431d676b82..5784c3559b10e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
@@ -73,7 +73,7 @@ public static Iterable<Object[]> parameters() {
             );
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "cartesian_point or cartesian_shape or string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
index e512334391bed..7799c3c756f23 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
@@ -11,6 +11,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -37,6 +38,13 @@ public static Iterable<Object[]> parameters() {
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
         TestCaseSupplier.forUnaryDatetime(suppliers, read, DataType.DATETIME, Instant::toEpochMilli, emptyList());
+        TestCaseSupplier.forUnaryDateNanos(
+            suppliers,
+            "ToDatetimeFromDateNanosEvaluator[field=" + read + "]",
+            DataType.DATETIME,
+            i -> DateUtils.toMilliSeconds(DateUtils.toLong(i)),
+            emptyList()
+        );
 
         TestCaseSupplier.forUnaryInt(
             suppliers,
@@ -162,7 +170,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "date_nanos or datetime or numeric or string");
     }
 
     private static String randomDateString(long from, long to) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java
index bd07141009d3e..4d52a470edb87 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java
@@ -89,7 +89,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
index d4d20629da09e..d5153019c1e41 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
@@ -139,7 +139,11 @@ public static Iterable<Object[]> parameters() {
             List.of()
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
+            true,
+            suppliers,
+            (v, p) -> "boolean or counter_double or counter_integer or counter_long or datetime or numeric or string"
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
index 7a3b83f3ab113..445c4b8f15d10 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
@@ -66,7 +66,7 @@ public static Iterable<Object[]> parameters() {
             );
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
index 831539852846c..38d871b17e8ba 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
@@ -66,7 +66,7 @@ public static Iterable<Object[]> parameters() {
                 List.of()
             );
         }
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or geo_shape or string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
index ffa94548f0a23..66358b7ba7440 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
@@ -63,7 +63,7 @@ public static Iterable<Object[]> parameters() {
         );
 
         // add null as parameter
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "ip or string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
index 7984c1e04effc..eb81d48e0c5be 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
@@ -268,7 +268,11 @@ public static Iterable<Object[]> parameters() {
             List.of()
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
+            true,
+            suppliers,
+            (v, p) -> "boolean or counter_integer or datetime or numeric or string"
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
index 27c69ae977f6b..4c2cf14af41e9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
@@ -11,6 +11,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -43,6 +44,7 @@ public static Iterable<Object[]> parameters() {
 
         // datetimes
         TestCaseSupplier.forUnaryDatetime(suppliers, read, DataType.LONG, Instant::toEpochMilli, List.of());
+        TestCaseSupplier.forUnaryDateNanos(suppliers, read, DataType.LONG, DateUtils::toLong, List.of());
         // random strings that don't look like a long
         TestCaseSupplier.forUnaryStrings(
             suppliers,
@@ -227,7 +229,11 @@ public static Iterable<Object[]> parameters() {
             l -> ((Integer) l).longValue(),
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
+            true,
+            suppliers,
+            (v, p) -> "boolean or counter_integer or counter_long or date_nanos or datetime or numeric or string"
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java
index 33e8eee7a8de4..6e99e720ad5ea 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java
@@ -70,7 +70,7 @@ public static Iterable<Object[]> parameters() {
             List.of()
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java
index 809b4ddaa78a4..0b101efa073d9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java
@@ -11,6 +11,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
@@ -87,6 +88,13 @@ public static Iterable<Object[]> parameters() {
             i -> new BytesRef(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(i.toEpochMilli())),
             List.of()
         );
+        TestCaseSupplier.forUnaryDateNanos(
+            suppliers,
+            "ToStringFromDateNanosEvaluator[field=" + read + "]",
+            DataType.KEYWORD,
+            i -> new BytesRef(DateFieldMapper.DEFAULT_DATE_TIME_NANOS_FORMATTER.formatNanos(DateUtils.toLong(i))),
+            List.of()
+        );
         TestCaseSupplier.forUnaryGeoPoint(
             suppliers,
             "ToStringFromGeoPointEvaluator[field=" + read + "]",
@@ -130,9 +138,12 @@ public static Iterable<Object[]> parameters() {
             v -> new BytesRef(v.toString()),
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> typeErrorString);
     }
 
+    private static String typeErrorString =
+        "boolean or cartesian_point or cartesian_shape or datetime or geo_point or geo_shape or ip or numeric or string or version";
+
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new ToString(source, args.get(0));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java
index a1fccac8edfd1..d8122aa73f81a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java
@@ -244,7 +244,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "boolean or datetime or numeric or string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
index 1c37afc1c0722..46a8086f9479c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
@@ -60,7 +60,7 @@ public static Iterable<Object[]> parameters() {
             );
         }
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string or version");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java
index 4af2ce1b7cb00..81f391d637317 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
 import java.time.ZonedDateTime;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Supplier;
 
@@ -35,76 +36,84 @@ public static Iterable<Object[]> parameters() {
         ZonedDateTime zdtStart = ZonedDateTime.parse("2023-12-04T10:15:30Z");
         ZonedDateTime zdtEnd = ZonedDateTime.parse("2023-12-05T10:45:00Z");
 
-        return parameterSuppliersFromTypedData(
-            List.of(
-                new TestCaseSupplier(
-                    "Date Diff In Seconds - OK",
-                    List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.KEYWORD, "unit"),
-                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
-                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
-                        ),
-                        "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], "
-                            + "endTimestamp=Attribute[channel=2]]",
-                        DataType.INTEGER,
-                        equalTo(88170)
-                    )
-                ),
-                new TestCaseSupplier(
-                    "Date Diff In Seconds with text- OK",
-                    List.of(DataType.TEXT, DataType.DATETIME, DataType.DATETIME),
-                    () -> new TestCaseSupplier.TestCase(
-                        List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.TEXT, "unit"),
-                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
-                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
-                        ),
-                        "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], "
-                            + "endTimestamp=Attribute[channel=2]]",
-                        DataType.INTEGER,
-                        equalTo(88170)
-                    )
-                ),
-                new TestCaseSupplier(
-                    "Date Diff Error Type unit",
-                    List.of(DataType.INTEGER, DataType.DATETIME, DataType.DATETIME),
-                    () -> TestCaseSupplier.TestCase.typeError(
-                        List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.INTEGER, "unit"),
-                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
-                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
-                        ),
-                        "first argument of [] must be [string], found value [unit] type [integer]"
-                    )
-                ),
-                new TestCaseSupplier(
-                    "Date Diff Error Type startTimestamp",
-                    List.of(DataType.TEXT, DataType.INTEGER, DataType.DATETIME),
-                    () -> TestCaseSupplier.TestCase.typeError(
-                        List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"),
-                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.INTEGER, "startTimestamp"),
-                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
-                        ),
-                        "second argument of [] must be [datetime], found value [startTimestamp] type [integer]"
-                    )
-                ),
-                new TestCaseSupplier(
-                    "Date Diff Error Type endTimestamp",
-                    List.of(DataType.TEXT, DataType.DATETIME, DataType.INTEGER),
-                    () -> TestCaseSupplier.TestCase.typeError(
-                        List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"),
-                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
-                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.INTEGER, "endTimestamp")
-                        ),
-                        "third argument of [] must be [datetime], found value [endTimestamp] type [integer]"
-                    )
+        List<TestCaseSupplier> suppliers = new ArrayList<>();
+        suppliers.add(
+            new TestCaseSupplier(
+                "Date Diff In Seconds - OK",
+                List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME),
+                () -> new TestCaseSupplier.TestCase(
+                    List.of(
+                        new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.KEYWORD, "unit"),
+                        new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
+                        new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
+                    ),
+                    "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], "
+                        + "endTimestamp=Attribute[channel=2]]",
+                    DataType.INTEGER,
+                    equalTo(88170)
                 )
             )
         );
+        suppliers.add(
+            new TestCaseSupplier(
+                "Date Diff In Seconds with text- OK",
+                List.of(DataType.TEXT, DataType.DATETIME, DataType.DATETIME),
+                () -> new TestCaseSupplier.TestCase(
+                    List.of(
+                        new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.TEXT, "unit"),
+                        new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
+                        new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
+                    ),
+                    "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], "
+                        + "endTimestamp=Attribute[channel=2]]",
+                    DataType.INTEGER,
+                    equalTo(88170)
+                )
+            )
+        );
+        suppliers.add(
+            new TestCaseSupplier(
+                "Date Diff Error Type unit",
+                List.of(DataType.INTEGER, DataType.DATETIME, DataType.DATETIME),
+                () -> TestCaseSupplier.TestCase.typeError(
+                    List.of(
+                        new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.INTEGER, "unit"),
+                        new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
+                        new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
+                    ),
+                    "first argument of [] must be [string], found value [unit] type [integer]"
+                )
+            )
+        );
+        suppliers.add(
+            new TestCaseSupplier(
+                "Date Diff Error Type startTimestamp",
+                List.of(DataType.TEXT, DataType.INTEGER, DataType.DATETIME),
+                () -> TestCaseSupplier.TestCase.typeError(
+                    List.of(
+                        new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"),
+                        new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.INTEGER, "startTimestamp"),
+                        new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
+                    ),
+                    "second argument of [] must be [datetime], found value [startTimestamp] type [integer]"
+                )
+            )
+        );
+        suppliers.add(
+            new TestCaseSupplier(
+                "Date Diff Error Type endTimestamp",
+                List.of(DataType.TEXT, DataType.DATETIME, DataType.INTEGER),
+                () -> TestCaseSupplier.TestCase.typeError(
+                    List.of(
+                        new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"),
+                        new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
+                        new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.INTEGER, "endTimestamp")
+                    ),
+                    "third argument of [] must be [datetime], found value [endTimestamp] type [integer]"
+                )
+            )
+        );
+        return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers));
     }
 
     public void testDateDiffFunction() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java
index bce3b7efebbb6..db375171b7224 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.time.Instant;
 import java.time.ZonedDateTime;
@@ -84,7 +84,12 @@ public static Iterable<Object[]> parameters() {
                         )
                         .withFoldingException(InvalidArgumentException.class, "invalid date field for []: not a unit")
                 )
-            )
+            ),
+            (v, p) -> switch (p) {
+                case 0 -> "string";
+                case 1 -> "datetime";
+                default -> "";
+            }
         );
     }
 
@@ -125,7 +130,7 @@ public void testInvalidChrono() {
     }
 
     @Override
-    protected Expression buildWithConfiguration(Source source, List<Expression> args, EsqlConfiguration configuration) {
+    protected Expression buildWithConfiguration(Source source, List<Expression> args, Configuration configuration) {
         return new DateExtract(source, args.get(0), args.get(1), configuration);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java
index b18748187709a..d132c2ef2b365 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.List;
 import java.util.function.Supplier;
@@ -58,12 +58,17 @@ public static Iterable<Object[]> parameters() {
                         equalTo(BytesRefs.toBytesRef("2023"))
                     )
                 )
-            )
+            ),
+            (v, p) -> switch (p) {
+                case 0 -> "string";
+                case 1 -> "datetime";
+                default -> "";
+            }
         );
     }
 
     @Override
-    protected Expression buildWithConfiguration(Source source, List<Expression> args, EsqlConfiguration configuration) {
+    protected Expression buildWithConfiguration(Source source, List<Expression> args, Configuration configuration) {
         return new DateFormat(source, args.get(0), args.get(1), configuration);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
index f0aa766fb1bf9..8da01fc1989ba 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
@@ -127,7 +127,8 @@ public static Iterable<Object[]> parameters() {
                                 + "failed to parse date field [not a date] with format [yyyy-MM-dd]"
                         )
                 )
-            )
+            ),
+            (v, p) -> "string"
         );
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
index 17d8cd6a57223..48b23ed5c8840 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
@@ -54,7 +54,11 @@ public static Iterable<Object[]> parameters() {
             ofDuration(Duration.ofSeconds(30), ts, "2023-02-17T10:25:30.00Z"),
             randomSecond()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) {
+            case 0 -> "dateperiod or timeduration";
+            case 1 -> "datetime";
+            default -> null;
+        });
     }
 
     public void testCreateRoundingDuration() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java
index 8edc21db427d2..ce6ee1702ee66 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java
@@ -10,15 +10,13 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.compute.data.Block;
-import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.hamcrest.Matcher;
 
 import java.util.List;
@@ -38,9 +36,10 @@ public static Iterable<Object[]> parameters() {
             List.of(
                 new TestCaseSupplier(
                     "Now Test",
+                    List.of(),
                     () -> new TestCaseSupplier.TestCase(
                         List.of(),
-                        matchesPattern("LiteralsEvaluator\\[lit=.*\\]"),
+                        matchesPattern("LiteralsEvaluator\\[lit=.*]"),
                         DataType.DATETIME,
                         equalTo(EsqlTestUtils.TEST_CFG.now().toInstant().toEpochMilli())
                     )
@@ -50,15 +49,10 @@ public static Iterable<Object[]> parameters() {
     }
 
     @Override
-    protected Expression buildWithConfiguration(Source source, List<Expression> args, EsqlConfiguration configuration) {
+    protected Expression buildWithConfiguration(Source source, List<Expression> args, Configuration configuration) {
         return new Now(Source.EMPTY, configuration);
     }
 
-    @Override
-    protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlock) {
-        assertThat(((LongBlock) value).asVector().getLong(0), equalTo(EsqlTestUtils.TEST_CFG.now().toInstant().toEpochMilli()));
-    }
-
     @Override
     protected Matcher<Object> allNullsMatcher() {
         return equalTo(EsqlTestUtils.TEST_CFG.now().toInstant().toEpochMilli());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java
index 3cdc54f240a96..e777a0ce587e0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java
@@ -84,7 +84,11 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) {
+            case 0 -> "ip";
+            case 1 -> "string";
+            default -> "";
+        });
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java
index 298bcb3f49466..5209d042b6408 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java
@@ -106,7 +106,11 @@ public static Iterable<Object[]> parameters() {
             })
         );
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) {
+            case 0 -> "ip";
+            case 1, 2 -> "integer";
+            default -> "";
+        });
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java
index b5923c7a5b214..493e9e0e9d900 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java
@@ -63,7 +63,7 @@ public static Iterable<Object[]> parameters() {
                 equalTo(Math.abs(arg))
             );
         }));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "numeric");
     }
 
     public AbsTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java
index 7c5cd87dfee39..278c9123e30b1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java
@@ -56,7 +56,7 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java
index 38e210d81e5fd..04fec5a20b438 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java
@@ -56,7 +56,7 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java
index 1144919094812..c475a75699da7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java
@@ -36,7 +36,7 @@ public static Iterable<Object[]> parameters() {
             Double.POSITIVE_INFINITY,
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java
index c9f7a1baeadbe..b51154515de82 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanTests.java
@@ -33,7 +33,7 @@ public static Iterable<Object[]> parameters() {
             Double.POSITIVE_INFINITY,
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java
index f644d8bc72dce..bfe35a08b8ba1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java
@@ -72,7 +72,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers = anyNullIsNull(true, suppliers);
 
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java
index 1572b928a0d75..ddc099a2ad0b1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java
@@ -66,7 +66,7 @@ public static Iterable<Object[]> parameters() {
             UNSIGNED_LONG_MAX,
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java
index dc5eec4f90d32..47dc99f2c13f9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosTests.java
@@ -33,7 +33,7 @@ public static Iterable<Object[]> parameters() {
             Double.POSITIVE_INFINITY,
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java
index 79557b15be08a..ad4208420f481 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshTests.java
@@ -61,7 +61,7 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java
index 763ad3a2b49c9..50ed71262e5df 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java
@@ -10,8 +10,6 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.compute.data.Block;
-import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -31,7 +29,7 @@ public ETests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupp
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("E Test", () -> {
+        return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("E Test", List.of(DataType.INTEGER), () -> {
             return new TestCaseSupplier.TestCase(
                 List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")),
                 "LiteralsEvaluator[lit=2.718281828459045]",
@@ -46,11 +44,6 @@ protected Expression build(Source source, List<Expression> args) {
         return new E(Source.EMPTY);
     }
 
-    @Override
-    protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlock) {
-        assertThat(((DoubleBlock) value).asVector().getDouble(0), equalTo(Math.E));
-    }
-
     @Override
     protected Matcher<Object> allNullsMatcher() {
         return equalTo(Math.E);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java
index bddb0a9a86a82..d42f4ffde0609 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java
@@ -75,7 +75,7 @@ public static Iterable<Object[]> parameters() {
 
         suppliers = anyNullIsNull(true, suppliers);
 
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java
index 269dabcc6b6b8..1d35e034de908 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java
@@ -50,7 +50,7 @@ public static Iterable<Object[]> parameters() {
             Double.POSITIVE_INFINITY,
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java
index ca0c8718f5ac0..44ad4547481d6 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java
@@ -124,7 +124,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
index 1c002e111e575..671cffe9e7f9e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
@@ -191,7 +191,7 @@ public static Iterable<Object[]> parameters() {
         suppliers = anyNullIsNull(true, suppliers);
 
         // Negative cases
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java
index 8e427fcbae2b8..e93bc7b43d98e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java
@@ -10,8 +10,6 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.compute.data.Block;
-import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -31,7 +29,7 @@ public PiTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSup
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Pi Test", () -> {
+        return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Pi Test", List.of(DataType.INTEGER), () -> {
             return new TestCaseSupplier.TestCase(
                 List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")),
                 "LiteralsEvaluator[lit=3.141592653589793]",
@@ -46,11 +44,6 @@ protected Expression build(Source source, List<Expression> args) {
         return new Pi(Source.EMPTY);
     }
 
-    @Override
-    protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlock) {
-        assertThat(((DoubleBlock) value).asVector().getDouble(0), equalTo(Math.PI));
-    }
-
     @Override
     protected Matcher<Object> allNullsMatcher() {
         return equalTo(Math.PI);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
index bea0f399233fd..9d8b87bab8878 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
@@ -77,7 +77,7 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java
index 21b44134458b7..8c612e5e664e0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java
@@ -72,7 +72,7 @@ public static Iterable<Object[]> parameters() {
 
         suppliers = anyNullIsNull(true, suppliers);
 
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java
index 7a1190d86c2bf..990356f8df6de 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java
@@ -33,7 +33,7 @@ public static Iterable<Object[]> parameters() {
             Double.POSITIVE_INFINITY,
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java
index b83519c6d1299..d24dcd1f18f8f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhTests.java
@@ -61,7 +61,7 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java
index 9c81bbdc3cd49..23f2adc6c02e0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java
@@ -109,7 +109,7 @@ public static Iterable<Object[]> parameters() {
                 "Line -1:-1: java.lang.ArithmeticException: Square root of negative"
             )
         );
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric"));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java
index 369c33a1291f1..995894fec5259 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanTests.java
@@ -33,7 +33,7 @@ public static Iterable<Object[]> parameters() {
             Double.POSITIVE_INFINITY,
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java
index 14fdcdca2fe96..73a86fd5a114c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhTests.java
@@ -33,7 +33,7 @@ public static Iterable<Object[]> parameters() {
             Double.POSITIVE_INFINITY,
             List.of()
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java
index 959db368ce348..1a622b25b3353 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java
@@ -10,8 +10,6 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.compute.data.Block;
-import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -31,7 +29,7 @@ public TauTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSu
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Tau Test", () -> {
+        return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Tau Test", List.of(DataType.INTEGER), () -> {
             return new TestCaseSupplier.TestCase(
                 List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")),
                 "LiteralsEvaluator[lit=6.283185307179586]",
@@ -46,11 +44,6 @@ protected Expression build(Source source, List<Expression> args) {
         return new Tau(Source.EMPTY);
     }
 
-    @Override
-    protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlock) {
-        assertThat(((DoubleBlock) value).asVector().getDouble(0), equalTo(Tau.TAU));
-    }
-
     @Override
     protected Matcher<Object> allNullsMatcher() {
         return equalTo(Math.PI * 2);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
index 212b66027d455..13920fac26f5b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
@@ -14,13 +14,13 @@
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.hamcrest.Matcher;
 
 import java.math.BigInteger;
@@ -36,6 +36,7 @@
 import java.util.stream.LongStream;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
 
@@ -389,6 +390,51 @@ protected static void dateTimes(
         }
     }
 
+    protected static void dateNanos(
+        List<TestCaseSupplier> cases,
+        String name,
+        String evaluatorName,
+        DataType expectedDataType,
+        BiFunction<Integer, LongStream, Matcher<Object>> matcher
+    ) {
+        if (EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG.isEnabled() == false) {
+            return;
+        }
+        cases.add(
+            new TestCaseSupplier(
+                name + "(epoch nanos)",
+                List.of(DataType.DATE_NANOS),
+                () -> new TestCaseSupplier.TestCase(
+                    List.of(new TestCaseSupplier.TypedData(List.of(0L), DataType.DATE_NANOS, "field")),
+                    evaluatorName + "[field=Attribute[channel=0]]",
+                    expectedDataType,
+                    matcher.apply(1, LongStream.of(0L))
+                )
+            )
+        );
+        cases.add(new TestCaseSupplier(name + "(date_nanos)", List.of(DataType.DATE_NANOS), () -> {
+            long data = randomLong();
+            return new TestCaseSupplier.TestCase(
+                List.of(new TestCaseSupplier.TypedData(List.of(data), DataType.DATE_NANOS, "field")),
+                evaluatorName + "[field=Attribute[channel=0]]",
+                expectedDataType,
+                matcher.apply(1, LongStream.of(data))
+            );
+        }));
+        for (Block.MvOrdering ordering : Block.MvOrdering.values()) {
+            cases.add(new TestCaseSupplier(name + "(<dates_nanos>) " + ordering, List.of(DataType.DATE_NANOS), () -> {
+                List<Long> mvData = randomList(1, 100, ESTestCase::randomLong);
+                putInOrder(mvData, ordering);
+                return new TestCaseSupplier.TestCase(
+                    List.of(new TestCaseSupplier.TypedData(mvData, DataType.DATE_NANOS, "field")),
+                    evaluatorName + "[field=Attribute[channel=0]]",
+                    expectedDataType,
+                    matcher.apply(mvData.size(), mvData.stream().mapToLong(Long::longValue))
+                );
+            }));
+        }
+    }
+
     /**
      * Build many test cases with {@code geo_point} values.
      * This assumes that the function consumes {@code geo_point} values and produces {@code geo_point} values.
@@ -616,13 +662,6 @@ private static <T extends Comparable<T>> void putInOrder(List<T> mvData, Block.M
 
     protected abstract Expression build(Source source, Expression field);
 
-    protected abstract DataType[] supportedTypes();
-
-    protected final DataType[] representableNumerics() {
-        // TODO numeric should only include representable numbers but that is a change for a followup
-        return DataType.types().stream().filter(DataType::isNumeric).filter(EsqlDataTypes::isRepresentable).toArray(DataType[]::new);
-    }
-
     protected DataType expectedType(List<DataType> argTypes) {
         return argTypes.get(0);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java
index 7039d9edf794b..37f4464c8b3ca 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
 import static org.hamcrest.Matchers.equalTo;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java
index 43c683040eac4..af046a5f39d81 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java
@@ -55,7 +55,7 @@ public static Iterable<Object[]> parameters() {
              */
             (size, data) -> avg.apply(size, data.mapToDouble(v -> unsignedLongToDouble(NumericUtils.asLongUnsigned(v))))
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "numeric");
     }
 
     @Override
@@ -63,11 +63,6 @@ protected Expression build(Source source, Expression field) {
         return new MvAvg(source, field);
     }
 
-    @Override
-    protected DataType[] supportedTypes() {
-        return representableNumerics();
-    }
-
     @Override
     protected DataType expectedType(List<DataType> argTypes) {
         return DataType.DOUBLE;  // Averages are always a double
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java
index 3c668b6f41d4b..4467b49cd674a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java
@@ -67,7 +67,7 @@ public static Iterable<Object[]> parameters() {
                 }
             }
         }
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java
index 8c8772f8ed4e2..51b15ead26c56 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java
@@ -36,11 +36,12 @@ public static Iterable<Object[]> parameters() {
         longs(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
         unsignedLongs(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
         dateTimes(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        dateNanos(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
         geoPoints(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
         cartesianPoints(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
         geoShape(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
         cartesianShape(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "");
     }
 
     @Override
@@ -48,11 +49,6 @@ protected Expression build(Source source, Expression field) {
         return new MvCount(source, field);
     }
 
-    @Override
-    protected DataType[] supportedTypes() {
-        return representableTypes();
-    }
-
     @Override
     protected DataType expectedType(List<DataType> argTypes) {
         return DataType.INTEGER;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
index f202a8033ffc9..d8d3b607efcc0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
@@ -49,7 +49,7 @@ public static Iterable<Object[]> parameters() {
 
         // TODO switch extraction to BigInteger so this just works.
         // unsignedLongs(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values));
-        return parameterSuppliersFromTypedData(cases);
+        return parameterSuppliersFromTypedData(anyNullIsNull(false, cases));
     }
 
     @Override
@@ -57,11 +57,6 @@ protected Expression build(Source source, Expression field) {
         return new MvDedupe(source, field);
     }
 
-    @Override
-    protected DataType[] supportedTypes() {
-        return representableTypes();
-    }
-
     @SuppressWarnings("unchecked")
     private static Matcher<Object> getMatcher(Stream<?> v) {
         Set<Object> values = v.collect(Collectors.toSet());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java
index 6e143d9175f41..f6ef06a84ac2d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java
@@ -37,11 +37,12 @@ public static Iterable<Object[]> parameters() {
         longs(cases, "mv_first", "MvFirst", DataType.LONG, (size, values) -> equalTo(values.findFirst().getAsLong()));
         unsignedLongs(cases, "mv_first", "MvFirst", DataType.UNSIGNED_LONG, (size, values) -> equalTo(values.findFirst().get()));
         dateTimes(cases, "mv_first", "MvFirst", DataType.DATETIME, (size, values) -> equalTo(values.findFirst().getAsLong()));
+        dateNanos(cases, "mv_first", "MvFirst", DataType.DATE_NANOS, (size, values) -> equalTo(values.findFirst().getAsLong()));
         geoPoints(cases, "mv_first", "MvFirst", DataType.GEO_POINT, (size, values) -> equalTo(values.findFirst().get()));
         cartesianPoints(cases, "mv_first", "MvFirst", DataType.CARTESIAN_POINT, (size, values) -> equalTo(values.findFirst().get()));
         geoShape(cases, "mv_first", "MvFirst", DataType.GEO_SHAPE, (size, values) -> equalTo(values.findFirst().get()));
         cartesianShape(cases, "mv_first", "MvFirst", DataType.CARTESIAN_SHAPE, (size, values) -> equalTo(values.findFirst().get()));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "");
     }
 
     @Override
@@ -49,11 +50,6 @@ protected Expression build(Source source, Expression field) {
         return new MvFirst(source, field);
     }
 
-    @Override
-    protected DataType[] supportedTypes() {
-        return representableTypes();
-    }
-
     @Override
     protected DataType expectedType(List<DataType> argTypes) {
         return argTypes.get(0);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java
index 83d94f2cc9884..09e483c3a43ee 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java
@@ -37,11 +37,12 @@ public static Iterable<Object[]> parameters() {
         longs(cases, "mv_last", "MvLast", DataType.LONG, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsLong()));
         unsignedLongs(cases, "mv_last", "MvLast", DataType.UNSIGNED_LONG, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
         dateTimes(cases, "mv_last", "MvLast", DataType.DATETIME, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsLong()));
+        dateNanos(cases, "mv_last", "MvLast", DataType.DATE_NANOS, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsLong()));
         geoPoints(cases, "mv_last", "MvLast", DataType.GEO_POINT, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
         cartesianPoints(cases, "mv_last", "MvLast", DataType.CARTESIAN_POINT, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
         geoShape(cases, "mv_last", "MvLast", DataType.GEO_SHAPE, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
         cartesianShape(cases, "mv_last", "MvLast", DataType.CARTESIAN_SHAPE, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "representable");
     }
 
     @Override
@@ -49,11 +50,6 @@ protected Expression build(Source source, Expression field) {
         return new MvLast(source, field);
     }
 
-    @Override
-    protected DataType[] supportedTypes() {
-        return representableTypes();
-    }
-
     @Override
     protected DataType expectedType(List<DataType> argTypes) {
         return argTypes.get(0);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxTests.java
index 63530234e53fa..a3ad1f2415e20 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxTests.java
@@ -38,16 +38,12 @@ public static Iterable<Object[]> parameters() {
         longs(cases, "mv_max", "MvMax", (size, values) -> equalTo(values.max().getAsLong()));
         unsignedLongs(cases, "mv_max", "MvMax", (size, values) -> equalTo(values.reduce(BigInteger::max).get()));
         dateTimes(cases, "mv_max", "MvMax", (size, values) -> equalTo(values.max().getAsLong()));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases);
+        dateNanos(cases, "mv_max", "MvMax", DataType.DATE_NANOS, (size, values) -> equalTo(values.max().getAsLong()));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "representableNonSpatial");
     }
 
     @Override
     protected Expression build(Source source, Expression field) {
         return new MvMax(source, field);
     }
-
-    @Override
-    protected DataType[] supportedTypes() {
-        return representableNonSpatialTypes();
-    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java
index f44f5d44e3f62..002aa77946bcf 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java
@@ -92,16 +92,11 @@ public static Iterable<Object[]> parameters() {
                 )
             )
         );
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "numeric");
     }
 
     @Override
     protected Expression build(Source source, Expression field) {
         return new MvMedian(source, field);
     }
-
-    @Override
-    protected DataType[] supportedTypes() {
-        return representableNumerics();
-    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinTests.java
index 5be67548f784e..a4d5a4004b840 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinTests.java
@@ -38,16 +38,12 @@ public static Iterable<Object[]> parameters() {
         longs(cases, "mv_min", "MvMin", (size, values) -> equalTo(values.min().getAsLong()));
         unsignedLongs(cases, "mv_min", "MvMin", (size, values) -> equalTo(values.reduce(BigInteger::min).get()));
         dateTimes(cases, "mv_min", "MvMin", (size, values) -> equalTo(values.min().getAsLong()));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases);
+        dateNanos(cases, "mv_min", "MvMin", DataType.DATE_NANOS, (size, values) -> equalTo(values.min().getAsLong()));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "representableNonSpatial");
     }
 
     @Override
     protected Expression build(Source source, Expression field) {
         return new MvMin(source, field);
     }
-
-    @Override
-    protected DataType[] supportedTypes() {
-        return representableNonSpatialTypes();
-    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumSerializationTests.java
new file mode 100644
index 0000000000000..dcb525c79b272
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumSerializationTests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
+
+import java.io.IOException;
+
+public class MvPSeriesWeightedSumSerializationTests extends AbstractExpressionSerializationTests<MvPSeriesWeightedSum> {
+    @Override
+    protected MvPSeriesWeightedSum createTestInstance() {
+        Source source = randomSource();
+        Expression field = randomChild();
+        Expression p = randomChild();
+
+        return new MvPSeriesWeightedSum(source, field, p);
+    }
+
+    @Override
+    protected MvPSeriesWeightedSum mutateInstance(MvPSeriesWeightedSum instance) throws IOException {
+        Source source = instance.source();
+        Expression field = instance.field();
+        Expression p = instance.p();
+
+        switch (between(0, 1)) {
+            case 0 -> field = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild);
+            case 1 -> p = randomValueOtherThan(p, AbstractExpressionSerializationTests::randomChild);
+
+        }
+        return new MvPSeriesWeightedSum(source, field, p);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java
new file mode 100644
index 0000000000000..156fc4bfe7c36
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class MvPSeriesWeightedSumTests extends AbstractScalarFunctionTestCase {
+    public MvPSeriesWeightedSumTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        List<TestCaseSupplier> cases = new ArrayList<>();
+        doubles(cases);
+
+        cases = randomizeBytesRefsOffset(cases);
+        cases = anyNullIsNull(
+            cases,
+            (nullPosition, nullValueDataType, original) -> nullValueDataType == DataType.NULL ? DataType.NULL : original.expectedType(),
+            (nullPosition, nullData, original) -> {
+                if (nullData.isForceLiteral()) {
+                    return equalTo("LiteralsEvaluator[lit=null]");
+                }
+                return nullData.type() == DataType.NULL ? equalTo("LiteralsEvaluator[lit=null]") : original;
+            }
+        );
+        cases = errorsForCasesWithoutExamples(cases, (valid, position) -> "double");
+
+        return parameterSuppliersFromTypedData(cases);
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new MvPSeriesWeightedSum(source, args.get(0), args.get(1));
+    }
+
+    private static void doubles(List<TestCaseSupplier> cases) {
+        cases.add(new TestCaseSupplier("most common scenario", List.of(DataType.DOUBLE, DataType.DOUBLE), () -> {
+            List<Double> field = randomList(1, 10, () -> randomDoubleBetween(1, 10, false));
+            double p = randomDoubleBetween(-10, 10, true);
+            return testCase(field, p);
+        }));
+
+        cases.add(new TestCaseSupplier("values between 0 and 1", List.of(DataType.DOUBLE, DataType.DOUBLE), () -> {
+            List<Double> field = randomList(1, 10, () -> randomDoubleBetween(0, 1, true));
+            double p = randomDoubleBetween(-10, 10, true);
+            return testCase(field, p);
+        }));
+
+        cases.add(new TestCaseSupplier("values between -1 and 0", List.of(DataType.DOUBLE, DataType.DOUBLE), () -> {
+            List<Double> field = randomList(1, 10, () -> randomDoubleBetween(-1, 0, true));
+            double p = randomDoubleBetween(-10, 10, true);
+            return testCase(field, p);
+        }));
+
+        cases.add(new TestCaseSupplier("values between 1 and Double.MAX_VALUE", List.of(DataType.DOUBLE, DataType.DOUBLE), () -> {
+            List<Double> field = randomList(1, 10, () -> randomDoubleBetween(1, Double.MAX_VALUE, true));
+            double p = randomDoubleBetween(-10, 10, true);
+            return testCase(field, p);
+        }));
+
+        cases.add(new TestCaseSupplier("values between -Double.MAX_VALUE and 1", List.of(DataType.DOUBLE, DataType.DOUBLE), () -> {
+            List<Double> field = randomList(1, 10, () -> randomDoubleBetween(-Double.MAX_VALUE, 1, true));
+            double p = randomDoubleBetween(-10, 10, true);
+            return testCase(field, p);
+        }));
+    }
+
+    private static TestCaseSupplier.TestCase testCase(List<Double> field, double p) {
+        double expectedResult = calcPSeriesWeightedSum(field, p);
+
+        TestCaseSupplier.TestCase testCase = new TestCaseSupplier.TestCase(
+            List.of(
+                new TestCaseSupplier.TypedData(field, DataType.DOUBLE, "field"),
+                new TestCaseSupplier.TypedData(p, DataType.DOUBLE, "p").forceLiteral()
+            ),
+            "MvPSeriesWeightedSumDoubleEvaluator[block=Attribute[channel=0], p=" + p + "]",
+            DataType.DOUBLE,
+            Double.isFinite(expectedResult) ? closeTo(expectedResult, Math.abs(expectedResult * .00000001)) : nullValue()
+        );
+        if (Double.isFinite(expectedResult) == false) {
+            return testCase.withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
+                .withWarning("Line -1:-1: java.lang.ArithmeticException: double overflow");
+        }
+        return testCase;
+    }
+
+    private static double calcPSeriesWeightedSum(List<Double> field, double p) {
+        double sum = 0;
+        for (int i = 0; i < field.size(); i++) {
+            double current = field.get(i) / Math.pow(i + 1, p);
+            sum += current;
+        }
+        return sum;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileSimpleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileSimpleTests.java
new file mode 100644
index 0000000000000..81ae8efb7aba7
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileSimpleTests.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.TestBlockFactory;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class MvPercentileSimpleTests extends ESTestCase {
+    public void testDoubleMvAsc() {
+        try (DoubleBlock.Builder builder = TestBlockFactory.getNonBreakingInstance().newDoubleBlockBuilder(10)) {
+            builder.beginPositionEntry();
+            builder.appendDouble(80);
+            builder.appendDouble(90);
+            builder.endPositionEntry();
+            builder.beginPositionEntry();
+            builder.appendDouble(-6.33);
+            builder.appendDouble(-3.34);
+            builder.appendDouble(-0.31);
+            builder.appendDouble(6.23);
+            builder.endPositionEntry();
+            builder.mvOrdering(Block.MvOrdering.SORTED_ASCENDING);
+            try (DoubleBlock block = builder.build()) {
+                MvPercentile.DoubleSortingScratch scratch = new MvPercentile.DoubleSortingScratch();
+                double p0 = MvPercentile.calculateDoublePercentile(block, block.getFirstValueIndex(0), block.getValueCount(0), 75, scratch);
+                double p1 = MvPercentile.calculateDoublePercentile(block, block.getFirstValueIndex(1), block.getValueCount(1), 75, scratch);
+                assertThat(p0, equalTo(87.5));
+                assertThat(p1, equalTo(1.325));
+            }
+        }
+    }
+
+    public void testDoubleRandomOrder() {
+        try (DoubleBlock.Builder builder = TestBlockFactory.getNonBreakingInstance().newDoubleBlockBuilder(10)) {
+            builder.beginPositionEntry();
+            builder.appendDouble(80);
+            builder.appendDouble(90);
+            builder.endPositionEntry();
+            builder.beginPositionEntry();
+            builder.appendDouble(-3.34);
+            builder.appendDouble(-6.33);
+            builder.appendDouble(6.23);
+            builder.appendDouble(-0.31);
+            builder.endPositionEntry();
+            try (DoubleBlock block = builder.build()) {
+                MvPercentile.DoubleSortingScratch scratch = new MvPercentile.DoubleSortingScratch();
+                double p0 = MvPercentile.calculateDoublePercentile(block, block.getFirstValueIndex(0), block.getValueCount(0), 75, scratch);
+                double p1 = MvPercentile.calculateDoublePercentile(block, block.getFirstValueIndex(1), block.getValueCount(1), 75, scratch);
+                assertThat(p0, equalTo(87.5));
+                assertThat(p1, equalTo(1.325));
+            }
+        }
+    }
+
+    public void testIntMvAsc() {
+        try (IntBlock.Builder builder = TestBlockFactory.getNonBreakingInstance().newIntBlockBuilder(10)) {
+            builder.beginPositionEntry();
+            builder.appendInt(80);
+            builder.appendInt(90);
+            builder.endPositionEntry();
+            builder.beginPositionEntry();
+            builder.appendInt(-6);
+            builder.appendInt(-3);
+            builder.appendInt(0);
+            builder.appendInt(6);
+            builder.endPositionEntry();
+            builder.mvOrdering(Block.MvOrdering.SORTED_ASCENDING);
+            try (IntBlock block = builder.build()) {
+                MvPercentile.IntSortingScratch scratch = new MvPercentile.IntSortingScratch();
+                int p0 = MvPercentile.calculateIntPercentile(block, block.getFirstValueIndex(0), block.getValueCount(0), 75, scratch);
+                int p1 = MvPercentile.calculateIntPercentile(block, block.getFirstValueIndex(1), block.getValueCount(1), 75, scratch);
+                assertThat(p0, equalTo(87));
+                assertThat(p1, equalTo(1));
+            }
+        }
+    }
+
+    public void testIntRandomOrder() {
+        try (IntBlock.Builder builder = TestBlockFactory.getNonBreakingInstance().newIntBlockBuilder(10)) {
+            builder.beginPositionEntry();
+            builder.appendInt(80);
+            builder.appendInt(90);
+            builder.endPositionEntry();
+            builder.beginPositionEntry();
+            builder.appendInt(-3);
+            builder.appendInt(-6);
+            builder.appendInt(6);
+            builder.appendInt(0);
+            builder.endPositionEntry();
+            try (IntBlock block = builder.build()) {
+                MvPercentile.IntSortingScratch scratch = new MvPercentile.IntSortingScratch();
+                int p0 = MvPercentile.calculateIntPercentile(block, block.getFirstValueIndex(0), block.getValueCount(0), 75, scratch);
+                int p1 = MvPercentile.calculateIntPercentile(block, block.getFirstValueIndex(1), block.getValueCount(1), 75, scratch);
+                assertThat(p0, equalTo(87));
+                assertThat(p1, equalTo(1));
+            }
+        }
+    }
+
+    public void testLongMvAsc() {
+        try (LongBlock.Builder builder = TestBlockFactory.getNonBreakingInstance().newLongBlockBuilder(10)) {
+            builder.beginPositionEntry();
+            builder.appendLong(80);
+            builder.appendLong(90);
+            builder.endPositionEntry();
+            builder.beginPositionEntry();
+            builder.appendLong(-6);
+            builder.appendLong(-3);
+            builder.appendLong(0);
+            builder.appendLong(6);
+            builder.endPositionEntry();
+            builder.mvOrdering(Block.MvOrdering.SORTED_ASCENDING);
+            try (LongBlock block = builder.build()) {
+                MvPercentile.LongSortingScratch scratch = new MvPercentile.LongSortingScratch();
+                long p0 = MvPercentile.calculateLongPercentile(block, block.getFirstValueIndex(0), block.getValueCount(0), 75, scratch);
+                long p1 = MvPercentile.calculateLongPercentile(block, block.getFirstValueIndex(1), block.getValueCount(1), 75, scratch);
+                assertThat(p0, equalTo(87L));
+                assertThat(p1, equalTo(1L));
+            }
+        }
+    }
+
+    public void testLongRandomOrder() {
+        try (LongBlock.Builder builder = TestBlockFactory.getNonBreakingInstance().newLongBlockBuilder(10)) {
+            builder.beginPositionEntry();
+            builder.appendLong(80);
+            builder.appendLong(90);
+            builder.endPositionEntry();
+            builder.beginPositionEntry();
+            builder.appendLong(-3);
+            builder.appendLong(-6);
+            builder.appendLong(6);
+            builder.appendLong(0);
+            builder.endPositionEntry();
+            try (LongBlock block = builder.build()) {
+                MvPercentile.LongSortingScratch scratch = new MvPercentile.LongSortingScratch();
+                long p0 = MvPercentile.calculateLongPercentile(block, block.getFirstValueIndex(0), block.getValueCount(0), 75, scratch);
+                long p1 = MvPercentile.calculateLongPercentile(block, block.getFirstValueIndex(1), block.getValueCount(1), 75, scratch);
+                assertThat(p0, equalTo(87L));
+                assertThat(p1, equalTo(1L));
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java
new file mode 100644
index 0000000000000..29cc959e6a943
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java
@@ -0,0 +1,481 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.StringUtils;
+import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.MultivalueTestCaseSupplier;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class MvPercentileTests extends AbstractScalarFunctionTestCase {
+    public MvPercentileTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        List<TestCaseSupplier> cases = new ArrayList<>();
+
+        var fieldSuppliers = Stream.of(
+            MultivalueTestCaseSupplier.intCases(Integer.MIN_VALUE, Integer.MAX_VALUE, true),
+            MultivalueTestCaseSupplier.longCases(Long.MIN_VALUE, Long.MAX_VALUE, true),
+            MultivalueTestCaseSupplier.doubleCases(-Double.MAX_VALUE, Double.MAX_VALUE, true)
+        ).flatMap(List::stream).toList();
+
+        var percentileSuppliers = Stream.of(
+            TestCaseSupplier.intCases(0, 100, true),
+            TestCaseSupplier.longCases(0, 100, true),
+            TestCaseSupplier.doubleCases(0, 100, true)
+        ).flatMap(List::stream).toList();
+
+        for (var fieldSupplier : fieldSuppliers) {
+            for (var percentileSupplier : percentileSuppliers) {
+                cases.add(makeSupplier(fieldSupplier, percentileSupplier));
+            }
+        }
+
+        for (var percentileType : List.of(INTEGER, LONG, DOUBLE)) {
+            cases.addAll(
+                List.of(
+                    // Doubles
+                    new TestCaseSupplier(
+                        "median double",
+                        List.of(DOUBLE, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10., 5., 10.), DOUBLE, "field"),
+                                percentileWithType(50, percentileType)
+                            ),
+                            evaluatorString(DOUBLE, percentileType),
+                            DOUBLE,
+                            equalTo(5.)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "single value double",
+                        List.of(DOUBLE, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(55.), DOUBLE, "field"),
+                                percentileWithType(randomIntBetween(0, 100), percentileType)
+                            ),
+                            evaluatorString(DOUBLE, percentileType),
+                            DOUBLE,
+                            equalTo(55.)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "p0 double",
+                        List.of(DOUBLE, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10., 5., 10.), DOUBLE, "field"),
+                                percentileWithType(0, percentileType)
+                            ),
+                            evaluatorString(DOUBLE, percentileType),
+                            DOUBLE,
+                            equalTo(-10.)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "p100 double",
+                        List.of(DOUBLE, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10., 5., 10.), DOUBLE, "field"),
+                                percentileWithType(100, percentileType)
+                            ),
+                            evaluatorString(DOUBLE, percentileType),
+                            DOUBLE,
+                            equalTo(10.)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "averaged double",
+                        List.of(DOUBLE, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10., 5., 10.), DOUBLE, "field"),
+                                percentileWithType(75, percentileType)
+                            ),
+                            evaluatorString(DOUBLE, percentileType),
+                            DOUBLE,
+                            equalTo(7.5)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "big double difference",
+                        List.of(DOUBLE, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-Double.MAX_VALUE, Double.MAX_VALUE), DOUBLE, "field"),
+                                percentileWithType(50, percentileType)
+                            ),
+                            evaluatorString(DOUBLE, percentileType),
+                            DOUBLE,
+                            closeTo(0, 0.0000001)
+                        )
+                    ),
+
+                    // Int
+                    new TestCaseSupplier(
+                        "median int",
+                        List.of(INTEGER, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10, 5, 10), INTEGER, "field"),
+                                percentileWithType(50, percentileType)
+                            ),
+                            evaluatorString(INTEGER, percentileType),
+                            INTEGER,
+                            equalTo(5)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "single value int",
+                        List.of(INTEGER, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(55), INTEGER, "field"),
+                                percentileWithType(randomIntBetween(0, 100), percentileType)
+                            ),
+                            evaluatorString(INTEGER, percentileType),
+                            INTEGER,
+                            equalTo(55)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "p0 int",
+                        List.of(INTEGER, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10, 5, 10), INTEGER, "field"),
+                                percentileWithType(0, percentileType)
+                            ),
+                            evaluatorString(INTEGER, percentileType),
+                            INTEGER,
+                            equalTo(-10)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "p100 int",
+                        List.of(INTEGER, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10, 5, 10), INTEGER, "field"),
+                                percentileWithType(100, percentileType)
+                            ),
+                            evaluatorString(INTEGER, percentileType),
+                            INTEGER,
+                            equalTo(10)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "averaged int",
+                        List.of(INTEGER, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10, 5, 10), INTEGER, "field"),
+                                percentileWithType(75, percentileType)
+                            ),
+                            evaluatorString(INTEGER, percentileType),
+                            INTEGER,
+                            equalTo(7)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "big int difference",
+                        List.of(INTEGER, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(Integer.MIN_VALUE, Integer.MAX_VALUE), INTEGER, "field"),
+                                percentileWithType(50, percentileType)
+                            ),
+                            evaluatorString(INTEGER, percentileType),
+                            INTEGER,
+                            equalTo(-1) // Negative max is 1 smaller than positive max
+                        )
+                    ),
+
+                    // Long
+                    new TestCaseSupplier(
+                        "median long",
+                        List.of(LONG, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10L, 5L, 10L), LONG, "field"),
+                                percentileWithType(50, percentileType)
+                            ),
+                            evaluatorString(LONG, percentileType),
+                            LONG,
+                            equalTo(5L)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "single value long",
+                        List.of(LONG, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(55L), LONG, "field"),
+                                percentileWithType(randomIntBetween(0, 100), percentileType)
+                            ),
+                            evaluatorString(LONG, percentileType),
+                            LONG,
+                            equalTo(55L)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "p0 long",
+                        List.of(LONG, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10L, 5L, 10L), LONG, "field"),
+                                percentileWithType(0, percentileType)
+                            ),
+                            evaluatorString(LONG, percentileType),
+                            LONG,
+                            equalTo(-10L)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "p100 long",
+                        List.of(LONG, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10L, 5L, 10L), LONG, "field"),
+                                percentileWithType(100, percentileType)
+                            ),
+                            evaluatorString(LONG, percentileType),
+                            LONG,
+                            equalTo(10L)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "averaged long",
+                        List.of(LONG, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(-10L, 5L, 10L), LONG, "field"),
+                                percentileWithType(75, percentileType)
+                            ),
+                            evaluatorString(LONG, percentileType),
+                            LONG,
+                            equalTo(7L)
+                        )
+                    ),
+                    new TestCaseSupplier(
+                        "big long difference",
+                        List.of(LONG, percentileType),
+                        () -> new TestCaseSupplier.TestCase(
+                            List.of(
+                                new TestCaseSupplier.TypedData(List.of(Long.MIN_VALUE, Long.MAX_VALUE), LONG, "field"),
+                                percentileWithType(50, percentileType)
+                            ),
+                            evaluatorString(LONG, percentileType),
+                            LONG,
+                            equalTo(0L)
+                        )
+                    )
+                )
+            );
+
+            for (var fieldType : List.of(INTEGER, LONG, DataType.DOUBLE)) {
+                cases.add(
+                    new TestCaseSupplier(
+                        "out of bounds percentile <" + fieldType + ", " + percentileType + ">",
+                        List.of(fieldType, percentileType),
+                        () -> {
+                            var percentile = numberWithType(
+                                randomBoolean() ? randomIntBetween(Integer.MIN_VALUE, -1) : randomIntBetween(101, Integer.MAX_VALUE),
+                                percentileType
+                            );
+                            return new TestCaseSupplier.TestCase(
+                                List.of(
+                                    new TestCaseSupplier.TypedData(numberWithType(0, fieldType), fieldType, "field"),
+                                    new TestCaseSupplier.TypedData(percentile, percentileType, "percentile")
+                                ),
+                                evaluatorString(fieldType, percentileType),
+                                fieldType,
+                                nullValue()
+                            ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
+                                .withWarning(
+                                    "Line -1:-1: java.lang.IllegalArgumentException: Percentile parameter must be "
+                                        + "a number between 0 and 100, found ["
+                                        + percentile.doubleValue()
+                                        + "]"
+                                );
+                        }
+                    )
+                );
+            }
+        }
+        cases.add(
+            new TestCaseSupplier(
+                "from example",
+                List.of(DOUBLE, INTEGER),
+                () -> new TestCaseSupplier.TestCase(
+                    List.of(
+                        new TestCaseSupplier.TypedData(List.of(-3.34, -6.33, 6.23, -0.31), DOUBLE, "field"),
+                        new TestCaseSupplier.TypedData(75, INTEGER, "percentile")
+                    ),
+                    evaluatorString(DOUBLE, INTEGER),
+                    DOUBLE,
+                    equalTo(1.325)
+                )
+            )
+        );
+
+        return parameterSuppliersFromTypedDataWithDefaultChecks(
+            (nullPosition, nullValueDataType, original) -> nullValueDataType == DataType.NULL && nullPosition == 0
+                ? DataType.NULL
+                : original.expectedType(),
+            (nullPosition, nullData, original) -> original,
+            cases,
+            (v, p) -> "numeric except unsigned_long"
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    private static TestCaseSupplier makeSupplier(
+        TestCaseSupplier.TypedDataSupplier fieldSupplier,
+        TestCaseSupplier.TypedDataSupplier percentileSupplier
+    ) {
+        return new TestCaseSupplier(
+            "field: " + fieldSupplier.name() + ", percentile: " + percentileSupplier.name(),
+            List.of(fieldSupplier.type(), percentileSupplier.type()),
+            () -> {
+                var fieldTypedData = fieldSupplier.get();
+                var percentileTypedData = percentileSupplier.get();
+
+                var values = (List<Number>) fieldTypedData.data();
+                var percentile = ((Number) percentileTypedData.data()).doubleValue();
+
+                var expected = calculatePercentile(values, percentile);
+
+                return new TestCaseSupplier.TestCase(
+                    List.of(fieldTypedData, percentileTypedData),
+                    evaluatorString(fieldSupplier.type(), percentileSupplier.type()),
+                    fieldSupplier.type(),
+                    expected instanceof Double expectedDouble
+                        ? closeTo(expectedDouble, Math.abs(expectedDouble * 0.0000001))
+                        : equalTo(expected)
+                );
+            }
+        );
+    }
+
+    private static Number calculatePercentile(List<Number> rawValues, double percentile) {
+        if (rawValues.isEmpty() || percentile < 0 || percentile > 100) {
+            return null;
+        }
+
+        if (rawValues.size() == 1) {
+            return rawValues.get(0);
+        }
+
+        int valueCount = rawValues.size();
+        var p = percentile / 100.0;
+        var index = p * (valueCount - 1);
+        var lowerIndex = (int) index;
+        var upperIndex = lowerIndex + 1;
+        var fraction = index - lowerIndex;
+
+        if (rawValues.get(0) instanceof Integer) {
+            var values = rawValues.stream().mapToInt(Number::intValue).sorted().toArray();
+
+            if (percentile == 0) {
+                return values[0];
+            } else if (percentile == 100) {
+                return values[valueCount - 1];
+            } else {
+                assert lowerIndex >= 0 && upperIndex < valueCount;
+                var difference = (long) values[upperIndex] - values[lowerIndex];
+                return values[lowerIndex] + (int) (fraction * difference);
+            }
+        }
+
+        if (rawValues.get(0) instanceof Long) {
+            var values = rawValues.stream().mapToLong(Number::longValue).sorted().toArray();
+
+            if (percentile == 0) {
+                return values[0];
+            } else if (percentile == 100) {
+                return values[valueCount - 1];
+            } else {
+                assert lowerIndex >= 0 && upperIndex < valueCount;
+                return calculatePercentile(fraction, new BigDecimal(values[lowerIndex]), new BigDecimal(values[upperIndex])).longValue();
+            }
+        }
+
+        if (rawValues.get(0) instanceof Double) {
+            var values = rawValues.stream().mapToDouble(Number::doubleValue).sorted().toArray();
+
+            if (percentile == 0) {
+                return values[0];
+            } else if (percentile == 100) {
+                return values[valueCount - 1];
+            } else {
+                assert lowerIndex >= 0 && upperIndex < valueCount;
+                return calculatePercentile(fraction, new BigDecimal(values[lowerIndex]), new BigDecimal(values[upperIndex])).doubleValue();
+            }
+        }
+
+        throw new IllegalArgumentException("Unsupported type: " + rawValues.get(0).getClass());
+    }
+
+    private static BigDecimal calculatePercentile(double fraction, BigDecimal lowerValue, BigDecimal upperValue) {
+        return lowerValue.add(new BigDecimal(fraction).multiply(upperValue.subtract(lowerValue)));
+    }
+
+    private static TestCaseSupplier.TypedData percentileWithType(Number value, DataType type) {
+        return new TestCaseSupplier.TypedData(numberWithType(value, type), type, "percentile");
+    }
+
+    private static Number numberWithType(Number value, DataType type) {
+        return switch (type) {
+            case INTEGER -> value.intValue();
+            case LONG -> value.longValue();
+            default -> value.doubleValue();
+        };
+    }
+
+    private static String evaluatorString(DataType fieldDataType, DataType percentileDataType) {
+        var fieldTypeName = StringUtils.underscoreToLowerCamelCase(fieldDataType.name());
+
+        fieldTypeName = fieldTypeName.substring(0, 1).toUpperCase(Locale.ROOT) + fieldTypeName.substring(1);
+
+        var percentileEvaluator = TestCaseSupplier.castToDoubleEvaluator("Attribute[channel=1]", percentileDataType);
+
+        return "MvPercentile" + fieldTypeName + "Evaluator[values=Attribute[channel=0], percentile=" + percentileEvaluator + "]";
+    }
+
+    @Override
+    protected final Expression build(Source source, List<Expression> args) {
+        return new MvPercentile(source, args.get(0), args.get(1));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
index 5684c68051446..e5bac422805af 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
 import static org.hamcrest.Matchers.equalTo;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
index 15c81557961f1..d07ed2aeae887 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
@@ -11,7 +11,6 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
@@ -25,7 +24,9 @@
 import java.util.List;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 public class MvSortTests extends AbstractScalarFunctionTestCase {
     public MvSortTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
@@ -40,6 +41,7 @@ public static Iterable<Object[]> parameters() {
         longs(suppliers);
         doubles(suppliers);
         bytesRefs(suppliers);
+        nulls(suppliers);
         return parameterSuppliersFromTypedData(suppliers);
     }
 
@@ -57,12 +59,11 @@ private static void booleans(List<TestCaseSupplier> suppliers) {
                     new TestCaseSupplier.TypedData(field, DataType.BOOLEAN, "field"),
                     new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
-                "MvSort" + ElementType.BOOLEAN + "[field=Attribute[channel=0], order=true]",
+                "MvSortBoolean[field=Attribute[channel=0], order=true]",
                 DataType.BOOLEAN,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
             );
         }));
-
     }
 
     private static void ints(List<TestCaseSupplier> suppliers) {
@@ -74,7 +75,7 @@ private static void ints(List<TestCaseSupplier> suppliers) {
                     new TestCaseSupplier.TypedData(field, DataType.INTEGER, "field"),
                     new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
-                "MvSort" + ElementType.INT + "[field=Attribute[channel=0], order=false]",
+                "MvSortInt[field=Attribute[channel=0], order=false]",
                 DataType.INTEGER,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
             );
@@ -90,7 +91,7 @@ private static void longs(List<TestCaseSupplier> suppliers) {
                     new TestCaseSupplier.TypedData(field, DataType.LONG, "field"),
                     new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
-                "MvSort" + ElementType.LONG + "[field=Attribute[channel=0], order=true]",
+                "MvSortLong[field=Attribute[channel=0], order=true]",
                 DataType.LONG,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
             );
@@ -104,7 +105,7 @@ private static void longs(List<TestCaseSupplier> suppliers) {
                     new TestCaseSupplier.TypedData(field, DataType.DATETIME, "field"),
                     new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
-                "MvSort" + ElementType.LONG + "[field=Attribute[channel=0], order=false]",
+                "MvSortLong[field=Attribute[channel=0], order=false]",
                 DataType.DATETIME,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
             );
@@ -120,7 +121,7 @@ private static void doubles(List<TestCaseSupplier> suppliers) {
                     new TestCaseSupplier.TypedData(field, DataType.DOUBLE, "field"),
                     new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
-                "MvSort" + ElementType.DOUBLE + "[field=Attribute[channel=0], order=true]",
+                "MvSortDouble[field=Attribute[channel=0], order=true]",
                 DataType.DOUBLE,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
             );
@@ -136,7 +137,7 @@ private static void bytesRefs(List<TestCaseSupplier> suppliers) {
                     new TestCaseSupplier.TypedData(field, DataType.KEYWORD, "field"),
                     new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
-                "MvSort" + ElementType.BYTES_REF + "[field=Attribute[channel=0], order=false]",
+                "MvSortBytesRef[field=Attribute[channel=0], order=false]",
                 DataType.KEYWORD,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
             );
@@ -150,7 +151,7 @@ private static void bytesRefs(List<TestCaseSupplier> suppliers) {
                     new TestCaseSupplier.TypedData(field, DataType.TEXT, "field"),
                     new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
-                "MvSort" + ElementType.BYTES_REF + "[field=Attribute[channel=0], order=true]",
+                "MvSortBytesRef[field=Attribute[channel=0], order=true]",
                 DataType.TEXT,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
             );
@@ -164,7 +165,7 @@ private static void bytesRefs(List<TestCaseSupplier> suppliers) {
                     new TestCaseSupplier.TypedData(field, DataType.IP, "field"),
                     new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
-                "MvSort" + ElementType.BYTES_REF + "[field=Attribute[channel=0], order=false]",
+                "MvSortBytesRef[field=Attribute[channel=0], order=false]",
                 DataType.IP,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
             );
@@ -178,14 +179,43 @@ private static void bytesRefs(List<TestCaseSupplier> suppliers) {
                     new TestCaseSupplier.TypedData(field, DataType.VERSION, "field"),
                     new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
-                "MvSort" + ElementType.BYTES_REF + "[field=Attribute[channel=0], order=true]",
+                "MvSortBytesRef[field=Attribute[channel=0], order=true]",
                 DataType.VERSION,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
             );
         }));
     }
 
+    private static void nulls(List<TestCaseSupplier> suppliers) {
+        List<TestCaseSupplier> extra = new ArrayList<>();
+        for (TestCaseSupplier s : suppliers) {
+            extra.add(new TestCaseSupplier("null <" + s.types().get(0) + ">, <keyword>", s.types(), () -> {
+                TestCaseSupplier.TestCase delegate = s.get();
+                return new TestCaseSupplier.TestCase(
+                    List.of(new TestCaseSupplier.TypedData(null, s.types().get(0), "field"), delegate.getData().get(1)),
+                    delegate.evaluatorToString(),
+                    delegate.expectedType(),
+                    nullValue()
+                );
+            }));
+        }
+        suppliers.addAll(extra);
+        suppliers.add(new TestCaseSupplier("<null>, <keyword>", List.of(DataType.NULL, DataType.KEYWORD), () -> {
+            BytesRef order = new BytesRef("ASC");
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(null, DataType.NULL, "field"),
+                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
+                ),
+                equalTo("LiteralsEvaluator[lit=null]"),
+                DataType.NULL,
+                nullValue()
+            );
+        }));
+    }
+
     public void testInvalidOrder() {
+        // TODO move to parameters
         String invalidOrder = randomAlphaOfLength(10);
         DriverContext driverContext = driverContext();
         IllegalArgumentException e = expectThrows(
@@ -200,9 +230,4 @@ public void testInvalidOrder() {
         );
         assertThat(e.getMessage(), equalTo("Invalid order value in [], expected one of [ASC, DESC] but got [" + invalidOrder + "]"));
     }
-
-    @Override
-    public void testSimpleWithNulls() {
-        assumeFalse("test case is invalid", false);
-    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java
index c1d820cd93931..26d113359a6f8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java
@@ -65,7 +65,7 @@ public static Iterable<Object[]> parameters() {
             data.add(asLongUnsigned(UNSIGNED_LONG_MAX));
             return data;
         }));
-        return parameterSuppliersFromTypedData(cases);
+        return parameterSuppliersFromTypedData(anyNullIsNull(false, cases));
     }
 
     private static TestCaseSupplier arithmeticExceptionCase(DataType dataType, Supplier<Object> dataSupplier) {
@@ -87,9 +87,4 @@ private static TestCaseSupplier arithmeticExceptionCase(DataType dataType, Suppl
     protected Expression build(Source source, Expression field) {
         return new MvSum(source, field);
     }
-
-    @Override
-    protected DataType[] supportedTypes() {
-        return representableNumerics();
-    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java
index e9f0fd5b51516..d415cb55ea632 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java
@@ -52,7 +52,7 @@ public static Iterable<Object[]> parameters() {
             }
         }
 
-        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "string"));
     }
 
     private static TestCaseSupplier supplier(DataType leftType, DataType rightType, DataType delimType) {
@@ -140,9 +140,4 @@ private static List<BytesRef> calculateExpected(List<BytesRef> left, List<BytesR
         }
         return expected;
     }
-
-    @Override
-    public void testSimpleWithNulls() {
-        assumeFalse("mv_zip returns null only if both left and right inputs are nulls", false);
-    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
index c779fa9e2789f..f760694391ee4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
@@ -37,6 +37,7 @@
 import java.util.function.Supplier;
 
 import static org.elasticsearch.compute.data.BlockUtils.toJavaObject;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.hamcrest.Matchers.equalTo;
 
 public class CoalesceTests extends AbstractScalarFunctionTestCase {
@@ -44,21 +45,17 @@ public CoalesceTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testC
         this.testCase = testCaseSupplier.get();
     }
 
-    /**
-     * Generate the test cases for this test. The tests don't actually include
-     * any nulls, but we insert those nulls in {@link #testSimpleWithNulls()}.
-     */
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        List<TestCaseSupplier> suppliers = new ArrayList<>();
+        List<TestCaseSupplier> noNullsSuppliers = new ArrayList<>();
         VaragsTestCaseBuilder builder = new VaragsTestCaseBuilder(type -> "Coalesce");
         builder.expectString(strings -> strings.filter(v -> v != null).findFirst());
         builder.expectLong(longs -> longs.filter(v -> v != null).findFirst());
         builder.expectInt(ints -> ints.filter(v -> v != null).findFirst());
         builder.expectBoolean(booleans -> booleans.filter(v -> v != null).findFirst());
-        suppliers.addAll(builder.suppliers());
-        addSpatialCombinations(suppliers);
-        suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.IP), () -> {
+        noNullsSuppliers.addAll(builder.suppliers());
+        addSpatialCombinations(noNullsSuppliers);
+        noNullsSuppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.IP), () -> {
             var first = randomBoolean() ? null : EsqlDataTypeConverter.stringToIP(NetworkAddress.format(randomIp(true)));
             var second = EsqlDataTypeConverter.stringToIP(NetworkAddress.format(randomIp(true)));
             return new TestCaseSupplier.TestCase(
@@ -71,7 +68,7 @@ public static Iterable<Object[]> parameters() {
                 equalTo(first == null ? second : first)
             );
         }));
-        suppliers.add(new TestCaseSupplier(List.of(DataType.VERSION, DataType.VERSION), () -> {
+        noNullsSuppliers.add(new TestCaseSupplier(List.of(DataType.VERSION, DataType.VERSION), () -> {
             var first = randomBoolean()
                 ? null
                 : EsqlDataTypeConverter.stringToVersion(randomInt(10) + "." + randomInt(10) + "." + randomInt(10));
@@ -86,7 +83,7 @@ public static Iterable<Object[]> parameters() {
                 equalTo(first == null ? second : first)
             );
         }));
-        suppliers.add(new TestCaseSupplier(List.of(DataType.DATETIME, DataType.DATETIME), () -> {
+        noNullsSuppliers.add(new TestCaseSupplier(List.of(DataType.DATETIME, DataType.DATETIME), () -> {
             Long firstDate = randomBoolean() ? null : ZonedDateTime.parse("2023-12-04T10:15:30Z").toInstant().toEpochMilli();
             Long secondDate = ZonedDateTime.parse("2023-12-05T10:45:00Z").toInstant().toEpochMilli();
             return new TestCaseSupplier.TestCase(
@@ -100,9 +97,54 @@ public static Iterable<Object[]> parameters() {
             );
         }));
 
+        List<TestCaseSupplier> suppliers = new ArrayList<>(noNullsSuppliers);
+        for (TestCaseSupplier s : noNullsSuppliers) {
+            for (int nullUpTo = 1; nullUpTo < s.types().size(); nullUpTo++) {
+                int finalNullUpTo = nullUpTo;
+
+                // Build cases with nulls up to a point
+                suppliers.add(
+                    new TestCaseSupplier(nullCaseName(s, nullUpTo, false), s.types(), () -> nullCase(s.get(), finalNullUpTo, false))
+                );
+
+                // Now do the same thing, but with the type forced to null
+                List<DataType> types = new ArrayList<>(s.types());
+                for (int i = 0; i < nullUpTo; i++) {
+                    types.set(i, DataType.NULL);
+                }
+                suppliers.add(new TestCaseSupplier(nullCaseName(s, nullUpTo, true), types, () -> nullCase(s.get(), finalNullUpTo, true)));
+            }
+        }
+
         return parameterSuppliersFromTypedData(suppliers);
     }
 
+    private static String nullCaseName(TestCaseSupplier supplier, int nullUpTo, boolean forceNullType) {
+        StringBuilder name = new StringBuilder();
+        for (int i = 0; i < nullUpTo; i++) {
+            name.append("null <").append((forceNullType ? DataType.NULL : supplier.types().get(i)).typeName()).append(">, ");
+        }
+        for (int i = nullUpTo; i < supplier.types().size(); i++) {
+            name.append('<').append(supplier.types().get(i).typeName()).append('>');
+            if (i != supplier.types().size() - 1) {
+                name.append(", ");
+            }
+        }
+        return name.toString();
+    }
+
+    private static TestCaseSupplier.TestCase nullCase(TestCaseSupplier.TestCase delegate, int nullUpTo, boolean forceNullType) {
+        List<TestCaseSupplier.TypedData> data = new ArrayList<>(delegate.getData());
+        for (int i = 0; i < nullUpTo; i++) {
+            data.set(i, new TestCaseSupplier.TypedData(null, forceNullType ? DataType.NULL : data.get(i).type(), data.get(i).name()));
+        }
+        Object expected = data.get(nullUpTo).data();
+        if (expected instanceof List<?> l && l.size() == 1) {
+            expected = l.get(0);
+        }
+        return new TestCaseSupplier.TestCase(data, delegate.evaluatorToString(), delegate.expectedType(), equalTo(expected));
+    }
+
     protected static void addSpatialCombinations(List<TestCaseSupplier> suppliers) {
         for (DataType dataType : List.of(DataType.GEO_POINT, DataType.GEO_SHAPE, DataType.CARTESIAN_POINT, DataType.CARTESIAN_SHAPE)) {
             TestCaseSupplier.TypedDataSupplier leftDataSupplier = SpatialRelatesFunctionTestCase.testCaseSupplier(dataType, false);
@@ -119,25 +161,6 @@ protected static void addSpatialCombinations(List<TestCaseSupplier> suppliers) {
         }
     }
 
-    @Override
-    protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlock) {
-        for (int i = 0; i < data.size(); i++) {
-            if (nullBlock == i) {
-                continue;
-            }
-            Object v = data.get(i);
-            if (v == null) {
-                continue;
-            }
-            if (v instanceof List<?> l && l.size() == 1) {
-                v = l.get(0);
-            }
-            assertThat(toJavaObject(value, 0), equalTo(v));
-            return;
-        }
-        assertThat(value.isNull(0), equalTo(true));
-    }
-
     @Override
     protected Coalesce build(Source source, List<Expression> args) {
         return new Coalesce(Source.EMPTY, args.get(0), args.subList(1, args.size()));
@@ -180,14 +203,14 @@ public void testCoalesceNullabilityIsUnknown() {
     public void testCoalesceKnownNullable() {
         List<Expression> sub = new ArrayList<>(testCase.getDataAsFields());
         sub.add(between(0, sub.size()), new Literal(Source.EMPTY, null, sub.get(0).dataType()));
-        Coalesce exp = build(Source.EMPTY, sub);
+        Coalesce exp = build(Source.EMPTY, testCase.getDataAsFields());
         // Still UNKNOWN - if it were TRUE then an optimizer would replace it with null
         assertThat(exp.nullable(), equalTo(Nullability.UNKNOWN));
     }
 
     public void testCoalesceNotNullable() {
         List<Expression> sub = new ArrayList<>(testCase.getDataAsFields());
-        sub.add(between(0, sub.size()), randomLiteral(sub.get(0).dataType()));
+        sub.add(between(0, sub.size()), randomLiteral(sub.get(sub.size() - 1).dataType()));
         Coalesce exp = build(Source.EMPTY, sub);
         // Known not to be nullable because it contains a non-null literal
         assertThat(exp.nullable(), equalTo(Nullability.FALSE));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java
index b99b47b6f505a..1ff9358b507c3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java
@@ -10,21 +10,19 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.compute.data.Block;
-import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.hamcrest.Matcher;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.hamcrest.Matchers.equalTo;
 
 public class IsNotNullTests extends AbstractScalarFunctionTestCase {
@@ -36,7 +34,7 @@ public IsNotNullTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> test
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
         for (DataType type : DataType.types()) {
-            if (false == EsqlDataTypes.isRepresentable(type)) {
+            if (false == DataType.isRepresentable(type)) {
                 continue;
             }
             if (type != DataType.NULL) {
@@ -69,11 +67,6 @@ public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(failureForCasesWithoutExamples(suppliers));
     }
 
-    @Override
-    protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlock) {
-        assertFalse(((BooleanBlock) value).asVector().getBoolean(0));
-    }
-
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new IsNotNull(Source.EMPTY, args.get(0));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java
index 7abfad39967a5..e80480a636a19 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java
@@ -10,21 +10,19 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.compute.data.Block;
-import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.hamcrest.Matcher;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.hamcrest.Matchers.equalTo;
 
 public class IsNullTests extends AbstractScalarFunctionTestCase {
@@ -36,7 +34,7 @@ public IsNullTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCas
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
         for (DataType type : DataType.types()) {
-            if (false == EsqlDataTypes.isRepresentable(type)) {
+            if (false == DataType.isRepresentable(type)) {
                 continue;
             }
             if (type != DataType.NULL) {
@@ -69,11 +67,6 @@ public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(failureForCasesWithoutExamples(suppliers));
     }
 
-    @Override
-    protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlock) {
-        assertTrue(((BooleanBlock) value).asVector().getBoolean(0));
-    }
-
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new IsNull(Source.EMPTY, args.get(0));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java
index 4ab1517d0c17a..0729d4854f65f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunctionTestCase.java
@@ -27,10 +27,10 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatialGeo;
 import static org.elasticsearch.xpack.esql.core.type.DataType.isString;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction.compatibleTypeNames;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatial;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isSpatialGeo;
 import static org.hamcrest.Matchers.equalTo;
 
 public abstract class BinarySpatialFunctionTestCase extends AbstractScalarFunctionTestCase {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java
index 71e73398ddcd4..96cddfdd64099 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java
@@ -36,7 +36,7 @@ public static Iterable<Object[]> parameters() {
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
         TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedEvaluator, DOUBLE, StXTests::valueOf, List.of());
         TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedEvaluator, DOUBLE, StXTests::valueOf, List.of());
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or cartesian_point");
     }
 
     private static double valueOf(BytesRef wkb) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java
index a30ae924754d6..165dbb2c0ab77 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java
@@ -36,7 +36,7 @@ public static Iterable<Object[]> parameters() {
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
         TestCaseSupplier.forUnaryGeoPoint(suppliers, expectedEvaluator, DOUBLE, StYTests::valueOf, List.of());
         TestCaseSupplier.forUnaryCartesianPoint(suppliers, expectedEvaluator, DOUBLE, StYTests::valueOf, List.of());
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or cartesian_point");
     }
 
     private static double valueOf(BytesRef wkb) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java
index a92f3ffb49533..f77a892d8682e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java
@@ -67,7 +67,7 @@ static Iterable<Object[]> parameters(String name, boolean trimLeading, boolean t
                 }));
             }
         }
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string");
     }
 
     private static TestCaseSupplier.TestCase testCase(String name, DataType type, String data, String expected) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
index c398faacb90d0..bbe92ae4a6618 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -52,11 +51,11 @@ public static Iterable<Object[]> parameters() {
         Set<DataType> supported = Set.of(DataType.NULL, DataType.KEYWORD, DataType.TEXT);
         List<Set<DataType>> supportedPerPosition = List.of(supported, supported);
         for (DataType lhs : DataType.types()) {
-            if (lhs == DataType.NULL || EsqlDataTypes.isRepresentable(lhs) == false) {
+            if (lhs == DataType.NULL || DataType.isRepresentable(lhs) == false) {
                 continue;
             }
             for (DataType rhs : DataType.types()) {
-                if (rhs == DataType.NULL || EsqlDataTypes.isRepresentable(rhs) == false) {
+                if (rhs == DataType.NULL || DataType.isRepresentable(rhs) == false) {
                     continue;
                 }
                 boolean lhsIsString = lhs == DataType.KEYWORD || lhs == DataType.TEXT;
@@ -65,7 +64,7 @@ public static Iterable<Object[]> parameters() {
                     continue;
                 }
 
-                suppliers.add(typeErrorSupplier(false, supportedPerPosition, List.of(lhs, rhs)));
+                suppliers.add(typeErrorSupplier(false, supportedPerPosition, List.of(lhs, rhs), (v, p) -> "string"));
             }
         }
         return parameterSuppliersFromTypedData(suppliers);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
index 5ae69b03ae882..6d086e2626cb6 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
 
+import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.function.Supplier;
@@ -32,99 +33,66 @@ public EndsWithTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testC
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new LinkedList<>();
-        suppliers.add(new TestCaseSupplier("ends_with empty suffix", () -> {
-            String str = randomAlphaOfLength(5);
-            String suffix = "";
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.KEYWORD, "suffix")
-                ),
-                "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataType.BOOLEAN,
-                equalTo(str.endsWith(suffix))
-            );
-        }));
-        suppliers.add(new TestCaseSupplier("ends_with empty str", () -> {
-            String str = "";
-            String suffix = randomAlphaOfLength(5);
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.KEYWORD, "suffix")
-                ),
-                "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataType.BOOLEAN,
-                equalTo(str.endsWith(suffix))
-            );
-        }));
-        suppliers.add(new TestCaseSupplier("ends_with one char suffix", () -> {
-            String str = randomAlphaOfLength(5);
-            String suffix = randomAlphaOfLength(1);
-            str = str + suffix;
-
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.KEYWORD, "suffix")
-                ),
-                "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataType.BOOLEAN,
-                equalTo(str.endsWith(suffix))
-            );
-        }));
-        suppliers.add(new TestCaseSupplier("ends_with no match suffix", () -> {
-            String str = randomAlphaOfLength(5);
-            String suffix = "no_match_suffix";
-            str = suffix + str;
-
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.KEYWORD, "suffix")
-                ),
-                "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataType.BOOLEAN,
-                equalTo(str.endsWith(suffix))
-            );
-        }));
-        suppliers.add(new TestCaseSupplier("ends_with randomized test", () -> {
-            String str = randomRealisticUnicodeOfLength(5);
-            String suffix = randomRealisticUnicodeOfLength(5);
-            str = str + suffix;
-
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.KEYWORD, "suffix")
-                ),
-                "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataType.BOOLEAN,
-                equalTo(str.endsWith(suffix))
-            );
-        }));
-        suppliers.add(new TestCaseSupplier("ends_with with text args", () -> {
-            String str = randomAlphaOfLength(5);
-            String suffix = randomAlphaOfLength(1);
-            str = str + suffix;
-
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.TEXT, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.TEXT, "suffix")
-                ),
-                "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataType.BOOLEAN,
-                equalTo(str.endsWith(suffix))
-            );
-        }));
-        return parameterSuppliersFromTypedData(suppliers);
+        for (DataType strType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) {
+            for (DataType suffixType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) {
+                suppliers.add(
+                    new TestCaseSupplier(
+                        "<" + strType + ">, empty <" + suffixType + ">",
+                        List.of(strType, suffixType),
+                        () -> testCase(strType, suffixType, randomAlphaOfLength(5), "", equalTo(true))
+                    )
+                );
+                suppliers.add(
+                    new TestCaseSupplier(
+                        "empty <" + strType + ">, <" + suffixType + ">",
+                        List.of(strType, suffixType),
+                        () -> testCase(strType, suffixType, "", randomAlphaOfLength(5), equalTo(false))
+                    )
+                );
+                suppliers.add(
+                    new TestCaseSupplier("<" + strType + ">, one char <" + suffixType + "> matches", List.of(strType, suffixType), () -> {
+                        String str = randomAlphaOfLength(5);
+                        String suffix = randomAlphaOfLength(1);
+                        str = str + suffix;
+                        return testCase(strType, suffixType, str, suffix, equalTo(true));
+                    })
+                );
+                suppliers.add(
+                    new TestCaseSupplier("<" + strType + ">, one char <" + suffixType + "> differs", List.of(strType, suffixType), () -> {
+                        String str = randomAlphaOfLength(5);
+                        String suffix = randomAlphaOfLength(1);
+                        str = str + randomValueOtherThan(suffix, () -> randomAlphaOfLength(1));
+                        return testCase(strType, suffixType, str, suffix, equalTo(false));
+                    })
+                );
+                suppliers.add(
+                    new TestCaseSupplier("random <" + strType + ">, random <" + suffixType + ">", List.of(strType, suffixType), () -> {
+                        String str = randomAlphaOfLength(5);
+                        String suffix = randomAlphaOfLength(3);
+                        return testCase(strType, suffixType, str, suffix, equalTo(str.endsWith(suffix)));
+                    })
+                );
+            }
+        }
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (valid, position) -> "string");
     }
 
-    private Matcher<Object> resultsMatcher(List<TestCaseSupplier.TypedData> typedData) {
-        String str = ((BytesRef) typedData.get(0).data()).utf8ToString();
-        String prefix = ((BytesRef) typedData.get(1).data()).utf8ToString();
-        return equalTo(str.endsWith(prefix));
+    private static TestCaseSupplier.TestCase testCase(
+        DataType strType,
+        DataType suffixType,
+        String str,
+        String suffix,
+        Matcher<Boolean> matcher
+    ) {
+        return new TestCaseSupplier.TestCase(
+            List.of(
+                new TestCaseSupplier.TypedData(new BytesRef(str), strType, "str"),
+                new TestCaseSupplier.TypedData(new BytesRef(suffix), suffixType, "suffix")
+            ),
+            "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
+            DataType.BOOLEAN,
+            matcher
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java
index 88ee7881e128a..627c46da025ea 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java
@@ -167,7 +167,11 @@ public static Iterable<Object[]> parameters() {
             );
         }));
 
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) {
+            case 0 -> "string";
+            case 1 -> "integer";
+            default -> "";
+        });
     }
 
     private static String unicodeLeftSubstring(String str, int length) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java
index a1451b6bedf7a..6ae5a9d961398 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java
@@ -49,7 +49,7 @@ public static Iterable<Object[]> parameters() {
         cases.addAll(makeTestCases("6 bytes, 2 code points", () -> "❗️", 2));
         cases.addAll(makeTestCases("100 random alpha", () -> randomAlphaOfLength(100), 100));
         cases.addAll(makeTestCases("100 random code points", () -> randomUnicodeOfCodepointLength(100), 100));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string");
     }
 
     private static List<TestCaseSupplier> makeTestCases(String title, Supplier<String> text, int expectedLength) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
index 13d8edf489a66..207125bed2a19 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
@@ -78,7 +78,12 @@ public static Iterable<Object[]> parameters() {
             }
         }
 
-        suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers));
+        suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), (v, p) -> {
+            if (p == 0 || p == 1) {
+                return "string";
+            }
+            return "integer";
+        });
 
         // Here follows some non-randomized examples that we want to cover on every run
         suppliers.add(supplier("a tiger", "a t", null, 1));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
index 0074f83b3bbce..5a34d850cffe3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
@@ -11,7 +11,6 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
@@ -19,13 +18,13 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Function;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
@@ -73,7 +72,7 @@ static Iterable<Object[]> parameters(Function<String, String> escapeString, Supp
             if (type == DataType.KEYWORD || type == DataType.TEXT || type == DataType.NULL) {
                 continue;
             }
-            if (EsqlDataTypes.isRepresentable(type) == false) {
+            if (DataType.isRepresentable(type) == false) {
                 continue;
             }
             cases.add(
@@ -140,22 +139,32 @@ private static void cases(List<TestCaseSupplier> cases, String title, Supplier<T
                     equalTo(expected)
                 );
             }));
+            cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, type), () -> {
+                TextAndPattern v = textAndPattern.get();
+                return new TestCaseSupplier.TestCase(
+                    List.of(
+                        new TestCaseSupplier.TypedData(new BytesRef(v.text), type, "e"),
+                        new TestCaseSupplier.TypedData(new BytesRef(v.pattern), type, "pattern").forceLiteral()
+                    ),
+                    startsWith("AutomataMatchEvaluator[input=Attribute[channel=0], pattern=digraph Automaton {\n"),
+                    DataType.BOOLEAN,
+                    equalTo(expected)
+                );
+            }));
         }
     }
 
-    @Override
-    protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlock) {
-        assumeFalse("generated test cases containing nulls by hand", true);
-    }
-
     @Override
     protected Expression build(Source source, List<Expression> args) {
         Expression expression = args.get(0);
         Literal pattern = (Literal) args.get(1);
-        Literal caseInsensitive = (Literal) args.get(2);
+        Literal caseInsensitive = args.size() > 2 ? (Literal) args.get(2) : null;
         String patternString = ((BytesRef) pattern.fold()).utf8ToString();
-        boolean caseInsensitiveBool = (boolean) caseInsensitive.fold();
+        boolean caseInsensitiveBool = caseInsensitive != null ? (boolean) caseInsensitive.fold() : false;
         logger.info("pattern={} caseInsensitive={}", patternString, caseInsensitiveBool);
-        return new RLike(source, expression, new RLikePattern(patternString), caseInsensitiveBool);
+
+        return caseInsensitiveBool
+            ? new RLike(source, expression, new RLikePattern(patternString), true)
+            : new RLike(source, expression, new RLikePattern(patternString));
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java
index 8d0368d1c618f..4d97a2f629c23 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java
@@ -107,7 +107,11 @@ public static Iterable<Object[]> parameters() {
         }));
 
         cases = anyNullIsNull(true, cases);
-        cases = errorsForCasesWithoutExamples(cases);
+        cases = errorsForCasesWithoutExamples(cases, (v, p) -> switch (p) {
+            case 0 -> "string";
+            case 1 -> "integer";
+            default -> "";
+        });
         return parameterSuppliersFromTypedData(cases);
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
index fe77b9dcdb075..bf1325854f1a2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
@@ -103,7 +103,7 @@ public static Iterable<Object[]> parameters() {
                     "Unclosed character class near index 0\n[\n^".replaceAll("\n", System.lineSeparator())
                 );
         }));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string");
     }
 
     private static TestCaseSupplier fixedCase(String name, String str, String oldStr, String newStr, String result) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java
index cc98edb85f547..a1ef77a62b67c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java
@@ -166,7 +166,11 @@ public static Iterable<Object[]> parameters() {
                 equalTo(new BytesRef(unicodeRightSubstring(text, length)))
             );
         }));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) {
+            case 0 -> "string";
+            case 1 -> "integer";
+            default -> throw new IllegalStateException("bad parameter number");
+        });
     }
 
     private static String unicodeRightSubstring(String str, int length) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java
index dd28b43bd66ed..b5560f37914a9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java
@@ -64,7 +64,7 @@ public static Iterable<Object[]> parameters() {
                 }));
             }
         }
-        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
index bd01f926d1571..8bc8cf3184a75 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
@@ -16,8 +16,9 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.hamcrest.Matcher;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.function.Supplier;
 
@@ -30,43 +31,28 @@ public StartsWithTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> tes
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
-        return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Starts with basic test", () -> {
-            String str = randomAlphaOfLength(5);
-            String prefix = randomAlphaOfLength(5);
-            if (randomBoolean()) {
-                str = prefix + str;
+        List<TestCaseSupplier> suppliers = new ArrayList<>();
+        for (DataType strType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) {
+            for (DataType prefixType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) {
+                suppliers.add(new TestCaseSupplier(List.of(strType, prefixType), () -> {
+                    String str = randomAlphaOfLength(5);
+                    String prefix = randomAlphaOfLength(5);
+                    if (randomBoolean()) {
+                        str = prefix + str;
+                    }
+                    return new TestCaseSupplier.TestCase(
+                        List.of(
+                            new TestCaseSupplier.TypedData(new BytesRef(str), strType, "str"),
+                            new TestCaseSupplier.TypedData(new BytesRef(prefix), prefixType, "prefix")
+                        ),
+                        "StartsWithEvaluator[str=Attribute[channel=0], prefix=Attribute[channel=1]]",
+                        DataType.BOOLEAN,
+                        equalTo(str.startsWith(prefix))
+                    );
+                }));
             }
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(prefix), DataType.KEYWORD, "prefix")
-                ),
-                "StartsWithEvaluator[str=Attribute[channel=0], prefix=Attribute[channel=1]]",
-                DataType.BOOLEAN,
-                equalTo(str.startsWith(prefix))
-            );
-        }), new TestCaseSupplier("Starts with basic test with text args", () -> {
-            String str = randomAlphaOfLength(5);
-            String prefix = randomAlphaOfLength(5);
-            if (randomBoolean()) {
-                str = prefix + str;
-            }
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.TEXT, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(prefix), DataType.TEXT, "prefix")
-                ),
-                "StartsWithEvaluator[str=Attribute[channel=0], prefix=Attribute[channel=1]]",
-                DataType.BOOLEAN,
-                equalTo(str.startsWith(prefix))
-            );
-        })));
-    }
-
-    private Matcher<Object> resultsMatcher(List<TestCaseSupplier.TypedData> typedData) {
-        String str = ((BytesRef) typedData.get(0).data()).utf8ToString();
-        String prefix = ((BytesRef) typedData.get(1).data()).utf8ToString();
-        return equalTo(str.startsWith(prefix));
+        }
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (valid, position) -> "string");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
index 1c49d3b408ad6..6b934aae775df 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
@@ -86,7 +86,12 @@ public static Iterable<Object[]> parameters() {
                         equalTo(new BytesRef(""))
                     );
                 })
-            )
+            ),
+            (v, p) -> switch (p) {
+                case 0 -> "string";
+                case 1, 2 -> "integer";
+                default -> "";
+            }
         );
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
index abb419e1e4a81..d5d5a0188e262 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
@@ -18,12 +18,12 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DateUtils;
+import org.elasticsearch.xpack.esql.core.util.DateUtils;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -47,18 +47,18 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10)));
 
         // add null as parameter
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string");
     }
 
     public void testRandomLocale() {
         String testString = randomAlphaOfLength(10);
-        EsqlConfiguration cfg = randomLocaleConfig();
+        Configuration cfg = randomLocaleConfig();
         ToLower func = new ToLower(Source.EMPTY, new Literal(Source.EMPTY, testString, DataType.KEYWORD), cfg);
         assertThat(BytesRefs.toBytesRef(testString.toLowerCase(cfg.locale())), equalTo(func.fold()));
     }
 
-    private EsqlConfiguration randomLocaleConfig() {
-        return new EsqlConfiguration(
+    private Configuration randomLocaleConfig() {
+        return new Configuration(
             DateUtils.UTC,
             randomLocale(random()),
             null,
@@ -73,7 +73,7 @@ private EsqlConfiguration randomLocaleConfig() {
     }
 
     @Override
-    protected Expression buildWithConfiguration(Source source, List<Expression> args, EsqlConfiguration configuration) {
+    protected Expression buildWithConfiguration(Source source, List<Expression> args, Configuration configuration) {
         return new ToLower(source, args.get(0), configuration);
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
index f101cacd73dc5..0bc3d8d90dbd9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
@@ -18,12 +18,12 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DateUtils;
+import org.elasticsearch.xpack.esql.core.util.DateUtils;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -47,18 +47,18 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10)));
 
         // add null as parameter
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string");
     }
 
     public void testRandomLocale() {
         String testString = randomAlphaOfLength(10);
-        EsqlConfiguration cfg = randomLocaleConfig();
+        Configuration cfg = randomLocaleConfig();
         ToUpper func = new ToUpper(Source.EMPTY, new Literal(Source.EMPTY, testString, DataType.KEYWORD), cfg);
         assertThat(BytesRefs.toBytesRef(testString.toUpperCase(cfg.locale())), equalTo(func.fold()));
     }
 
-    private EsqlConfiguration randomLocaleConfig() {
-        return new EsqlConfiguration(
+    private Configuration randomLocaleConfig() {
+        return new Configuration(
             DateUtils.UTC,
             randomLocale(random()),
             null,
@@ -73,7 +73,7 @@ private EsqlConfiguration randomLocaleConfig() {
     }
 
     @Override
-    protected Expression buildWithConfiguration(Source source, List<Expression> args, EsqlConfiguration configuration) {
+    protected Expression buildWithConfiguration(Source source, List<Expression> args, Configuration configuration) {
         return new ToUpper(source, args.get(0), configuration);
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java
index 06736db28b2cc..53b3a99f97b9c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java
@@ -11,7 +11,6 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
@@ -73,11 +72,6 @@ private static void addCases(List<TestCaseSupplier> suppliers) {
         }
     }
 
-    @Override
-    protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlock) {
-        assumeFalse("generated test cases containing nulls by hand", true);
-    }
-
     @Override
     protected Expression build(Source source, List<Expression> args) {
         Expression expression = args.get(0);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java
deleted file mode 100644
index 974c8703b2a09..0000000000000
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.expression.predicate.operator;
-
-import org.elasticsearch.compute.data.Block;
-import org.elasticsearch.xpack.esql.analysis.Verifier;
-import org.elasticsearch.xpack.esql.common.Failure;
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.Literal;
-import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
-import org.elasticsearch.xpack.esql.core.tree.Location;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
-import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.hamcrest.Matcher;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Locale;
-
-import static org.elasticsearch.compute.data.BlockUtils.toJavaObject;
-import static org.elasticsearch.xpack.esql.core.type.DataType.isNull;
-import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.commonType;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
-
-public abstract class AbstractBinaryOperatorTestCase extends AbstractScalarFunctionTestCase {
-
-    protected abstract Matcher<Object> resultsMatcher(List<TestCaseSupplier.TypedData> typedData);
-
-    /**
-     * Return a {@link Matcher} to validate the results of evaluating the function
-     *
-     * @param data a list of the parameters that were passed to the evaluator
-     * @return a matcher to validate correctness against the given data set
-     */
-    protected abstract Matcher<Object> resultMatcher(List<Object> data, DataType dataType);
-
-    protected boolean rhsOk(Object o) {
-        return true;
-    }
-
-    @Override
-    protected Expression build(Source source, List<Expression> args) {
-        return build(source, args.get(0), args.get(1));
-    }
-
-    protected abstract BinaryOperator<?, ?, ?, ?> build(Source source, Expression lhs, Expression rhs);
-
-    /**
-     * What type is acceptable for any of the function parameters.
-     * @param type The type to probe.
-     * @return True if the type is supported by the respective function.
-     */
-    protected abstract boolean supportsType(DataType type);
-
-    /**
-     * What combination of parameter types are acceptable by the function.
-     * @param lhsType Left argument type.
-     * @param rhsType Right argument type.
-     * @return True if the type combination is supported by the respective function.
-     */
-    protected boolean supportsTypes(DataType lhsType, DataType rhsType) {
-        if ((lhsType == DataType.UNSIGNED_LONG || rhsType == DataType.UNSIGNED_LONG) && lhsType != rhsType) {
-            // UL can only be operated on together with another UL, so skip non-UL&UL combinations
-            return false;
-        }
-        return supportsType(lhsType) && supportsType(rhsType);
-    }
-
-    public final void testApplyToAllTypes() {
-        // TODO replace with test cases
-        for (DataType lhsType : DataType.types()) {
-            for (DataType rhsType : DataType.types()) {
-                if (supportsTypes(lhsType, rhsType) == false) {
-                    continue;
-                }
-                Literal lhs = randomLiteral(lhsType);
-                Literal rhs = randomValueOtherThanMany(l -> rhsOk(l.value()) == false, () -> randomLiteral(rhsType));
-                Object result;
-                BinaryOperator<?, ?, ?, ?> op;
-                Source src = new Source(Location.EMPTY, lhsType.typeName() + " " + rhsType.typeName());
-                if (isRepresentable(lhsType) && isRepresentable(rhsType)) {
-                    op = build(src, field("lhs", lhsType), field("rhs", rhsType));
-                    try (Block block = evaluator(op).get(driverContext()).eval(row(Arrays.asList(lhs.value(), rhs.value())))) {
-                        result = toJavaObject(block, 0);
-                    }
-                } else {
-                    op = build(src, lhs, rhs);
-                    result = op.fold();
-                }
-                if (isNull(lhsType) || isNull(rhsType)) {
-                    assertThat(op.toString(), result, is(nullValue()));
-                } else if (result == null) {
-                    assertCriticalWarnings(
-                        "Line -1:-1: evaluation of [" + op + "] failed, treating result as null. Only first 20 failures recorded.",
-                        "Line -1:-1: java.lang.ArithmeticException: " + commonType(lhsType, rhsType).typeName() + " overflow"
-                    );
-                } else {
-                    // The type's currently only used for distinguishing between LONG and UNSIGNED_LONG. UL requires both operands be of
-                    // the same type, so either left or right type can be provided below. But otherwise the common type can be used
-                    // instead.
-                    assertThat(op.toString(), result, resultMatcher(List.of(lhs.value(), rhs.value()), lhsType));
-                }
-            }
-        }
-    }
-
-    public final void testResolveType() {
-        for (DataType lhsType : DataType.types()) {
-            if (isRepresentable(lhsType) == false) {
-                continue;
-            }
-            Literal lhs = randomLiteral(lhsType);
-            for (DataType rhsType : DataType.types()) {
-                if (isRepresentable(rhsType) == false) {
-                    continue;
-                }
-                Literal rhs = randomLiteral(rhsType);
-                BinaryOperator<?, ?, ?, ?> op = build(new Source(Location.EMPTY, lhsType.typeName() + " " + rhsType.typeName()), lhs, rhs);
-
-                if (lhsType == DataType.UNSIGNED_LONG || rhsType == DataType.UNSIGNED_LONG) {
-                    validateUnsignedLongType(op, lhsType, rhsType);
-                    continue;
-                }
-                validateType(op, lhsType, rhsType);
-            }
-        }
-    }
-
-    private void validateUnsignedLongType(BinaryOperator<?, ?, ?, ?> op, DataType lhsType, DataType rhsType) {
-        Failure fail = Verifier.validateUnsignedLongOperator(op);
-        if (lhsType == rhsType) {
-            assertThat(op.toString(), fail, nullValue());
-            return;
-        }
-        assertThat(op.toString(), fail, not(nullValue()));
-        assertThat(
-            op.toString(),
-            fail.message(),
-            equalTo(
-                String.format(
-                    Locale.ROOT,
-                    "first argument of [%s] is [%s] and second is [%s]. [unsigned_long] can only be operated on together "
-                        + "with another [unsigned_long]",
-                    op,
-                    lhsType.typeName(),
-                    rhsType.typeName()
-                )
-            )
-        );
-
-    }
-
-    protected abstract void validateType(BinaryOperator<?, ?, ?, ?> op, DataType lhsType, DataType rhsType);
-}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticTestCase.java
deleted file mode 100644
index 141fc24e73e18..0000000000000
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticTestCase.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic;
-
-import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.AbstractBinaryOperatorTestCase;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
-import org.hamcrest.Matcher;
-
-import java.util.List;
-import java.util.Locale;
-import java.util.function.Supplier;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
-
-public abstract class AbstractArithmeticTestCase extends AbstractBinaryOperatorTestCase {
-    protected Matcher<Object> resultMatcher(List<Object> data, DataType dataType) {
-        Number lhs = (Number) data.get(0);
-        Number rhs = (Number) data.get(1);
-        if (lhs instanceof Double || rhs instanceof Double) {
-            return equalTo(expectedValue(lhs.doubleValue(), rhs.doubleValue()));
-        }
-        if (lhs instanceof Long || rhs instanceof Long) {
-            if (dataType == DataType.UNSIGNED_LONG) {
-                return equalTo(expectedUnsignedLongValue(lhs.longValue(), rhs.longValue()));
-            }
-            return equalTo(expectedValue(lhs.longValue(), rhs.longValue()));
-        }
-        if (lhs instanceof Integer || rhs instanceof Integer) {
-            return equalTo(expectedValue(lhs.intValue(), rhs.intValue()));
-        }
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    protected Matcher<Object> resultsMatcher(List<TestCaseSupplier.TypedData> typedData) {
-        Number lhs = (Number) typedData.get(0).data();
-        Number rhs = (Number) typedData.get(1).data();
-        if (typedData.stream().anyMatch(t -> t.type().equals(DataType.DOUBLE))) {
-            return equalTo(expectedValue(lhs.doubleValue(), rhs.doubleValue()));
-        }
-        if (typedData.stream().anyMatch(t -> t.type().equals(DataType.UNSIGNED_LONG))) {
-            return equalTo(expectedUnsignedLongValue(lhs.longValue(), rhs.longValue()));
-        }
-        if (typedData.stream().anyMatch(t -> t.type().equals(DataType.LONG))) {
-            return equalTo(expectedValue(lhs.longValue(), rhs.longValue()));
-        }
-        if (typedData.stream().anyMatch(t -> t.type().equals(DataType.INTEGER))) {
-            return equalTo(expectedValue(lhs.intValue(), rhs.intValue()));
-        }
-        throw new UnsupportedOperationException();
-    }
-
-    protected abstract double expectedValue(double lhs, double rhs);
-
-    protected abstract int expectedValue(int lhs, int rhs);
-
-    protected abstract long expectedValue(long lhs, long rhs);
-
-    protected abstract long expectedUnsignedLongValue(long lhs, long rhs);
-
-    @Override
-    protected boolean supportsType(DataType type) {
-        return type.isNumeric() && EsqlDataTypes.isRepresentable(type);
-    }
-
-    @Override
-    protected void validateType(BinaryOperator<?, ?, ?, ?> op, DataType lhsType, DataType rhsType) {
-        if (DataType.isNullOrNumeric(lhsType) && DataType.isNullOrNumeric(rhsType)) {
-            assertTrue(op.toString(), op.typeResolved().resolved());
-            assertThat(op.toString(), op.dataType(), equalTo(expectedType(lhsType, rhsType)));
-            return;
-        }
-        assertFalse(op.toString(), op.typeResolved().resolved());
-        if (op instanceof Mul) {
-            // TODO why is mul different?
-            assertThat(
-                op.toString(),
-                op.typeResolved().message(),
-                equalTo(String.format(Locale.ROOT, "[*] has arguments with incompatible types [%s] and [%s]", lhsType, rhsType))
-            );
-            return;
-        }
-        assertThat(
-            op.toString(),
-            op.typeResolved().message(),
-            containsString(
-                String.format(Locale.ROOT, "argument of [%s %s] must be [numeric], found value []", lhsType.typeName(), rhsType.typeName())
-            )
-        );
-    }
-
-    protected DataType expectedType(DataType lhsType, DataType rhsType) {
-        if (lhsType == DataType.DOUBLE || rhsType == DataType.DOUBLE) {
-            return DataType.DOUBLE;
-        }
-        if (lhsType == DataType.UNSIGNED_LONG || rhsType == DataType.UNSIGNED_LONG) {
-            assertThat(lhsType, is(DataType.UNSIGNED_LONG));
-            assertThat(rhsType, is(DataType.UNSIGNED_LONG));
-            return DataType.UNSIGNED_LONG;
-        }
-        if (lhsType == DataType.LONG || rhsType == DataType.LONG) {
-            return DataType.LONG;
-        }
-        if (lhsType == DataType.INTEGER || rhsType == DataType.INTEGER) {
-            return DataType.INTEGER;
-        }
-        if (lhsType == DataType.NULL || rhsType == DataType.NULL) {
-            return DataType.NULL;
-        }
-        throw new UnsupportedOperationException();
-    }
-
-    static TestCaseSupplier arithmeticExceptionOverflowCase(
-        DataType dataType,
-        Supplier<Object> lhsSupplier,
-        Supplier<Object> rhsSupplier,
-        String evaluator
-    ) {
-        String typeNameOverflow = dataType.typeName().toLowerCase(Locale.ROOT) + " overflow";
-        return new TestCaseSupplier(
-            "<" + typeNameOverflow + ">",
-            List.of(dataType),
-            () -> new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(lhsSupplier.get(), dataType, "lhs"),
-                    new TestCaseSupplier.TypedData(rhsSupplier.get(), dataType, "rhs")
-                ),
-                evaluator + "[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
-                dataType,
-                is(nullValue())
-            ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
-                .withWarning("Line -1:-1: java.lang.ArithmeticException: " + typeNameOverflow)
-        );
-    }
-}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractDateTimeArithmeticTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractDateTimeArithmeticTestCase.java
deleted file mode 100644
index 8a27a289bb77f..0000000000000
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractDateTimeArithmeticTestCase.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic;
-
-import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
-import org.hamcrest.Matcher;
-
-import java.time.Duration;
-import java.time.Period;
-import java.time.temporal.TemporalAmount;
-import java.util.List;
-import java.util.Locale;
-
-import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime;
-import static org.elasticsearch.xpack.esql.core.type.DataType.isNull;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrTemporalAmount;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount;
-import static org.hamcrest.Matchers.equalTo;
-
-public abstract class AbstractDateTimeArithmeticTestCase extends AbstractArithmeticTestCase {
-
-    @Override
-    protected Matcher<Object> resultMatcher(List<Object> data, DataType dataType) {
-        Object lhs = data.get(0);
-        Object rhs = data.get(1);
-        if (lhs instanceof TemporalAmount || rhs instanceof TemporalAmount) {
-            Object expectedValue;
-            if (lhs instanceof TemporalAmount && rhs instanceof TemporalAmount) {
-                assertThat("temporal amounts of different kinds", lhs.getClass(), equalTo(rhs.getClass()));
-                if (lhs instanceof Period) {
-                    expectedValue = expectedValue((Period) lhs, (Period) rhs);
-                } else {
-                    expectedValue = expectedValue((Duration) lhs, (Duration) rhs);
-                }
-            } else if (lhs instanceof TemporalAmount lhsTemporal) {
-                expectedValue = expectedValue((long) rhs, lhsTemporal);
-            } else { // rhs instanceof TemporalAmount
-                expectedValue = expectedValue((long) lhs, (TemporalAmount) rhs);
-            }
-            return equalTo(expectedValue);
-        }
-        return super.resultMatcher(data, dataType);
-    }
-
-    protected abstract long expectedValue(long datetime, TemporalAmount temporalAmount);
-
-    protected abstract Period expectedValue(Period lhs, Period rhs);
-
-    protected abstract Duration expectedValue(Duration lhs, Duration rhs);
-
-    @Override
-    protected final boolean supportsType(DataType type) {
-        return EsqlDataTypes.isDateTimeOrTemporal(type) || super.supportsType(type);
-    }
-
-    @Override
-    protected void validateType(BinaryOperator<?, ?, ?, ?> op, DataType lhsType, DataType rhsType) {
-        if (isDateTime(lhsType) || isDateTime(rhsType)) {
-            String failureMessage = null;
-            if (isDateTime(lhsType) && isDateTime(rhsType)
-                || isNullOrTemporalAmount(lhsType) == false && isNullOrTemporalAmount(rhsType) == false) {
-                failureMessage = String.format(
-                    Locale.ROOT,
-                    "[%s] has arguments with incompatible types [%s] and [%s]",
-                    op.symbol(),
-                    lhsType,
-                    rhsType
-                );
-            } else if (op instanceof Sub && isDateTime(rhsType)) {
-                failureMessage = String.format(
-                    Locale.ROOT,
-                    "[%s] arguments are in unsupported order: cannot subtract a [DATETIME] value [%s] from a [%s] amount [%s]",
-                    op.symbol(),
-                    op.right().sourceText(),
-                    lhsType,
-                    op.left().sourceText()
-                );
-            }
-            assertTypeResolution(failureMessage, op, lhsType, rhsType);
-        } else if (isTemporalAmount(lhsType) || isTemporalAmount(rhsType)) {
-            String failureMessage = isNull(lhsType) || isNull(rhsType) || lhsType == rhsType
-                ? null
-                : String.format(Locale.ROOT, "[%s] has arguments with incompatible types [%s] and [%s]", op.symbol(), lhsType, rhsType);
-            assertTypeResolution(failureMessage, op, lhsType, rhsType);
-        } else {
-            super.validateType(op, lhsType, rhsType);
-        }
-    }
-
-    private void assertTypeResolution(String failureMessage, BinaryOperator<?, ?, ?, ?> op, DataType lhsType, DataType rhsType) {
-        if (failureMessage != null) {
-            assertFalse(op.toString(), op.typeResolved().resolved());
-            assertThat(op.toString(), op.typeResolved().message(), equalTo(failureMessage));
-        } else {
-            assertTrue(op.toString(), op.typeResolved().resolved());
-            assertThat(op.toString(), op.dataType(), equalTo(expectedType(lhsType, rhsType)));
-        }
-    }
-
-    @Override
-    protected DataType expectedType(DataType lhsType, DataType rhsType) {
-        if (isDateTime(lhsType) || isDateTime(rhsType)) {
-            return DataType.DATETIME;
-        } else if (isNullOrTemporalAmount(lhsType) || isNullOrTemporalAmount(rhsType)) {
-            if (isNull(lhsType)) {
-                return rhsType;
-            } else if (isNull(rhsType)) {
-                return lhsType;
-            } else if (lhsType == rhsType) {
-                return lhsType;
-            } // else: UnsupportedOperationException
-        }
-        return super.expectedType(lhsType, rhsType);
-    }
-}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
index d477c9924230d..8c31b4a65dd14 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
@@ -27,10 +27,9 @@
 import java.util.function.BinaryOperator;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.xpack.esql.core.type.DateUtils.asDateTime;
-import static org.elasticsearch.xpack.esql.core.type.DateUtils.asMillis;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.asDateTime;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.asMillis;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned;
-import static org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.AbstractArithmeticTestCase.arithmeticExceptionOverflowCase;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
@@ -222,7 +221,7 @@ public static Iterable<Object[]> parameters() {
         suppliers = errorsForCasesWithoutExamples(suppliers, AddTests::addErrorMessageString);
 
         // Cases that should generate warnings
-        suppliers.addAll(List.of(new TestCaseSupplier("MV", () -> {
+        suppliers.add(new TestCaseSupplier("MV", List.of(DataType.INTEGER, DataType.INTEGER), () -> {
             // Ensure we don't have an overflow
             int rhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
             int lhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
@@ -237,7 +236,7 @@ public static Iterable<Object[]> parameters() {
                 is(nullValue())
             ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
                 .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value");
-        })));
+        }));
         // exact math arithmetic exceptions
         suppliers.add(
             arithmeticExceptionOverflowCase(
@@ -285,7 +284,7 @@ public static Iterable<Object[]> parameters() {
 
     private static String addErrorMessageString(boolean includeOrdinal, List<Set<DataType>> validPerPosition, List<DataType> types) {
         try {
-            return typeErrorMessage(includeOrdinal, validPerPosition, types);
+            return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "datetime or numeric");
         } catch (IllegalStateException e) {
             // This means all the positional args were okay, so the expected error is from the combination
             return "[+] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]";
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java
index 7bc5b24651218..c6b607ded7999 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java
@@ -168,7 +168,7 @@ public static Iterable<Object[]> parameters() {
 
     private static String divErrorMessageString(boolean includeOrdinal, List<Set<DataType>> validPerPosition, List<DataType> types) {
         try {
-            return typeErrorMessage(includeOrdinal, validPerPosition, types);
+            return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "numeric");
         } catch (IllegalStateException e) {
             // This means all the positional args were okay, so the expected error is from the combination
             return "[/] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]";
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java
index 133324bafd134..65335cdeeb4e1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java
@@ -155,7 +155,7 @@ public static Iterable<Object[]> parameters() {
 
     private static String modErrorMessageString(boolean includeOrdinal, List<Set<DataType>> validPerPosition, List<DataType> types) {
         try {
-            return typeErrorMessage(includeOrdinal, validPerPosition, types);
+            return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "numeric");
         } catch (IllegalStateException e) {
             // This means all the positional args were okay, so the expected error is from the combination
             return "[%] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]";
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java
index 54786d11918b5..a2ca331cb261e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java
@@ -21,7 +21,6 @@
 import java.util.function.Supplier;
 
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned;
-import static org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.AbstractArithmeticTestCase.arithmeticExceptionOverflowCase;
 import static org.hamcrest.Matchers.equalTo;
 
 public class MulTests extends AbstractScalarFunctionTestCase {
@@ -125,7 +124,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        return parameterSuppliersFromTypedData(suppliers);
+        return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
index 7eadd74eaeb9e..3b789f3022cfc 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
@@ -26,6 +26,7 @@
 import java.util.function.Supplier;
 
 import static org.elasticsearch.compute.data.BlockUtils.toJavaObject;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.hamcrest.Matchers.equalTo;
 
 public class NegTests extends AbstractScalarFunctionTestCase {
@@ -110,7 +111,7 @@ public static Iterable<Object[]> parameters() {
                 equalTo(arg.negated())
             );
         })));
-        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers);
+        return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "numeric, date_period or time_duration");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
index e57b523542d37..39d55d1ba0b54 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
@@ -21,10 +21,10 @@
 import java.util.List;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.xpack.esql.core.type.DateUtils.asDateTime;
-import static org.elasticsearch.xpack.esql.core.type.DateUtils.asMillis;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.asDateTime;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.asMillis;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.ZERO_AS_UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.AbstractArithmeticTestCase.arithmeticExceptionOverflowCase;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
@@ -111,7 +111,7 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        suppliers.add(new TestCaseSupplier("Datetime - Period", () -> {
+        suppliers.add(new TestCaseSupplier("Datetime - Period", List.of(DataType.DATETIME, DataType.DATE_PERIOD), () -> {
             long lhs = (Long) randomLiteral(DataType.DATETIME).value();
             Period rhs = (Period) randomLiteral(DataType.DATE_PERIOD).value();
             return new TestCaseSupplier.TestCase(
@@ -124,7 +124,7 @@ public static Iterable<Object[]> parameters() {
                 equalTo(asMillis(asDateTime(lhs).minus(rhs)))
             );
         }));
-        suppliers.add(new TestCaseSupplier("Period - Period", () -> {
+        suppliers.add(new TestCaseSupplier("Period - Period", List.of(DataType.DATE_PERIOD, DataType.DATE_PERIOD), () -> {
             Period lhs = (Period) randomLiteral(DataType.DATE_PERIOD).value();
             Period rhs = (Period) randomLiteral(DataType.DATE_PERIOD).value();
             return new TestCaseSupplier.TestCase(
@@ -137,7 +137,7 @@ public static Iterable<Object[]> parameters() {
                 equalTo(lhs.minus(rhs))
             );
         }));
-        suppliers.add(new TestCaseSupplier("Datetime - Duration", () -> {
+        suppliers.add(new TestCaseSupplier("Datetime - Duration", List.of(DataType.DATETIME, DataType.TIME_DURATION), () -> {
             long lhs = (Long) randomLiteral(DataType.DATETIME).value();
             Duration rhs = (Duration) randomLiteral(DataType.TIME_DURATION).value();
             TestCaseSupplier.TestCase testCase = new TestCaseSupplier.TestCase(
@@ -151,7 +151,7 @@ public static Iterable<Object[]> parameters() {
             );
             return testCase;
         }));
-        suppliers.add(new TestCaseSupplier("Duration - Duration", () -> {
+        suppliers.add(new TestCaseSupplier("Duration - Duration", List.of(DataType.TIME_DURATION, DataType.TIME_DURATION), () -> {
             Duration lhs = (Duration) randomLiteral(DataType.TIME_DURATION).value();
             Duration rhs = (Duration) randomLiteral(DataType.TIME_DURATION).value();
             return new TestCaseSupplier.TestCase(
@@ -164,22 +164,6 @@ public static Iterable<Object[]> parameters() {
                 equalTo(lhs.minus(rhs))
             );
         }));
-        suppliers.add(new TestCaseSupplier("MV", () -> {
-            // Ensure we don't have an overflow
-            int rhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
-            int lhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
-            int lhs2 = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
-            return new TestCaseSupplier.TestCase(
-                List.of(
-                    new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataType.INTEGER, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, DataType.INTEGER, "rhs")
-                ),
-                "SubIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
-                DataType.INTEGER,
-                is(nullValue())
-            ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
-                .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value");
-        }));
         // exact math arithmetic exceptions
         suppliers.add(
             arithmeticExceptionOverflowCase(
@@ -221,6 +205,29 @@ public static Iterable<Object[]> parameters() {
                 "SubUnsignedLongsEvaluator"
             )
         );
+        suppliers = anyNullIsNull(suppliers, (nullPosition, nullValueDataType, original) -> {
+            if (nullValueDataType == DataType.NULL) {
+                return original.getData().get(nullPosition == 0 ? 1 : 0).type();
+            }
+            return original.expectedType();
+        }, (nullPosition, nullData, original) -> original);
+
+        suppliers.add(new TestCaseSupplier("MV", List.of(DataType.INTEGER, DataType.INTEGER), () -> {
+            // Ensure we don't have an overflow
+            int rhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
+            int lhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
+            int lhs2 = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataType.INTEGER, "lhs"),
+                    new TestCaseSupplier.TypedData(rhs, DataType.INTEGER, "rhs")
+                ),
+                "SubIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
+                DataType.INTEGER,
+                is(nullValue())
+            ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
+                .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value");
+        }));
 
         return parameterSuppliersFromTypedData(suppliers);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java
index d3539f4a56fe9..66edd56e32f20 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java
@@ -116,7 +116,6 @@ public static Iterable<Object[]> parameters() {
             )
         );
         // Datetime
-        // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long?
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 "EqualsLongsEvaluator",
@@ -131,6 +130,20 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
+        suppliers.addAll(
+            TestCaseSupplier.forBinaryNotCasting(
+                "EqualsLongsEvaluator",
+                "lhs",
+                "rhs",
+                Object::equals,
+                DataType.BOOLEAN,
+                TestCaseSupplier.dateNanosCases(),
+                TestCaseSupplier.dateNanosCases(),
+                List.of(),
+                false
+            )
+        );
+
         suppliers.addAll(
             TestCaseSupplier.stringCases(
                 Object::equals,
@@ -198,11 +211,15 @@ public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(
                 anyNullIsNull(true, suppliers),
-                AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators
+                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(o, v, t, (l, p) -> typeErrorString)
             )
         );
     }
 
+    private static String typeErrorString =
+        "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, "
+            + "unsigned_long or version";
+
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new Equals(source, args.get(0), args.get(1));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java
index b2174f7be1593..a4d1bf69796e0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java
@@ -106,7 +106,6 @@ public static Iterable<Object[]> parameters() {
             )
         );
         // Datetime
-        // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long?
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 "GreaterThanOrEqualLongsEvaluator",
@@ -121,6 +120,20 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
+        suppliers.addAll(
+            TestCaseSupplier.forBinaryNotCasting(
+                "GreaterThanOrEqualLongsEvaluator",
+                "lhs",
+                "rhs",
+                (l, r) -> ((Number) l).longValue() >= ((Number) r).longValue(),
+                DataType.BOOLEAN,
+                TestCaseSupplier.dateNanosCases(),
+                TestCaseSupplier.dateNanosCases(),
+                List.of(),
+                false
+            )
+        );
+
         suppliers.addAll(
             TestCaseSupplier.stringCases(
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) >= 0,
@@ -133,7 +146,12 @@ public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(
                 anyNullIsNull(true, suppliers),
-                AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators
+                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(
+                    o,
+                    v,
+                    t,
+                    (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version"
+                )
             )
         );
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java
index edb276e16dd99..d3fede5c2e2ce 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java
@@ -106,7 +106,6 @@ public static Iterable<Object[]> parameters() {
             )
         );
         // Datetime
-        // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long?
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 "GreaterThanLongsEvaluator",
@@ -121,6 +120,20 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
+        suppliers.addAll(
+            TestCaseSupplier.forBinaryNotCasting(
+                "GreaterThanLongsEvaluator",
+                "lhs",
+                "rhs",
+                (l, r) -> ((Number) l).longValue() > ((Number) r).longValue(),
+                DataType.BOOLEAN,
+                TestCaseSupplier.dateNanosCases(),
+                TestCaseSupplier.dateNanosCases(),
+                List.of(),
+                false
+            )
+        );
+
         suppliers.addAll(
             TestCaseSupplier.stringCases(
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) > 0,
@@ -133,7 +146,12 @@ public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(
                 anyNullIsNull(true, suppliers),
-                AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators
+                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(
+                    o,
+                    v,
+                    t,
+                    (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version"
+                )
             )
         );
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java
index 620f9b7675506..2a1dfb098a3a4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java
@@ -25,6 +25,7 @@
 import java.util.function.Supplier;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.of;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java
index d89421f579b08..3b8270c1576fd 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java
@@ -106,7 +106,6 @@ public static Iterable<Object[]> parameters() {
             )
         );
         // Datetime
-        // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long?
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 "LessThanOrEqualLongsEvaluator",
@@ -121,6 +120,20 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
+        suppliers.addAll(
+            TestCaseSupplier.forBinaryNotCasting(
+                "LessThanOrEqualLongsEvaluator",
+                "lhs",
+                "rhs",
+                (l, r) -> ((Number) l).longValue() <= ((Number) r).longValue(),
+                DataType.BOOLEAN,
+                TestCaseSupplier.dateNanosCases(),
+                TestCaseSupplier.dateNanosCases(),
+                List.of(),
+                false
+            )
+        );
+
         suppliers.addAll(
             TestCaseSupplier.stringCases(
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) <= 0,
@@ -133,7 +146,12 @@ public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(
                 anyNullIsNull(true, suppliers),
-                AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators
+                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(
+                    o,
+                    v,
+                    t,
+                    (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version"
+                )
             )
         );
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
index 9487d774ff221..647988fe35326 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
@@ -106,7 +106,20 @@ public static Iterable<Object[]> parameters() {
             )
         );
         // Datetime
-        // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long?
+        suppliers.addAll(
+            TestCaseSupplier.forBinaryNotCasting(
+                "LessThanLongsEvaluator",
+                "lhs",
+                "rhs",
+                (l, r) -> ((Number) l).longValue() < ((Number) r).longValue(),
+                DataType.BOOLEAN,
+                TestCaseSupplier.dateCases(),
+                TestCaseSupplier.dateCases(),
+                List.of(),
+                false
+            )
+        );
+
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 "LessThanLongsEvaluator",
@@ -133,7 +146,12 @@ public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(
                 anyNullIsNull(true, suppliers),
-                AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators
+                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(
+                    o,
+                    v,
+                    t,
+                    (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version"
+                )
             )
         );
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java
index e7d8c680ba5cc..53676a43b16a0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java
@@ -115,7 +115,6 @@ public static Iterable<Object[]> parameters() {
             )
         );
         // Datetime
-        // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long?
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 "NotEqualsLongsEvaluator",
@@ -129,6 +128,20 @@ public static Iterable<Object[]> parameters() {
                 false
             )
         );
+        // Datetime
+        suppliers.addAll(
+            TestCaseSupplier.forBinaryNotCasting(
+                "NotEqualsLongsEvaluator",
+                "lhs",
+                "rhs",
+                (l, r) -> false == l.equals(r),
+                DataType.BOOLEAN,
+                TestCaseSupplier.dateNanosCases(),
+                TestCaseSupplier.dateNanosCases(),
+                List.of(),
+                false
+            )
+        );
         suppliers.addAll(
             TestCaseSupplier.stringCases(
                 (l, r) -> false == l.equals(r),
@@ -192,11 +205,15 @@ public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(
                 anyNullIsNull(true, suppliers),
-                AbstractScalarFunctionTestCase::errorMessageStringForBinaryOperators
+                (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(o, v, t, (l, p) -> typeErrorString)
             )
         );
     }
 
+    private static String typeErrorString =
+        "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, "
+            + "unsigned_long or version";
+
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new NotEquals(source, args.get(0), args.get(1));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java
new file mode 100644
index 0000000000000..504cf4ec1cd12
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.index;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
+import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+import org.elasticsearch.xpack.esql.type.EsFieldTests;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import static org.elasticsearch.test.ByteSizeEqualsMatcher.byteSizeEquals;
+
+public class EsIndexSerializationTests extends AbstractWireSerializingTestCase<EsIndex> {
+    public static EsIndex randomEsIndex() {
+        String name = randomAlphaOfLength(5);
+        Map<String, EsField> mapping = randomMapping();
+        Set<String> concreteIndices = randomConcreteIndices();
+        return new EsIndex(name, mapping, concreteIndices);
+    }
+
+    private static Map<String, EsField> randomMapping() {
+        int size = between(0, 10);
+        Map<String, EsField> result = new HashMap<>(size);
+        while (result.size() < size) {
+            result.put(randomAlphaOfLength(5), EsFieldTests.randomAnyEsField(1));
+        }
+        return result;
+    }
+
+    private static Set<String> randomConcreteIndices() {
+        int size = between(0, 10);
+        Set<String> result = new HashSet<>(size);
+        while (result.size() < size) {
+            result.add(randomAlphaOfLength(5));
+        }
+        return result;
+    }
+
+    @Override
+    protected Writeable.Reader<EsIndex> instanceReader() {
+        return a -> new EsIndex(new PlanStreamInput(a, new PlanNameRegistry(), a.namedWriteableRegistry(), null));
+    }
+
+    @Override
+    protected Writeable.Writer<EsIndex> instanceWriter() {
+        return (out, idx) -> new PlanStreamOutput(out, new PlanNameRegistry(), null).writeWriteable(idx);
+    }
+
+    @Override
+    protected EsIndex createTestInstance() {
+        return randomEsIndex();
+    }
+
+    @Override
+    protected EsIndex mutateInstance(EsIndex instance) throws IOException {
+        String name = instance.name();
+        Map<String, EsField> mapping = instance.mapping();
+        Set<String> concreteIndices = instance.concreteIndices();
+        switch (between(0, 2)) {
+            case 0 -> name = randomValueOtherThan(name, () -> randomAlphaOfLength(5));
+            case 1 -> mapping = randomValueOtherThan(mapping, EsIndexSerializationTests::randomMapping);
+            case 2 -> concreteIndices = randomValueOtherThan(concreteIndices, EsIndexSerializationTests::randomConcreteIndices);
+            default -> throw new IllegalArgumentException();
+        }
+        return new EsIndex(name, mapping, concreteIndices);
+    }
+
+    /**
+     * Build an {@link EsIndex} with many conflicting fields across many indices.
+     */
+    public static EsIndex indexWithManyConflicts(boolean withParent) {
+        /*
+         * The number of fields with a mapping conflict.
+         */
+        int conflictingCount = 250;
+        /*
+         * The number of indices that map conflicting fields are "keyword".
+         * One other index will map the field as "text"
+         */
+        int keywordIndicesCount = 600;
+        /*
+         * The number of fields that don't have a mapping conflict.
+         */
+        int nonConflictingCount = 7000;
+
+        Set<String> keywordIndices = new TreeSet<>();
+        for (int i = 0; i < keywordIndicesCount; i++) {
+            keywordIndices.add(String.format(Locale.ROOT, ".ds-logs-apache.access-external-2024.08.09-%08d", i));
+        }
+
+        Set<String> textIndices = Set.of("logs-endpoint.events.imported");
+
+        Map<String, EsField> fields = new TreeMap<>();
+        for (int i = 0; i < conflictingCount; i++) {
+            String name = String.format(Locale.ROOT, "blah.blah.blah.blah.blah.blah.conflict.name%04d", i);
+            Map<String, Set<String>> conflicts = Map.of("text", textIndices, "keyword", keywordIndices);
+            fields.put(name, new InvalidMappedField(name, conflicts));
+        }
+        for (int i = 0; i < nonConflictingCount; i++) {
+            String name = String.format(Locale.ROOT, "blah.blah.blah.blah.blah.blah.nonconflict.name%04d", i);
+            fields.put(name, new EsField(name, DataType.KEYWORD, Map.of(), true));
+        }
+
+        if (withParent) {
+            EsField parent = new EsField("parent", DataType.OBJECT, Map.copyOf(fields), false);
+            fields.put("parent", parent);
+        }
+
+        TreeSet<String> concrete = new TreeSet<>();
+        concrete.addAll(keywordIndices);
+        concrete.addAll(textIndices);
+
+        return new EsIndex("name", fields, concrete);
+    }
+
+    /**
+     * Test the size of serializing an index with many conflicts at the root level.
+     * See {@link #testManyTypeConflicts(boolean, ByteSizeValue)} for more.
+     */
+    public void testManyTypeConflicts() throws IOException {
+        testManyTypeConflicts(false, ByteSizeValue.ofBytes(991027));
+        /*
+         * History:
+         *  953.7kb - shorten error messages for UnsupportedAttributes #111973
+         *  967.7kb - cache EsFields #112008 (little overhead of the cache)
+         */
+    }
+
+    /**
+     * Test the size of serializing an index with many conflicts inside a "parent" object.
+     * See {@link #testManyTypeConflicts(boolean, ByteSizeValue)} for more.
+     */
+    public void testManyTypeConflictsWithParent() throws IOException {
+        testManyTypeConflicts(true, ByteSizeValue.ofBytes(1374498));
+        /*
+         * History:
+         * 16.9mb - start
+         *  1.8mb - shorten error messages for UnsupportedAttributes #111973
+         *  1.3mb - cache EsFields #112008
+         */
+    }
+
+    /**
+     * Test the size of serializing an index with many conflicts. Callers of
+     * this method intentionally use a very precise size for the serialized
+     * data so a programmer making changes has to think when this size changes.
+     * <p>
+     *     In general, shrinking the over the wire size is great and the precise
+     *     size should just ratchet downwards. Small upwards movement is fine so
+     *     long as you understand why the change is happening and you think it's
+     *     worth it for the data node request for a big index to grow.
+     * </p>
+     * <p>
+     *     Large upwards movement in the size is not fine! Folks frequently make
+     *     requests across large clusters with many fields and these requests can
+     *     really clog up the network interface. Super large results here can make
+     *     ESQL impossible to use at all for big mappings with many conflicts.
+     * </p>
+     */
+    private void testManyTypeConflicts(boolean withParent, ByteSizeValue expected) throws IOException {
+        try (BytesStreamOutput out = new BytesStreamOutput(); var pso = new PlanStreamOutput(out, new PlanNameRegistry(), null)) {
+            indexWithManyConflicts(withParent).writeTo(pso);
+            assertThat(ByteSizeValue.ofBytes(out.bytes().length()), byteSizeEquals(expected));
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
index 55691526ea428..a3d1e70e558d6 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
@@ -12,8 +12,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.dissect.DissectParser;
-import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
@@ -22,42 +20,9 @@
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.ArithmeticOperation;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
-import org.elasticsearch.xpack.esql.core.type.KeywordEsField;
-import org.elasticsearch.xpack.esql.expression.Order;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
-import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
-import org.elasticsearch.xpack.esql.plan.logical.Dissect;
-import org.elasticsearch.xpack.esql.plan.logical.Enrich;
-import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
-import org.elasticsearch.xpack.esql.plan.logical.Eval;
-import org.elasticsearch.xpack.esql.plan.logical.Filter;
-import org.elasticsearch.xpack.esql.plan.logical.Grok;
-import org.elasticsearch.xpack.esql.plan.logical.Limit;
-import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.plan.logical.Lookup;
-import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
-import org.elasticsearch.xpack.esql.plan.logical.OrderBy;
-import org.elasticsearch.xpack.esql.plan.logical.Project;
-import org.elasticsearch.xpack.esql.plan.logical.TopN;
-import org.elasticsearch.xpack.esql.plan.logical.join.Join;
-import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
-import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.esql.plan.physical.DissectExec;
 import org.elasticsearch.xpack.esql.plan.physical.EnrichExec;
@@ -87,12 +52,9 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
 
 import static org.elasticsearch.test.ListMatcher.matchesList;
 import static org.elasticsearch.test.MapMatcher.assertMap;
-import static org.elasticsearch.xpack.esql.SerializationTestUtils.serializeDeserialize;
 import static org.hamcrest.Matchers.equalTo;
 
 public class PlanNamedTypesTests extends ESTestCase {
@@ -135,39 +97,6 @@ public void testPhysicalPlanEntries() {
         assertMap(actual, matchesList(expected));
     }
 
-    // List of known serializable logical plan nodes - this should be kept up to date or retrieved
-    // programmatically.
-    public static final List<Class<? extends LogicalPlan>> LOGICAL_PLAN_NODE_CLS = List.of(
-        Aggregate.class,
-        Dissect.class,
-        Enrich.class,
-        EsRelation.class,
-        EsqlProject.class,
-        Eval.class,
-        Filter.class,
-        Grok.class,
-        Join.class,
-        Limit.class,
-        LocalRelation.class,
-        Lookup.class,
-        MvExpand.class,
-        OrderBy.class,
-        Project.class,
-        TopN.class
-    );
-
-    // Tests that all logical plan nodes have a suitably named serialization entry.
-    public void testLogicalPlanEntries() {
-        var expected = LOGICAL_PLAN_NODE_CLS.stream().map(Class::getSimpleName).toList();
-        var actual = PlanNamedTypes.namedTypeEntries()
-            .stream()
-            .filter(e -> e.categoryClass().isAssignableFrom(LogicalPlan.class))
-            .map(PlanNameRegistry.Entry::name)
-            .sorted()
-            .toList();
-        assertMap(actual, matchesList(expected));
-    }
-
     // Tests that all names are unique - there should be a good reason if this is not the case.
     public void testUniqueNames() {
         var actual = PlanNamedTypes.namedTypeEntries().stream().map(PlanNameRegistry.Entry::name).distinct().toList();
@@ -193,95 +122,6 @@ public void testWrappedStreamSimple() throws IOException {
         assertThat(in.readVInt(), equalTo(11_345));
     }
 
-    public void testFieldSortSimple() throws IOException {
-        var orig = new EsQueryExec.FieldSort(field("val", DataType.LONG), Order.OrderDirection.ASC, Order.NullsPosition.FIRST);
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeFieldSort(out, orig);
-        var deser = PlanNamedTypes.readFieldSort(planStreamInput(bso));
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-    }
-
-    public void testEsIndexSimple() throws IOException {
-        var orig = new EsIndex("test*", Map.of("first_name", new KeywordEsField("first_name")), Set.of("test1", "test2"));
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeEsIndex(out, orig);
-        var deser = PlanNamedTypes.readEsIndex(planStreamInput(bso));
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-    }
-
-    public void testDissectParserSimple() throws IOException {
-        String pattern = "%{b} %{c}";
-        var orig = new Dissect.Parser(pattern, ",", new DissectParser(pattern, ","));
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeDissectParser(out, orig);
-        var deser = PlanNamedTypes.readDissectParser(planStreamInput(bso));
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-    }
-
-    public void testEsRelation() throws IOException {
-        var orig = new EsRelation(
-            Source.EMPTY,
-            randomEsIndex(),
-            List.of(randomFieldAttribute()),
-            randomFrom(IndexMode.values()),
-            randomBoolean()
-        );
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeEsRelation(out, orig);
-        var deser = PlanNamedTypes.readEsRelation(planStreamInput(bso));
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-    }
-
-    public void testEsqlProject() throws IOException {
-        var orig = new EsqlProject(
-            Source.EMPTY,
-            new EsRelation(Source.EMPTY, randomEsIndex(), List.of(randomFieldAttribute()), randomFrom(IndexMode.values()), randomBoolean()),
-            List.of(randomFieldAttribute())
-        );
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeEsqlProject(out, orig);
-        var deser = PlanNamedTypes.readEsqlProject(planStreamInput(bso));
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-    }
-
-    public void testMvExpand() throws IOException {
-        var esRelation = new EsRelation(
-            Source.EMPTY,
-            randomEsIndex(),
-            List.of(randomFieldAttribute()),
-            randomFrom(IndexMode.values()),
-            randomBoolean()
-        );
-        var orig = new MvExpand(Source.EMPTY, esRelation, randomFieldAttribute(), randomFieldAttribute());
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeMvExpand(out, orig);
-        var deser = PlanNamedTypes.readMvExpand(planStreamInput(bso));
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-    }
-
-    private static <T> void assertNamedType(Class<T> type, T origObj) {
-        var deserObj = serializeDeserialize(origObj, (o, v) -> o.writeNamed(type, origObj), i -> i.readNamed(type));
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(origObj, unused -> deserObj);
-    }
-
-    static EsIndex randomEsIndex() {
-        Set<String> concreteIndices = new TreeSet<>();
-        while (concreteIndices.size() < 2) {
-            concreteIndices.add(randomAlphaOfLengthBetween(1, 25));
-        }
-        return new EsIndex(
-            randomAlphaOfLengthBetween(1, 25),
-            Map.of(randomAlphaOfLengthBetween(1, 25), randomKeywordEsField()),
-            concreteIndices
-        );
-    }
-
     static FieldAttribute randomFieldAttributeOrNull() {
         return randomBoolean() ? randomFieldAttribute() : null;
     }
@@ -293,53 +133,12 @@ static FieldAttribute randomFieldAttribute() {
             randomAlphaOfLength(randomIntBetween(1, 25)), // name
             randomDataType(),
             randomEsField(),
-            randomStringOrNull(), // qualifier
             randomNullability(),
             nameIdOrNull(),
             randomBoolean() // synthetic
         );
     }
 
-    static KeywordEsField randomKeywordEsField() {
-        return new KeywordEsField(
-            randomAlphaOfLength(randomIntBetween(1, 25)), // name
-            randomProperties(),
-            randomBoolean(), // hasDocValues
-            randomIntBetween(1, 12), // precision
-            randomBoolean(), // normalized
-            randomBoolean() // alias
-        );
-    }
-
-    static EsqlBinaryComparison randomBinaryComparison() {
-        int v = randomIntBetween(0, 5);
-        var left = field(randomName(), randomDataType());
-        var right = field(randomName(), randomDataType());
-        return switch (v) {
-            case 0 -> new Equals(Source.EMPTY, left, right);
-            case 1 -> new NotEquals(Source.EMPTY, left, right);
-            case 2 -> new GreaterThan(Source.EMPTY, left, right);
-            case 3 -> new GreaterThanOrEqual(Source.EMPTY, left, right);
-            case 4 -> new LessThan(Source.EMPTY, left, right);
-            case 5 -> new LessThanOrEqual(Source.EMPTY, left, right);
-            default -> throw new AssertionError(v);
-        };
-    }
-
-    static ArithmeticOperation randomArithmeticOperation() {
-        int v = randomIntBetween(0, 4);
-        var left = field(randomName(), randomDataType());
-        var right = field(randomName(), randomDataType());
-        return switch (v) {
-            case 0 -> new Add(Source.EMPTY, left, right);
-            case 1 -> new Sub(Source.EMPTY, left, right);
-            case 2 -> new Mul(Source.EMPTY, left, right);
-            case 3 -> new Div(Source.EMPTY, left, right);
-            case 4 -> new Mod(Source.EMPTY, left, right);
-            default -> throw new AssertionError(v);
-        };
-    }
-
     static NameId nameIdOrNull() {
         return randomBoolean() ? new NameId() : null;
     }
@@ -354,7 +153,7 @@ static Nullability randomNullability() {
         };
     }
 
-    static EsField randomEsField() {
+    public static EsField randomEsField() {
         return randomEsField(0);
     }
 
@@ -368,10 +167,6 @@ static EsField randomEsField(int depth) {
         );
     }
 
-    static Map<String, EsField> randomProperties() {
-        return randomProperties(0);
-    }
-
     static Map<String, EsField> randomProperties(int depth) {
         if (depth > 2) {
             return Map.of(); // prevent infinite recursion (between EsField and properties)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java
index 55763d9ec6e7b..5c4ab6bf6bbdd 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -133,13 +133,13 @@ public void testSourceSerialization() {
 
         for (var delim : new String[] { "", "\r", "\n", "\r\n" }) {
             String query = queryFn.apply(delim);
-            EsqlConfiguration config = configuration(query);
+            Configuration config = configuration(query);
 
             LogicalPlan planIn = analyze(query);
             LogicalPlan planOut = serializeDeserialize(
                 planIn,
-                PlanStreamOutput::writeLogicalPlanNode,
-                PlanStreamInput::readLogicalPlanNode,
+                PlanStreamOutput::writeNamedWriteable,
+                in -> in.readNamedWriteable(LogicalPlan.class),
                 config
             );
             assertThat(planIn, equalTo(planOut));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java
index 00fb9d4943005..cdb6c5384e16a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java
@@ -18,17 +18,31 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xpack.esql.Column;
+import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
-import org.elasticsearch.xpack.esql.session.EsqlConfigurationSerializationTests;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.expression.function.MetadataAttributeTests;
+import org.elasticsearch.xpack.esql.expression.function.ReferenceAttributeTests;
+import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
+import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttributeTests;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class PlanStreamOutputTests extends ESTestCase {
@@ -37,11 +51,7 @@ public void testTransportVersion() throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         TransportVersion v1 = TransportVersionUtils.randomCompatibleVersion(random());
         out.setTransportVersion(v1);
-        PlanStreamOutput planOut = new PlanStreamOutput(
-            out,
-            PlanNameRegistry.INSTANCE,
-            randomBoolean() ? null : EsqlConfigurationSerializationTests.randomConfiguration()
-        );
+        PlanStreamOutput planOut = new PlanStreamOutput(out, PlanNameRegistry.INSTANCE, randomBoolean() ? null : randomConfiguration());
         assertThat(planOut.getTransportVersion(), equalTo(v1));
         TransportVersion v2 = TransportVersionUtils.randomCompatibleVersion(random());
         planOut.setTransportVersion(v2);
@@ -53,7 +63,7 @@ public void testWriteBlockFromConfig() throws IOException {
         String tableName = randomAlphaOfLength(5);
         String columnName = randomAlphaOfLength(10);
         try (Column c = randomColumn()) {
-            EsqlConfiguration configuration = randomConfiguration(Map.of(tableName, Map.of(columnName, c)));
+            Configuration configuration = randomConfiguration("query_" + randomAlphaOfLength(1), Map.of(tableName, Map.of(columnName, c)));
             try (
                 BytesStreamOutput out = new BytesStreamOutput();
                 PlanStreamOutput planStream = new PlanStreamOutput(out, PlanNameRegistry.INSTANCE, configuration)
@@ -71,7 +81,7 @@ public void testWriteBlockFromConfig() throws IOException {
 
     public void testWriteBlockOnce() throws IOException {
         try (Block b = randomColumn().values()) {
-            EsqlConfiguration configuration = EsqlConfigurationSerializationTests.randomConfiguration();
+            Configuration configuration = randomConfiguration();
             try (
                 BytesStreamOutput out = new BytesStreamOutput();
                 PlanStreamOutput planStream = new PlanStreamOutput(out, PlanNameRegistry.INSTANCE, configuration)
@@ -92,7 +102,7 @@ public void testWriteBlockOnce() throws IOException {
 
     public void testWriteBlockTwice() throws IOException {
         try (Block b = randomColumn().values()) {
-            EsqlConfiguration configuration = EsqlConfigurationSerializationTests.randomConfiguration();
+            Configuration configuration = randomConfiguration();
             try (
                 BytesStreamOutput out = new BytesStreamOutput();
                 PlanStreamOutput planStream = new PlanStreamOutput(out, PlanNameRegistry.INSTANCE, configuration)
@@ -113,8 +123,187 @@ public void testWriteBlockTwice() throws IOException {
         }
     }
 
-    private EsqlConfiguration randomConfiguration(Map<String, Map<String, Column>> tables) {
-        return EsqlConfigurationSerializationTests.randomConfiguration("query_" + randomAlphaOfLength(1), tables);
+    public void testWriteAttributeMultipleTimes() throws IOException {
+        Attribute attribute = randomAttribute();
+        Configuration configuration = randomConfiguration();
+        try (
+            BytesStreamOutput out = new BytesStreamOutput();
+            PlanStreamOutput planStream = new PlanStreamOutput(out, PlanNameRegistry.INSTANCE, configuration)
+        ) {
+            int occurrences = randomIntBetween(2, 150);
+            for (int i = 0; i < occurrences; i++) {
+                planStream.writeNamedWriteable(attribute);
+            }
+            int depth = 0;
+            Attribute parent = attribute;
+            while (parent != null) {
+                depth++;
+                parent = parent instanceof FieldAttribute f ? f.parent() : null;
+            }
+            assertThat(planStream.cachedAttributes.size(), is(depth));
+            try (PlanStreamInput in = new PlanStreamInput(out.bytes().streamInput(), PlanNameRegistry.INSTANCE, REGISTRY, configuration)) {
+                Attribute first = in.readNamedWriteable(Attribute.class);
+                for (int i = 1; i < occurrences; i++) {
+                    Attribute next = in.readNamedWriteable(Attribute.class);
+                    assertThat(first, sameInstance(next));
+                }
+                for (int i = 0; i < depth; i++) {
+                    assertThat(first, equalTo(attribute));
+                    first = first instanceof FieldAttribute f ? f.parent() : null;
+                    attribute = attribute instanceof FieldAttribute f ? f.parent() : null;
+                }
+                assertThat(first, is(nullValue()));
+                assertThat(attribute, is(nullValue()));
+            }
+        }
+    }
+
+    public void testWriteMultipleAttributes() throws IOException {
+        Configuration configuration = randomConfiguration();
+        try (
+            BytesStreamOutput out = new BytesStreamOutput();
+            PlanStreamOutput planStream = new PlanStreamOutput(out, PlanNameRegistry.INSTANCE, configuration)
+        ) {
+            List<Attribute> attrs = new ArrayList<>();
+            int occurrences = randomIntBetween(2, 300);
+            for (int i = 0; i < occurrences; i++) {
+                attrs.add(randomAttribute());
+            }
+
+            // send all the attributes, three times
+            for (int i = 0; i < 3; i++) {
+                for (Attribute attr : attrs) {
+                    planStream.writeNamedWriteable(attr);
+                }
+            }
+
+            try (PlanStreamInput in = new PlanStreamInput(out.bytes().streamInput(), PlanNameRegistry.INSTANCE, REGISTRY, configuration)) {
+                List<Attribute> readAttrs = new ArrayList<>();
+                for (int i = 0; i < occurrences; i++) {
+                    readAttrs.add(in.readNamedWriteable(Attribute.class));
+                    assertThat(readAttrs.get(i), equalTo(attrs.get(i)));
+                }
+                // two more times
+                for (int i = 0; i < 2; i++) {
+                    for (int j = 0; j < occurrences; j++) {
+                        Attribute attr = in.readNamedWriteable(Attribute.class);
+                        assertThat(attr, sameInstance(readAttrs.get(j)));
+                    }
+                }
+            }
+        }
+    }
+
+    public void testWriteMultipleAttributesWithSmallCache() throws IOException {
+        Configuration configuration = randomConfiguration();
+        try (
+            BytesStreamOutput out = new BytesStreamOutput();
+            PlanStreamOutput planStream = new PlanStreamOutput(out, PlanNameRegistry.INSTANCE, configuration, PlanNamedTypes::name, 10)
+        ) {
+            expectThrows(InvalidArgumentException.class, () -> {
+                for (int i = 0; i <= 10; i++) {
+                    planStream.writeNamedWriteable(randomAttribute());
+                }
+            });
+        }
+    }
+
+    public void testWriteEqualAttributesDifferentID() throws IOException {
+        Configuration configuration = randomConfiguration();
+        try (
+            BytesStreamOutput out = new BytesStreamOutput();
+            PlanStreamOutput planStream = new PlanStreamOutput(out, PlanNameRegistry.INSTANCE, configuration)
+        ) {
+
+            Attribute one = randomAttribute();
+            Attribute two = one.withId(new NameId());
+
+            planStream.writeNamedWriteable(one);
+            planStream.writeNamedWriteable(two);
+
+            try (PlanStreamInput in = new PlanStreamInput(out.bytes().streamInput(), PlanNameRegistry.INSTANCE, REGISTRY, configuration)) {
+                Attribute oneCopy = in.readNamedWriteable(Attribute.class);
+                Attribute twoCopy = in.readNamedWriteable(Attribute.class);
+
+                assertThat(oneCopy, equalTo(one));
+                assertThat(twoCopy, equalTo(two));
+
+                assertThat(oneCopy.id(), not(equalTo(twoCopy.id())));
+            }
+        }
+    }
+
+    public void testWriteDifferentAttributesSameID() throws IOException {
+        Configuration configuration = randomConfiguration();
+        try (
+            BytesStreamOutput out = new BytesStreamOutput();
+            PlanStreamOutput planStream = new PlanStreamOutput(out, PlanNameRegistry.INSTANCE, configuration)
+        ) {
+
+            Attribute one = randomAttribute();
+            Attribute two = randomAttribute().withId(one.id());
+
+            planStream.writeNamedWriteable(one);
+            planStream.writeNamedWriteable(two);
+
+            try (PlanStreamInput in = new PlanStreamInput(out.bytes().streamInput(), PlanNameRegistry.INSTANCE, REGISTRY, configuration)) {
+                Attribute oneCopy = in.readNamedWriteable(Attribute.class);
+                Attribute twoCopy = in.readNamedWriteable(Attribute.class);
+
+                assertThat(oneCopy, equalTo(one));
+                assertThat(twoCopy, equalTo(two));
+
+                assertThat(oneCopy, not(equalTo(twoCopy)));
+                assertThat(oneCopy.id(), equalTo(twoCopy.id()));
+            }
+        }
+    }
+
+    public void testWriteMultipleEsFields() throws IOException {
+        Configuration configuration = randomConfiguration();
+        try (
+            BytesStreamOutput out = new BytesStreamOutput();
+            PlanStreamOutput planStream = new PlanStreamOutput(out, PlanNameRegistry.INSTANCE, configuration)
+        ) {
+            List<EsField> fields = new ArrayList<>();
+            int occurrences = randomIntBetween(2, 300);
+            for (int i = 0; i < occurrences; i++) {
+                fields.add(PlanNamedTypesTests.randomEsField());
+            }
+
+            // send all the EsFields, three times
+            for (int i = 0; i < 3; i++) {
+                for (EsField attr : fields) {
+                    attr.writeTo(planStream);
+                }
+            }
+
+            try (PlanStreamInput in = new PlanStreamInput(out.bytes().streamInput(), PlanNameRegistry.INSTANCE, REGISTRY, configuration)) {
+                List<EsField> readFields = new ArrayList<>();
+                for (int i = 0; i < occurrences; i++) {
+                    readFields.add(EsField.readFrom(in));
+                    assertThat(readFields.get(i), equalTo(fields.get(i)));
+                }
+                // two more times
+                for (int i = 0; i < 2; i++) {
+                    for (int j = 0; j < occurrences; j++) {
+                        EsField attr = EsField.readFrom(in);
+                        assertThat(attr, sameInstance(readFields.get(j)));
+                    }
+                }
+            }
+        }
+    }
+
+    private static Attribute randomAttribute() {
+        return switch (randomInt(3)) {
+            case 0 -> PlanNamedTypesTests.randomFieldAttribute();
+            case 1 -> ReferenceAttributeTests.randomReferenceAttribute();
+            case 2 -> UnsupportedAttributeTests.randomUnsupportedAttribute();
+            case 3 -> MetadataAttributeTests.randomMetadataAttribute();
+            default -> throw new IllegalArgumentException();
+
+        };
     }
 
     private static final int LEN = 10000;
@@ -133,5 +322,13 @@ private Column randomColumn() {
         BigArrays.NON_RECYCLING_INSTANCE
     );
 
-    private static final NamedWriteableRegistry REGISTRY = new NamedWriteableRegistry(Block.getNamedWriteables());
+    private static final NamedWriteableRegistry REGISTRY;
+
+    static {
+        List<NamedWriteableRegistry.Entry> writeables = new ArrayList<>();
+        writeables.addAll(Block.getNamedWriteables());
+        writeables.addAll(Attribute.getNamedWriteables());
+        writeables.add(UnsupportedAttribute.ENTRY);
+        REGISTRY = new NamedWriteableRegistry(new ArrayList<>(new HashSet<>(writeables)));
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java
index 2049fd5592d82..a3104b32dee34 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.optimizer;
 
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.test.ESTestCase;
@@ -22,8 +23,6 @@
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -32,6 +31,8 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
@@ -228,6 +229,16 @@ public MockFieldAttributeCommand(Source source, LogicalPlan child, FieldAttribut
             this.field = field;
         }
 
+        @Override
+        public void writeTo(StreamOutput out) {
+            throw new UnsupportedOperationException("not serialized");
+        }
+
+        @Override
+        public String getWriteableName() {
+            throw new UnsupportedOperationException("not serialized");
+        }
+
         @Override
         public UnaryPlan replaceChild(LogicalPlan newChild) {
             return new MockFieldAttributeCommand(source(), newChild, field);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
index d1a352589263a..ee66e3043b62f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
@@ -10,18 +10,21 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.apache.lucene.search.IndexSearcher;
-import org.elasticsearch.common.network.NetworkAddress;
+import org.elasticsearch.Build;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperServiceTestCase;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.EsqlTestUtils.TestSearchStats;
+import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
@@ -29,44 +32,43 @@
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec;
 import org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.Stat;
 import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize;
+import org.elasticsearch.xpack.esql.plan.physical.EvalExec;
 import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec;
 import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec;
 import org.elasticsearch.xpack.esql.plan.physical.LimitExec;
 import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.plan.physical.ProjectExec;
+import org.elasticsearch.xpack.esql.plan.physical.TopNExec;
 import org.elasticsearch.xpack.esql.planner.FilterTests;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
 import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.stats.Metrics;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
 import org.junit.Before;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import static java.util.Arrays.asList;
-import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping;
@@ -91,7 +93,7 @@ public class LocalPhysicalPlanOptimizerTests extends MapperServiceTestCase {
     private static final int KEYWORD_EST = EstimatesRowSize.estimateSize(DataType.KEYWORD);
 
     private TestPlannerOptimizer plannerOptimizer;
-    private final EsqlConfiguration config;
+    private final Configuration config;
     private final SearchStats IS_SV_STATS = new TestSearchStats() {
         @Override
         public boolean isSingleValue(String field) {
@@ -111,7 +113,7 @@ private static List<Tuple<String, Map<String, Object>>> settings() {
         return asList(new Tuple<>("default", Map.of()));
     }
 
-    public LocalPhysicalPlanOptimizerTests(String name, EsqlConfiguration config) {
+    public LocalPhysicalPlanOptimizerTests(String name, Configuration config) {
         this.config = config;
     }
 
@@ -231,11 +233,12 @@ public void testCountFieldWithEval() {
 
     // optimized doesn't know yet how to push down count over field
     public void testCountOneFieldWithFilter() {
-        var plan = plannerOptimizer.plan("""
+        String query = """
             from test
             | where salary > 1000
             | stats c = count(salary)
-            """, IS_SV_STATS);
+            """;
+        var plan = plannerOptimizer.plan(query, IS_SV_STATS);
 
         var limit = as(plan, LimitExec.class);
         var agg = as(limit.child(), AggregateExec.class);
@@ -249,7 +252,7 @@ public void testCountOneFieldWithFilter() {
         Source source = new Source(2, 8, "salary > 1000");
         var exists = QueryBuilders.existsQuery("salary");
         assertThat(stat.query(), is(exists));
-        var range = wrapWithSingleQuery(QueryBuilders.rangeQuery("salary").gt(1000), "salary", source);
+        var range = wrapWithSingleQuery(query, QueryBuilders.rangeQuery("salary").gt(1000), "salary", source);
         var expected = QueryBuilders.boolQuery().must(range).must(exists);
         assertThat(expected.toString(), is(esStatsQuery.query().toString()));
     }
@@ -340,11 +343,12 @@ public void testCountMultipleFieldsWithFilter() {
     }
 
     public void testAnotherCountAllWithFilter() {
-        var plan = plannerOptimizer.plan("""
+        String query = """
             from test
             | where emp_no > 10010
             | stats c = count()
-            """, IS_SV_STATS);
+            """;
+        var plan = plannerOptimizer.plan(query, IS_SV_STATS);
 
         var limit = as(plan, LimitExec.class);
         var agg = as(limit.child(), AggregateExec.class);
@@ -355,7 +359,7 @@ public void testAnotherCountAllWithFilter() {
         assertThat(esStatsQuery.limit(), is(nullValue()));
         assertThat(Expressions.names(esStatsQuery.output()), contains("count", "seen"));
         var source = ((SingleValueQuery.Builder) esStatsQuery.query()).source();
-        var expected = wrapWithSingleQuery(QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", source);
+        var expected = wrapWithSingleQuery(query, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", source);
         assertThat(expected.toString(), is(esStatsQuery.query().toString()));
     }
 
@@ -369,6 +373,101 @@ public void testMultiCountAllWithFilter() {
         assertThat(plan.anyMatch(EsQueryExec.class::isInstance), is(true));
     }
 
+    /**
+     * Expecting
+     * LimitExec[1000[INTEGER]]
+     * \_ExchangeExec[[],false]
+     *   \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na
+     * me{f}#6, long_noidx{f}#11, salary{f}#7]]
+     *     \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3]
+     *       \_EsQueryExec[test], indexMode[standard], query[{"query_string":{"query":"\"last_name: Smith\""
+     */
+    public void testMatchCommand() {
+        assumeTrue("skipping because MATCH_COMMAND is not enabled", EsqlCapabilities.Cap.MATCH_COMMAND.isEnabled());
+        var plan = plannerOptimizer.plan("""
+            from test
+            | match "last_name: Smith"
+            """, IS_SV_STATS);
+
+        var limit = as(plan, LimitExec.class);
+        var exchange = as(limit.child(), ExchangeExec.class);
+        var project = as(exchange.child(), ProjectExec.class);
+        var field = as(project.child(), FieldExtractExec.class);
+        var query = as(field.child(), EsQueryExec.class);
+        assertThat(query.limit().fold(), is(1000));
+        var expected = QueryBuilders.queryStringQuery("last_name: Smith");
+        assertThat(query.query().toString(), is(expected.toString()));
+    }
+
+    /**
+     * LimitExec[1000[INTEGER]]
+     * \_ExchangeExec[[],false]
+     *   \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n
+     * ame{f}#7, long_noidx{f}#12, salary{f}#8]]
+     *     \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen]
+     *       \_EsQueryExec[test], indexMode[standard],
+     *          query[{"bool":{ "must":[{
+     *            "esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}}}},
+     *            {"query_string":{"query":"last_name: Smith","fields":[]}}],"boost":1.0}
+     *          }]
+     */
+    public void testMatchCommandWithWhereClause() {
+        assumeTrue("skipping because MATCH_COMMAND is not enabled", EsqlCapabilities.Cap.MATCH_COMMAND.isEnabled());
+        String queryText = """
+            from test
+            | where emp_no > 10010
+            | match "last_name: Smith"
+            """;
+        var plan = plannerOptimizer.plan(queryText, IS_SV_STATS);
+
+        var limit = as(plan, LimitExec.class);
+        var exchange = as(limit.child(), ExchangeExec.class);
+        var project = as(exchange.child(), ProjectExec.class);
+        var field = as(project.child(), FieldExtractExec.class);
+        var query = as(field.child(), EsQueryExec.class);
+        assertThat(query.limit().fold(), is(1000));
+
+        Source source = new Source(2, 8, "emp_no > 10010");
+        var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", source);
+        var queryString = QueryBuilders.queryStringQuery("last_name: Smith");
+        var expected = QueryBuilders.boolQuery().must(range).must(queryString);
+        assertThat(query.query().toString(), is(expected.toString()));
+    }
+
+    /**
+     * TopNExec[[Order[emp_no{f}#3,ASC,LAST]],1000[INTEGER],0]
+     * \_ExchangeExec[[],false]
+     *   \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n
+     * ame{f}#7, long_noidx{f}#12, salary{f}#8]]
+     *     \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen]
+     *       \_EsQueryExec[test],
+     *          query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},
+     *          {"query_string":{"query":"John","fields":[]}}],"boost":1.0}}]
+     *          sort[[FieldSort[field=emp_no{f}#3, direction=ASC, nulls=LAST]]]
+     */
+    public void testMatchCommandWithMultipleMatches() {
+        assumeTrue("skipping because MATCH_COMMAND is not enabled", EsqlCapabilities.Cap.MATCH_COMMAND.isEnabled());
+        var plan = plannerOptimizer.plan("""
+            from test
+            | match "last_name: Smith"
+            | sort emp_no
+            | MATCH "John"
+            """, IS_SV_STATS);
+
+        var limit = as(plan, TopNExec.class);
+        var exchange = as(limit.child(), ExchangeExec.class);
+        var project = as(exchange.child(), ProjectExec.class);
+        var field = as(project.child(), FieldExtractExec.class);
+        var query = as(field.child(), EsQueryExec.class);
+        assertThat(query.limit().fold(), is(1000));
+
+        Source source = new Source(2, 8, "emp_no > 10010");
+        var queryString1 = QueryBuilders.queryStringQuery("last_name: Smith");
+        var queryString2 = QueryBuilders.queryStringQuery("John");
+        var expected = QueryBuilders.boolQuery().must(queryString1).must(queryString2);
+        assertThat(query.query().toString(), is(expected.toString()));
+    }
+
     // optimizer doesn't know yet how to break down different multi count
     public void testCountFieldsAndAllWithFilter() {
         var plan = plannerOptimizer.plan("""
@@ -405,7 +504,7 @@ public boolean exists(String field) {
         assertThat(agg.getMode(), is(FINAL));
         assertThat(Expressions.names(agg.aggregates()), contains("c"));
         var exchange = as(agg.child(), ExchangeExec.class);
-        assertThat(exchange.isInBetweenAggs(), is(true));
+        assertThat(exchange.inBetweenAggs(), is(true));
         var localSource = as(exchange.child(), LocalSourceExec.class);
         assertThat(Expressions.names(localSource.output()), contains("count", "seen"));
     }
@@ -567,44 +666,6 @@ public void testIsNotNull_TextField_Pushdown_WithCount() {
         assertThat(stat.query(), is(QueryBuilders.existsQuery("job")));
     }
 
-    /**
-     * Expects
-     * LimitExec[1000[INTEGER]]
-     * \_ExchangeExec[[],false]
-     *   \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9,
-     *     half_float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18,
-     *     unsigned_long{f}#16, version{f}#19, wildcard{f}#20]]
-     *     \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..][]
-     *       \_EsQueryExec[test], query[{"esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.0/24"],"boost":1.0}},"source":
-     *         "cidr_match(ip, \"127.0.0.0/24\")@1:19"}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[389]
-     */
-    public void testCidrMatchPushdownFilter() {
-        var allTypeMappingAnalyzer = makeAnalyzer("mapping-ip.json", new EnrichResolution());
-        final String fieldName = "ip_addr";
-
-        int cidrBlockCount = randomIntBetween(1, 10);
-        ArrayList<String> cidrBlocks = new ArrayList<>();
-        for (int i = 0; i < cidrBlockCount; i++) {
-            cidrBlocks.add(randomCidrBlock());
-        }
-        String cidrBlocksString = cidrBlocks.stream().map((s) -> "\"" + s + "\"").collect(Collectors.joining(","));
-        String cidrMatch = format(null, "cidr_match({}, {})", fieldName, cidrBlocksString);
-
-        var query = "from test | where " + cidrMatch;
-        var plan = plannerOptimizer.plan(query, EsqlTestUtils.TEST_SEARCH_STATS, allTypeMappingAnalyzer);
-
-        var limit = as(plan, LimitExec.class);
-        var exchange = as(limit.child(), ExchangeExec.class);
-        var project = as(exchange.child(), ProjectExec.class);
-        var field = as(project.child(), FieldExtractExec.class);
-        var queryExec = as(field.child(), EsQueryExec.class);
-        assertThat(queryExec.limit().fold(), is(1000));
-
-        var expectedInnerQuery = QueryBuilders.termsQuery(fieldName, cidrBlocks);
-        var expectedQuery = wrapWithSingleQuery(expectedInnerQuery, fieldName, new Source(1, 18, cidrMatch));
-        assertThat(queryExec.query().toString(), is(expectedQuery.toString()));
-    }
-
     private record OutOfRangeTestCase(String fieldName, String tooLow, String tooHigh) {};
 
     public void testOutOfRangeFilterPushdown() {
@@ -746,8 +807,79 @@ public void testMissingFieldsDoNotGetExtracted() {
         assertThat(Expressions.names(fields), contains("_meta_field", "gender", "job", "job.raw", "languages", "long_noidx"));
     }
 
-    private QueryBuilder wrapWithSingleQuery(QueryBuilder inner, String fieldName, Source source) {
-        return FilterTests.singleValueQuery(inner, fieldName, source);
+    /**
+     * Expects
+     * LimitExec[1000[INTEGER]]
+     * \_ExchangeExec[[],false]
+     *   \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n
+     * ame{f}#7, long_noidx{f}#12, salary{f}#8]]
+     *     \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen..]
+     *       \_EsQueryExec[test], indexMode[standard], query[{"match":{"first_name":{"query":"Anna"}}}][_doc{f}#13], limit[1000], sort[]
+     *       estimatedRowSize[324]
+     */
+    public void testSingleMatchFilterPushdown() {
+        assumeTrue("Match operator is available just for snapshots", Build.current().isSnapshot());
+
+        var plan = plannerOptimizer.plan("""
+            from test
+            | where first_name match "Anna"
+            """);
+
+        var limit = as(plan, LimitExec.class);
+        var exchange = as(limit.child(), ExchangeExec.class);
+        var project = as(exchange.child(), ProjectExec.class);
+        var fieldExtract = as(project.child(), FieldExtractExec.class);
+        var actualLuceneQuery = as(fieldExtract.child(), EsQueryExec.class).query();
+
+        var expectedLuceneQuery = new MatchQueryBuilder("first_name", "Anna");
+        assertThat(actualLuceneQuery, equalTo(expectedLuceneQuery));
+    }
+
+    /**
+     * Expects
+     * EvalExec[[CONCAT([65 6d 70 5f 6e 6f 3a 20][KEYWORD],TOSTRING(emp_no{f}#12),[2c 20 6e 61 6d 65 3a 20][KEYWORD],first_nam
+     * e{f}#13,[20][KEYWORD],last_name{f}#16) AS description]]
+     * \_TopNExec[[Order[emp_no{f}#12,ASC,LAST]],1000[INTEGER],50]
+     *   \_ExchangeExec[[],false]
+     *     \_ProjectExec[[_meta_field{f}#18, emp_no{f}#12, first_name{f}#13, gender{f}#14, job{f}#19, job.raw{f}#20, languages{f}#15, l
+     * ast_name{f}#16, long_noidx{f}#21, salary{f}#17]]
+     *       \_FieldExtractExec[_meta_field{f}#18, emp_no{f}#12, first_name{f}#13, ..]
+     *         \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"bool":{"should":[{"match":{"first_name":{"query":"Anna"}}}
+     *         ,{"match":{"first_name":{"query":"Anneke"}}}],"boost":1.0}},{"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":
+     *         {"gt":10000,"boost":1.0}}},"source":"emp_no > 10000@4:9"}},{"match":{"last_name":{"query":"Xinglin"}}}],"boost":1.0}}]
+     *         [_doc{f}#22], limit[1000], sort[[FieldSort[field=emp_no{f}#12, direction=ASC, nulls=LAST]]] estimatedRowSize[336]
+     */
+    public void testMultipleMatchFilterPushdown() {
+        assumeTrue("Match operator is available just for snapshots", Build.current().isSnapshot());
+
+        String query = """
+            from test
+            | where first_name match "Anna" OR first_name match "Anneke"
+            | sort emp_no
+            | where emp_no > 10000
+            | eval description = concat("emp_no: ", to_str(emp_no), ", name: ", first_name, " ", last_name)
+            | where last_name match "Xinglin"
+            """;
+        var plan = plannerOptimizer.plan(query);
+
+        var eval = as(plan, EvalExec.class);
+        var topNExec = as(eval.child(), TopNExec.class);
+        var exchange = as(topNExec.child(), ExchangeExec.class);
+        var project = as(exchange.child(), ProjectExec.class);
+        var fieldExtract = as(project.child(), FieldExtractExec.class);
+        var actualLuceneQuery = as(fieldExtract.child(), EsQueryExec.class).query();
+
+        Source filterSource = new Source(4, 8, "emp_no > 10000");
+        var expectedLuceneQuery = new BoolQueryBuilder().must(
+            new BoolQueryBuilder().should(new MatchQueryBuilder("first_name", "Anna")).should(new MatchQueryBuilder("first_name", "Anneke"))
+        )
+            .must(wrapWithSingleQuery(query, QueryBuilders.rangeQuery("emp_no").gt(10000), "emp_no", filterSource))
+            .must(new MatchQueryBuilder("last_name", "Xinglin"));
+        assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString()));
+    }
+
+    private QueryBuilder wrapWithSingleQuery(String query, QueryBuilder inner, String fieldName, Source source) {
+        return FilterTests.singleValueQuery(query, inner, fieldName, source);
     }
 
     private Stat queryStatsFor(PhysicalPlan plan) {
@@ -765,13 +897,4 @@ private Stat queryStatsFor(PhysicalPlan plan) {
     protected List<String> filteredWarnings() {
         return withDefaultLimitWarning(super.filteredWarnings());
     }
-
-    private String randomCidrBlock() {
-        boolean ipv4 = randomBoolean();
-
-        String address = NetworkAddress.format(randomIp(ipv4));
-        int cidrPrefixLength = ipv4 ? randomIntBetween(0, 32) : randomIntBetween(0, 128);
-
-        return format(null, "{}/{}", address, cidrPrefixLength);
-    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
index d61a49ce1122f..74f95e3defbd3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.compute.aggregation.QuantileStates;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.dissect.DissectParser;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
@@ -21,6 +22,7 @@
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
 import org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils;
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
+import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
@@ -40,8 +42,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
@@ -66,6 +66,7 @@
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Values;
 import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToString;
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateExtract;
@@ -106,12 +107,19 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.optimizer.rules.LiteralsOnTheRight;
+import org.elasticsearch.xpack.esql.optimizer.rules.OptimizerRules;
 import org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineFilters;
 import org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineLimits;
+import org.elasticsearch.xpack.esql.optimizer.rules.PushDownEnrich;
+import org.elasticsearch.xpack.esql.optimizer.rules.PushDownEval;
+import org.elasticsearch.xpack.esql.optimizer.rules.PushDownRegexExtract;
 import org.elasticsearch.xpack.esql.optimizer.rules.SplitInWithFoldableValue;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.parser.ParsingException;
+import org.elasticsearch.xpack.esql.plan.GeneratingPlan;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Dissect;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
@@ -119,6 +127,7 @@
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.plan.logical.Grok;
+import org.elasticsearch.xpack.esql.plan.logical.InlineStats;
 import org.elasticsearch.xpack.esql.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
@@ -140,6 +149,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 
 import static java.util.Arrays.asList;
@@ -157,6 +167,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.localSource;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.referenceAttribute;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.elasticsearch.xpack.esql.analysis.Analyzer.NO_FIELDS;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze;
@@ -188,6 +199,7 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
@@ -1021,7 +1033,7 @@ public void testPushDownDissectPastProject() {
 
         var keep = as(plan, Project.class);
         var dissect = as(keep.child(), Dissect.class);
-        assertThat(dissect.extractedFields(), contains(new ReferenceAttribute(Source.EMPTY, "y", DataType.KEYWORD)));
+        assertThat(dissect.extractedFields(), contains(referenceAttribute("y", DataType.KEYWORD)));
     }
 
     public void testPushDownGrokPastProject() {
@@ -1034,7 +1046,7 @@ public void testPushDownGrokPastProject() {
 
         var keep = as(plan, Project.class);
         var grok = as(keep.child(), Grok.class);
-        assertThat(grok.extractedFields(), contains(new ReferenceAttribute(Source.EMPTY, "y", DataType.KEYWORD)));
+        assertThat(grok.extractedFields(), contains(referenceAttribute("y", DataType.KEYWORD)));
     }
 
     public void testPushDownFilterPastProjectUsingEval() {
@@ -1760,7 +1772,7 @@ public void testDontPruneSameFieldDifferentDirectionSortClauses() {
             contains(
                 new Order(
                     EMPTY,
-                    new ReferenceAttribute(EMPTY, "e", INTEGER, null, Nullability.TRUE, null, false),
+                    new ReferenceAttribute(EMPTY, "e", INTEGER, Nullability.TRUE, null, false),
                     Order.OrderDirection.ASC,
                     Order.NullsPosition.LAST
                 ),
@@ -1803,7 +1815,7 @@ public void testPruneRedundantSortClauses() {
             contains(
                 new Order(
                     EMPTY,
-                    new ReferenceAttribute(EMPTY, "e", INTEGER, null, Nullability.TRUE, null, false),
+                    new ReferenceAttribute(EMPTY, "e", INTEGER, Nullability.TRUE, null, false),
                     Order.OrderDirection.ASC,
                     Order.NullsPosition.LAST
                 ),
@@ -3503,6 +3515,49 @@ public void testBucketWithAggExpression() {
         assertThat(agg.groupings().get(0), is(ref));
     }
 
+    public void testBucketWithNonFoldingArgs() {
+        assertThat(
+            typesError("from types | stats max(integer) by bucket(date, integer, \"2000-01-01\", \"2000-01-02\")"),
+            containsString(
+                "second argument of [bucket(date, integer, \"2000-01-01\", \"2000-01-02\")] must be a constant, " + "received [integer]"
+            )
+        );
+
+        assertThat(
+            typesError("from types | stats max(integer) by bucket(date, 2, date, \"2000-01-02\")"),
+            containsString("third argument of [bucket(date, 2, date, \"2000-01-02\")] must be a constant, " + "received [date]")
+        );
+
+        assertThat(
+            typesError("from types | stats max(integer) by bucket(date, 2, \"2000-01-02\", date)"),
+            containsString("fourth argument of [bucket(date, 2, \"2000-01-02\", date)] must be a constant, " + "received [date]")
+        );
+
+        assertThat(
+            typesError("from types | stats max(integer) by bucket(integer, long, 4, 5)"),
+            containsString("second argument of [bucket(integer, long, 4, 5)] must be a constant, " + "received [long]")
+        );
+
+        assertThat(
+            typesError("from types | stats max(integer) by bucket(integer, 3, long, 5)"),
+            containsString("third argument of [bucket(integer, 3, long, 5)] must be a constant, " + "received [long]")
+        );
+
+        assertThat(
+            typesError("from types | stats max(integer) by bucket(integer, 3, 4, long)"),
+            containsString("fourth argument of [bucket(integer, 3, 4, long)] must be a constant, " + "received [long]")
+        );
+    }
+
+    private String typesError(String query) {
+        VerificationException e = expectThrows(VerificationException.class, () -> planTypes(query));
+        String message = e.getMessage();
+        assertTrue(message.startsWith("Found "));
+        String pattern = "\nline ";
+        int index = message.indexOf(pattern);
+        return message.substring(index + pattern.length());
+    }
+
     /**
      * Expects
      * Project[[x{r}#5]]
@@ -4111,14 +4166,7 @@ public void testPlanSanityCheck() throws Exception {
         var invalidPlan = new OrderBy(
             limit.source(),
             limit,
-            asList(
-                new Order(
-                    limit.source(),
-                    salary,
-                    org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.ASC,
-                    org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.FIRST
-                )
-            )
+            asList(new Order(limit.source(), salary, Order.OrderDirection.ASC, Order.NullsPosition.FIRST))
         );
 
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> logicalOptimizer.optimize(invalidPlan));
@@ -4194,20 +4242,20 @@ public void testPushdownWithOverwrittenName() {
             assertThat(topN.order().size(), is(3));
 
             var firstOrder = as(topN.order().get(0), Order.class);
-            assertThat(firstOrder.direction(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.ASC));
-            assertThat(firstOrder.nullsPosition(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.FIRST));
+            assertThat(firstOrder.direction(), equalTo(Order.OrderDirection.ASC));
+            assertThat(firstOrder.nullsPosition(), equalTo(Order.NullsPosition.FIRST));
             var renamed_emp_no = as(firstOrder.child(), ReferenceAttribute.class);
             assertThat(renamed_emp_no.toString(), startsWith("$$emp_no$temp_name"));
 
             var secondOrder = as(topN.order().get(1), Order.class);
-            assertThat(secondOrder.direction(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.DESC));
-            assertThat(secondOrder.nullsPosition(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.LAST));
+            assertThat(secondOrder.direction(), equalTo(Order.OrderDirection.DESC));
+            assertThat(secondOrder.nullsPosition(), equalTo(Order.NullsPosition.LAST));
             var renamed_salary = as(secondOrder.child(), ReferenceAttribute.class);
             assertThat(renamed_salary.toString(), startsWith("$$salary$temp_name"));
 
             var thirdOrder = as(topN.order().get(2), Order.class);
-            assertThat(thirdOrder.direction(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.ASC));
-            assertThat(thirdOrder.nullsPosition(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.LAST));
+            assertThat(thirdOrder.direction(), equalTo(Order.OrderDirection.ASC));
+            assertThat(thirdOrder.nullsPosition(), equalTo(Order.NullsPosition.LAST));
             var renamed_emp_no2 = as(thirdOrder.child(), ReferenceAttribute.class);
             assertThat(renamed_emp_no2.toString(), startsWith("$$emp_no$temp_name"));
 
@@ -4254,6 +4302,210 @@ public void testPushdownWithOverwrittenName() {
         }
     }
 
+    record PushdownShadowingGeneratingPlanTestCase(
+        BiFunction<LogicalPlan, Attribute, LogicalPlan> applyLogicalPlan,
+        OptimizerRules.OptimizerRule<? extends LogicalPlan> rule
+    ) {};
+
+    static PushdownShadowingGeneratingPlanTestCase[] PUSHDOWN_SHADOWING_GENERATING_PLAN_TEST_CASES = {
+        // | EVAL y = to_integer(x), y = y + 1
+        new PushdownShadowingGeneratingPlanTestCase((plan, attr) -> {
+            Alias y1 = new Alias(EMPTY, "y", new ToInteger(EMPTY, attr));
+            Alias y2 = new Alias(EMPTY, "y", new Add(EMPTY, y1.toAttribute(), new Literal(EMPTY, 1, INTEGER)));
+            return new Eval(EMPTY, plan, List.of(y1, y2));
+        }, new PushDownEval()),
+        // | DISSECT x "%{y} %{y}"
+        new PushdownShadowingGeneratingPlanTestCase(
+            (plan, attr) -> new Dissect(
+                EMPTY,
+                plan,
+                attr,
+                new Dissect.Parser("%{y} %{y}", ",", new DissectParser("%{y} %{y}", ",")),
+                List.of(new ReferenceAttribute(EMPTY, "y", KEYWORD), new ReferenceAttribute(EMPTY, "y", KEYWORD))
+            ),
+            new PushDownRegexExtract()
+        ),
+        // | GROK x "%{WORD:y} %{WORD:y}"
+        new PushdownShadowingGeneratingPlanTestCase(
+            (plan, attr) -> new Grok(EMPTY, plan, attr, Grok.pattern(EMPTY, "%{WORD:y} %{WORD:y}")),
+            new PushDownRegexExtract()
+        ),
+        // | ENRICH some_policy ON x WITH y = some_enrich_idx_field, y = some_other_enrich_idx_field
+        new PushdownShadowingGeneratingPlanTestCase(
+            (plan, attr) -> new Enrich(
+                EMPTY,
+                plan,
+                Enrich.Mode.ANY,
+                new Literal(EMPTY, "some_policy", KEYWORD),
+                attr,
+                null,
+                Map.of(),
+                List.of(
+                    new Alias(EMPTY, "y", new ReferenceAttribute(EMPTY, "some_enrich_idx_field", KEYWORD)),
+                    new Alias(EMPTY, "y", new ReferenceAttribute(EMPTY, "some_other_enrich_idx_field", KEYWORD))
+                )
+            ),
+            new PushDownEnrich()
+        ) };
+
+    /**
+     * Consider
+     *
+     * Eval[[TO_INTEGER(x{r}#2) AS y, y{r}#4 + 1[INTEGER] AS y]]
+     * \_Project[[y{r}#3, x{r}#2]]
+     * \_Row[[1[INTEGER] AS x, 2[INTEGER] AS y]]
+     *
+     * We can freely push down the Eval without renaming, but need to update the Project's references.
+     *
+     * Project[[x{r}#2, y{r}#6 AS y]]
+     * \_Eval[[TO_INTEGER(x{r}#2) AS y, y{r}#4 + 1[INTEGER] AS y]]
+     * \_Row[[1[INTEGER] AS x, 2[INTEGER] AS y]]
+     *
+     * And similarly for dissect, grok and enrich.
+     */
+    public void testPushShadowingGeneratingPlanPastProject() {
+        Alias x = new Alias(EMPTY, "x", new Literal(EMPTY, "1", KEYWORD));
+        Alias y = new Alias(EMPTY, "y", new Literal(EMPTY, "2", KEYWORD));
+        LogicalPlan initialRow = new Row(EMPTY, List.of(x, y));
+        LogicalPlan initialProject = new Project(EMPTY, initialRow, List.of(y.toAttribute(), x.toAttribute()));
+
+        for (PushdownShadowingGeneratingPlanTestCase testCase : PUSHDOWN_SHADOWING_GENERATING_PLAN_TEST_CASES) {
+            LogicalPlan initialPlan = testCase.applyLogicalPlan.apply(initialProject, x.toAttribute());
+            @SuppressWarnings("unchecked")
+            List<Attribute> initialGeneratedExprs = ((GeneratingPlan) initialPlan).generatedAttributes();
+            LogicalPlan optimizedPlan = testCase.rule.apply(initialPlan);
+
+            Failures inconsistencies = LogicalVerifier.INSTANCE.verify(optimizedPlan);
+            assertFalse(inconsistencies.hasFailures());
+
+            Project project = as(optimizedPlan, Project.class);
+            LogicalPlan pushedDownGeneratingPlan = project.child();
+
+            List<? extends NamedExpression> projections = project.projections();
+            @SuppressWarnings("unchecked")
+            List<Attribute> newGeneratedExprs = ((GeneratingPlan) pushedDownGeneratingPlan).generatedAttributes();
+            assertEquals(newGeneratedExprs, initialGeneratedExprs);
+            // The rightmost generated attribute makes it into the final output as "y".
+            Attribute rightmostGenerated = newGeneratedExprs.get(newGeneratedExprs.size() - 1);
+
+            assertThat(Expressions.names(projections), contains("x", "y"));
+            assertThat(projections, everyItem(instanceOf(ReferenceAttribute.class)));
+            ReferenceAttribute yShadowed = as(projections.get(1), ReferenceAttribute.class);
+            assertTrue(yShadowed.semanticEquals(rightmostGenerated));
+        }
+    }
+
+    /**
+     * Consider
+     *
+     * Eval[[TO_INTEGER(x{r}#2) AS y, y{r}#4 + 1[INTEGER] AS y]]
+     * \_Project[[x{r}#2, y{r}#3, y{r}#3 AS z]]
+     * \_Row[[1[INTEGER] AS x, 2[INTEGER] AS y]]
+     *
+     * To push down the Eval, we must not shadow the reference y{r}#3, so we rename.
+     *
+     * Project[[x{r}#2, y{r}#3 AS z, $$y$temp_name$10{r}#12 AS y]]
+     * Eval[[TO_INTEGER(x{r}#2) AS $$y$temp_name$10, $$y$temp_name$10{r}#11 + 1[INTEGER] AS $$y$temp_name$10]]
+     * \_Row[[1[INTEGER] AS x, 2[INTEGER] AS y]]
+     *
+     * And similarly for dissect, grok and enrich.
+     */
+    public void testPushShadowingGeneratingPlanPastRenamingProject() {
+        Alias x = new Alias(EMPTY, "x", new Literal(EMPTY, "1", KEYWORD));
+        Alias y = new Alias(EMPTY, "y", new Literal(EMPTY, "2", KEYWORD));
+        LogicalPlan initialRow = new Row(EMPTY, List.of(x, y));
+        LogicalPlan initialProject = new Project(
+            EMPTY,
+            initialRow,
+            List.of(x.toAttribute(), y.toAttribute(), new Alias(EMPTY, "z", y.toAttribute()))
+        );
+
+        for (PushdownShadowingGeneratingPlanTestCase testCase : PUSHDOWN_SHADOWING_GENERATING_PLAN_TEST_CASES) {
+            LogicalPlan initialPlan = testCase.applyLogicalPlan.apply(initialProject, x.toAttribute());
+            @SuppressWarnings("unchecked")
+            List<Attribute> initialGeneratedExprs = ((GeneratingPlan) initialPlan).generatedAttributes();
+            LogicalPlan optimizedPlan = testCase.rule.apply(initialPlan);
+
+            Failures inconsistencies = LogicalVerifier.INSTANCE.verify(optimizedPlan);
+            assertFalse(inconsistencies.hasFailures());
+
+            Project project = as(optimizedPlan, Project.class);
+            LogicalPlan pushedDownGeneratingPlan = project.child();
+
+            List<? extends NamedExpression> projections = project.projections();
+            @SuppressWarnings("unchecked")
+            List<Attribute> newGeneratedExprs = ((GeneratingPlan) pushedDownGeneratingPlan).generatedAttributes();
+            List<String> newNames = Expressions.names(newGeneratedExprs);
+            assertThat(newNames.size(), equalTo(initialGeneratedExprs.size()));
+            assertThat(newNames, everyItem(startsWith("$$y$temp_name$")));
+            // The rightmost generated attribute makes it into the final output as "y".
+            Attribute rightmostGeneratedWithNewName = newGeneratedExprs.get(newGeneratedExprs.size() - 1);
+
+            assertThat(Expressions.names(projections), contains("x", "z", "y"));
+            assertThat(projections.get(0), instanceOf(ReferenceAttribute.class));
+            Alias zAlias = as(projections.get(1), Alias.class);
+            ReferenceAttribute yRenamed = as(zAlias.child(), ReferenceAttribute.class);
+            assertEquals(yRenamed.name(), "y");
+            Alias yAlias = as(projections.get(2), Alias.class);
+            ReferenceAttribute yTempRenamed = as(yAlias.child(), ReferenceAttribute.class);
+            assertTrue(yTempRenamed.semanticEquals(rightmostGeneratedWithNewName));
+        }
+    }
+
+    /**
+     * Consider
+     *
+     * Eval[[TO_INTEGER(x{r}#2) AS y, y{r}#3 + 1[INTEGER] AS y]]
+     * \_Project[[y{r}#1, y{r}#1 AS x]]
+     * \_Row[[2[INTEGER] AS y]]
+     *
+     * To push down the Eval, we must not shadow the reference y{r}#1, so we rename.
+     * Additionally, the rename "y AS x" needs to be propagated into the Eval.
+     *
+     * Project[[y{r}#1 AS x, $$y$temp_name$10{r}#12 AS y]]
+     * Eval[[TO_INTEGER(y{r}#1) AS $$y$temp_name$10, $$y$temp_name$10{r}#11 + 1[INTEGER] AS $$y$temp_name$10]]
+     * \_Row[[2[INTEGER] AS y]]
+     *
+     * And similarly for dissect, grok and enrich.
+     */
+    public void testPushShadowingGeneratingPlanPastRenamingProjectWithResolution() {
+        Alias y = new Alias(EMPTY, "y", new Literal(EMPTY, "2", KEYWORD));
+        Alias yAliased = new Alias(EMPTY, "x", y.toAttribute());
+        LogicalPlan initialRow = new Row(EMPTY, List.of(y));
+        LogicalPlan initialProject = new Project(EMPTY, initialRow, List.of(y.toAttribute(), yAliased));
+
+        for (PushdownShadowingGeneratingPlanTestCase testCase : PUSHDOWN_SHADOWING_GENERATING_PLAN_TEST_CASES) {
+            LogicalPlan initialPlan = testCase.applyLogicalPlan.apply(initialProject, yAliased.toAttribute());
+            @SuppressWarnings("unchecked")
+            List<Attribute> initialGeneratedExprs = ((GeneratingPlan) initialPlan).generatedAttributes();
+            LogicalPlan optimizedPlan = testCase.rule.apply(initialPlan);
+
+            // This ensures that our generating plan doesn't use invalid references, resp. that any rename from the Project has
+            // been propagated into the generating plan.
+            Failures inconsistencies = LogicalVerifier.INSTANCE.verify(optimizedPlan);
+            assertFalse(inconsistencies.hasFailures());
+
+            Project project = as(optimizedPlan, Project.class);
+            LogicalPlan pushedDownGeneratingPlan = project.child();
+
+            List<? extends NamedExpression> projections = project.projections();
+            @SuppressWarnings("unchecked")
+            List<Attribute> newGeneratedExprs = ((GeneratingPlan) pushedDownGeneratingPlan).generatedAttributes();
+            List<String> newNames = Expressions.names(newGeneratedExprs);
+            assertThat(newNames.size(), equalTo(initialGeneratedExprs.size()));
+            assertThat(newNames, everyItem(startsWith("$$y$temp_name$")));
+            // The rightmost generated attribute makes it into the final output as "y".
+            Attribute rightmostGeneratedWithNewName = newGeneratedExprs.get(newGeneratedExprs.size() - 1);
+
+            assertThat(Expressions.names(projections), contains("x", "y"));
+            Alias yRenamed = as(projections.get(0), Alias.class);
+            assertTrue(yRenamed.child().semanticEquals(y.toAttribute()));
+            Alias yTempRenamed = as(projections.get(1), Alias.class);
+            ReferenceAttribute yTemp = as(yTempRenamed.child(), ReferenceAttribute.class);
+            assertTrue(yTemp.semanticEquals(rightmostGeneratedWithNewName));
+        }
+    }
+
     /**
      * Expects
      * Project[[min{r}#4, languages{f}#11]]
@@ -4275,8 +4527,8 @@ public void testReplaceSortByExpressionsWithStats() {
         assertThat(topN.order().size(), is(1));
 
         var order = as(topN.order().get(0), Order.class);
-        assertThat(order.direction(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.ASC));
-        assertThat(order.nullsPosition(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.LAST));
+        assertThat(order.direction(), equalTo(Order.OrderDirection.ASC));
+        assertThat(order.nullsPosition(), equalTo(Order.NullsPosition.LAST));
         var expression = as(order.child(), ReferenceAttribute.class);
         assertThat(expression.toString(), startsWith("$$order_by$0$"));
 
@@ -4291,6 +4543,31 @@ public void testReplaceSortByExpressionsWithStats() {
         as(aggregate.child(), EsRelation.class);
     }
 
+    /**
+     * Expects
+     * Limit[1000[INTEGER]]
+     * \_InlineStats[[emp_no % 2{r}#6],[COUNT(salary{f}#12) AS c, emp_no % 2{r}#6]]
+     *   \_Eval[[emp_no{f}#7 % 2[INTEGER] AS emp_no % 2]]
+     *     \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
+     */
+    public void testInlinestatsNestedExpressionsInGroups() {
+        var plan = optimizedPlan("""
+            FROM test
+            | INLINESTATS c = COUNT(salary) by emp_no % 2
+            """);
+
+        var limit = as(plan, Limit.class);
+        var agg = as(limit.child(), InlineStats.class);
+        var groupings = agg.groupings();
+        var aggs = agg.aggregates();
+        var ref = as(groupings.get(0), ReferenceAttribute.class);
+        assertThat(aggs.get(1), is(ref));
+        var eval = as(agg.child(), Eval.class);
+        assertThat(eval.fields(), hasSize(1));
+        assertThat(eval.fields().get(0).toAttribute(), is(ref));
+        assertThat(eval.fields().get(0).name(), is("emp_no % 2"));
+    }
+
     /**
      * Expects
      *
@@ -4314,14 +4591,14 @@ public void testReplaceSortByExpressionsMultipleSorts() {
         assertThat(topN.order().size(), is(2));
 
         var order = as(topN.order().get(0), Order.class);
-        assertThat(order.direction(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.ASC));
-        assertThat(order.nullsPosition(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.LAST));
+        assertThat(order.direction(), equalTo(Order.OrderDirection.ASC));
+        assertThat(order.nullsPosition(), equalTo(Order.NullsPosition.LAST));
         ReferenceAttribute expression = as(order.child(), ReferenceAttribute.class);
         assertThat(expression.toString(), startsWith("$$order_by$0$"));
 
         order = as(topN.order().get(1), Order.class);
-        assertThat(order.direction(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.DESC));
-        assertThat(order.nullsPosition(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.FIRST));
+        assertThat(order.direction(), equalTo(Order.OrderDirection.DESC));
+        assertThat(order.nullsPosition(), equalTo(Order.NullsPosition.FIRST));
         FieldAttribute empNo = as(order.child(), FieldAttribute.class);
         assertThat(empNo.name(), equalTo("emp_no"));
 
@@ -4391,14 +4668,14 @@ public void testReplaceSortByExpressions() {
             assertThat(topN.order().size(), is(2));
 
             var firstOrderExpr = as(topN.order().get(0), Order.class);
-            assertThat(firstOrderExpr.direction(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.ASC));
-            assertThat(firstOrderExpr.nullsPosition(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.LAST));
+            assertThat(firstOrderExpr.direction(), equalTo(Order.OrderDirection.ASC));
+            assertThat(firstOrderExpr.nullsPosition(), equalTo(Order.NullsPosition.LAST));
             var renamedEmpNoSalaryExpression = as(firstOrderExpr.child(), ReferenceAttribute.class);
             assertThat(renamedEmpNoSalaryExpression.toString(), startsWith("$$order_by$0$"));
 
             var secondOrderExpr = as(topN.order().get(1), Order.class);
-            assertThat(secondOrderExpr.direction(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.DESC));
-            assertThat(secondOrderExpr.nullsPosition(), equalTo(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.FIRST));
+            assertThat(secondOrderExpr.direction(), equalTo(Order.OrderDirection.DESC));
+            assertThat(secondOrderExpr.nullsPosition(), equalTo(Order.NullsPosition.FIRST));
             var renamedNegatedSalaryExpression = as(secondOrderExpr.child(), ReferenceAttribute.class);
             assertThat(renamedNegatedSalaryExpression.toString(), startsWith("$$order_by$1$"));
 
@@ -4592,11 +4869,6 @@ public void testSimplyComparisonArithmeticWithUnfoldedProd() {
         assertSemanticMatching("integer * integer >= 3", "((integer * integer + 1) * 2 - 4) * 4 >= 16");
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108524")
-    public void testSimplifyComparisionArithmetics_floatDivision() {
-        doTestSimplifyComparisonArithmetics("2 / float < 4", "float", GT, .5);
-    }
-
     public void testSimplifyComparisonArithmeticWithMultipleOps() {
         // i >= 3
         doTestSimplifyComparisonArithmetics("((integer + 1) * 2 - 4) * 4 >= 16", "integer", GTE, 3);
@@ -4614,7 +4886,6 @@ public void testSimplifyComparisonArithmeticWithConjunction() {
         doTestSimplifyComparisonArithmetics("12 * (-integer - 5) == -120 AND integer < 6 ", "integer", EQ, 5);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108525")
     public void testSimplifyComparisonArithmeticWithDisjunction() {
         doTestSimplifyComparisonArithmetics("12 * (-integer - 5) >= -120 OR integer < 5", "integer", LTE, 5);
     }
@@ -4664,6 +4935,7 @@ public void testSimplifyComparisonArithmeticSkippedOnDiv() {
 
     public void testSimplifyComparisonArithmeticSkippedOnResultingFloatLiteral() {
         assertNotSimplified("integer * 2 " + randomBinaryComparison() + " 3");
+        assertNotSimplified("float * 4.0 " + randomBinaryComparison() + " 1");
     }
 
     public void testSimplifyComparisonArithmeticSkippedOnFloatFieldWithPlusMinus() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
index a99ce5d873b44..5b8a465b05f1c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
@@ -41,16 +41,14 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
+import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
@@ -70,6 +68,8 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.parser.ParsingException;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
@@ -110,7 +110,7 @@
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
 import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery;
 import org.elasticsearch.xpack.esql.querydsl.query.SpatialRelatesQuery;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
 import org.junit.Before;
 
@@ -139,7 +139,6 @@
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze;
 import static org.elasticsearch.xpack.esql.core.expression.Expressions.name;
 import static org.elasticsearch.xpack.esql.core.expression.Expressions.names;
-import static org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.ASC;
 import static org.elasticsearch.xpack.esql.core.expression.function.scalar.FunctionTestUtils.l;
 import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
@@ -180,7 +179,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
     private TestDataSource countriesBbox;
     private TestDataSource countriesBboxWeb;
 
-    private final EsqlConfiguration config;
+    private final Configuration config;
 
     private record TestDataSource(Map<String, EsField> mapping, EsIndex index, Analyzer analyzer) {}
 
@@ -196,7 +195,7 @@ private static List<Tuple<String, Map<String, Object>>> settings() {
         return asList(new Tuple<>("default", Map.of()));
     }
 
-    public PhysicalPlanOptimizerTests(String name, EsqlConfiguration config) {
+    public PhysicalPlanOptimizerTests(String name, Configuration config) {
         this.config = config;
     }
 
@@ -522,7 +521,7 @@ public void testExtractorForField() {
         assertThat(source.limit(), is(l(10)));
         assertThat(source.sorts().size(), is(1));
         FieldSort order = source.sorts().get(0);
-        assertThat(order.direction(), is(ASC));
+        assertThat(order.direction(), is(Order.OrderDirection.ASC));
         assertThat(name(order.field()), is("last_name"));
         // last name is keyword, salary, emp_no, doc id, segment, forwards and backwards doc id maps are all ints
         int estimatedSize = KEYWORD_EST + Integer.BYTES * 6;
@@ -1192,7 +1191,7 @@ public void testDoNotAliasesDefinedAfterTheExchange() throws Exception {
         assertThat(source.limit(), is(l(1)));
         assertThat(source.sorts().size(), is(1));
         FieldSort order = source.sorts().get(0);
-        assertThat(order.direction(), is(ASC));
+        assertThat(order.direction(), is(Order.OrderDirection.ASC));
         assertThat(name(order.field()), is("salary"));
         // ints for doc id, segment id, forwards and backwards mapping, languages, and salary
         assertThat(source.estimatedRowSize(), equalTo(Integer.BYTES * 6));
@@ -4549,7 +4548,7 @@ private static QueryBuilder findQueryBuilder(BoolQueryBuilder booleanQuery, Stri
 
     private void assertFieldExtractionWithDocValues(FieldExtractExec extract, DataType dataType, String... fieldNames) {
         extract.attributesToExtract().forEach(attr -> {
-            String name = attr.qualifiedName();
+            String name = attr.name();
             if (asList(fieldNames).contains(name)) {
                 assertThat("Expected field '" + name + "' to use doc-values", extract.hasDocValuesAttribute(attr), equalTo(true));
                 assertThat("Expected field '" + name + "' to have data type " + dataType, attr.dataType(), equalTo(dataType));
@@ -4604,7 +4603,7 @@ static PhysicalPlan localRelationshipAlignment(PhysicalPlan l) {
         // handle local reduction alignment
         return l.transformUp(ExchangeExec.class, exg -> {
             PhysicalPlan pl = exg;
-            if (exg.isInBetweenAggs() && exg.child() instanceof LocalSourceExec lse) {
+            if (exg.inBetweenAggs() && exg.child() instanceof LocalSourceExec lse) {
                 var output = exg.output();
                 if (lse.output().equals(output) == false) {
                     pl = exg.replaceChild(new LocalSourceExec(lse.source(), output, lse.supplier()));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java
index 9e4e0e0d485d5..1d25146ee4e2d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.planner.Mapper;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
 
 public class TestPlannerOptimizer {
@@ -25,9 +25,9 @@ public class TestPlannerOptimizer {
     private final PhysicalPlanOptimizer physicalPlanOptimizer;
     private final EsqlFunctionRegistry functionRegistry;
     private final Mapper mapper;
-    private final EsqlConfiguration config;
+    private final Configuration config;
 
-    public TestPlannerOptimizer(EsqlConfiguration config, Analyzer analyzer) {
+    public TestPlannerOptimizer(Configuration config, Analyzer analyzer) {
         this.analyzer = analyzer;
         this.config = config;
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineBinaryComparisonsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineBinaryComparisonsTests.java
new file mode 100644
index 0000000000000..3c98b7fa23e8b
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineBinaryComparisonsTests.java
@@ -0,0 +1,400 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.optimizer.rules;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.EsqlTestUtils;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
+
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.FIVE;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.FOUR;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.L;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.ONE;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.SIX;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.THREE;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.TWO;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.equalsOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.fieldAttribute;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOrEqualOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOrEqualOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf;
+import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
+import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
+import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+
+public class CombineBinaryComparisonsTests extends ESTestCase {
+
+    private static final Expression DUMMY_EXPRESSION =
+        new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 0);
+
+    public void testCombineBinaryComparisonsNotComparable() {
+        FieldAttribute fa = getFieldAttribute();
+        LessThanOrEqual lte = lessThanOrEqualOf(fa, SIX);
+        LessThan lt = lessThanOf(fa, FALSE);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+
+        And and = new And(EMPTY, lte, lt);
+        Expression exp = rule.rule(and);
+        assertEquals(exp, and);
+    }
+
+    // a <= 6 AND a < 5 -> a < 5
+    public void testCombineBinaryComparisonsUpper() {
+        FieldAttribute fa = getFieldAttribute();
+        LessThanOrEqual lte = lessThanOrEqualOf(fa, SIX);
+        LessThan lt = lessThanOf(fa, FIVE);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+
+        Expression exp = rule.rule(new And(EMPTY, lte, lt));
+        assertEquals(LessThan.class, exp.getClass());
+        LessThan r = (LessThan) exp;
+        assertEquals(FIVE, r.right());
+    }
+
+    // 6 <= a AND 5 < a -> 6 <= a
+    public void testCombineBinaryComparisonsLower() {
+        FieldAttribute fa = getFieldAttribute();
+        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, SIX);
+        GreaterThan gt = greaterThanOf(fa, FIVE);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+
+        Expression exp = rule.rule(new And(EMPTY, gte, gt));
+        assertEquals(GreaterThanOrEqual.class, exp.getClass());
+        GreaterThanOrEqual r = (GreaterThanOrEqual) exp;
+        assertEquals(SIX, r.right());
+    }
+
+    // 5 <= a AND 5 < a -> 5 < a
+    public void testCombineBinaryComparisonsInclude() {
+        FieldAttribute fa = getFieldAttribute();
+        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, FIVE);
+        GreaterThan gt = greaterThanOf(fa, FIVE);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+
+        Expression exp = rule.rule(new And(EMPTY, gte, gt));
+        assertEquals(GreaterThan.class, exp.getClass());
+        GreaterThan r = (GreaterThan) exp;
+        assertEquals(FIVE, r.right());
+    }
+
+    // 3 <= a AND 4 < a AND a <= 7 AND a < 6 -> 4 < a AND a < 6
+    public void testCombineMultipleBinaryComparisons() {
+        FieldAttribute fa = getFieldAttribute();
+        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, THREE);
+        GreaterThan gt = greaterThanOf(fa, FOUR);
+        LessThanOrEqual lte = lessThanOrEqualOf(fa, L(7));
+        LessThan lt = lessThanOf(fa, SIX);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+
+        Expression exp = rule.rule(new And(EMPTY, gte, new And(EMPTY, gt, new And(EMPTY, lt, lte))));
+        assertEquals(And.class, exp.getClass());
+        And and = (And) exp;
+        assertEquals(gt, and.left());
+        assertEquals(lt, and.right());
+    }
+
+    // 3 <= a AND TRUE AND 4 < a AND a != 5 AND a <= 7 -> 4 < a AND a <= 7 AND a != 5 AND TRUE
+    public void testCombineMixedMultipleBinaryComparisons() {
+        FieldAttribute fa = getFieldAttribute();
+        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, THREE);
+        GreaterThan gt = greaterThanOf(fa, FOUR);
+        LessThanOrEqual lte = lessThanOrEqualOf(fa, L(7));
+        Expression ne = notEqualsOf(fa, FIVE);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+
+        // TRUE AND a != 5 AND 4 < a <= 7
+        Expression exp = rule.rule(new And(EMPTY, gte, new And(EMPTY, TRUE, new And(EMPTY, gt, new And(EMPTY, ne, lte)))));
+        assertEquals(And.class, exp.getClass());
+        And and = ((And) exp);
+        assertEquals(And.class, and.right().getClass());
+        And right = (And) and.right();
+        assertEquals(gt, right.left());
+        assertEquals(lte, right.right());
+        assertEquals(And.class, and.left().getClass());
+        And left = (And) and.left();
+        assertEquals(TRUE, left.left());
+        assertEquals(ne, left.right());
+    }
+
+    // 1 <= a AND a < 5 -> 1 <= a AND a < 5
+    public void testCombineComparisonsIntoRange() {
+        FieldAttribute fa = getFieldAttribute();
+        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, ONE);
+        LessThan lt = lessThanOf(fa, FIVE);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(new And(EMPTY, gte, lt));
+        assertEquals(And.class, exp.getClass());
+
+        And and = (And) exp;
+        assertEquals(gte, and.left());
+        assertEquals(lt, and.right());
+    }
+
+    // a != 2 AND a > 3 -> a > 3
+    public void testCombineBinaryComparisonsConjunction_Neq2AndGt3() {
+        FieldAttribute fa = getFieldAttribute();
+
+        NotEquals neq = notEqualsOf(fa, TWO);
+        GreaterThan gt = greaterThanOf(fa, THREE);
+        And and = new And(EMPTY, neq, gt);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(and);
+        assertEquals(gt, exp);
+    }
+
+    // a != 2 AND a >= 2 -> a > 2
+    public void testCombineBinaryComparisonsConjunction_Neq2AndGte2() {
+        FieldAttribute fa = getFieldAttribute();
+
+        NotEquals neq = notEqualsOf(fa, TWO);
+        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, TWO);
+        And and = new And(EMPTY, neq, gte);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(and);
+        assertEquals(GreaterThan.class, exp.getClass());
+        GreaterThan gt = (GreaterThan) exp;
+        assertEquals(TWO, gt.right());
+    }
+
+    // a != 2 AND a >= 1 -> nop
+    public void testCombineBinaryComparisonsConjunction_Neq2AndGte1() {
+        FieldAttribute fa = getFieldAttribute();
+
+        NotEquals neq = notEqualsOf(fa, TWO);
+        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, ONE);
+        And and = new And(EMPTY, neq, gte);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(and);
+        assertEquals(And.class, exp.getClass()); // can't optimize
+    }
+
+    // a != 2 AND a <= 3 -> nop
+    public void testCombineBinaryComparisonsConjunction_Neq2AndLte3() {
+        FieldAttribute fa = getFieldAttribute();
+
+        NotEquals neq = notEqualsOf(fa, TWO);
+        LessThanOrEqual lte = lessThanOrEqualOf(fa, THREE);
+        And and = new And(EMPTY, neq, lte);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(and);
+        assertEquals(and, exp); // can't optimize
+    }
+
+    // a != 2 AND a <= 2 -> a < 2
+    public void testCombineBinaryComparisonsConjunction_Neq2AndLte2() {
+        FieldAttribute fa = getFieldAttribute();
+
+        NotEquals neq = notEqualsOf(fa, TWO);
+        LessThanOrEqual lte = lessThanOrEqualOf(fa, TWO);
+        And and = new And(EMPTY, neq, lte);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(and);
+        assertEquals(LessThan.class, exp.getClass());
+        LessThan lt = (LessThan) exp;
+        assertEquals(TWO, lt.right());
+    }
+
+    // a != 2 AND a <= 1 -> a <= 1
+    public void testCombineBinaryComparisonsConjunction_Neq2AndLte1() {
+        FieldAttribute fa = getFieldAttribute();
+
+        NotEquals neq = notEqualsOf(fa, TWO);
+        LessThanOrEqual lte = lessThanOrEqualOf(fa, ONE);
+        And and = new And(EMPTY, neq, lte);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(and);
+        assertEquals(lte, exp);
+    }
+
+    // Disjunction
+
+    public void testCombineBinaryComparisonsDisjunctionNotComparable() {
+        FieldAttribute fa = getFieldAttribute();
+
+        GreaterThan gt1 = greaterThanOf(fa, ONE);
+        GreaterThan gt2 = greaterThanOf(fa, FALSE);
+
+        Or or = new Or(EMPTY, gt1, gt2);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(or);
+        assertEquals(exp, or);
+    }
+
+    // 2 < a OR 1 < a OR 3 < a -> 1 < a
+    public void testCombineBinaryComparisonsDisjunctionLowerBound() {
+        FieldAttribute fa = getFieldAttribute();
+
+        GreaterThan gt1 = greaterThanOf(fa, ONE);
+        GreaterThan gt2 = greaterThanOf(fa, TWO);
+        GreaterThan gt3 = greaterThanOf(fa, THREE);
+
+        Or or = new Or(EMPTY, gt1, new Or(EMPTY, gt2, gt3));
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(or);
+        assertEquals(GreaterThan.class, exp.getClass());
+
+        GreaterThan gt = (GreaterThan) exp;
+        assertEquals(ONE, gt.right());
+    }
+
+    // 2 < a OR 1 < a OR 3 <= a -> 1 < a
+    public void testCombineBinaryComparisonsDisjunctionIncludeLowerBounds() {
+        FieldAttribute fa = getFieldAttribute();
+
+        GreaterThan gt1 = greaterThanOf(fa, ONE);
+        GreaterThan gt2 = greaterThanOf(fa, TWO);
+        GreaterThanOrEqual gte3 = greaterThanOrEqualOf(fa, THREE);
+
+        Or or = new Or(EMPTY, new Or(EMPTY, gt1, gt2), gte3);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(or);
+        assertEquals(GreaterThan.class, exp.getClass());
+
+        GreaterThan gt = (GreaterThan) exp;
+        assertEquals(ONE, gt.right());
+    }
+
+    // a < 1 OR a < 2 OR a < 3 -> a < 3
+    public void testCombineBinaryComparisonsDisjunctionUpperBound() {
+        FieldAttribute fa = getFieldAttribute();
+
+        LessThan lt1 = lessThanOf(fa, ONE);
+        LessThan lt2 = lessThanOf(fa, TWO);
+        LessThan lt3 = lessThanOf(fa, THREE);
+
+        Or or = new Or(EMPTY, new Or(EMPTY, lt1, lt2), lt3);
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(or);
+        assertEquals(LessThan.class, exp.getClass());
+
+        LessThan lt = (LessThan) exp;
+        assertEquals(THREE, lt.right());
+    }
+
+    // a < 2 OR a <= 2 OR a < 1 -> a <= 2
+    public void testCombineBinaryComparisonsDisjunctionIncludeUpperBounds() {
+        FieldAttribute fa = getFieldAttribute();
+
+        LessThan lt1 = lessThanOf(fa, ONE);
+        LessThan lt2 = lessThanOf(fa, TWO);
+        LessThanOrEqual lte2 = lessThanOrEqualOf(fa, TWO);
+
+        Or or = new Or(EMPTY, lt2, new Or(EMPTY, lte2, lt1));
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(or);
+        assertEquals(LessThanOrEqual.class, exp.getClass());
+
+        LessThanOrEqual lte = (LessThanOrEqual) exp;
+        assertEquals(TWO, lte.right());
+    }
+
+    // a < 2 OR 3 < a OR a < 1 OR 4 < a -> a < 2 OR 3 < a
+    public void testCombineBinaryComparisonsDisjunctionOfLowerAndUpperBounds() {
+        FieldAttribute fa = getFieldAttribute();
+
+        LessThan lt1 = lessThanOf(fa, ONE);
+        LessThan lt2 = lessThanOf(fa, TWO);
+
+        GreaterThan gt3 = greaterThanOf(fa, THREE);
+        GreaterThan gt4 = greaterThanOf(fa, FOUR);
+
+        Or or = new Or(EMPTY, new Or(EMPTY, lt2, gt3), new Or(EMPTY, lt1, gt4));
+
+        CombineBinaryComparisons rule = new CombineBinaryComparisons();
+        Expression exp = rule.rule(or);
+        assertEquals(Or.class, exp.getClass());
+
+        Or ro = (Or) exp;
+
+        assertEquals(LessThan.class, ro.left().getClass());
+        LessThan lt = (LessThan) ro.left();
+        assertEquals(TWO, lt.right());
+        assertEquals(GreaterThan.class, ro.right().getClass());
+        GreaterThan gt = (GreaterThan) ro.right();
+        assertEquals(THREE, gt.right());
+    }
+
+    // (a = 1 AND b = 3 AND c = 4) OR (a = 2 AND b = 3 AND c = 4) -> (b = 3 AND c = 4) AND (a = 1 OR a = 2)
+    public void testBooleanSimplificationCommonExpressionSubstraction() {
+        FieldAttribute fa = EsqlTestUtils.getFieldAttribute("a");
+        FieldAttribute fb = EsqlTestUtils.getFieldAttribute("b");
+        FieldAttribute fc = EsqlTestUtils.getFieldAttribute("c");
+
+        Expression a1 = equalsOf(fa, ONE);
+        Expression a2 = equalsOf(fa, TWO);
+        And common = new And(EMPTY, equalsOf(fb, THREE), equalsOf(fc, FOUR));
+        And left = new And(EMPTY, a1, common);
+        And right = new And(EMPTY, a2, common);
+        Or or = new Or(EMPTY, left, right);
+
+        Expression exp = new BooleanSimplification().rule(or);
+        assertEquals(new And(EMPTY, common, new Or(EMPTY, a1, a2)), exp);
+    }
+
+    public void testBinaryComparisonAndOutOfRangeNotEqualsDifferentFields() {
+        FieldAttribute doubleOne = fieldAttribute("double", DOUBLE);
+        FieldAttribute doubleTwo = fieldAttribute("double2", DOUBLE);
+        FieldAttribute intOne = fieldAttribute("int", INTEGER);
+        FieldAttribute datetimeOne = fieldAttribute("datetime", INTEGER);
+        FieldAttribute keywordOne = fieldAttribute("keyword", KEYWORD);
+        FieldAttribute keywordTwo = fieldAttribute("keyword2", KEYWORD);
+
+        List<And> testCases = asList(
+            // double > 10 AND integer != -10
+            new And(EMPTY, greaterThanOf(doubleOne, L(10)), notEqualsOf(intOne, L(-10))),
+            // keyword > '5' AND keyword2 != '48'
+            new And(EMPTY, greaterThanOf(keywordOne, L("5")), notEqualsOf(keywordTwo, L("48"))),
+            // keyword != '2021' AND datetime <= '2020-12-04T17:48:22.954240Z'
+            new And(EMPTY, notEqualsOf(keywordOne, L("2021")), lessThanOrEqualOf(datetimeOne, L("2020-12-04T17:48:22.954240Z"))),
+            // double > 10.1 AND double2 != -10.1
+            new And(EMPTY, greaterThanOf(doubleOne, L(10.1d)), notEqualsOf(doubleTwo, L(-10.1d)))
+        );
+
+        for (And and : testCases) {
+            CombineBinaryComparisons rule = new CombineBinaryComparisons();
+            Expression exp = rule.rule(and);
+            assertEquals("Rule should not have transformed [" + and.nodeString() + "]", and, exp);
+        }
+    }
+
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsTests.java
new file mode 100644
index 0000000000000..2060327f1e18d
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsTests.java
@@ -0,0 +1,281 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.optimizer.rules;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
+import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+import org.elasticsearch.xpack.esql.plan.logical.Filter;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.FIVE;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.FOUR;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.ONE;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.SIX;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.THREE;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.TWO;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.equalsOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation;
+import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
+import static org.hamcrest.Matchers.contains;
+
+public class CombineDisjunctionsTests extends ESTestCase {
+
+    public void testTwoEqualsWithOr() {
+        FieldAttribute fa = getFieldAttribute();
+
+        Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO));
+        Expression e = new CombineDisjunctions().rule(or);
+        assertEquals(In.class, e.getClass());
+        In in = (In) e;
+        assertEquals(fa, in.value());
+        assertThat(in.list(), contains(ONE, TWO));
+    }
+
+    public void testTwoEqualsWithSameValue() {
+        FieldAttribute fa = getFieldAttribute();
+
+        Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, ONE));
+        Expression e = new CombineDisjunctions().rule(or);
+        assertEquals(Equals.class, e.getClass());
+        Equals eq = (Equals) e;
+        assertEquals(fa, eq.left());
+        assertEquals(ONE, eq.right());
+    }
+
+    public void testOneEqualsOneIn() {
+        FieldAttribute fa = getFieldAttribute();
+
+        Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, List.of(TWO)));
+        Expression e = new CombineDisjunctions().rule(or);
+        assertEquals(In.class, e.getClass());
+        In in = (In) e;
+        assertEquals(fa, in.value());
+        assertThat(in.list(), contains(ONE, TWO));
+    }
+
+    public void testOneEqualsOneInWithSameValue() {
+        FieldAttribute fa = getFieldAttribute();
+
+        Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, asList(ONE, TWO)));
+        Expression e = new CombineDisjunctions().rule(or);
+        assertEquals(In.class, e.getClass());
+        In in = (In) e;
+        assertEquals(fa, in.value());
+        assertThat(in.list(), contains(ONE, TWO));
+    }
+
+    public void testSingleValueInToEquals() {
+        FieldAttribute fa = getFieldAttribute();
+
+        Equals equals = equalsOf(fa, ONE);
+        Or or = new Or(EMPTY, equals, new In(EMPTY, fa, List.of(ONE)));
+        Expression e = new CombineDisjunctions().rule(or);
+        assertEquals(equals, e);
+    }
+
+    public void testEqualsBehindAnd() {
+        FieldAttribute fa = getFieldAttribute();
+
+        And and = new And(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO));
+        Filter dummy = new Filter(EMPTY, relation(), and);
+        LogicalPlan transformed = new CombineDisjunctions().apply(dummy);
+        assertSame(dummy, transformed);
+        assertEquals(and, ((Filter) transformed).condition());
+    }
+
+    public void testTwoEqualsDifferentFields() {
+        FieldAttribute fieldOne = getFieldAttribute("ONE");
+        FieldAttribute fieldTwo = getFieldAttribute("TWO");
+
+        Or or = new Or(EMPTY, equalsOf(fieldOne, ONE), equalsOf(fieldTwo, TWO));
+        Expression e = new CombineDisjunctions().rule(or);
+        assertEquals(or, e);
+    }
+
+    public void testMultipleIn() {
+        FieldAttribute fa = getFieldAttribute();
+
+        Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), new In(EMPTY, fa, List.of(TWO)));
+        Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE)));
+        Expression e = new CombineDisjunctions().rule(secondOr);
+        assertEquals(In.class, e.getClass());
+        In in = (In) e;
+        assertEquals(fa, in.value());
+        assertThat(in.list(), contains(ONE, TWO, THREE));
+    }
+
+    public void testOrWithNonCombinableExpressions() {
+        FieldAttribute fa = getFieldAttribute();
+
+        Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), lessThanOf(fa, TWO));
+        Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE)));
+        Expression e = new CombineDisjunctions().rule(secondOr);
+        assertEquals(Or.class, e.getClass());
+        Or or = (Or) e;
+        assertEquals(or.left(), firstOr.right());
+        assertEquals(In.class, or.right().getClass());
+        In in = (In) or.right();
+        assertEquals(fa, in.value());
+        assertThat(in.list(), contains(ONE, THREE));
+    }
+
+    public void testCombineCIDRMatch() {
+        FieldAttribute faa = getFieldAttribute("a");
+        FieldAttribute fab = getFieldAttribute("b");
+
+        List<Expression> ipa1 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP));
+        List<Expression> ipa2 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP));
+        List<Expression> ipb1 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP));
+        List<Expression> ipb2 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP));
+
+        List<Expression> ipa = new HashSet<>(CollectionUtils.combine(ipa1, ipa2)).stream().toList();
+        List<Expression> ipb = new HashSet<>(CollectionUtils.combine(ipb1, ipb2)).stream().toList();
+
+        List<Expression> cidrs = new ArrayList<>(4);
+        cidrs.add(new CIDRMatch(EMPTY, faa, ipa1));
+        cidrs.add(new CIDRMatch(EMPTY, fab, ipb1));
+        cidrs.add(new CIDRMatch(EMPTY, faa, ipa2));
+        cidrs.add(new CIDRMatch(EMPTY, fab, ipb2));
+        Or oldOr = (Or) Predicates.combineOr(cidrs);
+        Expression e = new CombineDisjunctions().rule(oldOr);
+        assertEquals(Or.class, e.getClass());
+        Or newOr = (Or) e;
+        assertEquals(CIDRMatch.class, newOr.left().getClass());
+        assertEquals(CIDRMatch.class, newOr.right().getClass());
+
+        CIDRMatch cm1 = (CIDRMatch) newOr.left();
+        CIDRMatch cm2 = (CIDRMatch) newOr.right();
+
+        if (cm1.ipField() == faa) {
+            assertEquals(fab, cm2.ipField());
+            assertTrue(cm1.matches().size() == ipa.size() && cm1.matches().containsAll(ipa) && ipa.containsAll(cm1.matches()));
+            assertTrue(cm2.matches().size() == ipb.size() && cm2.matches().containsAll(ipb) && ipb.containsAll(cm2.matches()));
+        } else {
+            assertEquals(fab, cm1.ipField());
+            assertTrue(cm1.matches().size() == ipb.size() && cm1.matches().containsAll(ipb) && ipb.containsAll(cm1.matches()));
+            assertTrue(cm2.matches().size() == ipa.size() && cm2.matches().containsAll(ipa) && ipa.containsAll(cm2.matches()));
+        }
+    }
+
+    public void testCombineCIDRMatchEqualsIns() {
+        FieldAttribute fa_cidr_a = getFieldAttribute("cidr_a");
+        FieldAttribute fa_cidr_b = getFieldAttribute("cidr_b");
+        FieldAttribute fa_in_a = getFieldAttribute("in_a");
+        FieldAttribute fa_eqin_a = getFieldAttribute("eqin_a");
+        FieldAttribute fa_eq_a = getFieldAttribute("eq_a");
+
+        List<Expression> ipa1 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP));
+        List<Expression> ipa2 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP));
+        List<Expression> ipb1 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP));
+        List<Expression> ipb2 = randomList(1, 5, () -> new Literal(EMPTY, randomLiteral(DataType.IP).value(), DataType.IP));
+
+        List<Expression> ipa = new HashSet<>(CollectionUtils.combine(ipa1, ipa2)).stream().toList();
+        List<Expression> ipb = new HashSet<>(CollectionUtils.combine(ipb1, ipb2)).stream().toList();
+
+        List<Expression> all = new ArrayList<>(10);
+        all.add(new CIDRMatch(EMPTY, fa_cidr_a, ipa1));
+        all.add(new CIDRMatch(EMPTY, fa_cidr_b, ipb1));
+        all.add(new CIDRMatch(EMPTY, fa_cidr_a, ipa2));
+        all.add(new CIDRMatch(EMPTY, fa_cidr_b, ipb2));
+
+        all.add(new In(EMPTY, fa_in_a, List.of(ONE, TWO, THREE)));
+        all.add(new In(EMPTY, fa_eqin_a, List.of(FOUR, FIVE, SIX)));
+        all.add(new In(EMPTY, fa_in_a, List.of(THREE, FOUR, FIVE)));
+
+        all.add(new Equals(EMPTY, fa_eq_a, ONE));
+        all.add(new Equals(EMPTY, fa_eqin_a, ONE));
+        all.add(new Equals(EMPTY, fa_eq_a, SIX));
+
+        Or oldOr = (Or) Predicates.combineOr(all);
+
+        Expression e = new CombineDisjunctions().rule(oldOr);
+        assertEquals(Or.class, e.getClass());
+        Or newOr = (Or) e;
+        assertEquals(Or.class, newOr.left().getClass());
+        Or newOr1 = (Or) newOr.left();
+        assertEquals(CIDRMatch.class, newOr.right().getClass());
+        CIDRMatch cidr2 = (CIDRMatch) newOr.right();
+        assertEquals(Or.class, e.getClass());
+
+        assertEquals(Or.class, newOr1.left().getClass());
+        Or newOr2 = (Or) newOr1.left();
+        assertEquals(In.class, newOr2.left().getClass());
+        In in1 = (In) newOr2.left();
+        assertEquals(In.class, newOr2.right().getClass());
+        In in2 = (In) newOr2.right();
+        assertEquals(Or.class, newOr1.right().getClass());
+        Or newOr3 = (Or) newOr1.right();
+        assertEquals(In.class, newOr3.left().getClass());
+        In in3 = (In) newOr3.left();
+        assertEquals(CIDRMatch.class, newOr3.right().getClass());
+        CIDRMatch cidr1 = (CIDRMatch) newOr3.right();
+
+        if (cidr1.ipField() == fa_cidr_a) {
+            assertEquals(fa_cidr_b, cidr2.ipField());
+            assertTrue(cidr1.matches().size() == ipa.size() && cidr1.matches().containsAll(ipa) && ipa.containsAll(cidr1.matches()));
+            assertTrue(cidr2.matches().size() == ipb.size() && cidr2.matches().containsAll(ipb) && ipb.containsAll(cidr2.matches()));
+        } else {
+            assertEquals(fa_cidr_b, cidr1.ipField());
+            assertTrue(cidr1.matches().size() == ipb.size() && cidr1.matches().containsAll(ipb) && ipb.containsAll(cidr1.matches()));
+            assertTrue(cidr2.matches().size() == ipa.size() && cidr2.matches().containsAll(ipa) && ipa.containsAll(cidr2.matches()));
+        }
+
+        if (in1.value() == fa_in_a) {
+            assertTrue(in1.list().size() == 5 && in1.list().containsAll(List.of(ONE, TWO, THREE, FOUR, FIVE)));
+            if (in2.value() == fa_eqin_a) {
+                assertEquals(in3.value(), fa_eq_a);
+                assertTrue(in3.list().size() == 2 && in3.list().containsAll(List.of(ONE, SIX)));
+                assertTrue(in2.list().size() == 4 && in2.list().containsAll(List.of(ONE, FOUR, FIVE, SIX)));
+            } else {
+                assertEquals(in2.value(), fa_eq_a);
+                assertTrue(in2.list().size() == 2 && in2.list().containsAll(List.of(ONE, SIX)));
+                assertTrue(in3.list().size() == 4 && in3.list().containsAll(List.of(ONE, FOUR, FIVE, SIX)));
+            }
+        } else if (in2.value() == fa_in_a) {
+            assertTrue(in2.list().size() == 5 && in2.list().containsAll(List.of(ONE, TWO, THREE, FOUR, FIVE)));
+            if (in1.value() == fa_eqin_a) {
+                assertEquals(in3.value(), fa_eq_a);
+                assertTrue(in3.list().size() == 2 && in3.list().containsAll(List.of(ONE, SIX)));
+                assertTrue(in1.list().size() == 4 && in1.list().containsAll(List.of(ONE, FOUR, FIVE, SIX)));
+            } else {
+                assertEquals(in3.value(), fa_eq_a);
+                assertTrue(in3.list().size() == 2 && in3.list().containsAll(List.of(ONE, SIX)));
+                assertTrue(in1.list().size() == 4 && in1.list().containsAll(List.of(ONE, FOUR, FIVE, SIX)));
+            }
+        } else {
+            assertTrue(in3.list().size() == 5 && in3.list().containsAll(List.of(ONE, TWO, THREE, FOUR, FIVE)));
+            if (in1.value() == fa_eqin_a) {
+                assertEquals(in2.value(), fa_eq_a);
+                assertTrue(in2.list().size() == 2 && in2.list().containsAll(List.of(ONE, SIX)));
+                assertTrue(in1.list().size() == 4 && in1.list().containsAll(List.of(ONE, FOUR, FIVE, SIX)));
+            } else {
+                assertEquals(in1.value(), fa_eq_a);
+                assertTrue(in1.list().size() == 2 && in1.list().containsAll(List.of(ONE, SIX)));
+                assertTrue(in2.list().size() == 4 && in2.list().containsAll(List.of(ONE, FOUR, FIVE, SIX)));
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToInTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToInTests.java
deleted file mode 100644
index 7bc2d69cb56e6..0000000000000
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToInTests.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.optimizer.rules;
-
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
-import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
-import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
-import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
-import org.elasticsearch.xpack.esql.plan.logical.Filter;
-import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
-
-import java.util.List;
-
-import static java.util.Arrays.asList;
-import static org.elasticsearch.xpack.esql.EsqlTestUtils.ONE;
-import static org.elasticsearch.xpack.esql.EsqlTestUtils.THREE;
-import static org.elasticsearch.xpack.esql.EsqlTestUtils.TWO;
-import static org.elasticsearch.xpack.esql.EsqlTestUtils.equalsOf;
-import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute;
-import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf;
-import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation;
-import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
-import static org.hamcrest.Matchers.contains;
-
-public class CombineDisjunctionsToInTests extends ESTestCase {
-    public void testTwoEqualsWithOr() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(In.class, e.getClass());
-        In in = (In) e;
-        assertEquals(fa, in.value());
-        assertThat(in.list(), contains(ONE, TWO));
-    }
-
-    public void testTwoEqualsWithSameValue() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, ONE));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(Equals.class, e.getClass());
-        Equals eq = (Equals) e;
-        assertEquals(fa, eq.left());
-        assertEquals(ONE, eq.right());
-    }
-
-    public void testOneEqualsOneIn() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, List.of(TWO)));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(In.class, e.getClass());
-        In in = (In) e;
-        assertEquals(fa, in.value());
-        assertThat(in.list(), contains(ONE, TWO));
-    }
-
-    public void testOneEqualsOneInWithSameValue() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, asList(ONE, TWO)));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(In.class, e.getClass());
-        In in = (In) e;
-        assertEquals(fa, in.value());
-        assertThat(in.list(), contains(ONE, TWO));
-    }
-
-    public void testSingleValueInToEquals() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Equals equals = equalsOf(fa, ONE);
-        Or or = new Or(EMPTY, equals, new In(EMPTY, fa, List.of(ONE)));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(equals, e);
-    }
-
-    public void testEqualsBehindAnd() {
-        FieldAttribute fa = getFieldAttribute();
-
-        And and = new And(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO));
-        Filter dummy = new Filter(EMPTY, relation(), and);
-        LogicalPlan transformed = new CombineDisjunctionsToIn().apply(dummy);
-        assertSame(dummy, transformed);
-        assertEquals(and, ((Filter) transformed).condition());
-    }
-
-    public void testTwoEqualsDifferentFields() {
-        FieldAttribute fieldOne = getFieldAttribute("ONE");
-        FieldAttribute fieldTwo = getFieldAttribute("TWO");
-
-        Or or = new Or(EMPTY, equalsOf(fieldOne, ONE), equalsOf(fieldTwo, TWO));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(or, e);
-    }
-
-    public void testMultipleIn() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), new In(EMPTY, fa, List.of(TWO)));
-        Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE)));
-        Expression e = new CombineDisjunctionsToIn().rule(secondOr);
-        assertEquals(In.class, e.getClass());
-        In in = (In) e;
-        assertEquals(fa, in.value());
-        assertThat(in.list(), contains(ONE, TWO, THREE));
-    }
-
-    public void testOrWithNonCombinableExpressions() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), lessThanOf(fa, TWO));
-        Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE)));
-        Expression e = new CombineDisjunctionsToIn().rule(secondOr);
-        assertEquals(Or.class, e.getClass());
-        Or or = (Or) e;
-        assertEquals(or.left(), firstOr.right());
-        assertEquals(In.class, or.right().getClass());
-        In in = (In) or.right();
-        assertEquals(fa, in.value());
-        assertThat(in.list(), contains(ONE, THREE));
-    }
-}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java
index d575ba1fcb55a..697dfcf0a8e6b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
-import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
@@ -55,10 +54,6 @@ static UnresolvedAttribute attribute(String name) {
         return new UnresolvedAttribute(EMPTY, name);
     }
 
-    static ReferenceAttribute referenceAttribute(String name, DataType type) {
-        return new ReferenceAttribute(EMPTY, name, type);
-    }
-
     static Literal integer(int i) {
         return new Literal(EMPTY, i, DataType.INTEGER);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java
new file mode 100644
index 0000000000000..18d8bc9fb0a75
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.parser;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+
+public class GrammarInDevelopmentParsingTests extends ESTestCase {
+
+    public void testDevelopmentInline() throws Exception {
+        parse("row a = 1 | inlinestats b = min(a) by c, d.e", "inlinestats");
+    }
+
+    public void testDevelopmentLookup() throws Exception {
+        parse("row a = 1 | lookup \"foo\" on j", "lookup");
+    }
+
+    public void testDevelopmentMetrics() throws Exception {
+        parse("metrics foo", "metrics");
+    }
+
+    public void testDevelopmentMatch() throws Exception {
+        parse("row a = 1 | match foo", "match");
+    }
+
+    void parse(String query, String errorMessage) {
+        ParsingException pe = expectThrows(ParsingException.class, () -> parser().createStatement(query));
+        assertThat(pe.getMessage(), containsString("mismatched input '" + errorMessage + "'"));
+        // check the parser eliminated the DEV_ tokens from the message
+        assertThat(pe.getMessage(), not(containsString("DEV_")));
+    }
+
+    private EsqlParser parser() {
+        EsqlParser parser = new EsqlParser();
+        EsqlConfig config = parser.config();
+        assumeTrue(" requires snapshot builds", config.devVersion);
+
+        // manually disable dev mode (make it production)
+        config.setDevVersion(false);
+        return parser;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
index 2a4c5eeeeaaea..6980171a7bcd7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
@@ -18,12 +18,12 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
+import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.StringQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
-import org.elasticsearch.xpack.esql.core.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
@@ -33,6 +33,7 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.esql.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Dissect;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
@@ -50,11 +51,13 @@
 import org.elasticsearch.xpack.esql.plan.logical.Row;
 import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.referenceAttribute;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
@@ -310,6 +313,7 @@ public void testAggsWithGroupKeyAsAgg() throws Exception {
     }
 
     public void testInlineStatsWithGroups() {
+        assumeTrue("INLINESTATS requires snapshot builds", Build.current().isSnapshot());
         assertEquals(
             new InlineStats(
                 EMPTY,
@@ -326,6 +330,7 @@ public void testInlineStatsWithGroups() {
     }
 
     public void testInlineStatsWithoutGroups() {
+        assumeTrue("INLINESTATS requires snapshot builds", Build.current().isSnapshot());
         assertEquals(
             new InlineStats(
                 EMPTY,
@@ -786,6 +791,11 @@ public void testGrokPattern() {
             "line 1:22: Invalid GROK pattern [%{NUMBER:foo} %{WORD:foo}]:"
                 + " the attribute [foo] is defined multiple times with different types"
         );
+
+        expectError(
+            "row a = \"foo\" | GROK a \"(?P<justification>.+)\"",
+            "line 1:18: Invalid grok pattern [(?P<justification>.+)]: [undefined group option]"
+        );
     }
 
     public void testLikeRLike() {
@@ -962,6 +972,17 @@ public void testInputParams() {
         assertThat(alias.child().fold(), is(11));
     }
 
+    public void testMatchCommand() throws IOException {
+        assumeTrue("Match command available just for snapshots", Build.current().isSnapshot());
+        String queryString = "field: value";
+        assertEquals(
+            new Filter(EMPTY, PROCESSING_CMD_INPUT, new StringQueryPredicate(EMPTY, queryString, null)),
+            processingCommand("match \"" + queryString + "\"")
+        );
+
+        expectError("from a | match an unquoted string", "mismatched input 'an' expecting QUOTED_STRING");
+    }
+
     public void testMissingInputParams() {
         expectError("row x = ?, y = ?", List.of(new QueryParam(null, 1, INTEGER)), "Not enough actual parameters 1");
     }
@@ -998,7 +1019,7 @@ public void testInvalidNamedParams() {
             "Unknown query parameter [n2], did you mean any of [n3, n1]?"
         );
 
-        expectError("from test | where x < ?_1", List.of(new QueryParam("_1", 5, INTEGER)), "extraneous input '_1' expecting <EOF>");
+        expectError("from test | where x < ?@1", List.of(new QueryParam("@1", 5, INTEGER)), "extraneous input '@1' expecting <EOF>");
 
         expectError("from test | where x < ?#1", List.of(new QueryParam("#1", 5, INTEGER)), "token recognition error at: '#'");
 
@@ -1007,6 +1028,10 @@ public void testInvalidNamedParams() {
             List.of(new QueryParam("n_1", 5, INTEGER), new QueryParam("n_2", 5, INTEGER)),
             "extraneous input '?' expecting <EOF>"
         );
+
+        expectError("from test | where x < ?Å", List.of(new QueryParam("Å", 5, INTEGER)), "line 1:24: token recognition error at: 'Å'");
+
+        expectError("from test | eval x = ?Å", List.of(new QueryParam("Å", 5, INTEGER)), "line 1:23: token recognition error at: 'Å'");
     }
 
     public void testPositionalParams() {
@@ -1048,12 +1073,6 @@ public void testInvalidPositionalParams() {
                 + "line 1:35: No parameter is defined for position 2, did you mean position 1?"
         );
 
-        expectError(
-            "from test | where x < ?0 and y < ?2",
-            List.of(new QueryParam(null, 5, INTEGER)),
-            "No parameter is defined for position 2, did you mean position 1"
-        );
-
         expectError(
             "from test | where x < ?0",
             List.of(new QueryParam(null, 5, INTEGER), new QueryParam(null, 10, INTEGER)),
@@ -1086,6 +1105,18 @@ public void testParamInWhere() {
         assertThat(limit.children().get(0).children().size(), equalTo(1));
         assertThat(limit.children().get(0).children().get(0), instanceOf(UnresolvedRelation.class));
 
+        plan = statement("from test | where x < ?_n1 |  limit 10", new QueryParams(List.of(new QueryParam("_n1", 5, INTEGER))));
+        assertThat(plan, instanceOf(Limit.class));
+        limit = (Limit) plan;
+        assertThat(limit.limit(), instanceOf(Literal.class));
+        assertThat(((Literal) limit.limit()).value(), equalTo(10));
+        assertThat(limit.children().size(), equalTo(1));
+        assertThat(limit.children().get(0), instanceOf(Filter.class));
+        w = (Filter) limit.children().get(0);
+        assertThat(((Literal) w.condition().children().get(1)).value(), equalTo(5));
+        assertThat(limit.children().get(0).children().size(), equalTo(1));
+        assertThat(limit.children().get(0).children().get(0), instanceOf(UnresolvedRelation.class));
+
         plan = statement("from test | where x < ?1 |  limit 10", new QueryParams(List.of(new QueryParam(null, 5, INTEGER))));
         assertThat(plan, instanceOf(Limit.class));
         limit = (Limit) plan;
@@ -1097,6 +1128,18 @@ public void testParamInWhere() {
         assertThat(((Literal) w.condition().children().get(1)).value(), equalTo(5));
         assertThat(limit.children().get(0).children().size(), equalTo(1));
         assertThat(limit.children().get(0).children().get(0), instanceOf(UnresolvedRelation.class));
+
+        plan = statement("from test | where x < ?__1 |  limit 10", new QueryParams(List.of(new QueryParam("__1", 5, INTEGER))));
+        assertThat(plan, instanceOf(Limit.class));
+        limit = (Limit) plan;
+        assertThat(limit.limit(), instanceOf(Literal.class));
+        assertThat(((Literal) limit.limit()).value(), equalTo(10));
+        assertThat(limit.children().size(), equalTo(1));
+        assertThat(limit.children().get(0), instanceOf(Filter.class));
+        w = (Filter) limit.children().get(0);
+        assertThat(((Literal) w.condition().children().get(1)).value(), equalTo(5));
+        assertThat(limit.children().get(0).children().size(), equalTo(1));
+        assertThat(limit.children().get(0).children().get(0), instanceOf(UnresolvedRelation.class));
     }
 
     public void testParamInEval() {
@@ -1140,6 +1183,26 @@ public void testParamInEval() {
         assertThat(f.children().size(), equalTo(1));
         assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class));
 
+        plan = statement(
+            "from test | where x < ?_n1 | eval y = ?_n2 + ?_n3 |  limit 10",
+            new QueryParams(
+                List.of(new QueryParam("_n1", 5, INTEGER), new QueryParam("_n2", -1, INTEGER), new QueryParam("_n3", 100, INTEGER))
+            )
+        );
+        assertThat(plan, instanceOf(Limit.class));
+        limit = (Limit) plan;
+        assertThat(limit.limit(), instanceOf(Literal.class));
+        assertThat(((Literal) limit.limit()).value(), equalTo(10));
+        assertThat(limit.children().size(), equalTo(1));
+        assertThat(limit.children().get(0), instanceOf(Eval.class));
+        eval = (Eval) limit.children().get(0);
+        assertThat(((Literal) ((Add) eval.fields().get(0).child()).left()).value(), equalTo(-1));
+        assertThat(((Literal) ((Add) eval.fields().get(0).child()).right()).value(), equalTo(100));
+        f = (Filter) eval.children().get(0);
+        assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5));
+        assertThat(f.children().size(), equalTo(1));
+        assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class));
+
         plan = statement(
             "from test | where x < ?1 | eval y = ?2 + ?1 |  limit 10",
             new QueryParams(List.of(new QueryParam(null, 5, INTEGER), new QueryParam(null, -1, INTEGER)))
@@ -1157,6 +1220,24 @@ public void testParamInEval() {
         assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5));
         assertThat(f.children().size(), equalTo(1));
         assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class));
+
+        plan = statement(
+            "from test | where x < ?_1 | eval y = ?_2 + ?_1 |  limit 10",
+            new QueryParams(List.of(new QueryParam("_1", 5, INTEGER), new QueryParam("_2", -1, INTEGER)))
+        );
+        assertThat(plan, instanceOf(Limit.class));
+        limit = (Limit) plan;
+        assertThat(limit.limit(), instanceOf(Literal.class));
+        assertThat(((Literal) limit.limit()).value(), equalTo(10));
+        assertThat(limit.children().size(), equalTo(1));
+        assertThat(limit.children().get(0), instanceOf(Eval.class));
+        eval = (Eval) limit.children().get(0);
+        assertThat(((Literal) ((Add) eval.fields().get(0).child()).left()).value(), equalTo(-1));
+        assertThat(((Literal) ((Add) eval.fields().get(0).child()).right()).value(), equalTo(5));
+        f = (Filter) eval.children().get(0);
+        assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5));
+        assertThat(f.children().size(), equalTo(1));
+        assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class));
     }
 
     public void testParamInAggFunction() {
@@ -1210,6 +1291,31 @@ public void testParamInAggFunction() {
         assertThat(f.children().size(), equalTo(1));
         assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class));
 
+        plan = statement(
+            "from test | where x < ?_n1 | eval y = ?_n2 + ?_n3 |  stats count(?_n4) by z",
+            new QueryParams(
+                List.of(
+                    new QueryParam("_n1", 5, INTEGER),
+                    new QueryParam("_n2", -1, INTEGER),
+                    new QueryParam("_n3", 100, INTEGER),
+                    new QueryParam("_n4", "*", KEYWORD)
+                )
+            )
+        );
+        assertThat(plan, instanceOf(Aggregate.class));
+        agg = (Aggregate) plan;
+        assertThat(((Literal) agg.aggregates().get(0).children().get(0).children().get(0)).value(), equalTo("*"));
+        assertThat(agg.child(), instanceOf(Eval.class));
+        assertThat(agg.children().size(), equalTo(1));
+        assertThat(agg.children().get(0), instanceOf(Eval.class));
+        eval = (Eval) agg.children().get(0);
+        assertThat(((Literal) ((Add) eval.fields().get(0).child()).left()).value(), equalTo(-1));
+        assertThat(((Literal) ((Add) eval.fields().get(0).child()).right()).value(), equalTo(100));
+        f = (Filter) eval.children().get(0);
+        assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5));
+        assertThat(f.children().size(), equalTo(1));
+        assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class));
+
         plan = statement(
             "from test | where x < ?1 | eval y = ?2 + ?1 |  stats count(?3) by z",
             new QueryParams(
@@ -1229,6 +1335,26 @@ public void testParamInAggFunction() {
         assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5));
         assertThat(f.children().size(), equalTo(1));
         assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class));
+
+        plan = statement(
+            "from test | where x < ?_1 | eval y = ?_2 + ?_1 |  stats count(?_3) by z",
+            new QueryParams(
+                List.of(new QueryParam("_1", 5, INTEGER), new QueryParam("_2", -1, INTEGER), new QueryParam("_3", "*", KEYWORD))
+            )
+        );
+        assertThat(plan, instanceOf(Aggregate.class));
+        agg = (Aggregate) plan;
+        assertThat(((Literal) agg.aggregates().get(0).children().get(0).children().get(0)).value(), equalTo("*"));
+        assertThat(agg.child(), instanceOf(Eval.class));
+        assertThat(agg.children().size(), equalTo(1));
+        assertThat(agg.children().get(0), instanceOf(Eval.class));
+        eval = (Eval) agg.children().get(0);
+        assertThat(((Literal) ((Add) eval.fields().get(0).child()).left()).value(), equalTo(-1));
+        assertThat(((Literal) ((Add) eval.fields().get(0).child()).right()).value(), equalTo(5));
+        f = (Filter) eval.children().get(0);
+        assertThat(((Literal) f.condition().children().get(1)).value(), equalTo(5));
+        assertThat(f.children().size(), equalTo(1));
+        assertThat(f.children().get(0), instanceOf(UnresolvedRelation.class));
     }
 
     public void testParamMixed() {
@@ -1245,24 +1371,36 @@ public void testParamMixed() {
         );
 
         expectError(
-            "from test | where x < ?1 | eval y = ?n2 + ?n3 |  limit ?n4",
+            "from test | where x < ? | eval y = ?_n2 + ?n3 |  limit ?_4",
             List.of(
                 new QueryParam("n1", 5, INTEGER),
-                new QueryParam("n2", -1, INTEGER),
+                new QueryParam("_n2", -1, INTEGER),
                 new QueryParam("n3", 100, INTEGER),
                 new QueryParam("n4", 10, INTEGER)
             ),
+            "Inconsistent parameter declaration, "
+                + "use one of positional, named or anonymous params but not a combination of named and anonymous"
+        );
+
+        expectError(
+            "from test | where x < ?1 | eval y = ?n2 + ?_n3 |  limit ?n4",
+            List.of(
+                new QueryParam("n1", 5, INTEGER),
+                new QueryParam("n2", -1, INTEGER),
+                new QueryParam("_n3", 100, INTEGER),
+                new QueryParam("n4", 10, INTEGER)
+            ),
             "Inconsistent parameter declaration, "
                 + "use one of positional, named or anonymous params but not a combination of named and positional"
         );
 
         expectError(
-            "from test | where x < ? | eval y = ?2 + ?n3 |  limit ?n4",
+            "from test | where x < ? | eval y = ?2 + ?n3 |  limit ?_n4",
             List.of(
                 new QueryParam("n1", 5, INTEGER),
                 new QueryParam("n2", -1, INTEGER),
                 new QueryParam("n3", 100, INTEGER),
-                new QueryParam("n4", 10, INTEGER)
+                new QueryParam("_n4", 10, INTEGER)
             ),
             "Inconsistent parameter declaration, "
                 + "use one of positional, named or anonymous params but not a combination of positional and anonymous"
@@ -1515,6 +1653,22 @@ public void testSimpleMetricsWithStats() {
         );
     }
 
+    public void testInvalidAlias() {
+        expectError("row Å = 1", "line 1:5: token recognition error at: 'Å'");
+        expectError("from test | eval Å = 1", "line 1:18: token recognition error at: 'Å'");
+        expectError("from test | where Å == 1", "line 1:19: token recognition error at: 'Å'");
+        expectError("from test | keep Å", "line 1:18: token recognition error at: 'Å'");
+        expectError("from test | drop Å", "line 1:18: token recognition error at: 'Å'");
+        expectError("from test | sort Å", "line 1:18: token recognition error at: 'Å'");
+        expectError("from test | rename Å as A", "line 1:20: token recognition error at: 'Å'");
+        expectError("from test | rename A as Å", "line 1:25: token recognition error at: 'Å'");
+        expectError("from test | rename Å as Å", "line 1:20: token recognition error at: 'Å'");
+        expectError("from test | stats Å = count(*)", "line 1:19: token recognition error at: 'Å'");
+        expectError("from test | stats count(Å)", "line 1:25: token recognition error at: 'Å'");
+        expectError("from test | eval A = coalesce(Å, null)", "line 1:31: token recognition error at: 'Å'");
+        expectError("from test | eval A = coalesce(\"Å\", Å)", "line 1:36: token recognition error at: 'Å'");
+    }
+
     private LogicalPlan unresolvedRelation(String index) {
         return new UnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, index), false, List.of(), IndexMode.STANDARD, null);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/AbstractNodeSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/AbstractNodeSerializationTests.java
new file mode 100644
index 0000000000000..e6f5d6e4fac70
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/AbstractNodeSerializationTests.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.test.AbstractWireTestCase;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.tree.Node;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests;
+import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+import org.elasticsearch.xpack.esql.session.Configuration;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
+
+/**
+ * Superclass for serialization tests for all {@link Node} subclasses
+ * @param <T>
+ */
+public abstract class AbstractNodeSerializationTests<T extends Node<? super T>> extends AbstractWireTestCase<T> {
+    /**
+     * We use a single random config for all serialization because it's pretty
+     * heavy to build, especially in {@link #testConcurrentSerialization()}.
+     */
+    private Configuration config;
+
+    public static Source randomSource() {
+        int lineNumber = between(0, EXAMPLE_QUERY.length - 1);
+        String line = EXAMPLE_QUERY[lineNumber];
+        int offset = between(0, line.length() - 2);
+        int length = between(1, line.length() - offset - 1);
+        String text = line.substring(offset, offset + length);
+        return new Source(lineNumber + 1, offset, text);
+    }
+
+    public static List<Attribute> randomFieldAttributes(int min, int max, boolean onlyRepresentable) {
+        return randomList(min, max, () -> FieldAttributeTests.createFieldAttribute(0, onlyRepresentable));
+    }
+
+    @Override
+    protected final T copyInstance(T instance, TransportVersion version) throws IOException {
+        return copyInstance(
+            instance,
+            getNamedWriteableRegistry(),
+            (out, v) -> new PlanStreamOutput(out, new PlanNameRegistry(), configuration()).writeNamedWriteable(v),
+            in -> {
+                PlanStreamInput pin = new PlanStreamInput(in, new PlanNameRegistry(), in.namedWriteableRegistry(), configuration());
+                @SuppressWarnings("unchecked")
+                T deser = (T) pin.readNamedWriteable(categoryClass());
+                if (alwaysEmptySource()) {
+                    assertThat(deser.source(), sameInstance(Source.EMPTY));
+                } else {
+                    assertThat(deser.source(), equalTo(instance.source()));
+                }
+                return deser;
+            },
+            version
+        );
+    }
+
+    protected abstract Class<? extends Node<?>> categoryClass();
+
+    protected boolean alwaysEmptySource() {
+        return false;
+    }
+
+    public final Configuration configuration() {
+        return config;
+    }
+
+    private static final String[] EXAMPLE_QUERY = new String[] {
+        "I am the very model of a modern Major-Gineral,",
+        "I've information vegetable, animal, and mineral,",
+        "I know the kings of England, and I quote the fights historical",
+        "From Marathon to Waterloo, in order categorical;",
+        "I'm very well acquainted, too, with matters mathematical,",
+        "I understand equations, both the simple and quadratical,",
+        "About binomial theorem I'm teeming with a lot o' news,",
+        "With many cheerful facts about the square of the hypotenuse." };
+
+    @Before
+    public void initConfig() {
+        config = randomConfiguration(String.join("\n", EXAMPLE_QUERY), Map.of());
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/AbstractLogicalPlanSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/AbstractLogicalPlanSerializationTests.java
new file mode 100644
index 0000000000000..6936c96a143d4
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/AbstractLogicalPlanSerializationTests.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.tree.Node;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
+import org.elasticsearch.xpack.esql.plan.AbstractNodeSerializationTests;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelationSerializationTests;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class AbstractLogicalPlanSerializationTests<T extends LogicalPlan> extends AbstractNodeSerializationTests<T> {
+    public static LogicalPlan randomChild(int depth) {
+        if (randomBoolean() && depth < 4) {
+            // TODO more random options
+            return LookupSerializationTests.randomLookup(depth + 1);
+        }
+        return randomBoolean() ? EsRelationSerializationTests.randomEsRelation() : LocalRelationSerializationTests.randomLocalRelation();
+    }
+
+    @Override
+    protected final NamedWriteableRegistry getNamedWriteableRegistry() {
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.addAll(LogicalPlan.getNamedWriteables());
+        entries.addAll(AggregateFunction.getNamedWriteables());
+        entries.addAll(Expression.getNamedWriteables());
+        entries.addAll(Attribute.getNamedWriteables());
+        entries.addAll(Block.getNamedWriteables());
+        entries.addAll(NamedExpression.getNamedWriteables());
+        return new NamedWriteableRegistry(entries);
+    }
+
+    @Override
+    protected final Class<? extends Node<?>> categoryClass() {
+        return LogicalPlan.class;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/AggregateSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/AggregateSerializationTests.java
new file mode 100644
index 0000000000000..01f797491103c
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/AggregateSerializationTests.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Alias;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.Max;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.Min;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.Sum;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.Top;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.Values;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class AggregateSerializationTests extends AbstractLogicalPlanSerializationTests<Aggregate> {
+    @Override
+    protected Aggregate createTestInstance() {
+        Source source = randomSource();
+        LogicalPlan child = randomChild(0);
+        Aggregate.AggregateType aggregateType = randomFrom(Aggregate.AggregateType.values());
+        List<Expression> groupings = randomFieldAttributes(0, 5, false).stream().map(a -> (Expression) a).toList();
+        List<? extends NamedExpression> aggregates = randomAggregates();
+        return new Aggregate(source, child, aggregateType, groupings, aggregates);
+    }
+
+    public static List<? extends NamedExpression> randomAggregates() {
+        int size = between(1, 5);
+        List<NamedExpression> result = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            Expression agg = switch (between(0, 5)) {
+                case 0 -> new Max(randomSource(), FieldAttributeTests.createFieldAttribute(1, true));
+                case 1 -> new Min(randomSource(), FieldAttributeTests.createFieldAttribute(1, true));
+                case 2 -> new Count(randomSource(), FieldAttributeTests.createFieldAttribute(1, true));
+                case 3 -> new Top(
+                    randomSource(),
+                    FieldAttributeTests.createFieldAttribute(1, true),
+                    new Literal(randomSource(), between(1, 5), DataType.INTEGER),
+                    new Literal(randomSource(), randomFrom("ASC", "DESC"), DataType.KEYWORD)
+                );
+                case 4 -> new Values(randomSource(), FieldAttributeTests.createFieldAttribute(1, true));
+                case 5 -> new Sum(randomSource(), FieldAttributeTests.createFieldAttribute(1, true));
+                default -> throw new IllegalArgumentException();
+            };
+            result.add(new Alias(randomSource(), randomAlphaOfLength(5), agg));
+        }
+        return result;
+    }
+
+    @Override
+    protected Aggregate mutateInstance(Aggregate instance) throws IOException {
+        LogicalPlan child = instance.child();
+        Aggregate.AggregateType aggregateType = instance.aggregateType();
+        List<Expression> groupings = instance.groupings();
+        List<? extends NamedExpression> aggregates = instance.aggregates();
+        switch (between(0, 3)) {
+            case 0 -> child = randomValueOtherThan(child, () -> randomChild(0));
+            case 1 -> aggregateType = randomValueOtherThan(aggregateType, () -> randomFrom(Aggregate.AggregateType.values()));
+            case 2 -> groupings = randomValueOtherThan(
+                groupings,
+                () -> randomFieldAttributes(0, 5, false).stream().map(a -> (Expression) a).toList()
+            );
+            case 3 -> aggregates = randomValueOtherThan(aggregates, AggregateSerializationTests::randomAggregates);
+        }
+        return new Aggregate(instance.source(), child, aggregateType, groupings, aggregates);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/DissectSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/DissectSerializationTests.java
new file mode 100644
index 0000000000000..eb0127fdddc80
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/DissectSerializationTests.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.dissect.DissectParser;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests;
+
+import java.io.IOException;
+import java.util.List;
+
+public class DissectSerializationTests extends AbstractLogicalPlanSerializationTests<Dissect> {
+    @Override
+    protected Dissect createTestInstance() {
+        Source source = randomSource();
+        LogicalPlan child = randomChild(0);
+        Expression input = FieldAttributeTests.createFieldAttribute(1, false);
+        Dissect.Parser parser = randomParser();
+        List<Attribute> extracted = randomFieldAttributes(0, 4, false);
+        return new Dissect(source, child, input, parser, extracted);
+    }
+
+    public static Dissect.Parser randomParser() {
+        String suffix = randomAlphaOfLength(5);
+        String pattern = "%{b} %{c}" + suffix;
+        return new Dissect.Parser(pattern, ",", new DissectParser(pattern, ","));
+    }
+
+    @Override
+    protected Dissect mutateInstance(Dissect instance) throws IOException {
+        LogicalPlan child = instance.child();
+        Expression input = instance.input();
+        Dissect.Parser parser = randomParser();
+        List<Attribute> extracted = randomFieldAttributes(0, 4, false);
+        switch (between(0, 3)) {
+            case 0 -> child = randomValueOtherThan(child, () -> randomChild(0));
+            case 1 -> input = randomValueOtherThan(input, () -> FieldAttributeTests.createFieldAttribute(1, false));
+            case 2 -> parser = randomValueOtherThan(parser, DissectSerializationTests::randomParser);
+            case 3 -> extracted = randomValueOtherThan(extracted, () -> randomFieldAttributes(0, 4, false));
+        }
+        return new Dissect(instance.source(), child, input, parser, extracted);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EnrichSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EnrichSerializationTests.java
new file mode 100644
index 0000000000000..52c2035702354
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EnrichSerializationTests.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class EnrichSerializationTests extends AbstractLogicalPlanSerializationTests<Enrich> {
+    @Override
+    protected Enrich createTestInstance() {
+        Source source = randomSource();
+        LogicalPlan child = randomChild(0);
+        Enrich.Mode mode = randomFrom(Enrich.Mode.values());
+        Expression policyName = randomPolicyName();
+        NamedExpression matchField = FieldAttributeTests.createFieldAttribute(1, false);
+        EnrichPolicy policy = randomEnrichPolicy();
+        Map<String, String> concreteIndices = randomConcreteIndices();
+        List<NamedExpression> enrichFields = randomFieldAttributes(0, 10, false).stream().map(f -> (NamedExpression) f).toList();
+        return new Enrich(source, child, mode, policyName, matchField, policy, concreteIndices, enrichFields);
+    }
+
+    private static Expression randomPolicyName() {
+        return new Literal(randomSource(), randomAlphaOfLength(5), DataType.KEYWORD);
+    }
+
+    private static EnrichPolicy randomEnrichPolicy() {
+        String type = randomFrom(EnrichPolicy.SUPPORTED_POLICY_TYPES);
+        EnrichPolicy.QuerySource query = randomBoolean() ? randomQuerySource() : null;
+        List<String> indices = randomList(0, 5, () -> randomAlphaOfLength(5));
+        String matchField = randomAlphaOfLength(5);
+        List<String> enrichFields = randomList(0, 5, () -> randomAlphaOfLength(5));
+        return new EnrichPolicy(type, query, indices, matchField, enrichFields);
+    }
+
+    private static EnrichPolicy.QuerySource randomQuerySource() {
+        QueryBuilder queryBuilder = randomBoolean()
+            ? new MatchAllQueryBuilder()
+            : new TermQueryBuilder(randomAlphaOfLength(4), randomAlphaOfLength(4));
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        try (XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON, out)) {
+            XContentBuilder content = queryBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+            content.flush();
+            return new EnrichPolicy.QuerySource(new BytesArray(out.toByteArray()), content.contentType());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static Map<String, String> randomConcreteIndices() {
+        int size = between(0, 5);
+        Map<String, String> result = new HashMap<>();
+        while (result.size() < size) {
+            result.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
+        }
+        return result;
+    }
+
+    @Override
+    protected Enrich mutateInstance(Enrich instance) throws IOException {
+        Source source = instance.source();
+        LogicalPlan child = instance.child();
+        Enrich.Mode mode = instance.mode();
+        Expression policyName = instance.policyName();
+        NamedExpression matchField = instance.matchField();
+        EnrichPolicy policy = instance.policy();
+        Map<String, String> concreteIndices = instance.concreteIndices();
+        List<NamedExpression> enrichFields = instance.enrichFields();
+        switch (between(0, 6)) {
+            case 0 -> child = randomValueOtherThan(child, () -> randomChild(0));
+            case 1 -> mode = randomValueOtherThan(mode, () -> randomFrom(Enrich.Mode.values()));
+            case 2 -> policyName = randomValueOtherThan(policyName, EnrichSerializationTests::randomPolicyName);
+            case 3 -> matchField = randomValueOtherThan(matchField, () -> FieldAttributeTests.createFieldAttribute(1, false));
+            case 4 -> policy = randomValueOtherThan(policy, EnrichSerializationTests::randomEnrichPolicy);
+            case 5 -> concreteIndices = randomValueOtherThan(concreteIndices, EnrichSerializationTests::randomConcreteIndices);
+            case 6 -> enrichFields = randomValueOtherThan(
+                enrichFields,
+                () -> randomFieldAttributes(0, 10, false).stream().map(f -> (NamedExpression) f).toList()
+            );
+            default -> throw new IllegalArgumentException();
+        }
+        return new Enrich(source, child, mode, policyName, matchField, policy, concreteIndices, enrichFields);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EsRelationSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EsRelationSerializationTests.java
new file mode 100644
index 0000000000000..fa3038b24e8eb
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EsRelationSerializationTests.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.EsIndexSerializationTests;
+
+import java.io.IOException;
+import java.util.List;
+
+public class EsRelationSerializationTests extends AbstractLogicalPlanSerializationTests<EsRelation> {
+    public static EsRelation randomEsRelation() {
+        Source source = randomSource();
+        EsIndex index = EsIndexSerializationTests.randomEsIndex();
+        List<Attribute> attributes = randomFieldAttributes(0, 10, false);
+        IndexMode indexMode = randomFrom(IndexMode.values());
+        boolean frozen = randomBoolean();
+        return new EsRelation(source, index, attributes, indexMode, frozen);
+    }
+
+    @Override
+    protected EsRelation createTestInstance() {
+        return randomEsRelation();
+    }
+
+    @Override
+    protected EsRelation mutateInstance(EsRelation instance) throws IOException {
+        EsIndex index = instance.index();
+        List<Attribute> attributes = instance.output();
+        IndexMode indexMode = instance.indexMode();
+        boolean frozen = instance.frozen();
+        switch (between(0, 3)) {
+            case 0 -> index = randomValueOtherThan(index, EsIndexSerializationTests::randomEsIndex);
+            case 1 -> attributes = randomValueOtherThan(attributes, () -> randomFieldAttributes(0, 10, false));
+            case 2 -> indexMode = randomValueOtherThan(indexMode, () -> randomFrom(IndexMode.values()));
+            case 3 -> frozen = false == frozen;
+            default -> throw new IllegalArgumentException();
+        }
+        return new EsRelation(instance.source(), index, attributes, indexMode, frozen);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EvalSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EvalSerializationTests.java
new file mode 100644
index 0000000000000..5f5c4886dcfd1
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/EvalSerializationTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Alias;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.AliasTests;
+
+import java.io.IOException;
+import java.util.List;
+
+public class EvalSerializationTests extends AbstractLogicalPlanSerializationTests<Eval> {
+    @Override
+    protected Eval createTestInstance() {
+        Source source = randomSource();
+        LogicalPlan child = randomChild(0);
+        List<Alias> fields = randomList(0, 10, AliasTests::randomAlias);
+        return new Eval(source, child, fields);
+    }
+
+    @Override
+    protected Eval mutateInstance(Eval instance) throws IOException {
+        LogicalPlan child = instance.child();
+        List<Alias> fields = instance.fields();
+        if (randomBoolean()) {
+            child = randomValueOtherThan(child, () -> randomChild(0));
+        } else {
+            fields = randomValueOtherThan(fields, () -> randomList(0, 10, AliasTests::randomAlias));
+        }
+        return new Eval(instance.source(), child, fields);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/FilterSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/FilterSerializationTests.java
new file mode 100644
index 0000000000000..21eb36356e8f0
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/FilterSerializationTests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests;
+
+import java.io.IOException;
+
+public class FilterSerializationTests extends AbstractLogicalPlanSerializationTests<Filter> {
+    @Override
+    protected Filter createTestInstance() {
+        LogicalPlan child = randomChild(0);
+        Expression condition = FieldAttributeTests.createFieldAttribute(3, false);
+        return new Filter(randomSource(), child, condition);
+    }
+
+    @Override
+    protected Filter mutateInstance(Filter instance) throws IOException {
+        LogicalPlan child = instance.child();
+        Expression condition = instance.condition();
+        if (randomBoolean()) {
+            child = randomValueOtherThan(child, () -> randomChild(0));
+        } else {
+            condition = randomValueOtherThan(condition, () -> FieldAttributeTests.createFieldAttribute(3, false));
+        }
+        return new Filter(instance.source(), child, condition);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/GrokSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/GrokSerializationTests.java
new file mode 100644
index 0000000000000..770b7983fd782
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/GrokSerializationTests.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests;
+import org.elasticsearch.xpack.esql.expression.function.ReferenceAttributeTests;
+
+import java.io.IOException;
+import java.util.List;
+
+public class GrokSerializationTests extends AbstractLogicalPlanSerializationTests<Grok> {
+    @Override
+    protected Grok createTestInstance() {
+        Source source = randomSource();
+        LogicalPlan child = randomChild(0);
+        Expression inputExpr = FieldAttributeTests.createFieldAttribute(3, false);
+        String pattern = randomAlphaOfLength(5);
+        List<Attribute> extracted = randomList(1, 10, ReferenceAttributeTests::randomReferenceAttribute);
+        return new Grok(source, child, inputExpr, Grok.pattern(source, pattern), extracted);
+    }
+
+    @Override
+    protected Grok mutateInstance(Grok instance) throws IOException {
+        LogicalPlan child = instance.child();
+        Expression inputExpr = instance.input();
+        String pattern = instance.parser().pattern();
+        List<Attribute> extracted = instance.extractedFields();
+        switch (between(0, 2)) {
+            case 0 -> child = randomValueOtherThan(child, () -> randomChild(0));
+            case 1 -> inputExpr = randomValueOtherThan(inputExpr, () -> FieldAttributeTests.createFieldAttribute(0, false));
+            case 2 -> pattern = randomValueOtherThan(pattern, () -> randomAlphaOfLength(5));
+        }
+        return new Grok(instance.source(), child, inputExpr, Grok.pattern(instance.source(), pattern), extracted);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/InlineStatsSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/InlineStatsSerializationTests.java
new file mode 100644
index 0000000000000..5366fca1fbd71
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/InlineStatsSerializationTests.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.io.IOException;
+import java.util.List;
+
+public class InlineStatsSerializationTests extends AbstractLogicalPlanSerializationTests<InlineStats> {
+    @Override
+    protected InlineStats createTestInstance() {
+        Source source = randomSource();
+        LogicalPlan child = randomChild(0);
+        List<Expression> groupings = randomFieldAttributes(0, 5, false).stream().map(a -> (Expression) a).toList();
+        List<? extends NamedExpression> aggregates = randomFieldAttributes(0, 5, false).stream().map(a -> (NamedExpression) a).toList();
+        return new InlineStats(source, child, groupings, aggregates);
+    }
+
+    @Override
+    protected InlineStats mutateInstance(InlineStats instance) throws IOException {
+        LogicalPlan child = instance.child();
+        List<Expression> groupings = instance.groupings();
+        List<? extends NamedExpression> aggregates = instance.aggregates();
+        switch (between(0, 2)) {
+            case 0 -> child = randomValueOtherThan(child, () -> randomChild(0));
+            case 1 -> groupings = randomValueOtherThan(
+                groupings,
+                () -> randomFieldAttributes(0, 5, false).stream().map(a -> (Expression) a).toList()
+            );
+            case 2 -> aggregates = randomValueOtherThan(
+                aggregates,
+                () -> randomFieldAttributes(0, 5, false).stream().map(a -> (NamedExpression) a).toList()
+            );
+        }
+        return new InlineStats(instance.source(), child, groupings, aggregates);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/JoinSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/JoinSerializationTests.java
new file mode 100644
index 0000000000000..1a7f29303e635
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/JoinSerializationTests.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinType;
+
+import java.io.IOException;
+import java.util.List;
+
+public class JoinSerializationTests extends AbstractLogicalPlanSerializationTests<Join> {
+    @Override
+    protected Join createTestInstance() {
+        Source source = randomSource();
+        LogicalPlan left = randomChild(0);
+        LogicalPlan right = randomChild(0);
+        JoinConfig config = randomJoinConfig();
+        return new Join(source, left, right, config);
+    }
+
+    private static JoinConfig randomJoinConfig() {
+        JoinType type = randomFrom(JoinType.values());
+        List<Attribute> matchFields = randomFieldAttributes(1, 10, false);
+        List<Attribute> leftFields = randomFieldAttributes(1, 10, false);
+        List<Attribute> rightFields = randomFieldAttributes(1, 10, false);
+        return new JoinConfig(type, matchFields, leftFields, rightFields);
+    }
+
+    @Override
+    protected Join mutateInstance(Join instance) throws IOException {
+        LogicalPlan left = instance.left();
+        LogicalPlan right = instance.right();
+        JoinConfig config = instance.config();
+        switch (between(0, 2)) {
+            case 0 -> left = randomValueOtherThan(left, () -> randomChild(0));
+            case 1 -> right = randomValueOtherThan(right, () -> randomChild(0));
+            case 2 -> config = randomValueOtherThan(config, JoinSerializationTests::randomJoinConfig);
+        }
+        return new Join(instance.source(), left, right, config);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/LimitSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/LimitSerializationTests.java
new file mode 100644
index 0000000000000..5d994eb2880ba
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/LimitSerializationTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests;
+
+import java.io.IOException;
+
+public class LimitSerializationTests extends AbstractLogicalPlanSerializationTests<Limit> {
+    @Override
+    protected Limit createTestInstance() {
+        Source source = randomSource();
+        Expression limit = FieldAttributeTests.createFieldAttribute(0, false);
+        LogicalPlan child = randomChild(0);
+        return new Limit(source, limit, child);
+    }
+
+    @Override
+    protected Limit mutateInstance(Limit instance) throws IOException {
+        Expression limit = instance.limit();
+        LogicalPlan child = instance.child();
+        if (randomBoolean()) {
+            limit = randomValueOtherThan(limit, () -> FieldAttributeTests.createFieldAttribute(0, false));
+        } else {
+            child = randomValueOtherThan(child, () -> randomChild(0));
+        }
+        return new Limit(instance.source(), limit, child);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/LookupSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/LookupSerializationTests.java
new file mode 100644
index 0000000000000..53f36e124ebb0
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/LookupSerializationTests.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelationSerializationTests;
+
+import java.io.IOException;
+import java.util.List;
+
+public class LookupSerializationTests extends AbstractLogicalPlanSerializationTests<Lookup> {
+    public static Lookup randomLookup(int depth) {
+        Source source = randomSource();
+        LogicalPlan child = randomChild(depth);
+        Expression tableName = AbstractExpressionSerializationTests.randomChild();
+        List<Attribute> matchFields = randomFieldAttributes(1, 10, false);
+        LocalRelation localRelation = randomBoolean() ? null : LocalRelationSerializationTests.randomLocalRelation();
+        return new Lookup(source, child, tableName, matchFields, localRelation);
+    }
+
+    @Override
+    protected Lookup createTestInstance() {
+        return randomLookup(0);
+    }
+
+    @Override
+    protected Lookup mutateInstance(Lookup instance) throws IOException {
+        Source source = instance.source();
+        LogicalPlan child = instance.child();
+        Expression tableName = instance.tableName();
+        List<Attribute> matchFields = instance.matchFields();
+        LocalRelation localRelation = instance.localRelation();
+        switch (between(0, 3)) {
+            case 0 -> child = randomValueOtherThan(child, () -> randomChild(0));
+            case 1 -> tableName = randomValueOtherThan(tableName, AbstractExpressionSerializationTests::randomChild);
+            case 2 -> matchFields = randomValueOtherThan(matchFields, () -> randomFieldAttributes(1, 10, false));
+            case 3 -> localRelation = randomValueOtherThan(
+                localRelation,
+                () -> randomBoolean() ? null : LocalRelationSerializationTests.randomLocalRelation()
+            );
+        }
+        return new Lookup(source, child, tableName, matchFields, localRelation);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/MvExpandSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/MvExpandSerializationTests.java
new file mode 100644
index 0000000000000..08e59095161ed
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/MvExpandSerializationTests.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests;
+import org.elasticsearch.xpack.esql.expression.function.ReferenceAttributeTests;
+
+import java.io.IOException;
+
+public class MvExpandSerializationTests extends AbstractLogicalPlanSerializationTests<MvExpand> {
+    @Override
+    protected MvExpand createTestInstance() {
+        Source source = randomSource();
+        LogicalPlan child = randomChild(0);
+        NamedExpression target = FieldAttributeTests.createFieldAttribute(0, false);
+        Attribute expanded = ReferenceAttributeTests.randomReferenceAttribute();
+        return new MvExpand(source, child, target, expanded);
+    }
+
+    @Override
+    protected MvExpand mutateInstance(MvExpand instance) throws IOException {
+        LogicalPlan child = instance.child();
+        NamedExpression target = instance.target();
+        Attribute expanded = instance.expanded();
+        switch (between(0, 2)) {
+            case 0 -> child = randomValueOtherThan(child, () -> randomChild(0));
+            case 1 -> target = randomValueOtherThan(target, () -> FieldAttributeTests.createFieldAttribute(0, false));
+            case 2 -> expanded = randomValueOtherThan(expanded, ReferenceAttributeTests::randomReferenceAttribute);
+        }
+        return new MvExpand(instance.source(), child, target, expanded);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/OrderBySerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/OrderBySerializationTests.java
new file mode 100644
index 0000000000000..95d262664e86a
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/OrderBySerializationTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.Order;
+import org.elasticsearch.xpack.esql.expression.OrderSerializationTests;
+
+import java.io.IOException;
+import java.util.List;
+
+public class OrderBySerializationTests extends AbstractLogicalPlanSerializationTests<OrderBy> {
+    @Override
+    protected OrderBy createTestInstance() {
+        Source source = randomSource();
+        LogicalPlan child = randomChild(0);
+        List<Order> orders = randomList(1, 10, OrderSerializationTests::randomOrder);
+        return new OrderBy(source, child, orders);
+    }
+
+    @Override
+    protected OrderBy mutateInstance(OrderBy instance) throws IOException {
+        LogicalPlan child = instance.child();
+        List<Order> orders = instance.order();
+        if (randomBoolean()) {
+            child = randomValueOtherThan(child, () -> randomChild(0));
+        } else {
+            orders = randomValueOtherThan(orders, () -> randomList(1, 10, OrderSerializationTests::randomOrder));
+        }
+        return new OrderBy(instance.source(), child, orders);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/PhasedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/PhasedTests.java
new file mode 100644
index 0000000000000..5e45de6c77c42
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/PhasedTests.java
@@ -0,0 +1,161 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.expression.Alias;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class PhasedTests extends ESTestCase {
+    public void testZeroLayers() {
+        EsRelation relation = new EsRelation(Source.synthetic("relation"), new EsIndex("foo", Map.of()), IndexMode.STANDARD, false);
+        relation.setOptimized();
+        assertThat(Phased.extractFirstPhase(relation), nullValue());
+    }
+
+    public void testOneLayer() {
+        EsRelation relation = new EsRelation(Source.synthetic("relation"), new EsIndex("foo", Map.of()), IndexMode.STANDARD, false);
+        LogicalPlan orig = new Dummy(Source.synthetic("orig"), relation);
+        orig.setOptimized();
+        assertThat(Phased.extractFirstPhase(orig), sameInstance(relation));
+        LogicalPlan finalPhase = Phased.applyResultsFromFirstPhase(
+            orig,
+            List.of(new ReferenceAttribute(Source.EMPTY, "foo", DataType.KEYWORD)),
+            List.of()
+        );
+        assertThat(
+            finalPhase,
+            equalTo(new Row(orig.source(), List.of(new Alias(orig.source(), "foo", new Literal(orig.source(), "foo", DataType.KEYWORD)))))
+        );
+        finalPhase.setOptimized();
+        assertThat(Phased.extractFirstPhase(finalPhase), nullValue());
+    }
+
+    public void testTwoLayer() {
+        EsRelation relation = new EsRelation(Source.synthetic("relation"), new EsIndex("foo", Map.of()), IndexMode.STANDARD, false);
+        LogicalPlan inner = new Dummy(Source.synthetic("inner"), relation);
+        LogicalPlan orig = new Dummy(Source.synthetic("outer"), inner);
+        orig.setOptimized();
+        assertThat(
+            "extractFirstPhase should call #firstPhase on the earliest child in the plan",
+            Phased.extractFirstPhase(orig),
+            sameInstance(relation)
+        );
+        LogicalPlan secondPhase = Phased.applyResultsFromFirstPhase(
+            orig,
+            List.of(new ReferenceAttribute(Source.EMPTY, "foo", DataType.KEYWORD)),
+            List.of()
+        );
+        secondPhase.setOptimized();
+        assertThat(
+            "applyResultsFromFirstPhase should call #nextPhase one th earliest child in the plan",
+            secondPhase,
+            equalTo(
+                new Dummy(
+                    Source.synthetic("outer"),
+                    new Row(orig.source(), List.of(new Alias(orig.source(), "foo", new Literal(orig.source(), "foo", DataType.KEYWORD))))
+                )
+            )
+        );
+
+        assertThat(Phased.extractFirstPhase(secondPhase), sameInstance(secondPhase.children().get(0)));
+        LogicalPlan finalPhase = Phased.applyResultsFromFirstPhase(
+            secondPhase,
+            List.of(new ReferenceAttribute(Source.EMPTY, "foo", DataType.KEYWORD)),
+            List.of()
+        );
+        finalPhase.setOptimized();
+        assertThat(
+            finalPhase,
+            equalTo(new Row(orig.source(), List.of(new Alias(orig.source(), "foo", new Literal(orig.source(), "foo", DataType.KEYWORD)))))
+        );
+
+        assertThat(Phased.extractFirstPhase(finalPhase), nullValue());
+    }
+
+    public class Dummy extends UnaryPlan implements Phased {
+        Dummy(Source source, LogicalPlan child) {
+            super(source, child);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            throw new UnsupportedOperationException("not serialized");
+        }
+
+        @Override
+        public String getWriteableName() {
+            throw new UnsupportedOperationException("not serialized");
+        }
+
+        @Override
+        public boolean expressionsResolved() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        protected NodeInfo<? extends LogicalPlan> info() {
+            return NodeInfo.create(this, Dummy::new, child());
+        }
+
+        @Override
+        public int hashCode() {
+            return child().hashCode();
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj instanceof Dummy == false) {
+                return false;
+            }
+            Dummy other = (Dummy) obj;
+            return child().equals(other.child());
+        }
+
+        @Override
+        public UnaryPlan replaceChild(LogicalPlan newChild) {
+            return new Dummy(source(), newChild);
+        }
+
+        @Override
+        public List<Attribute> output() {
+            return child().output();
+        }
+
+        @Override
+        public LogicalPlan firstPhase() {
+            return child();
+        }
+
+        @Override
+        public LogicalPlan nextPhase(List<Attribute> schema, List<Page> firstPhaseResult) {
+            // Replace myself with a dummy "row" command
+            return new Row(
+                source(),
+                schema.stream().map(a -> new Alias(source(), a.name(), new Literal(source(), a.name(), DataType.KEYWORD))).toList()
+            );
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/ProjectSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/ProjectSerializationTests.java
new file mode 100644
index 0000000000000..cf9a3d3f17b12
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/ProjectSerializationTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.io.IOException;
+import java.util.List;
+
+public class ProjectSerializationTests extends AbstractLogicalPlanSerializationTests<Project> {
+    @Override
+    protected Project createTestInstance() {
+        Source source = randomSource();
+        LogicalPlan child = randomChild(0);
+        List<? extends NamedExpression> projections = randomFieldAttributes(0, 10, false);
+        return new Project(source, child, projections);
+    }
+
+    @Override
+    protected Project mutateInstance(Project instance) throws IOException {
+        LogicalPlan child = instance.child();
+        List<? extends NamedExpression> projections = instance.projections();
+        if (randomBoolean()) {
+            child = randomValueOtherThan(child, () -> randomChild(0));
+        } else {
+            projections = randomValueOtherThan(projections, () -> randomFieldAttributes(0, 10, false));
+        }
+        return new Project(instance.source(), child, projections);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/TopNSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/TopNSerializationTests.java
new file mode 100644
index 0000000000000..6c3863582f215
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/TopNSerializationTests.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
+import org.elasticsearch.xpack.esql.expression.Order;
+import org.elasticsearch.xpack.esql.expression.OrderSerializationTests;
+
+import java.io.IOException;
+import java.util.List;
+
+public class TopNSerializationTests extends AbstractLogicalPlanSerializationTests<TopN> {
+    public static TopN randomTopN(int depth) {
+        Source source = randomSource();
+        LogicalPlan child = randomChild(depth);
+        List<Order> order = randomOrders();
+        Expression limit = AbstractExpressionSerializationTests.randomChild();
+        return new TopN(source, child, order, limit);
+    }
+
+    private static List<Order> randomOrders() {
+        return randomList(1, 10, OrderSerializationTests::randomOrder);
+    }
+
+    @Override
+    protected TopN createTestInstance() {
+        return randomTopN(0);
+    }
+
+    @Override
+    protected TopN mutateInstance(TopN instance) throws IOException {
+        Source source = instance.source();
+        LogicalPlan child = instance.child();
+        List<Order> order = instance.order();
+        Expression limit = instance.limit();
+        switch (between(0, 2)) {
+            case 0 -> child = randomValueOtherThan(child, () -> randomChild(0));
+            case 1 -> order = randomValueOtherThan(order, TopNSerializationTests::randomOrders);
+            case 2 -> limit = randomValueOtherThan(limit, AbstractExpressionSerializationTests::randomChild);
+        }
+        return new TopN(source, child, order, limit);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/EsqlProjectSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/EsqlProjectSerializationTests.java
new file mode 100644
index 0000000000000..7e5e368fff77f
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/EsqlProjectSerializationTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical.local;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.plan.logical.AbstractLogicalPlanSerializationTests;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+
+import java.io.IOException;
+import java.util.List;
+
+public class EsqlProjectSerializationTests extends AbstractLogicalPlanSerializationTests<EsqlProject> {
+    @Override
+    protected EsqlProject createTestInstance() {
+        LogicalPlan child = randomChild(0);
+        List<Attribute> projections = randomFieldAttributes(1, 10, false);
+        return new EsqlProject(randomSource(), child, projections);
+    }
+
+    @Override
+    protected EsqlProject mutateInstance(EsqlProject instance) throws IOException {
+        LogicalPlan child = instance.child();
+        List<? extends NamedExpression> projections = instance.projections();
+        if (randomBoolean()) {
+            child = randomValueOtherThan(child, () -> randomChild(0));
+        } else {
+            projections = randomValueOtherThan(projections, () -> randomFieldAttributes(1, 10, false));
+        }
+        return new EsqlProject(instance.source(), child, projections);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelationSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelationSerializationTests.java
new file mode 100644
index 0000000000000..647e7c358eddb
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelationSerializationTests.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical.local;
+
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.xpack.esql.TestBlockFactory;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.plan.logical.AbstractLogicalPlanSerializationTests;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral;
+
+public class LocalRelationSerializationTests extends AbstractLogicalPlanSerializationTests<LocalRelation> {
+    public static LocalRelation randomLocalRelation() {
+        Source source = randomSource();
+        List<Attribute> output = randomFieldAttributes(1, 10, true);
+        LocalSupplier supplier = randomLocalSupplier(output);
+        return new LocalRelation(source, output, supplier);
+    }
+
+    private static LocalSupplier randomLocalSupplier(List<Attribute> attributes) {
+        Block[] blocks = new Block[attributes.size()];
+        for (int b = 0; b < blocks.length; b++) {
+            blocks[b] = BlockUtils.constantBlock(
+                TestBlockFactory.getNonBreakingInstance(),
+                randomLiteral(attributes.get(b).dataType()).value(),
+                1
+            );
+        }
+        return LocalSupplier.of(blocks);
+    }
+
+    @Override
+    protected LocalRelation createTestInstance() {
+        return randomLocalRelation();
+    }
+
+    @Override
+    protected LocalRelation mutateInstance(LocalRelation instance) throws IOException {
+        /*
+         * There are two ways we could mutate this. Either we mutate just
+         * the data, or we mutate the attributes and the data. Some attributes
+         * don't *allow* for us to mutate the data. For example, if the attributes
+         * are all NULL typed. In that case we can't mutate the data.
+         *
+         * So we flip a coin. If that lands on true, we *try* to modify that data.
+         * If that spits out the same data - or if the coin lands on false - we'll
+         * modify the attributes and the data.
+         */
+        if (randomBoolean()) {
+            List<Attribute> output = instance.output();
+            LocalSupplier supplier = randomLocalSupplier(output);
+            if (supplier.equals(instance.supplier()) == false) {
+                return new LocalRelation(instance.source(), output, supplier);
+            }
+        }
+        List<Attribute> output = randomValueOtherThan(instance.output(), () -> randomFieldAttributes(1, 10, true));
+        LocalSupplier supplier = randomLocalSupplier(output);
+        return new LocalRelation(instance.source(), output, supplier);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/AbstractPhysicalPlanSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/AbstractPhysicalPlanSerializationTests.java
new file mode 100644
index 0000000000000..4b74114a0e01c
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/AbstractPhysicalPlanSerializationTests.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.physical;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.tree.Node;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
+import org.elasticsearch.xpack.esql.plan.AbstractNodeSerializationTests;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.plan.physical.AggregateExecSerializationTests.randomAggregateExec;
+import static org.elasticsearch.xpack.esql.plan.physical.DissectExecSerializationTests.randomDissectExec;
+import static org.elasticsearch.xpack.esql.plan.physical.EsSourceExecSerializationTests.randomEsSourceExec;
+import static org.elasticsearch.xpack.esql.plan.physical.ExchangeExecSerializationTests.randomExchangeExec;
+import static org.elasticsearch.xpack.esql.plan.physical.ExchangeSinkExecSerializationTests.randomExchangeSinkExec;
+import static org.elasticsearch.xpack.esql.plan.physical.ExchangeSourceExecSerializationTests.randomExchangeSourceExec;
+
+public abstract class AbstractPhysicalPlanSerializationTests<T extends PhysicalPlan> extends AbstractNodeSerializationTests<T> {
+    public static PhysicalPlan randomChild(int depth) {
+        if (randomBoolean() && depth < 4) {
+            switch (between(0, 3)) {
+                case 0 -> randomDissectExec(depth + 1);
+                case 1 -> randomExchangeExec(depth + 1);
+                case 2 -> randomExchangeSinkExec(depth + 1);
+                case 3 -> randomAggregateExec(depth + 1);
+            }
+        }
+        return randomBoolean() ? randomExchangeSourceExec() : randomEsSourceExec();
+    }
+
+    public static Integer randomEstimatedRowSize() {
+        return randomBoolean() ? null : between(0, Integer.MAX_VALUE);
+    }
+
+    @Override
+    protected final NamedWriteableRegistry getNamedWriteableRegistry() {
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.addAll(PhysicalPlan.getNamedWriteables());
+        entries.addAll(LogicalPlan.getNamedWriteables());
+        entries.addAll(AggregateFunction.getNamedWriteables());
+        entries.addAll(Expression.getNamedWriteables());
+        entries.addAll(Attribute.getNamedWriteables());
+        entries.addAll(Block.getNamedWriteables());
+        entries.addAll(NamedExpression.getNamedWriteables());
+        entries.addAll(new SearchModule(Settings.EMPTY, List.of()).getNamedWriteables()); // Query builders
+        entries.add(Add.ENTRY); // Used by the eval tests
+        return new NamedWriteableRegistry(entries);
+    }
+
+    @Override
+    protected final Class<? extends Node<?>> categoryClass() {
+        return PhysicalPlan.class;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExecSerializationTests.java
new file mode 100644
index 0000000000000..ca9dd2045004e
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/AggregateExecSerializationTests.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.physical;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.plan.logical.AggregateSerializationTests;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.plan.logical.AbstractLogicalPlanSerializationTests.randomFieldAttributes;
+
+public class AggregateExecSerializationTests extends AbstractPhysicalPlanSerializationTests<AggregateExec> {
+    public static AggregateExec randomAggregateExec(int depth) {
+        Source source = randomSource();
+        PhysicalPlan child = randomChild(depth);
+        List<Expression> groupings = randomFieldAttributes(0, 5, false).stream().map(a -> (Expression) a).toList();
+        List<? extends NamedExpression> aggregates = AggregateSerializationTests.randomAggregates();
+        AggregateExec.Mode mode = randomFrom(AggregateExec.Mode.values());
+        Integer estimatedRowSize = randomEstimatedRowSize();
+        return new AggregateExec(source, child, groupings, aggregates, mode, estimatedRowSize);
+    }
+
+    @Override
+    protected AggregateExec createTestInstance() {
+        return randomAggregateExec(0);
+    }
+
+    @Override
+    protected AggregateExec mutateInstance(AggregateExec instance) throws IOException {
+        PhysicalPlan child = instance.child();
+        List<? extends Expression> groupings = instance.groupings();
+        List<? extends NamedExpression> aggregates = instance.aggregates();
+        AggregateExec.Mode mode = instance.getMode();
+        Integer estimatedRowSize = instance.estimatedRowSize();
+        switch (between(0, 4)) {
+            case 0 -> child = randomValueOtherThan(child, () -> randomChild(0));
+            case 1 -> groupings = randomValueOtherThan(groupings, () -> randomFieldAttributes(0, 5, false));
+            case 2 -> aggregates = randomValueOtherThan(aggregates, AggregateSerializationTests::randomAggregates);
+            case 3 -> mode = randomValueOtherThan(mode, () -> randomFrom(AggregateExec.Mode.values()));
+            case 4 -> estimatedRowSize = randomValueOtherThan(
+                estimatedRowSize,
+                AbstractPhysicalPlanSerializationTests::randomEstimatedRowSize
+            );
+            default -> throw new IllegalStateException();
+        }
+        return new AggregateExec(instance.source(), child, groupings, aggregates, mode, estimatedRowSize);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/DissectExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/DissectExecSerializationTests.java
new file mode 100644
index 0000000000000..3cad0b88a0837
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/DissectExecSerializationTests.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.physical;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests;
+import org.elasticsearch.xpack.esql.plan.logical.Dissect;
+import org.elasticsearch.xpack.esql.plan.logical.DissectSerializationTests;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.plan.logical.AbstractLogicalPlanSerializationTests.randomFieldAttributes;
+
+public class DissectExecSerializationTests extends AbstractPhysicalPlanSerializationTests<DissectExec> {
+    public static DissectExec randomDissectExec(int depth) {
+        Source source = randomSource();
+        PhysicalPlan child = randomChild(depth);
+        Expression inputExpression = FieldAttributeTests.createFieldAttribute(1, false);
+        Dissect.Parser parser = DissectSerializationTests.randomParser();
+        List<Attribute> extracted = randomFieldAttributes(0, 4, false);
+        return new DissectExec(source, child, inputExpression, parser, extracted);
+    }
+
+    @Override
+    protected DissectExec createTestInstance() {
+        return randomDissectExec(0);
+    }
+
+    @Override
+    protected DissectExec mutateInstance(DissectExec instance) throws IOException {
+        PhysicalPlan child = instance.child();
+        Expression inputExpression = FieldAttributeTests.createFieldAttribute(1, false);
+        Dissect.Parser parser = DissectSerializationTests.randomParser();
+        List<Attribute> extracted = randomFieldAttributes(0, 4, false);
+        switch (between(0, 3)) {
+            case 0 -> child = randomValueOtherThan(child, () -> randomChild(0));
+            case 1 -> inputExpression = randomValueOtherThan(inputExpression, () -> FieldAttributeTests.createFieldAttribute(1, false));
+            case 2 -> parser = randomValueOtherThan(parser, DissectSerializationTests::randomParser);
+            case 3 -> extracted = randomValueOtherThan(extracted, () -> randomFieldAttributes(0, 4, false));
+            default -> throw new IllegalStateException();
+        }
+        return new DissectExec(instance.source(), child, inputExpression, parser, extracted);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExecSerializationTests.java
new file mode 100644
index 0000000000000..6bb5111b154e6
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExecSerializationTests.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.physical;
+
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.Order;
+import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.EsIndexSerializationTests;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.plan.logical.AbstractLogicalPlanSerializationTests.randomFieldAttributes;
+
+public class EsQueryExecSerializationTests extends AbstractPhysicalPlanSerializationTests<EsQueryExec> {
+    public static EsQueryExec randomEsQueryExec() {
+        Source source = randomSource();
+        EsIndex index = EsIndexSerializationTests.randomEsIndex();
+        IndexMode indexMode = randomFrom(IndexMode.values());
+        List<Attribute> attrs = randomFieldAttributes(1, 10, false);
+        QueryBuilder query = randomQuery();
+        Expression limit = new Literal(randomSource(), between(0, Integer.MAX_VALUE), DataType.INTEGER);
+        List<EsQueryExec.FieldSort> sorts = randomFieldSorts();
+        Integer estimatedRowSize = randomEstimatedRowSize();
+        return new EsQueryExec(source, index, indexMode, attrs, query, limit, sorts, estimatedRowSize);
+    }
+
+    public static QueryBuilder randomQuery() {
+        return randomBoolean() ? new MatchAllQueryBuilder() : new TermQueryBuilder(randomAlphaOfLength(4), randomAlphaOfLength(4));
+    }
+
+    public static List<EsQueryExec.FieldSort> randomFieldSorts() {
+        return randomList(0, 4, EsQueryExecSerializationTests::randomFieldSort);
+    }
+
+    public static EsQueryExec.FieldSort randomFieldSort() {
+        FieldAttribute field = FieldAttributeTests.createFieldAttribute(0, false);
+        Order.OrderDirection direction = randomFrom(Order.OrderDirection.values());
+        Order.NullsPosition nulls = randomFrom(Order.NullsPosition.values());
+        return new EsQueryExec.FieldSort(field, direction, nulls);
+    }
+
+    @Override
+    protected EsQueryExec createTestInstance() {
+        return randomEsQueryExec();
+    }
+
+    @Override
+    protected EsQueryExec mutateInstance(EsQueryExec instance) throws IOException {
+        EsIndex index = instance.index();
+        IndexMode indexMode = instance.indexMode();
+        List<Attribute> attrs = instance.attrs();
+        QueryBuilder query = instance.query();
+        Expression limit = instance.limit();
+        List<EsQueryExec.FieldSort> sorts = instance.sorts();
+        Integer estimatedRowSize = instance.estimatedRowSize();
+        switch (between(0, 6)) {
+            case 0 -> index = randomValueOtherThan(index, EsIndexSerializationTests::randomEsIndex);
+            case 1 -> indexMode = randomValueOtherThan(indexMode, () -> randomFrom(IndexMode.values()));
+            case 2 -> attrs = randomValueOtherThan(attrs, () -> randomFieldAttributes(1, 10, false));
+            case 3 -> query = randomValueOtherThan(query, EsQueryExecSerializationTests::randomQuery);
+            case 4 -> limit = randomValueOtherThan(
+                limit,
+                () -> new Literal(randomSource(), between(0, Integer.MAX_VALUE), DataType.INTEGER)
+            );
+            case 5 -> sorts = randomValueOtherThan(sorts, EsQueryExecSerializationTests::randomFieldSorts);
+            case 6 -> estimatedRowSize = randomValueOtherThan(
+                estimatedRowSize,
+                AbstractPhysicalPlanSerializationTests::randomEstimatedRowSize
+            );
+        }
+        return new EsQueryExec(instance.source(), index, indexMode, attrs, query, limit, sorts, estimatedRowSize);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExecSerializationTests.java
new file mode 100644
index 0000000000000..253127cc7ee95
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EsSourceExecSerializationTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.physical;
+
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.EsIndexSerializationTests;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.plan.logical.AbstractLogicalPlanSerializationTests.randomFieldAttributes;
+
+public class EsSourceExecSerializationTests extends AbstractPhysicalPlanSerializationTests<EsSourceExec> {
+    public static EsSourceExec randomEsSourceExec() {
+        Source source = randomSource();
+        EsIndex index = EsIndexSerializationTests.randomEsIndex();
+        List<Attribute> attributes = randomFieldAttributes(1, 10, false);
+        QueryBuilder query = new TermQueryBuilder(randomAlphaOfLength(5), randomAlphaOfLength(5));
+        IndexMode indexMode = randomFrom(IndexMode.values());
+        return new EsSourceExec(source, index, attributes, query, indexMode);
+    }
+
+    @Override
+    protected EsSourceExec createTestInstance() {
+        return randomEsSourceExec();
+    }
+
+    @Override
+    protected EsSourceExec mutateInstance(EsSourceExec instance) throws IOException {
+        EsIndex index = instance.index();
+        List<Attribute> attributes = instance.output();
+        QueryBuilder query = instance.query();
+        IndexMode indexMode = instance.indexMode();
+        switch (between(0, 3)) {
+            case 0 -> index = randomValueOtherThan(index, EsIndexSerializationTests::randomEsIndex);
+            case 1 -> attributes = randomValueOtherThan(attributes, () -> randomFieldAttributes(1, 10, false));
+            case 2 -> query = randomValueOtherThan(query, () -> new TermQueryBuilder(randomAlphaOfLength(5), randomAlphaOfLength(5)));
+            case 3 -> indexMode = randomValueOtherThan(indexMode, () -> randomFrom(IndexMode.values()));
+            default -> throw new IllegalStateException();
+        }
+        return new EsSourceExec(instance.source(), index, attributes, query, indexMode);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EvalExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EvalExecSerializationTests.java
new file mode 100644
index 0000000000000..45baf4822b1d2
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/EvalExecSerializationTests.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.physical;
+
+import org.elasticsearch.xpack.esql.core.expression.Alias;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.FieldAttributeTests;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
+
+import java.io.IOException;
+import java.util.List;
+
+public class EvalExecSerializationTests extends AbstractPhysicalPlanSerializationTests<EvalExec> {
+    public static EvalExec randomEvalExec(int depth) {
+        Source source = randomSource();
+        PhysicalPlan child = randomChild(depth);
+        List<Alias> fields = randomFields();
+        return new EvalExec(source, child, fields);
+    }
+
+    public static List<Alias> randomFields() {
+        return randomList(1, 10, EvalExecSerializationTests::randomField);
+    }
+
+    public static Alias randomField() {
+        Expression child = new Add(
+            randomSource(),
+            FieldAttributeTests.createFieldAttribute(0, true),
+            FieldAttributeTests.createFieldAttribute(0, true)
+        );
+        return new Alias(randomSource(), randomAlphaOfLength(5), child);
+    }
+
+    @Override
+    protected EvalExec createTestInstance() {
+        return randomEvalExec(0);
+    }
+
+    @Override
+    protected EvalExec mutateInstance(EvalExec instance) throws IOException {
+        PhysicalPlan child = instance.child();
+        List<Alias> fields = instance.fields();
+        if (randomBoolean()) {
+            child = randomValueOtherThan(child, () -> randomChild(0));
+        } else {
+            fields = randomValueOtherThan(fields, EvalExecSerializationTests::randomFields);
+        }
+        return new EvalExec(instance.source(), child, fields);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeExecSerializationTests.java
new file mode 100644
index 0000000000000..572eac0ff409c
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeExecSerializationTests.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.physical;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.io.IOException;
+import java.util.List;
+
+public class ExchangeExecSerializationTests extends AbstractPhysicalPlanSerializationTests<ExchangeExec> {
+    static ExchangeExec randomExchangeExec(int depth) {
+        Source source = randomSource();
+        List<Attribute> output = randomFieldAttributes(1, 5, false);
+        boolean inBetweenAggs = randomBoolean();
+        PhysicalPlan child = randomChild(depth);
+        return new ExchangeExec(source, output, inBetweenAggs, child);
+    }
+
+    @Override
+    protected ExchangeExec createTestInstance() {
+        return randomExchangeExec(0);
+    }
+
+    @Override
+    protected ExchangeExec mutateInstance(ExchangeExec instance) throws IOException {
+        List<Attribute> output = instance.output();
+        boolean inBetweenAggs = instance.inBetweenAggs();
+        PhysicalPlan child = instance.child();
+        switch (between(0, 2)) {
+            case 0 -> output = randomValueOtherThan(output, () -> randomFieldAttributes(1, 5, false));
+            case 1 -> inBetweenAggs = false == inBetweenAggs;
+            case 2 -> child = randomValueOtherThan(child, () -> randomChild(0));
+        }
+        return new ExchangeExec(instance.source(), output, inBetweenAggs, child);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java
new file mode 100644
index 0000000000000..be30ac20df64e
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.physical;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.xpack.esql.analysis.Analyzer;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.EsIndexSerializationTests;
+import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
+import org.elasticsearch.xpack.esql.plan.logical.Limit;
+import org.elasticsearch.xpack.esql.plan.logical.Project;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.test.ByteSizeEqualsMatcher.byteSizeEquals;
+import static org.hamcrest.Matchers.equalTo;
+
+public class ExchangeSinkExecSerializationTests extends AbstractPhysicalPlanSerializationTests<ExchangeSinkExec> {
+    static ExchangeSinkExec randomExchangeSinkExec(int depth) {
+        Source source = randomSource();
+        List<Attribute> output = randomFieldAttributes(1, 5, false);
+        boolean intermediateAgg = randomBoolean();
+        PhysicalPlan child = randomChild(depth);
+        return new ExchangeSinkExec(source, output, intermediateAgg, child);
+    }
+
+    @Override
+    protected ExchangeSinkExec createTestInstance() {
+        return randomExchangeSinkExec(0);
+    }
+
+    @Override
+    protected ExchangeSinkExec mutateInstance(ExchangeSinkExec instance) throws IOException {
+        List<Attribute> output = instance.output();
+        boolean intermediateAgg = instance.isIntermediateAgg();
+        PhysicalPlan child = instance.child();
+        switch (between(0, 2)) {
+            case 0 -> output = randomValueOtherThan(output, () -> randomFieldAttributes(1, 5, false));
+            case 1 -> intermediateAgg = false == intermediateAgg;
+            case 2 -> child = randomValueOtherThan(child, () -> randomChild(0));
+        }
+        return new ExchangeSinkExec(instance.source(), output, intermediateAgg, child);
+    }
+
+    @Override
+    protected boolean alwaysEmptySource() {
+        return true;
+    }
+
+    /**
+     * Test the size of serializing a plan with many conflicts.
+     * See {@link #testManyTypeConflicts(boolean, ByteSizeValue)} for more.
+     */
+    public void testManyTypeConflicts() throws IOException {
+        testManyTypeConflicts(false, ByteSizeValue.ofBytes(1897374));
+        /*
+         * History:
+         *  2.3mb - shorten error messages for UnsupportedAttributes #111973
+         *  1.8mb - cache EsFields #112008
+         */
+    }
+
+    /**
+     * Test the size of serializing a plan with many conflicts.
+     * See {@link #testManyTypeConflicts(boolean, ByteSizeValue)} for more.
+     */
+    public void testManyTypeConflictsWithParent() throws IOException {
+        testManyTypeConflicts(true, ByteSizeValue.ofBytes(3271487));
+        /*
+         * History:
+         *  2 gb+ - start
+         * 43.3mb - Cache attribute subclasses #111447
+         *  5.6mb - shorten error messages for UnsupportedAttributes #111973
+         *  3.1mb - cache EsFields #112008
+         */
+    }
+
+    /**
+     * Test the size of serializing a plan with many conflicts. Callers of
+     * this method intentionally use a very precise size for the serialized
+     * data so a programmer making changes has to think when this size changes.
+     * <p>
+     *     In general, shrinking the over the wire size is great and the precise
+     *     size should just ratchet downwards. Small upwards movement is fine so
+     *     long as you understand why the change is happening and you think it's
+     *     worth it for the data node request for a big index to grow.
+     * </p>
+     * <p>
+     *     Large upwards movement in the size is not fine! Folks frequently make
+     *     requests across large clusters with many fields and these requests can
+     *     really clog up the network interface. Super large results here can make
+     *     ESQL impossible to use at all for big mappings with many conflicts.
+     * </p>
+     */
+    private void testManyTypeConflicts(boolean withParent, ByteSizeValue expected) throws IOException {
+        EsIndex index = EsIndexSerializationTests.indexWithManyConflicts(withParent);
+        List<Attribute> attributes = Analyzer.mappingAsAttributes(randomSource(), index.mapping());
+        EsRelation relation = new EsRelation(randomSource(), index, attributes, IndexMode.STANDARD);
+        Limit limit = new Limit(randomSource(), new Literal(randomSource(), 10, DataType.INTEGER), relation);
+        Project project = new Project(randomSource(), limit, limit.output());
+        FragmentExec fragmentExec = new FragmentExec(project);
+        ExchangeSinkExec exchangeSinkExec = new ExchangeSinkExec(randomSource(), fragmentExec.output(), false, fragmentExec);
+        try (
+            BytesStreamOutput out = new BytesStreamOutput();
+            PlanStreamOutput pso = new PlanStreamOutput(out, new PlanNameRegistry(), configuration())
+        ) {
+            pso.writePhysicalPlanNode(exchangeSinkExec);
+            assertThat(ByteSizeValue.ofBytes(out.bytes().length()), byteSizeEquals(expected));
+            try (
+                PlanStreamInput psi = new PlanStreamInput(
+                    out.bytes().streamInput(),
+                    new PlanNameRegistry(),
+                    getNamedWriteableRegistry(),
+                    configuration()
+                )
+            ) {
+                assertThat(psi.readPhysicalPlanNode(), equalTo(exchangeSinkExec));
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSourceExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSourceExecSerializationTests.java
new file mode 100644
index 0000000000000..a1a890f3addd6
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSourceExecSerializationTests.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.physical;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.io.IOException;
+import java.util.List;
+
+public class ExchangeSourceExecSerializationTests extends AbstractPhysicalPlanSerializationTests<ExchangeSourceExec> {
+    static ExchangeSourceExec randomExchangeSourceExec() {
+        Source source = randomSource();
+        List<Attribute> output = randomFieldAttributes(1, 5, false);
+        boolean intermediateAgg = randomBoolean();
+        return new ExchangeSourceExec(source, output, intermediateAgg);
+    }
+
+    @Override
+    protected ExchangeSourceExec createTestInstance() {
+        return randomExchangeSourceExec();
+    }
+
+    @Override
+    protected ExchangeSourceExec mutateInstance(ExchangeSourceExec instance) throws IOException {
+        List<Attribute> output = instance.output();
+        boolean intermediateAgg = instance.isIntermediateAgg();
+        if (randomBoolean()) {
+            output = randomValueOtherThan(output, () -> randomFieldAttributes(1, 5, false));
+        } else {
+            intermediateAgg = false == intermediateAgg;
+        }
+        return new ExchangeSourceExec(instance.source(), output, intermediateAgg);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
index fcb7d02460e94..bfad9fd3d634c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
@@ -49,7 +49,7 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 
 import java.time.Duration;
 import java.time.ZoneOffset;
@@ -65,7 +65,7 @@ public class EvalMapperTests extends ESTestCase {
     private static final FieldAttribute LONG = field("long", DataType.LONG);
     private static final FieldAttribute DATE = field("date", DataType.DATETIME);
 
-    private static final EsqlConfiguration TEST_CONFIG = new EsqlConfiguration(
+    private static final Configuration TEST_CONFIG = new Configuration(
         ZoneOffset.UTC,
         Locale.US,
         "test",
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java
index 78e3ee134b6d7..fcb27c3ba5673 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java
@@ -21,12 +21,15 @@
 import org.elasticsearch.xpack.esql.SerializationTestUtils;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.Queries;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
+import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerContext;
@@ -35,6 +38,7 @@
 import org.elasticsearch.xpack.esql.plan.physical.FragmentExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
@@ -45,6 +49,7 @@
 
 import static java.util.Arrays.asList;
 import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
+import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
@@ -52,7 +57,6 @@
 import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.FILTER;
 import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.MUST;
 import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.SHOULD;
-import static org.elasticsearch.xpack.esql.core.util.SourceUtils.writeSource;
 import static org.hamcrest.Matchers.nullValue;
 
 public class FilterTests extends ESTestCase {
@@ -65,14 +69,13 @@ public class FilterTests extends ESTestCase {
     private static Analyzer analyzer;
     private static LogicalPlanOptimizer logicalOptimizer;
     private static PhysicalPlanOptimizer physicalPlanOptimizer;
-    private static Map<String, EsField> mapping;
     private static Mapper mapper;
 
     @BeforeClass
     public static void init() {
         parser = new EsqlParser();
 
-        mapping = loadMapping("mapping-basic.json");
+        Map<String, EsField> mapping = loadMapping("mapping-basic.json");
         EsIndex test = new EsIndex("test", mapping, Set.of("test"));
         IndexResolution getIndexResult = IndexResolution.valid(test);
         logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG));
@@ -100,13 +103,14 @@ public void testTimestampRequestFilterNoQueryFilter() {
     public void testTimestampNoRequestFilterQueryFilter() {
         var value = 10;
 
-        var plan = plan(LoggerMessageFormat.format(null, """
+        String query = LoggerMessageFormat.format(null, """
              FROM test
             |WHERE {} > {}
-            """, EMP_NO, value), null);
+            """, EMP_NO, value);
+        var plan = plan(query, null);
 
         var filter = filterQueryForTransportNodes(plan);
-        var expected = singleValueQuery(rangeQuery(EMP_NO).gt(value), EMP_NO, ((SingleValueQuery.Builder) filter).source());
+        var expected = singleValueQuery(query, rangeQuery(EMP_NO).gt(value), EMP_NO, ((SingleValueQuery.Builder) filter).source());
         assertEquals(expected.toString(), filter.toString());
     }
 
@@ -114,14 +118,16 @@ public void testTimestampRequestFilterQueryFilter() {
         var value = 10;
         var restFilter = restFilterQuery(EMP_NO);
 
-        var plan = plan(LoggerMessageFormat.format(null, """
+        String query = LoggerMessageFormat.format(null, """
              FROM test
             |WHERE {} > 10
-            """, EMP_NO, value), restFilter);
+            """, EMP_NO, value);
+        var plan = plan(query, restFilter);
 
         var filter = filterQueryForTransportNodes(plan);
         var builder = ((BoolQueryBuilder) filter).filter().get(1);
         var queryFilter = singleValueQuery(
+            query,
             rangeQuery(EMP_NO).gt(value).includeUpper(false),
             EMP_NO,
             ((SingleValueQuery.Builder) builder).source()
@@ -135,15 +141,16 @@ public void testTimestampRequestFilterQueryFilterWithConjunction() {
         var highValue = 100;
         var restFilter = restFilterQuery(EMP_NO);
 
-        var plan = plan(LoggerMessageFormat.format(null, """
+        String query = LoggerMessageFormat.format(null, """
              FROM test
             |WHERE {} > {} AND {} < {}
-            """, EMP_NO, lowValue, EMP_NO, highValue), restFilter);
+            """, EMP_NO, lowValue, EMP_NO, highValue);
+        var plan = plan(query, restFilter);
 
         var filter = filterQueryForTransportNodes(plan);
         var musts = ((BoolQueryBuilder) ((BoolQueryBuilder) filter).filter().get(1)).must();
-        var left = singleValueQuery(rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(0)).source());
-        var right = singleValueQuery(rangeQuery(EMP_NO).lt(highValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(1)).source());
+        var left = singleValueQuery(query, rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(0)).source());
+        var right = singleValueQuery(query, rangeQuery(EMP_NO).lt(highValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(1)).source());
         var must = Queries.combine(MUST, asList(left, right));
         var expected = Queries.combine(FILTER, asList(restFilter, must));
         assertEquals(expected.toString(), filter.toString());
@@ -169,15 +176,16 @@ public void testTimestampRequestFilterQueryFilterWithDisjunctionOnSameField() {
         var highValue = 100;
         var restFilter = restFilterQuery(EMP_NO);
 
-        var plan = plan(LoggerMessageFormat.format(null, """
+        String query = LoggerMessageFormat.format(null, """
              FROM test
             |WHERE {} > {} OR {} < {}
-            """, EMP_NO, lowValue, EMP_NO, highValue), restFilter);
+            """, EMP_NO, lowValue, EMP_NO, highValue);
+        var plan = plan(query, restFilter);
 
         var filter = filterQueryForTransportNodes(plan);
         var shoulds = ((BoolQueryBuilder) ((BoolQueryBuilder) filter).filter().get(1)).should();
-        var left = singleValueQuery(rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) shoulds.get(0)).source());
-        var right = singleValueQuery(rangeQuery(EMP_NO).lt(highValue), EMP_NO, ((SingleValueQuery.Builder) shoulds.get(1)).source());
+        var left = singleValueQuery(query, rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) shoulds.get(0)).source());
+        var right = singleValueQuery(query, rangeQuery(EMP_NO).lt(highValue), EMP_NO, ((SingleValueQuery.Builder) shoulds.get(1)).source());
         var should = Queries.combine(SHOULD, asList(left, right));
         var expected = Queries.combine(FILTER, asList(restFilter, should));
         assertEquals(expected.toString(), filter.toString());
@@ -189,15 +197,16 @@ public void testTimestampRequestFilterQueryFilterWithMultiConjunction() {
         var eqValue = 1234;
         var restFilter = restFilterQuery(EMP_NO);
 
-        var plan = plan(LoggerMessageFormat.format(null, """
+        String query = LoggerMessageFormat.format(null, """
              FROM test
             |WHERE {} > {} AND {} == {} AND {} < {}
-            """, EMP_NO, lowValue, OTHER_FIELD, eqValue, EMP_NO, highValue), restFilter);
+            """, EMP_NO, lowValue, OTHER_FIELD, eqValue, EMP_NO, highValue);
+        var plan = plan(query, restFilter);
 
         var filter = filterQueryForTransportNodes(plan);
         var musts = ((BoolQueryBuilder) ((BoolQueryBuilder) filter).filter().get(1)).must();
-        var left = singleValueQuery(rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(0)).source());
-        var right = singleValueQuery(rangeQuery(EMP_NO).lt(highValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(1)).source());
+        var left = singleValueQuery(query, rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(0)).source());
+        var right = singleValueQuery(query, rangeQuery(EMP_NO).lt(highValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(1)).source());
         var must = Queries.combine(MUST, asList(left, right));
         var expected = Queries.combine(FILTER, asList(restFilter, must));
         assertEquals(expected.toString(), filter.toString());
@@ -210,16 +219,17 @@ public void testTimestampRequestFilterQueryMultipleFilters() {
 
         var restFilter = restFilterQuery(EMP_NO);
 
-        var plan = plan(LoggerMessageFormat.format(null, """
+        String query = LoggerMessageFormat.format(null, """
              FROM test
             |WHERE {} > {}
             |EVAL {} = {}
             |WHERE {} > {}
-            """, EMP_NO, lowValue, EMP_NO, eqValue, EMP_NO, highValue), restFilter);
+            """, EMP_NO, lowValue, EMP_NO, eqValue, EMP_NO, highValue);
+        var plan = plan(query, restFilter);
 
         var filter = filterQueryForTransportNodes(plan);
         var builder = ((BoolQueryBuilder) filter).filter().get(1);
-        var queryFilter = singleValueQuery(rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) builder).source());
+        var queryFilter = singleValueQuery(query, rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) builder).source());
         var expected = Queries.combine(FILTER, asList(restFilter, queryFilter));
         assertEquals(expected.toString(), filter.toString());
     }
@@ -265,21 +275,25 @@ public void testTimestampAsFunctionArgumentInsideExpression() {
      * Ugly hack to create a QueryBuilder for SingleValueQuery.
      * For some reason however the queryName is set to null on range queries when deserializing.
      */
-    public static QueryBuilder singleValueQuery(QueryBuilder inner, String field, Source source) {
+    public static QueryBuilder singleValueQuery(String query, QueryBuilder inner, String field, Source source) {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
+            Configuration config = randomConfiguration(query, Map.of());
+
             // emulate SingleValueQuery writeTo
             out.writeFloat(AbstractQueryBuilder.DEFAULT_BOOST);
             out.writeOptionalString(null);
             out.writeNamedWriteable(inner);
             out.writeString(field);
-            writeSource(out, source);
+            source.writeTo(new PlanStreamOutput(out, new PlanNameRegistry(), config));
 
             StreamInput in = new NamedWriteableAwareStreamInput(
                 ByteBufferStreamInput.wrap(BytesReference.toBytes(out.bytes())),
                 SerializationTestUtils.writableRegistry()
             );
 
-            Object obj = SingleValueQuery.ENTRY.reader.read(in);
+            Object obj = SingleValueQuery.ENTRY.reader.read(
+                new PlanStreamInput(in, new PlanNameRegistry(), in.namedWriteableRegistry(), config)
+            );
             return (QueryBuilder) obj;
         } catch (IOException e) {
             throw new UncheckedIOException(e);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
index 5ecdb4864a9cc..81c012bb95fd8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
@@ -33,16 +33,16 @@
 import org.elasticsearch.xpack.esql.TestBlockFactory;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
-import org.elasticsearch.xpack.esql.core.expression.Order;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
+import org.elasticsearch.xpack.esql.expression.Order;
+import org.elasticsearch.xpack.esql.index.EsIndex;
 import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.hamcrest.Matcher;
 import org.junit.After;
 
@@ -135,8 +135,8 @@ private LocalExecutionPlanner planner() throws IOException {
         );
     }
 
-    private EsqlConfiguration config() {
-        return new EsqlConfiguration(
+    private Configuration config() {
+        return new Configuration(
             randomZone(),
             randomLocale(random()),
             "test_user",
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java
index 5954836096ffa..7c64796abf521 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
 import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
 import org.elasticsearch.xpack.esql.analysis.Verifier;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.optimizer.TestPlannerOptimizer;
 import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
@@ -31,10 +31,13 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.matchesRegex;
 
+//@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug")
 public class QueryTranslatorTests extends ESTestCase {
 
     private static TestPlannerOptimizer plannerOptimizer;
 
+    private static TestPlannerOptimizer plannerOptimizerIPs;
+
     private static Analyzer makeAnalyzer(String mappingFileName) {
         var mapping = loadMapping(mappingFileName);
         EsIndex test = new EsIndex("test", mapping, Set.of("test"));
@@ -49,6 +52,7 @@ private static Analyzer makeAnalyzer(String mappingFileName) {
     @BeforeClass
     public static void init() {
         plannerOptimizer = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer("mapping-all-types.json"));
+        plannerOptimizerIPs = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer("mapping-hosts.json"));
     }
 
     @Override
@@ -63,6 +67,13 @@ public void assertQueryTranslation(String query, Matcher<String> translationMatc
         assertThat(translatedQuery, translationMatcher);
     }
 
+    public void assertQueryTranslationIPs(String query, Matcher<String> translationMatcher) {
+        PhysicalPlan optimized = plannerOptimizerIPs.plan(query);
+        EsQueryExec eqe = (EsQueryExec) optimized.collectLeaves().get(0);
+        final String translatedQuery = eqe.query().toString().replaceAll("\\s+", "");
+        assertThat(translatedQuery, translationMatcher);
+    }
+
     public void testBinaryComparisons() {
         assertQueryTranslation("""
             FROM test | WHERE 10 < integer""", containsString("""
@@ -102,54 +113,181 @@ public void testBinaryComparisons() {
     }
 
     public void testRanges() {
-        // Note: Currently binary comparisons are not combined into range queries, so we get bool queries with multiple
-        // one-sided ranges for now.
-
-        // Once we combine binary comparisons, this query should be trivial.
+        // ORs
         assertQueryTranslation("""
             FROM test | WHERE 10 < integer OR integer < 12""", matchesRegex("""
             .*should.*""" + """
             esql_single_value":\\{"field":"integer".*"range":\\{"integer":\\{"gt":10,.*""" + """
             esql_single_value":\\{"field":"integer".*"range":\\{"integer":\\{"lt":12.*"""));
 
+        assertQueryTranslation("""
+            FROM test | WHERE 10 <= integer OR integer < 12""", matchesRegex("""
+            .*should.*""" + """
+            esql_single_value":\\{"field":"integer".*"range":\\{"integer":\\{"gte":10,.*""" + """
+            esql_single_value":\\{"field":"integer".*"range":\\{"integer":\\{"lt":12.*"""));
+
+        assertQueryTranslation("""
+            FROM test | WHERE 10 < integer OR integer <= 12""", matchesRegex("""
+            .*should.*""" + """
+            esql_single_value":\\{"field":"integer".*"range":\\{"integer":\\{"gt":10,.*""" + """
+            esql_single_value":\\{"field":"integer".*"range":\\{"integer":\\{"lte":12.*"""));
+
+        assertQueryTranslation("""
+            FROM test | WHERE 10 <= integer OR integer <= 12""", matchesRegex("""
+            .*should.*""" + """
+            esql_single_value":\\{"field":"integer".*"range":\\{"integer":\\{"gte":10,.*""" + """
+            esql_single_value":\\{"field":"integer".*"range":\\{"integer":\\{"lte":12.*"""));
+
+        // ANDs
         assertQueryTranslation("""
             FROM test | WHERE 10 < integer AND integer < 12""", matchesRegex("""
-            .*must.*esql_single_value":\\{"field":"integer\"""" + """
-            .*"range":\\{"integer":\\{"gt":10,.*"range":\\{"integer":\\{"lt":12.*"""));
+            \\{"esql_single_value":\\{"field":"integer","next":\\{"range":\\{"integer":\\{"gt":10,"lt":12.*"""));
 
         assertQueryTranslation("""
             FROM test | WHERE 10 <= integer AND integer <= 12""", matchesRegex("""
-            .*must.*esql_single_value":\\{"field":"integer\"""" + """
-            .*"range":\\{"integer":\\{"gte":10,.*"range":\\{"integer":\\{"lte":12.*"""));
+            \\{"esql_single_value":\\{"field":"integer","next":\\{"range":\\{"integer":\\{"gte":10,"lte":12.*"""));
+
+        assertQueryTranslation("""
+            FROM test | WHERE 10 <= integer AND integer < 12""", matchesRegex("""
+            \\{"esql_single_value":\\{"field":"integer","next":\\{"range":\\{"integer":\\{"gte":10,"lt":12.*"""));
 
         assertQueryTranslation("""
             FROM test | WHERE 10.9 < double AND double < 12.1""", matchesRegex("""
-            .*must.*esql_single_value":\\{"field":"double\"""" + """
-            .*"range":\\{"double":\\{"gt":10.9,.*"range":\\{"double":\\{"lt":12.1.*"""));
+            \\{"esql_single_value":\\{"field":"double","next":\\{"range":\\{"double":\\{"gt":10.9,"lt":12.1.*"""));
 
         assertQueryTranslation("""
             FROM test | WHERE 10.9 <= double AND double <= 12.1""", matchesRegex("""
-            .*must.*esql_single_value":\\{"field":"double\"""" + """
-            .*"range":\\{"double":\\{"gte":10.9,.*"range":\\{"double":\\{"lte":12.1.*"""));
+            \\{"esql_single_value":\\{"field":"double","next":\\{"range":\\{"double":\\{"gte":10.9,"lte":12.1.*"""));
 
         assertQueryTranslation("""
-            FROM test | WHERE "2007-12-03T10:15:30+01:00" < date AND date < "2024-01-01T10:15:30+01:00\"""", matchesRegex("""
-            .*must.*esql_single_value":\\{"field":"date\"""" + """
-            .*"range":\\{"date":\\{"gt":\"2007-12-03T09:15:30.000Z\",.*"range":\\{"date":\\{"lt":\"2024-01-01T09:15:30.000Z\".*"""));
+            FROM test | WHERE 10.9 < double AND double <= 12.1""", matchesRegex("""
+            \\{"esql_single_value":\\{"field":"double","next":\\{"range":\\{"double":\\{"gt":10.9,"lte":12.1.*"""));
 
         assertQueryTranslation("""
-            FROM test | WHERE "2007-12-03T10:15:30+01:00" <= date AND date <= "2024-01-01T10:15:30+01:00\"""", matchesRegex("""
-            .*must.*esql_single_value":\\{"field":"date\"""" + """
-            .*"range":\\{"date":\\{"gte":\"2007-12-03T09:15:30.000Z\",.*"range":\\{"date":\\{"lte":\"2024-01-01T09:15:30.000Z\".*"""));
+            FROM test | WHERE 2147483648::unsigned_long < unsigned_long AND unsigned_long < 2147483650::unsigned_long""", matchesRegex("""
+            \\{"esql_single_value":\\{"field":"unsigned_long".*\\{"range":\\{"unsigned_long":\\{"gt":2147483648,"lt":2147483650.*"""));
 
         assertQueryTranslation("""
-            FROM test | WHERE 2147483648::unsigned_long < unsigned_long AND unsigned_long < 2147483650::unsigned_long""", matchesRegex("""
-            .*must.*esql_single_value":\\{"field":"unsigned_long\"""" + """
-            .*"range":\\{"unsigned_long":\\{"gt":2147483648,.*"range":\\{"unsigned_long":\\{"lt":2147483650,.*"""));
+            FROM test | WHERE 2147483648::unsigned_long <= unsigned_long AND unsigned_long <= 2147483650::unsigned_long""", matchesRegex("""
+            \\{"esql_single_value":\\{"field":"unsigned_long".*\\{"range":\\{"unsigned_long":\\{"gte":2147483648,"lte":2147483650.*"""));
 
         assertQueryTranslation("""
             FROM test | WHERE 2147483648::unsigned_long <= unsigned_long AND unsigned_long <= 2147483650::unsigned_long""", matchesRegex("""
-            .*must.*esql_single_value":\\{"field":"unsigned_long\"""" + """
-            .*"range":\\{"unsigned_long":\\{"gte":2147483648,.*"range":\\{"unsigned_long":\\{"lte":2147483650,.*"""));
+            \\{"esql_single_value":\\{"field":"unsigned_long".*\\{"range":\\{"unsigned_long":\\{"gte":2147483648,"lte":2147483650.*"""));
+
+        // mixed ANDs and NotEquals
+        assertQueryTranslation("""
+            FROM test | WHERE 10 < integer AND integer < 12 AND integer > 0 AND integer != 5""", matchesRegex("""
+            .*bool.*must.*""" + """
+            esql_single_value":\\{"field":"integer","next":\\{"bool":""" + """
+            .*must_not.*\\[\\{"term":\\{"integer":\\{"value":5.*""" + """
+            esql_single_value":\\{"field":"integer","next":\\{"range":\\{"integer":\\{"gt":10,"lt":12.*"""));
+
+        // multiple Ranges
+        assertQueryTranslation("""
+            FROM test | WHERE 10 < integer AND double < 1.0 AND integer < 12 AND double > -1.0""", matchesRegex("""
+            .*bool.*must.*""" + """
+            esql_single_value":\\{"field":"integer","next":\\{"range":\\{"integer":\\{"gt":10,"lt":12.*""" + """
+            esql_single_value":\\{"field":"double","next":\\{"range":\\{"double":\\{"gt":-1.0,"lt":1.0.*"""));
+
+        assertQueryTranslation("""
+            FROM test | WHERE "2007-12-03T10:15:30Z" <= date AND date <= "2024-01-01T10:15:30\"""", containsString("""
+            "esql_single_value":{"field":"date","next":{"range":{"date":{"gte":"2007-12-03T10:15:30.000Z","lte":"2024-01-01T10:15:30.000Z",\
+            "time_zone":"Z","format":"strict_date_optional_time","boost":1.0}}}"""));
+
+        assertQueryTranslation("""
+            FROM test | WHERE "2007-12-03T10:15:30" <= date AND date <= "2024-01-01T10:15:30Z\"""", containsString("""
+            "esql_single_value":{"field":"date","next":{"range":{"date":{"gte":"2007-12-03T10:15:30.000Z","lte":"2024-01-01T10:15:30.000Z",\
+            "time_zone":"Z","format":"strict_date_optional_time","boost":1.0}}}"""));
+
+        // various timezones
+        assertQueryTranslation("""
+            FROM test | WHERE "2007-12-03T10:15:30+01:00" < date AND date < "2024-01-01T10:15:30+01:00\"""", containsString("""
+            "esql_single_value":{"field":"date","next":{"range":{"date":{"gt":"2007-12-03T09:15:30.000Z","lt":"2024-01-01T09:15:30.000Z",\
+            "time_zone":"Z","format":"strict_date_optional_time","boost":1.0}}}"""));
+
+        assertQueryTranslation("""
+            FROM test | WHERE "2007-12-03T10:15:30-01:00" <= date AND date <= "2024-01-01T10:15:30+01:00\"""", containsString("""
+            "esql_single_value":{"field":"date","next":{"range":{"date":{"gte":"2007-12-03T11:15:30.000Z","lte":"2024-01-01T09:15:30.000Z",\
+            "time_zone":"Z","format":"strict_date_optional_time","boost":1.0}}}"""));
+
+        assertQueryTranslation("""
+            FROM test | WHERE "2007-12-03T10:15:30" <= date AND date <= "2024-01-01T10:15:30+01:00\"""", containsString("""
+            "esql_single_value":{"field":"date","next":{"range":{"date":{"gte":"2007-12-03T10:15:30.000Z","lte":"2024-01-01T09:15:30.000Z",\
+            "time_zone":"Z","format":"strict_date_optional_time","boost":1.0}}}"""));
+    }
+
+    public void testIPs() {
+        // Nothing to combine
+        assertQueryTranslationIPs("""
+            FROM hosts | WHERE CIDR_MATCH(ip0, "127.0.0.1") OR CIDR_MATCH(ip0, "127.0.0.3") AND CIDR_MATCH(ip1, "fe80::cae2:65ff:fece:fec0")
+            """, matchesRegex("""
+            .*bool.*should.*""" + """
+            esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1".*""" + """
+            .*bool.*must.*""" + """
+            esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.3".*""" + """
+            esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["fe80::cae2:65ff:fece:fec0".*"""));
+
+        // ANDs
+        assertQueryTranslationIPs("""
+            FROM hosts | WHERE ip1 >= "127.0.0.1" AND ip1 <= "128.0.0.1" \
+            AND ip0 > "127.0.0.1" AND  ip0 < "128.0.0.1\"""", matchesRegex("""
+            .*bool.*must.*""" + """
+            esql_single_value":\\{"field":"ip1".*"range":\\{"ip1":\\{"gte":"127.0.0.1","lte":"128.0.0.1".*""" + """
+            esql_single_value":\\{"field":"ip0".*"range":\\{"ip0":\\{"gt":"127.0.0.1","lt":"128.0.0.1".*"""));
+
+        // ORs - Combine Equals, In and CIDRMatch on IP type
+        assertQueryTranslationIPs("""
+            FROM hosts | WHERE host == "alpha" OR host == "gamma" OR CIDR_MATCH(ip1, "127.0.0.2/32") OR CIDR_MATCH(ip1, "127.0.0.3/32") \
+            OR card IN ("eth0", "eth1") OR card == "lo0" OR CIDR_MATCH(ip0, "127.0.0.1") OR \
+            CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRegex("""
+            .*bool.*should.*""" + """
+            esql_single_value":\\{"field":"host".*"terms":\\{"host":\\["alpha","gamma","beta".*""" + """
+            esql_single_value":\\{"field":"card".*"terms":\\{"card":\\["eth0","eth1","lo0".*""" + """
+            esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.2/32","127.0.0.3/32".*""" + """
+            esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","fe80::cae2:65ff:fece:feb9".*"""));
+
+        assertQueryTranslationIPs("""
+            FROM hosts | WHERE host == "alpha" OR host == "gamma" OR CIDR_MATCH(ip1, "127.0.0.2/32") OR CIDR_MATCH(ip1, "127.0.0.3/32") \
+            OR card IN ("eth0", "eth1") OR card == "lo0" OR CIDR_MATCH(ip0, "127.0.0.1") OR \
+            CIDR_MATCH(ip0, "127.0.0.0/24", "172.0.0.0/31") OR CIDR_MATCH(ip0, "127.0.1.0/24") OR \
+            CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:fec0", "172.0.2.0/24") OR \
+            CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRegex("""
+            .*bool.*should.*""" + """
+            esql_single_value":\\{"field":"host".*"terms":\\{"host":\\["alpha","gamma","beta".*""" + """
+            esql_single_value":\\{"field":"card".*"terms":\\{"card":\\["eth0","eth1","lo0".*""" + """
+            esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.2/32","127.0.0.3/32".*""" + """
+            esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","127.0.0.0/24","172.0.0.0/31","127.0.1.0/24",\
+            "fe80::cae2:65ff:fece:fec0","172.0.2.0/24","fe80::cae2:65ff:fece:feb9".*"""));
+
+        assertQueryTranslationIPs("""
+            FROM hosts | WHERE host == "alpha" OR host == "gamma" OR ip1 IN ("127.0.0.2"::ip) OR CIDR_MATCH(ip1, "127.0.0.3/32") \
+            OR card IN ("eth0", "eth1") OR card == "lo0" OR ip0 IN ("127.0.0.1"::ip, "128.0.0.1"::ip) \
+            OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRegex("""
+            .*bool.*should.*""" + """
+            esql_single_value":\\{"field":"host".*"terms":\\{"host":\\["alpha","gamma","beta".*""" + """
+            esql_single_value":\\{"field":"card".*"terms":\\{"card":\\["eth0","eth1","lo0".*""" + """
+            esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.3/32","127.0.0.2".*""" + """
+            esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","128.0.0.1","fe80::cae2:65ff:fece:feb9".*"""));
+
+        assertQueryTranslationIPs("""
+            FROM hosts | WHERE host == "alpha" OR host == "gamma" OR ip1 == "127.0.0.2"::ip OR CIDR_MATCH(ip1, "127.0.0.3/32") \
+            OR card IN ("eth0", "eth1") OR card == "lo0" OR ip0 IN ("127.0.0.1"::ip, "128.0.0.1"::ip) \
+            OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRegex("""
+            .*bool.*should.*""" + """
+            esql_single_value":\\{"field":"host".*"terms":\\{"host":\\["alpha","gamma","beta".*""" + """
+            esql_single_value":\\{"field":"card".*"terms":\\{"card":\\["eth0","eth1","lo0".*""" + """
+            esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.3/32","127.0.0.2".*""" + """
+            esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","128.0.0.1","fe80::cae2:65ff:fece:feb9".*"""));
+
+        assertQueryTranslationIPs("""
+            FROM hosts | WHERE host == "alpha" OR host == "gamma" OR ip1 == "127.0.0.2" OR CIDR_MATCH(ip1, "127.0.0.3/32") \
+            OR card IN ("eth0", "eth1") OR card == "lo0" OR ip0 IN ("127.0.0.1"::ip, "128.0.0.1"::ip) \
+            OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRegex("""
+            .*bool.*should.*""" + """
+            esql_single_value":\\{"field":"host".*"terms":\\{"host":\\["alpha","gamma","beta".*""" + """
+            esql_single_value":\\{"field":"card".*"terms":\\{"card":\\["eth0","eth1","lo0".*""" + """
+            esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.3/32","127.0.0.2".*""" + """
+            esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","128.0.0.1","fe80::cae2:65ff:fece:feb9".*"""));
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java
index b08a2798bc509..0cd1fa11a7499 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java
@@ -39,7 +39,6 @@
 import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec;
 import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.LocalExecutionPlannerContext;
 import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.PhysicalOperation;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.List;
 import java.util.Random;
@@ -323,7 +322,7 @@ private Block extractBlockForColumn(
     }
 
     private boolean shouldMapToDocValues(DataType dataType, MappedFieldType.FieldExtractPreference extractPreference) {
-        return extractPreference == DOC_VALUES && EsqlDataTypes.isSpatialPoint(dataType);
+        return extractPreference == DOC_VALUES && DataType.isSpatialPoint(dataType);
     }
 
     private static class TestBlockCopier {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
index c93f3b9e0e350..26529a3605d38 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ComputeListenerTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.compute.operator.DriverProfile;
+import org.elasticsearch.compute.operator.DriverSleeps;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.TaskCancellationService;
@@ -92,7 +93,17 @@ private ComputeResponse randomResponse() {
         int numProfiles = randomIntBetween(0, 2);
         List<DriverProfile> profiles = new ArrayList<>(numProfiles);
         for (int i = 0; i < numProfiles; i++) {
-            profiles.add(new DriverProfile(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), List.of()));
+            profiles.add(
+                new DriverProfile(
+                    randomNonNegativeLong(),
+                    randomNonNegativeLong(),
+                    randomNonNegativeLong(),
+                    randomNonNegativeLong(),
+                    randomNonNegativeLong(),
+                    List.of(),
+                    DriverSleeps.empty()
+                )
+            );
         }
         return new ComputeResponse(profiles);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
index 06c6b5de3cdea..2dc3a43cf3d71 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.plugin;
 
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
@@ -19,10 +20,10 @@
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
-import org.elasticsearch.xpack.esql.core.index.EsIndex;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerContext;
@@ -31,7 +32,6 @@
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.planner.Mapper;
-import org.elasticsearch.xpack.esql.session.EsqlConfigurationSerializationTests;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -39,6 +39,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
+import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomTables;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_CFG;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyPolicyResolution;
@@ -84,11 +86,13 @@ protected DataNodeRequest createTestInstance() {
         );
         DataNodeRequest request = new DataNodeRequest(
             sessionId,
-            EsqlConfigurationSerializationTests.randomConfiguration(query, EsqlConfigurationSerializationTests.randomTables()),
+            randomConfiguration(query, randomTables()),
             randomAlphaOfLength(10),
             shardIds,
             aliasFilters,
-            physicalPlan
+            physicalPlan,
+            generateRandomStringArray(10, 10, false, false),
+            IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())
         );
         request.setParentTask(randomAlphaOfLength(10), randomNonNegativeLong());
         return request;
@@ -96,7 +100,7 @@ protected DataNodeRequest createTestInstance() {
 
     @Override
     protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException {
-        return switch (between(0, 6)) {
+        return switch (between(0, 8)) {
             case 0 -> {
                 var request = new DataNodeRequest(
                     randomAlphaOfLength(20),
@@ -104,7 +108,9 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.clusterAlias(),
                     in.shardIds(),
                     in.aliasFilters(),
-                    in.plan()
+                    in.plan(),
+                    in.indices(),
+                    in.indicesOptions()
                 );
                 request.setParentTask(in.getParentTask());
                 yield request;
@@ -112,11 +118,13 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
             case 1 -> {
                 var request = new DataNodeRequest(
                     in.sessionId(),
-                    EsqlConfigurationSerializationTests.randomConfiguration(),
+                    randomConfiguration(),
                     in.clusterAlias(),
                     in.shardIds(),
                     in.aliasFilters(),
-                    in.plan()
+                    in.plan(),
+                    in.indices(),
+                    in.indicesOptions()
                 );
                 request.setParentTask(in.getParentTask());
                 yield request;
@@ -129,7 +137,9 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.clusterAlias(),
                     shardIds,
                     in.aliasFilters(),
-                    in.plan()
+                    in.plan(),
+                    in.indices(),
+                    in.indicesOptions()
                 );
                 request.setParentTask(in.getParentTask());
                 yield request;
@@ -154,7 +164,9 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.clusterAlias(),
                     in.shardIds(),
                     in.aliasFilters(),
-                    mapAndMaybeOptimize(parse(newQuery))
+                    mapAndMaybeOptimize(parse(newQuery)),
+                    in.indices(),
+                    in.indicesOptions()
                 );
                 request.setParentTask(in.getParentTask());
                 yield request;
@@ -172,7 +184,9 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.clusterAlias(),
                     in.shardIds(),
                     aliasFilters,
-                    in.plan()
+                    in.plan(),
+                    in.indices(),
+                    in.indicesOptions()
                 );
                 request.setParentTask(request.getParentTask());
                 yield request;
@@ -184,7 +198,9 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     in.clusterAlias(),
                     in.shardIds(),
                     in.aliasFilters(),
-                    in.plan()
+                    in.plan(),
+                    in.indices(),
+                    in.indicesOptions()
                 );
                 request.setParentTask(
                     randomValueOtherThan(request.getParentTask().getNodeId(), () -> randomAlphaOfLength(10)),
@@ -200,7 +216,42 @@ protected DataNodeRequest mutateInstance(DataNodeRequest in) throws IOException
                     clusterAlias,
                     in.shardIds(),
                     in.aliasFilters(),
-                    in.plan()
+                    in.plan(),
+                    in.indices(),
+                    in.indicesOptions()
+                );
+                request.setParentTask(request.getParentTask());
+                yield request;
+            }
+            case 7 -> {
+                var indices = randomValueOtherThan(in.indices(), () -> generateRandomStringArray(10, 10, false, false));
+                var request = new DataNodeRequest(
+                    in.sessionId(),
+                    in.configuration(),
+                    in.clusterAlias(),
+                    in.shardIds(),
+                    in.aliasFilters(),
+                    in.plan(),
+                    indices,
+                    in.indicesOptions()
+                );
+                request.setParentTask(request.getParentTask());
+                yield request;
+            }
+            case 8 -> {
+                var indicesOptions = randomValueOtherThan(
+                    in.indicesOptions(),
+                    () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())
+                );
+                var request = new DataNodeRequest(
+                    in.sessionId(),
+                    in.configuration(),
+                    in.clusterAlias(),
+                    in.shardIds(),
+                    in.aliasFilters(),
+                    in.plan(),
+                    in.indices(),
+                    indicesOptions
                 );
                 request.setParentTask(request.getParentTask());
                 yield request;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuerySerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuerySerializationTests.java
index a3bf34ad38b8e..5ba03a3fd89fb 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuerySerializationTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuerySerializationTests.java
@@ -7,16 +7,30 @@
 
 package org.elasticsearch.xpack.esql.querydsl.query;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.test.AbstractWireTestCase;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+import org.elasticsearch.xpack.esql.session.Configuration;
+import org.junit.Before;
 
+import java.io.IOException;
 import java.util.List;
 
-public class SingleValueQuerySerializationTests extends AbstractWireSerializingTestCase<SingleValueQuery.Builder> {
+import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
+
+public class SingleValueQuerySerializationTests extends AbstractWireTestCase<SingleValueQuery.Builder> {
+    /**
+     * We use a single random config for all serialization because it's pretty
+     * heavy to build, especially in {@link #testConcurrentSerialization()}.
+     */
+    private Configuration config;
+
     @Override
     protected SingleValueQuery.Builder createTestInstance() {
         return new SingleValueQuery.Builder(randomQuery(), randomFieldName(), Source.EMPTY);
@@ -48,14 +62,31 @@ protected SingleValueQuery.Builder mutateInstance(SingleValueQuery.Builder insta
     }
 
     @Override
-    protected Writeable.Reader<SingleValueQuery.Builder> instanceReader() {
-        return SingleValueQuery.Builder::new;
+    protected final SingleValueQuery.Builder copyInstance(SingleValueQuery.Builder instance, TransportVersion version) throws IOException {
+        return copyInstance(
+            instance,
+            getNamedWriteableRegistry(),
+            (out, v) -> new PlanStreamOutput(out, new PlanNameRegistry(), config).writeNamedWriteable(v),
+            in -> {
+                PlanStreamInput pin = new PlanStreamInput(in, new PlanNameRegistry(), in.namedWriteableRegistry(), config);
+                return (SingleValueQuery.Builder) pin.readNamedWriteable(QueryBuilder.class);
+            },
+            version
+        );
     }
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         return new NamedWriteableRegistry(
-            List.of(new NamedWriteableRegistry.Entry(QueryBuilder.class, TermQueryBuilder.NAME, TermQueryBuilder::new))
+            List.of(
+                SingleValueQuery.ENTRY,
+                new NamedWriteableRegistry.Entry(QueryBuilder.class, TermQueryBuilder.NAME, TermQueryBuilder::new)
+            )
         );
     }
+
+    @Before
+    public void initConfig() {
+        this.config = randomConfiguration();
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/ConfigurationSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/ConfigurationSerializationTests.java
new file mode 100644
index 0000000000000..8f0a4227e60ef
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/ConfigurationSerializationTests.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.session;
+
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockStreamInput;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.esql.Column;
+import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
+
+import java.time.ZoneId;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
+import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomTables;
+
+public class ConfigurationSerializationTests extends AbstractWireSerializingTestCase<Configuration> {
+
+    @Override
+    protected Writeable.Reader<Configuration> instanceReader() {
+        return in -> new Configuration(
+            new BlockStreamInput(in, new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE))
+        );
+    }
+
+    @Override
+    protected Configuration createTestInstance() {
+        return randomConfiguration();
+    }
+
+    @Override
+    protected Configuration mutateInstance(Configuration in) {
+        ZoneId zoneId = in.zoneId();
+        Locale locale = in.locale();
+        String username = in.username();
+        String clusterName = in.clusterName();
+        QueryPragmas pragmas = in.pragmas();
+        int resultTruncationMaxSize = in.resultTruncationMaxSize();
+        int resultTruncationDefaultSize = in.resultTruncationDefaultSize();
+        String query = in.query();
+        boolean profile = in.profile();
+        Map<String, Map<String, Column>> tables = in.tables();
+        switch (between(0, 9)) {
+            case 0 -> zoneId = randomValueOtherThan(zoneId, () -> randomZone().normalized());
+            case 1 -> locale = randomValueOtherThan(in.locale(), () -> randomLocale(random()));
+            case 2 -> username = randomAlphaOfLength(15);
+            case 3 -> clusterName = randomAlphaOfLength(15);
+            case 4 -> pragmas = new QueryPragmas(
+                Settings.builder().put(QueryPragmas.EXCHANGE_BUFFER_SIZE.getKey(), between(1, 10)).build()
+            );
+            case 5 -> resultTruncationMaxSize += randomIntBetween(3, 10);
+            case 6 -> resultTruncationDefaultSize += randomIntBetween(3, 10);
+            case 7 -> query += randomAlphaOfLength(2);
+            case 8 -> profile = false == profile;
+            case 9 -> {
+                while (true) {
+                    Map<String, Map<String, Column>> newTables = null;
+                    try {
+                        newTables = randomTables();
+                        if (false == tables.equals(newTables)) {
+                            tables = newTables;
+                            newTables = null;
+                            break;
+                        }
+                    } finally {
+                        if (newTables != null) {
+                            Releasables.close(
+                                Releasables.wrap(
+                                    Iterators.flatMap(
+                                        newTables.values().iterator(),
+                                        columns -> Iterators.map(columns.values().iterator(), Column::values)
+                                    )
+                                )
+                            );
+                        }
+                    }
+                }
+            }
+        }
+        return new Configuration(
+            zoneId,
+            locale,
+            username,
+            clusterName,
+            pragmas,
+            resultTruncationMaxSize,
+            resultTruncationDefaultSize,
+            query,
+            profile,
+            tables
+        );
+
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Block.getNamedWriteables());
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlConfigurationSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlConfigurationSerializationTests.java
deleted file mode 100644
index fe3bb0f064732..0000000000000
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlConfigurationSerializationTests.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.session;
-
-import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.breaker.NoopCircuitBreaker;
-import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.compute.data.Block;
-import org.elasticsearch.compute.data.BlockFactory;
-import org.elasticsearch.compute.data.BlockStreamInput;
-import org.elasticsearch.compute.data.BlockUtils;
-import org.elasticsearch.compute.data.ElementType;
-import org.elasticsearch.compute.lucene.DataPartitioning;
-import org.elasticsearch.core.Releasables;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.esql.Column;
-import org.elasticsearch.xpack.esql.action.ParseTables;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
-import org.elasticsearch.xpack.esql.planner.PlannerUtils;
-import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
-
-import java.time.ZoneId;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-
-import static org.elasticsearch.xpack.esql.session.EsqlConfiguration.QUERY_COMPRESS_THRESHOLD_CHARS;
-
-public class EsqlConfigurationSerializationTests extends AbstractWireSerializingTestCase<EsqlConfiguration> {
-
-    @Override
-    protected Writeable.Reader<EsqlConfiguration> instanceReader() {
-        return in -> new EsqlConfiguration(
-            new BlockStreamInput(in, new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE))
-        );
-    }
-
-    private static QueryPragmas randomQueryPragmas() {
-        return new QueryPragmas(
-            Settings.builder().put(QueryPragmas.DATA_PARTITIONING.getKey(), randomFrom(DataPartitioning.values())).build()
-        );
-    }
-
-    public static EsqlConfiguration randomConfiguration() {
-        int len = randomIntBetween(1, 300) + (frequently() ? 0 : QUERY_COMPRESS_THRESHOLD_CHARS);
-        return randomConfiguration(randomRealisticUnicodeOfLength(len), randomTables());
-    }
-
-    public static EsqlConfiguration randomConfiguration(String query, Map<String, Map<String, Column>> tables) {
-        var zoneId = randomZone();
-        var locale = randomLocale(random());
-        var username = randomAlphaOfLengthBetween(1, 10);
-        var clusterName = randomAlphaOfLengthBetween(3, 10);
-        var truncation = randomNonNegativeInt();
-        var defaultTruncation = randomNonNegativeInt();
-        boolean profile = randomBoolean();
-
-        return new EsqlConfiguration(
-            zoneId,
-            locale,
-            username,
-            clusterName,
-            randomQueryPragmas(),
-            truncation,
-            defaultTruncation,
-            query,
-            profile,
-            tables
-        );
-    }
-
-    public static Map<String, Map<String, Column>> randomTables() {
-        if (randomBoolean()) {
-            return Map.of();
-        }
-        int count = between(1, 10);
-        Map<String, Map<String, Column>> tables = new HashMap<>(count);
-        try {
-            for (int i = 0; i < 10; i++) {
-                tables.put(randomAlphaOfLength(i + 1), randomColumns());
-            }
-            return tables;
-        } finally {
-            if (tables.size() != count) {
-                Releasables.close(
-                    Releasables.wrap(
-                        Iterators.flatMap(tables.values().iterator(), columns -> Iterators.map(columns.values().iterator(), Column::values))
-                    )
-                );
-            }
-        }
-    }
-
-    static Map<String, Column> randomColumns() {
-        int count = between(1, 10);
-        Map<String, Column> columns = new HashMap<>(count);
-        int positions = between(1, 10_000);
-        try {
-            for (int i = 0; i < count; i++) {
-                String name = randomAlphaOfLength(i + 1);
-                DataType dataType = randomFrom(ParseTables.SUPPORTED_TYPES);
-                ElementType type = PlannerUtils.toElementType(dataType);
-                try (
-                    Block.Builder builder = type.newBlockBuilder(
-                        positions,
-                        new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE)
-                    )
-                ) {
-                    for (int p = 0; p < positions; p++) {
-                        BlockUtils.appendValue(builder, AbstractFunctionTestCase.randomLiteral(dataType).value(), type);
-                    }
-                    columns.put(name, new Column(dataType, builder.build()));
-                }
-            }
-            return columns;
-        } finally {
-            if (columns.size() != count) {
-                Releasables.close(Releasables.wrap(Iterators.map(columns.values().iterator(), Column::values)));
-            }
-        }
-    }
-
-    @Override
-    protected EsqlConfiguration createTestInstance() {
-        return randomConfiguration();
-    }
-
-    @Override
-    protected EsqlConfiguration mutateInstance(EsqlConfiguration in) {
-        ZoneId zoneId = in.zoneId();
-        Locale locale = in.locale();
-        String username = in.username();
-        String clusterName = in.clusterName();
-        QueryPragmas pragmas = in.pragmas();
-        int resultTruncationMaxSize = in.resultTruncationMaxSize();
-        int resultTruncationDefaultSize = in.resultTruncationDefaultSize();
-        String query = in.query();
-        boolean profile = in.profile();
-        Map<String, Map<String, Column>> tables = in.tables();
-        switch (between(0, 9)) {
-            case 0 -> zoneId = randomValueOtherThan(zoneId, () -> randomZone().normalized());
-            case 1 -> locale = randomValueOtherThan(in.locale(), () -> randomLocale(random()));
-            case 2 -> username = randomAlphaOfLength(15);
-            case 3 -> clusterName = randomAlphaOfLength(15);
-            case 4 -> pragmas = new QueryPragmas(
-                Settings.builder().put(QueryPragmas.EXCHANGE_BUFFER_SIZE.getKey(), between(1, 10)).build()
-            );
-            case 5 -> resultTruncationMaxSize += randomIntBetween(3, 10);
-            case 6 -> resultTruncationDefaultSize += randomIntBetween(3, 10);
-            case 7 -> query += randomAlphaOfLength(2);
-            case 8 -> profile = false == profile;
-            case 9 -> {
-                while (true) {
-                    Map<String, Map<String, Column>> newTables = null;
-                    try {
-                        newTables = randomTables();
-                        if (false == tables.equals(newTables)) {
-                            tables = newTables;
-                            newTables = null;
-                            break;
-                        }
-                    } finally {
-                        if (newTables != null) {
-                            Releasables.close(
-                                Releasables.wrap(
-                                    Iterators.flatMap(
-                                        newTables.values().iterator(),
-                                        columns -> Iterators.map(columns.values().iterator(), Column::values)
-                                    )
-                                )
-                            );
-                        }
-                    }
-                }
-            }
-        }
-        return new EsqlConfiguration(
-            zoneId,
-            locale,
-            username,
-            clusterName,
-            pragmas,
-            resultTruncationMaxSize,
-            resultTruncationDefaultSize,
-            query,
-            profile,
-            tables
-        );
-
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(Block.getNamedWriteables());
-    }
-}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
index a4d75c1eb85c3..d3795c9e9d953 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.fieldcaps.IndexFieldCapabilities;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -26,7 +27,6 @@
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.session.IndexResolver;
 import org.elasticsearch.xpack.esql.session.Result;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
 import org.junit.After;
 import org.junit.Before;
 import org.mockito.stubbing.Answer;
@@ -75,7 +75,7 @@ public void testFailedMetric() {
         String[] indices = new String[] { "test" };
 
         Client qlClient = mock(Client.class);
-        IndexResolver idxResolver = new IndexResolver(qlClient, EsqlDataTypeRegistry.INSTANCE);
+        IndexResolver idxResolver = new IndexResolver(qlClient);
         // simulate a valid field_caps response so we can parse and correctly analyze de query
         FieldCapabilitiesResponse fieldCapabilitiesResponse = mock(FieldCapabilitiesResponse.class);
         when(fieldCapabilitiesResponse.getIndices()).thenReturn(indices);
@@ -89,7 +89,7 @@ public void testFailedMetric() {
         }).when(qlClient).execute(eq(EsqlResolveFieldsAction.TYPE), any(), any());
 
         Client esqlClient = mock(Client.class);
-        IndexResolver indexResolver = new IndexResolver(esqlClient, EsqlDataTypeRegistry.INSTANCE);
+        IndexResolver indexResolver = new IndexResolver(esqlClient);
         doAnswer((Answer<Void>) invocation -> {
             @SuppressWarnings("unchecked")
             ActionListener<FieldCapabilitiesResponse> listener = (ActionListener<FieldCapabilitiesResponse>) invocation.getArguments()[2];
@@ -152,7 +152,8 @@ private List<FieldCapabilitiesIndexResponse> indexFieldCapabilities(String[] ind
                         Map.entry("foo", new IndexFieldCapabilities("foo", "integer", false, true, true, false, null, Map.of())),
                         Map.entry("bar", new IndexFieldCapabilities("bar", "long", false, true, true, false, null, Map.of()))
                     ),
-                    true
+                    true,
+                    IndexMode.STANDARD
                 )
             );
         }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
index 3036a729aa298..323bfebabe688 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
@@ -16,12 +16,10 @@
 import org.elasticsearch.dissect.DissectParser;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
-import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
-import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttributeTests;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedNamedExpression;
@@ -29,7 +27,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.FullTextPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.LikePattern;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.AbstractNodeTestCase;
 import org.elasticsearch.xpack.esql.core.tree.Node;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -39,6 +36,7 @@
 import org.elasticsearch.xpack.esql.core.tree.SourceTests;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow;
@@ -46,12 +44,14 @@
 import org.elasticsearch.xpack.esql.plan.logical.Dissect;
 import org.elasticsearch.xpack.esql.plan.logical.Grok;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.plan.logical.PhasedTests;
 import org.elasticsearch.xpack.esql.plan.logical.join.JoinType;
 import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.Stat;
 import org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType;
 import org.elasticsearch.xpack.esql.plan.physical.OutputExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.mockito.exceptions.base.MockitoException;
 
 import java.io.IOException;
@@ -85,6 +85,7 @@
 import java.util.jar.JarInputStream;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
 import static org.mockito.Mockito.mock;
 
 /**
@@ -115,7 +116,7 @@ public class EsqlNodeSubclassTests<T extends B, B extends Node<B>> extends NodeS
     private static final Predicate<String> CLASSNAME_FILTER = className -> {
         boolean esqlCore = className.startsWith("org.elasticsearch.xpack.esql.core") != false;
         boolean esqlProper = className.startsWith("org.elasticsearch.xpack.esql") != false;
-        return esqlCore || esqlProper;
+        return (esqlCore || esqlProper) && className.equals(PhasedTests.Dummy.class.getName()) == false;
     };
 
     /**
@@ -126,7 +127,7 @@ public class EsqlNodeSubclassTests<T extends B, B extends Node<B>> extends NodeS
     @SuppressWarnings("rawtypes")
     public static List<Object[]> nodeSubclasses() throws IOException {
         return subclassesOf(Node.class, CLASSNAME_FILTER).stream()
-            .filter(c -> testClassFor(c) == null)
+            .filter(c -> testClassFor(c) == null || c != PhasedTests.Dummy.class)
             .map(c -> new Object[] { c })
             .toList();
     }
@@ -480,7 +481,7 @@ public void accept(Page page) {
             return randomZone();
         }
         if (argClass == Configuration.class) {
-            return EsqlTestUtils.randomConfiguration();
+            return randomConfiguration();
         }
         try {
             return mock(argClass);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/AbstractEsFieldTypeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/AbstractEsFieldTypeTests.java
new file mode 100644
index 0000000000000..9b2bf03b5c8aa
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/AbstractEsFieldTypeTests.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.type;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.AbstractWireTestCase;
+import org.elasticsearch.xpack.esql.EsqlTestUtils;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+public abstract class AbstractEsFieldTypeTests<T extends EsField> extends AbstractWireTestCase<EsField> {
+    public static EsField randomAnyEsField(int maxDepth) {
+        return switch (between(0, 5)) {
+            case 0 -> EsFieldTests.randomEsField(maxDepth);
+            case 1 -> DateEsFieldTests.randomDateEsField(maxDepth);
+            case 2 -> InvalidMappedFieldTests.randomInvalidMappedField(maxDepth);
+            case 3 -> KeywordEsFieldTests.randomKeywordEsField(maxDepth);
+            case 4 -> TextEsFieldTests.randomTextEsField(maxDepth);
+            case 5 -> UnsupportedEsFieldTests.randomUnsupportedEsField(maxDepth);
+            default -> throw new IllegalArgumentException();
+        };
+    }
+
+    @Override
+    protected abstract T createTestInstance();
+
+    protected abstract T mutate(T instance);
+
+    @Override
+    protected EsField copyInstance(EsField instance, TransportVersion version) throws IOException {
+        NamedWriteableRegistry namedWriteableRegistry = getNamedWriteableRegistry();
+        try (
+            BytesStreamOutput output = new BytesStreamOutput();
+            var pso = new PlanStreamOutput(output, new PlanNameRegistry(), EsqlTestUtils.TEST_CFG)
+        ) {
+            pso.setTransportVersion(version);
+            instance.writeTo(pso);
+            try (
+                StreamInput in1 = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry);
+                var psi = new PlanStreamInput(in1, new PlanNameRegistry(), in1.namedWriteableRegistry(), EsqlTestUtils.TEST_CFG)
+            ) {
+                psi.setTransportVersion(version);
+                return EsField.readFrom(psi);
+            }
+        }
+    }
+
+    /**
+     * Generate sub-properties.
+     * @param maxDepth the maximum number of levels of properties to make
+     */
+    static Map<String, EsField> randomProperties(int maxDepth) {
+        if (maxDepth < 0) {
+            throw new IllegalArgumentException("depth must be >= 0");
+        }
+        if (maxDepth == 0 || randomBoolean()) {
+            return Map.of();
+        }
+        int targetSize = between(1, 5);
+        Map<String, EsField> properties = new TreeMap<>();
+        while (properties.size() < targetSize) {
+            properties.put(randomAlphaOfLength(properties.size() + 1), randomAnyEsField(maxDepth - 1));
+        }
+        return properties;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected final T mutateInstance(EsField instance) throws IOException {
+        return mutate((T) instance);
+    }
+
+    @Override
+    protected final NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(List.of());
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DataTypeConversionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/DataTypeConversionTests.java
similarity index 93%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DataTypeConversionTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/DataTypeConversionTests.java
index dee41e089de13..871bf632adcc6 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DataTypeConversionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/DataTypeConversionTests.java
@@ -4,13 +4,16 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.esql.core.type;
+package org.elasticsearch.xpack.esql.type;
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Location;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.Converter;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypeConverter;
 import org.elasticsearch.xpack.versionfield.Version;
 
 import java.math.BigDecimal;
@@ -32,9 +35,8 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
-import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.commonType;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.converterFor;
-import static org.elasticsearch.xpack.esql.core.type.DateUtils.asDateTime;
+import static org.elasticsearch.xpack.esql.core.util.DateUtils.asDateTime;
 
 public class DataTypeConversionTests extends ESTestCase {
 
@@ -56,7 +58,7 @@ public void testConversionToString() {
             assertNull(conversion.convert(null));
             assertEquals("1973-11-29T21:33:09.101Z", conversion.convert(asDateTime(123456789101L)));
             assertEquals("1966-02-02T02:26:50.899Z", conversion.convert(asDateTime(-123456789101L)));
-            assertEquals("2020-05-01T10:20:30.123456789Z", conversion.convert(DateUtils.asDateTime("2020-05-01T10:20:30.123456789Z")));
+            assertEquals("2020-05-01T10:20:30.123456789Z", conversion.convert(asDateTime("2020-05-01T10:20:30.123456789Z")));
         }
     }
 
@@ -102,7 +104,7 @@ public void testConversionToLong() {
             assertEquals(123456789101L, conversion.convert(asDateTime(123456789101L)));
             assertEquals(-123456789101L, conversion.convert(asDateTime(-123456789101L)));
             // Nanos are ignored, only millis are used
-            assertEquals(1588328430123L, conversion.convert(DateUtils.asDateTime("2020-05-01T10:20:30.123456789Z")));
+            assertEquals(1588328430123L, conversion.convert(asDateTime("2020-05-01T10:20:30.123456789Z")));
         }
         {
             Converter conversion = converterFor(KEYWORD, to);
@@ -213,7 +215,7 @@ public void testConversionToFloat() {
             assertEquals(1.23456789101E11f, (float) conversion.convert(asDateTime(123456789101L)), 0);
             assertEquals(-1.23456789101E11f, (float) conversion.convert(asDateTime(-123456789101L)), 0);
             // Nanos are ignored, only millis are used
-            assertEquals(1.5883284E12f, conversion.convert(DateUtils.asDateTime("2020-05-01T10:20:30.123456789Z")));
+            assertEquals(1.5883284E12f, conversion.convert(asDateTime("2020-05-01T10:20:30.123456789Z")));
         }
         {
             Converter conversion = converterFor(KEYWORD, to);
@@ -260,7 +262,7 @@ public void testConversionToDouble() {
             assertEquals(1.23456789101E11, (double) conversion.convert(asDateTime(123456789101L)), 0);
             assertEquals(-1.23456789101E11, (double) conversion.convert(asDateTime(-123456789101L)), 0);
             // Nanos are ignored, only millis are used
-            assertEquals(1.588328430123E12, conversion.convert(DateUtils.asDateTime("2020-05-01T10:20:30.123456789Z")));
+            assertEquals(1.588328430123E12, conversion.convert(asDateTime("2020-05-01T10:20:30.123456789Z")));
         }
         {
             Converter conversion = converterFor(KEYWORD, to);
@@ -429,7 +431,7 @@ public void testConversionToInt() {
             assertEquals(223456789, conversion.convert(asDateTime(223456789L)));
             assertEquals(-123456789, conversion.convert(asDateTime(-123456789L)));
             // Nanos are ignored, only millis are used
-            assertEquals(62123, conversion.convert(DateUtils.asDateTime("1970-01-01T00:01:02.123456789Z")));
+            assertEquals(62123, conversion.convert(asDateTime("1970-01-01T00:01:02.123456789Z")));
             Exception e = expectThrows(InvalidArgumentException.class, () -> conversion.convert(asDateTime(Long.MAX_VALUE)));
             assertEquals("[" + Long.MAX_VALUE + "] out of [integer] range", e.getMessage());
         }
@@ -462,7 +464,7 @@ public void testConversionToShort() {
             assertEquals((short) 12345, conversion.convert(asDateTime(12345L)));
             assertEquals((short) -12345, conversion.convert(asDateTime(-12345L)));
             // Nanos are ignored, only millis are used
-            assertEquals((short) 1123, conversion.convert(DateUtils.asDateTime("1970-01-01T00:00:01.123456789Z")));
+            assertEquals((short) 1123, conversion.convert(asDateTime("1970-01-01T00:00:01.123456789Z")));
             Exception e = expectThrows(InvalidArgumentException.class, () -> conversion.convert(asDateTime(Integer.MAX_VALUE)));
             assertEquals("[" + Integer.MAX_VALUE + "] out of [short] range", e.getMessage());
         }
@@ -495,7 +497,7 @@ public void testConversionToByte() {
             assertEquals((byte) 123, conversion.convert(asDateTime(123L)));
             assertEquals((byte) -123, conversion.convert(asDateTime(-123L)));
             // Nanos are ignored, only millis are used
-            assertEquals((byte) 123, conversion.convert(DateUtils.asDateTime("1970-01-01T00:00:00.123456789Z")));
+            assertEquals((byte) 123, conversion.convert(asDateTime("1970-01-01T00:00:00.123456789Z")));
             Exception e = expectThrows(InvalidArgumentException.class, () -> conversion.convert(asDateTime(Integer.MAX_VALUE)));
             assertEquals("[" + Integer.MAX_VALUE + "] out of [byte] range", e.getMessage());
         }
@@ -519,25 +521,6 @@ public void testConversionToIdentity() {
         assertEquals(10, conversion.convert(10));
     }
 
-    public void testCommonType() {
-        assertEquals(BOOLEAN, commonType(BOOLEAN, NULL));
-        assertEquals(BOOLEAN, commonType(NULL, BOOLEAN));
-        assertEquals(BOOLEAN, commonType(BOOLEAN, BOOLEAN));
-        assertEquals(NULL, commonType(NULL, NULL));
-        assertEquals(INTEGER, commonType(INTEGER, KEYWORD));
-        assertEquals(LONG, commonType(TEXT, LONG));
-        assertEquals(SHORT, commonType(SHORT, BYTE));
-        assertEquals(FLOAT, commonType(BYTE, FLOAT));
-        assertEquals(FLOAT, commonType(FLOAT, INTEGER));
-        assertEquals(UNSIGNED_LONG, commonType(UNSIGNED_LONG, LONG));
-        assertEquals(DOUBLE, commonType(DOUBLE, FLOAT));
-        assertEquals(FLOAT, commonType(FLOAT, UNSIGNED_LONG));
-
-        // strings
-        assertEquals(TEXT, commonType(TEXT, KEYWORD));
-        assertEquals(TEXT, commonType(KEYWORD, TEXT));
-    }
-
     public void testEsDataTypes() {
         for (DataType type : DataType.types()) {
             assertEquals(type, DataType.fromTypeName(type.typeName()));
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DateEsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/DateEsFieldTests.java
similarity index 89%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DateEsFieldTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/DateEsFieldTests.java
index dea03ee8a8cdf..bf0494d5fd043 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DateEsFieldTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/DateEsFieldTests.java
@@ -5,7 +5,10 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core.type;
+package org.elasticsearch.xpack.esql.type;
+
+import org.elasticsearch.xpack.esql.core.type.DateEsField;
+import org.elasticsearch.xpack.esql.core.type.EsField;
 
 import java.util.Map;
 
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsFieldTests.java
similarity index 91%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsFieldTests.java
index e72ae0c5c0cda..e824b4de03e26 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsFieldTests.java
@@ -5,7 +5,10 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core.type;
+package org.elasticsearch.xpack.esql.type;
+
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.EsField;
 
 import java.util.Map;
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java
new file mode 100644
index 0000000000000..8ad083683f696
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.type;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOC_DATA_TYPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.HALF_FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
+import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.PARTIAL_AGG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.SCALED_FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.SOURCE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TSID_DATA_TYPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTimeOrTemporal;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isString;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.commonType;
+
+public class EsqlDataTypeConverterTests extends ESTestCase {
+
+    public void testNanoTimeToString() {
+        long expected = randomLong();
+        long actual = EsqlDataTypeConverter.dateNanosToLong(EsqlDataTypeConverter.nanoTimeToString(expected));
+        assertEquals(expected, actual);
+    }
+
+    public void testCommonTypeNull() {
+        for (DataType dataType : DataType.values()) {
+            assertEqualsCommonType(dataType, NULL, dataType);
+        }
+    }
+
+    public void testCommonTypeStrings() {
+        List<DataType> STRINGS = Arrays.stream(DataType.values()).filter(DataType::isString).toList();
+        for (DataType dataType1 : STRINGS) {
+            for (DataType dataType2 : DataType.values()) {
+                if (dataType2 == NULL) {
+                    assertEqualsCommonType(dataType1, NULL, dataType1);
+                } else if ((isString(dataType1) && isString(dataType2))) {
+                    if (dataType1 == dataType2) {
+                        assertEqualsCommonType(dataType1, dataType2, dataType1);
+                    } else {
+                        assertEqualsCommonType(dataType1, dataType2, TEXT);
+                    }
+                } else {
+                    assertNullCommonType(dataType1, dataType2);
+                }
+            }
+        }
+    }
+
+    public void testCommonTypeDateTimeIntervals() {
+        List<DataType> DATE_TIME_INTERVALS = Arrays.stream(DataType.values()).filter(DataType::isDateTimeOrTemporal).toList();
+        for (DataType dataType1 : DATE_TIME_INTERVALS) {
+            for (DataType dataType2 : DataType.values()) {
+                if (dataType2 == NULL) {
+                    assertEqualsCommonType(dataType1, NULL, dataType1);
+                } else if (isDateTimeOrTemporal(dataType2)) {
+                    if (isDateTime(dataType1) || isDateTime(dataType2)) {
+                        assertEqualsCommonType(dataType1, dataType2, DATETIME);
+                    } else if (dataType1 == dataType2) {
+                        assertEqualsCommonType(dataType1, dataType2, dataType1);
+                    } else {
+                        assertNullCommonType(dataType1, dataType2);
+                    }
+                } else {
+                    assertNullCommonType(dataType1, dataType2);
+                }
+            }
+        }
+    }
+
+    public void testCommonTypeNumeric() {
+        // whole numbers
+        commonNumericType(BYTE, List.of(NULL, BYTE));
+        commonNumericType(SHORT, List.of(NULL, BYTE, SHORT));
+        commonNumericType(INTEGER, List.of(NULL, BYTE, SHORT, INTEGER));
+        commonNumericType(LONG, List.of(NULL, BYTE, SHORT, INTEGER, LONG));
+        commonNumericType(UNSIGNED_LONG, List.of(NULL, BYTE, SHORT, INTEGER, LONG, UNSIGNED_LONG));
+        // floats
+        commonNumericType(HALF_FLOAT, List.of(NULL, BYTE, SHORT, INTEGER, LONG, UNSIGNED_LONG, HALF_FLOAT, FLOAT));
+        commonNumericType(FLOAT, List.of(NULL, BYTE, SHORT, INTEGER, LONG, UNSIGNED_LONG, FLOAT, HALF_FLOAT));
+        commonNumericType(DOUBLE, List.of(NULL, BYTE, SHORT, INTEGER, LONG, UNSIGNED_LONG, HALF_FLOAT, FLOAT, DOUBLE, SCALED_FLOAT));
+        commonNumericType(SCALED_FLOAT, List.of(NULL, BYTE, SHORT, INTEGER, LONG, UNSIGNED_LONG, HALF_FLOAT, FLOAT, SCALED_FLOAT, DOUBLE));
+    }
+
+    /**
+     * The first argument and the second argument(s) have the first argument as a common type.
+     */
+    private static void commonNumericType(DataType numericType, List<DataType> lowerTypes) {
+        List<DataType> NUMERICS = Arrays.stream(DataType.values()).filter(DataType::isNumeric).toList();
+        List<DataType> DOUBLES = Arrays.stream(DataType.values()).filter(DataType::isRationalNumber).toList();
+        for (DataType dataType : DataType.values()) {
+            if (DOUBLES.containsAll(List.of(numericType, dataType)) && (dataType.estimatedSize().equals(numericType.estimatedSize()))) {
+                assertEquals(numericType, commonType(dataType, numericType));
+            } else if (lowerTypes.contains(dataType)) {
+                assertEqualsCommonType(numericType, dataType, numericType);
+            } else if (NUMERICS.contains(dataType)) {
+                assertEqualsCommonType(numericType, dataType, dataType);
+            } else {
+                assertNullCommonType(numericType, dataType);
+            }
+        }
+    }
+
+    public void testCommonTypeMiscellaneous() {
+        List<DataType> MISCELLANEOUS = List.of(
+            COUNTER_INTEGER,
+            COUNTER_LONG,
+            COUNTER_DOUBLE,
+            UNSUPPORTED,
+            OBJECT,
+            SOURCE,
+            DATE_NANOS,
+            DOC_DATA_TYPE,
+            TSID_DATA_TYPE,
+            PARTIAL_AGG,
+            IP,
+            VERSION,
+            GEO_POINT,
+            GEO_SHAPE,
+            CARTESIAN_POINT,
+            CARTESIAN_SHAPE,
+            BOOLEAN
+        );
+        for (DataType dataType1 : MISCELLANEOUS) {
+            for (DataType dataType2 : DataType.values()) {
+                if (dataType2 == NULL || dataType1 == dataType2) {
+                    assertEqualsCommonType(dataType1, dataType2, dataType1);
+                } else {
+                    assertNullCommonType(dataType1, dataType2);
+                }
+            }
+        }
+    }
+
+    private static void assertEqualsCommonType(DataType dataType1, DataType dataType2, DataType commonType) {
+        assertEquals(commonType, commonType(dataType1, dataType2));
+        assertEquals(commonType, commonType(dataType2, dataType1));
+    }
+
+    private static void assertNullCommonType(DataType dataType1, DataType dataType2) {
+        assertNull(commonType(dataType1, dataType2));
+        assertNull(commonType(dataType2, dataType1));
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java
index ad7be1e38681f..e4e10a5c6af19 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java
@@ -9,11 +9,12 @@
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.fieldcaps.IndexFieldCapabilities;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.session.IndexResolver;
 
 import java.util.List;
@@ -46,12 +47,14 @@ private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType,
                 idx,
                 idx,
                 Map.of(field, new IndexFieldCapabilities(field, esTypeName, false, true, true, false, metricType, Map.of())),
-                true
+                true,
+                IndexMode.TIME_SERIES
             )
         );
 
         FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of());
-        IndexResolution resolution = new IndexResolver(null, EsqlDataTypeRegistry.INSTANCE).mergedMappings("idx-*", caps);
+        // IndexResolver uses EsqlDataTypeRegistry directly
+        IndexResolution resolution = new IndexResolver(null).mergedMappings("idx-*", caps);
         EsField f = resolution.get().mapping().get(field);
         assertThat(f.getDataType(), equalTo(expected));
     }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/InvalidMappedFieldTests.java
similarity index 90%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedFieldTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/InvalidMappedFieldTests.java
index 47a99329d0222..c66088b0695d4 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedFieldTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/InvalidMappedFieldTests.java
@@ -5,7 +5,10 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core.type;
+package org.elasticsearch.xpack.esql.type;
+
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
 
 import java.util.Map;
 
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/KeywordEsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/KeywordEsFieldTests.java
similarity index 92%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/KeywordEsFieldTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/KeywordEsFieldTests.java
index a5d3b8329b2df..ef04f0e27c096 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/KeywordEsFieldTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/KeywordEsFieldTests.java
@@ -5,9 +5,11 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core.type;
+package org.elasticsearch.xpack.esql.type;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.type.KeywordEsField;
 
 import java.util.Map;
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java
index 0fa8719f17744..d4ca40b75d2f3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java
@@ -9,13 +9,14 @@
 
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.test.AbstractNamedWriteableTestCase;
+import org.elasticsearch.test.AbstractWireTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField;
 import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint;
@@ -32,8 +33,7 @@
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
-import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
-import org.elasticsearch.xpack.esql.session.EsqlConfigurationSerializationTests;
+import org.elasticsearch.xpack.esql.session.Configuration;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -42,6 +42,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration;
 import static org.elasticsearch.xpack.esql.core.type.DataType.isString;
 
 /**
@@ -57,13 +58,13 @@
  * These differences can be minimized once Expression is fully supported in the new serialization approach, and the esql and esql.core
  * modules are merged, or at least the relevant classes are moved.
  */
-public class MultiTypeEsFieldTests extends AbstractNamedWriteableTestCase<MultiTypeEsField> {
+public class MultiTypeEsFieldTests extends AbstractWireTestCase<MultiTypeEsField> {
 
-    private EsqlConfiguration config;
+    private Configuration config;
 
     @Before
     public void initConfig() {
-        config = EsqlConfigurationSerializationTests.randomConfiguration();
+        config = randomConfiguration();
     }
 
     @Override
@@ -94,26 +95,19 @@ protected MultiTypeEsField mutateInstance(MultiTypeEsField instance) throws IOEx
     protected final NamedWriteableRegistry getNamedWriteableRegistry() {
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(UnaryScalarFunction.getNamedWriteables());
         entries.addAll(Attribute.getNamedWriteables());
-        entries.addAll(EsField.getNamedWriteables());
-        entries.add(MultiTypeEsField.ENTRY);
         entries.addAll(Expression.getNamedWriteables());
         return new NamedWriteableRegistry(entries);
     }
 
-    @Override
-    protected final Class<MultiTypeEsField> categoryClass() {
-        return MultiTypeEsField.class;
-    }
-
     @Override
     protected final MultiTypeEsField copyInstance(MultiTypeEsField instance, TransportVersion version) throws IOException {
         return copyInstance(
             instance,
             getNamedWriteableRegistry(),
-            (out, v) -> new PlanStreamOutput(out, new PlanNameRegistry(), config).writeNamedWriteable(v),
+            (out, v) -> v.writeTo(new PlanStreamOutput(out, new PlanNameRegistry(), config)),
             in -> {
                 PlanStreamInput pin = new PlanStreamInput(in, new PlanNameRegistry(), in.namedWriteableRegistry(), config);
-                return (MultiTypeEsField) pin.readNamedWriteable(EsField.class);
+                return EsField.readFrom(pin);
             },
             version
         );
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TextEsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/TextEsFieldTests.java
similarity index 90%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TextEsFieldTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/TextEsFieldTests.java
index 817dd7cd27094..9af3b7376f2b2 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TextEsFieldTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/TextEsFieldTests.java
@@ -5,7 +5,10 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core.type;
+package org.elasticsearch.xpack.esql.type;
+
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.type.TextEsField;
 
 import java.util.Map;
 
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/UnsupportedEsFieldTests.java
similarity index 91%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsFieldTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/UnsupportedEsFieldTests.java
index e05d8ca10425e..a89ca9481b7e1 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsFieldTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/UnsupportedEsFieldTests.java
@@ -5,7 +5,10 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core.type;
+package org.elasticsearch.xpack.esql.type;
+
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
 
 import java.util.Map;
 
diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java
index 9c840ba36749a..40e41cb18bdf8 100644
--- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java
+++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.indices.ExecutorNames;
 import org.elasticsearch.indices.SystemDataStreamDescriptor;
@@ -300,6 +301,7 @@ public void cleanUpFeature(ClusterService clusterService, Client client, ActionL
         if (dataStreamDescriptors.isEmpty() == false) {
             try {
                 Request request = new Request(
+                    TimeValue.THIRTY_SECONDS /* TODO should we wait longer? */,
                     dataStreamDescriptors.stream().map(SystemDataStreamDescriptor::getDataStreamName).toArray(String[]::new)
                 );
                 request.indicesOptions(
diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsAction.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsAction.java
index a87297702bd30..75c29b6635a46 100644
--- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsAction.java
+++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsAction.java
@@ -26,15 +26,16 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
@@ -170,7 +171,7 @@ public LocalAction(
             final IndexNameExpressionResolver resolver,
             final ThreadPool threadPool
         ) {
-            super(NAME, actionFilters, transportService.getTaskManager());
+            super(NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
             this.clusterService = clusterService;
             this.client = client;
             this.resolver = resolver;
diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsShardAction.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsShardAction.java
index 206a632dcd0e4..ef05cd7fe8db2 100644
--- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsShardAction.java
+++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsShardAction.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -29,6 +28,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportDeleteSecretAction.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportDeleteSecretAction.java
index 520efe269eb96..b154826e3e8f0 100644
--- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportDeleteSecretAction.java
+++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportDeleteSecretAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportGetSecretAction.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportGetSecretAction.java
index 4c8311924ab4b..7dcb796dd41a4 100644
--- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportGetSecretAction.java
+++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportGetSecretAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportPostSecretAction.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportPostSecretAction.java
index c87c4b58559ea..777bbcc61fec4 100644
--- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportPostSecretAction.java
+++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/TransportPostSecretAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java
index 36d4751423113..ad9900b5b0164 100644
--- a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java
+++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.shard.IndexLongFieldRange;
+import org.elasticsearch.indices.DateFieldRangeInfo;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.protocol.xpack.frozen.FreezeRequest;
@@ -44,6 +45,7 @@
 import java.time.Instant;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING;
@@ -76,8 +78,15 @@ public void testTimestampRangeRecalculatedOnStalePrimaryAllocation() throws IOEx
 
         createIndex("index", 1, 1);
 
-        final DocWriteResponse indexResponse = prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567Z")
-            .get();
+        String timestampVal = "2010-01-06T02:03:04.567Z";
+        String eventIngestedVal = "2010-01-06T02:03:05.567Z";  // one second later
+
+        final DocWriteResponse indexResponse = prepareIndex("index").setSource(
+            DataStream.TIMESTAMP_FIELD_NAME,
+            timestampVal,
+            IndexMetadata.EVENT_INGESTED_FIELD_NAME,
+            eventIngestedVal
+        ).get();
 
         ensureGreen("index");
 
@@ -117,13 +126,23 @@ public void testTimestampRangeRecalculatedOnStalePrimaryAllocation() throws IOEx
         assertThat(timestampFieldRange, not(sameInstance(IndexLongFieldRange.UNKNOWN)));
         assertThat(timestampFieldRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
         assertTrue(timestampFieldRange.isComplete());
-        assertThat(timestampFieldRange.getMin(), equalTo(Instant.parse("2010-01-06T02:03:04.567Z").toEpochMilli()));
-        assertThat(timestampFieldRange.getMax(), equalTo(Instant.parse("2010-01-06T02:03:04.567Z").toEpochMilli()));
+        assertThat(timestampFieldRange.getMin(), equalTo(Instant.parse(timestampVal).toEpochMilli()));
+        assertThat(timestampFieldRange.getMax(), equalTo(Instant.parse(timestampVal).toEpochMilli()));
 
-        assertThat(indexMetadata.getEventIngestedRange(), sameInstance(IndexLongFieldRange.UNKNOWN));
+        IndexLongFieldRange eventIngestedFieldRange = clusterAdmin().prepareState()
+            .get()
+            .getState()
+            .metadata()
+            .index("index")
+            .getEventIngestedRange();
+        assertThat(eventIngestedFieldRange, not(sameInstance(IndexLongFieldRange.UNKNOWN)));
+        assertThat(eventIngestedFieldRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
+        assertTrue(eventIngestedFieldRange.isComplete());
+        assertThat(eventIngestedFieldRange.getMin(), equalTo(Instant.parse(eventIngestedVal).toEpochMilli()));
+        assertThat(eventIngestedFieldRange.getMax(), equalTo(Instant.parse(eventIngestedVal).toEpochMilli()));
     }
 
-    public void testTimestampFieldTypeExposedByAllIndicesServices() throws Exception {
+    public void testTimestampAndEventIngestedFieldTypeExposedByAllIndicesServices() throws Exception {
         internalCluster().startNodes(between(2, 4));
 
         final String locale;
@@ -181,11 +200,11 @@ public void testTimestampFieldTypeExposedByAllIndicesServices() throws Exception
 
         ensureGreen("index");
         if (randomBoolean()) {
-            prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, date).get();
+            prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, date, IndexMetadata.EVENT_INGESTED_FIELD_NAME, date).get();
         }
 
         for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) {
-            assertNull(indicesService.getTimestampFieldType(index));
+            assertNull(indicesService.getTimestampFieldTypeInfo(index));
         }
 
         assertAcked(
@@ -193,15 +212,129 @@ public void testTimestampFieldTypeExposedByAllIndicesServices() throws Exception
         );
         ensureGreen("index");
         for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) {
-            final PlainActionFuture<DateFieldMapper.DateFieldType> timestampFieldTypeFuture = new PlainActionFuture<>();
+            final PlainActionFuture<Map<String, DateFieldMapper.DateFieldType>> future = new PlainActionFuture<>();
             assertBusy(() -> {
-                final DateFieldMapper.DateFieldType timestampFieldType = indicesService.getTimestampFieldType(index);
+                DateFieldRangeInfo timestampsFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(index);
+                DateFieldMapper.DateFieldType timestampFieldType = timestampsFieldTypeInfo.timestampFieldType();
+                DateFieldMapper.DateFieldType eventIngestedFieldType = timestampsFieldTypeInfo.eventIngestedFieldType();
+                assertNotNull(eventIngestedFieldType);
                 assertNotNull(timestampFieldType);
-                timestampFieldTypeFuture.onResponse(timestampFieldType);
+                future.onResponse(
+                    Map.of(
+                        DataStream.TIMESTAMP_FIELD_NAME,
+                        timestampFieldType,
+                        IndexMetadata.EVENT_INGESTED_FIELD_NAME,
+                        eventIngestedFieldType
+                    )
+                );
+            });
+            assertTrue(future.isDone());
+            assertThat(future.get().get(DataStream.TIMESTAMP_FIELD_NAME).dateTimeFormatter().locale().toString(), equalTo(locale));
+            assertThat(future.get().get(DataStream.TIMESTAMP_FIELD_NAME).dateTimeFormatter().parseMillis(date), equalTo(1580817683000L));
+            assertThat(future.get().get(IndexMetadata.EVENT_INGESTED_FIELD_NAME).dateTimeFormatter().locale().toString(), equalTo(locale));
+            assertThat(
+                future.get().get(IndexMetadata.EVENT_INGESTED_FIELD_NAME).dateTimeFormatter().parseMillis(date),
+                equalTo(1580817683000L)
+            );
+        }
+
+        assertAcked(
+            client().execute(
+                FreezeIndexAction.INSTANCE,
+                new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index").setFreeze(false)
+            ).actionGet()
+        );
+        ensureGreen("index");
+        for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) {
+            assertNull(indicesService.getTimestampFieldTypeInfo(index));
+        }
+    }
+
+    public void testTimestampOrEventIngestedFieldTypeExposedByAllIndicesServices() throws Exception {
+        internalCluster().startNodes(between(2, 4));
+
+        final String locale;
+        final String date;
+
+        switch (between(1, 3)) {
+            case 1 -> {
+                locale = "";
+                date = "04 Feb 2020 12:01:23Z";
+            }
+            case 2 -> {
+                locale = "en_GB";
+                date = "04 Feb 2020 12:01:23Z";
+            }
+            case 3 -> {
+                locale = "fr_FR";
+                date = "04 févr. 2020 12:01:23Z";
+            }
+            default -> throw new AssertionError("impossible");
+        }
+
+        String timeField = randomFrom(IndexMetadata.EVENT_INGESTED_FIELD_NAME, DataStream.TIMESTAMP_FIELD_NAME);
+        assertAcked(
+            prepareCreate("index").setSettings(
+                Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            )
+                .setMapping(
+                    jsonBuilder().startObject()
+                        .startObject("_doc")
+                        .startObject("properties")
+                        .startObject(timeField)
+                        .field("type", "date")
+                        .field("format", "dd LLL yyyy HH:mm:ssX")
+                        .field("locale", locale)
+                        .endObject()
+                        .endObject()
+                        .endObject()
+                        .endObject()
+                )
+        );
+
+        final Index index = clusterAdmin().prepareState()
+            .clear()
+            .setIndices("index")
+            .setMetadata(true)
+            .get()
+            .getState()
+            .metadata()
+            .index("index")
+            .getIndex();
+
+        ensureGreen("index");
+        if (randomBoolean()) {
+            prepareIndex("index").setSource(timeField, date).get();
+        }
+
+        for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) {
+            assertNull(indicesService.getTimestampFieldTypeInfo(index));
+        }
+
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index")).actionGet()
+        );
+        ensureGreen("index");
+        for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) {
+            // final PlainActionFuture<DateFieldMapper.DateFieldType> timestampFieldTypeFuture = new PlainActionFuture<>();
+            final PlainActionFuture<Map<String, DateFieldMapper.DateFieldType>> future = new PlainActionFuture<>();
+            assertBusy(() -> {
+                DateFieldRangeInfo timestampsFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(index);
+                DateFieldMapper.DateFieldType timestampFieldType = timestampsFieldTypeInfo.timestampFieldType();
+                DateFieldMapper.DateFieldType eventIngestedFieldType = timestampsFieldTypeInfo.eventIngestedFieldType();
+                if (timeField == DataStream.TIMESTAMP_FIELD_NAME) {
+                    assertNotNull(timestampFieldType);
+                    assertNull(eventIngestedFieldType);
+                    future.onResponse(Map.of(timeField, timestampFieldType));
+                } else {
+                    assertNull(timestampFieldType);
+                    assertNotNull(eventIngestedFieldType);
+                    future.onResponse(Map.of(timeField, eventIngestedFieldType));
+                }
             });
-            assertTrue(timestampFieldTypeFuture.isDone());
-            assertThat(timestampFieldTypeFuture.get().dateTimeFormatter().locale().toString(), equalTo(locale));
-            assertThat(timestampFieldTypeFuture.get().dateTimeFormatter().parseMillis(date), equalTo(1580817683000L));
+            assertTrue(future.isDone());
+            assertThat(future.get().get(timeField).dateTimeFormatter().locale().toString(), equalTo(locale));
+            assertThat(future.get().get(timeField).dateTimeFormatter().parseMillis(date), equalTo(1580817683000L));
         }
 
         assertAcked(
@@ -212,7 +345,7 @@ public void testTimestampFieldTypeExposedByAllIndicesServices() throws Exception
         );
         ensureGreen("index");
         for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) {
-            assertNull(indicesService.getTimestampFieldType(index));
+            assertNull(indicesService.getTimestampFieldTypeInfo(index));
         }
     }
 
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndicesInfoTransportAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndicesInfoTransportAction.java
index c59ca10061117..6676637c0d03a 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndicesInfoTransportAction.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndicesInfoTransportAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.frozen;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndicesUsageTransportAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndicesUsageTransportAction.java
index 07af26058e64b..e7eededb05d80 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndicesUsageTransportAction.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndicesUsageTransportAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.engine.frozen.FrozenEngine;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java
index 58e1eb28952b0..39e992b0d103c 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/action/TransportFreezeIndexAction.java
@@ -30,13 +30,13 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.frozen.FrozenEngine;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.frozen.FreezeRequest;
 import org.elasticsearch.protocol.xpack.frozen.FreezeResponse;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/geoip-enterprise-downloader/src/test/java/org/elasticsearch/xpack/geoip/EnterpriseGeoIpDownloaderLicenseListenerTests.java b/x-pack/plugin/geoip-enterprise-downloader/src/test/java/org/elasticsearch/xpack/geoip/EnterpriseGeoIpDownloaderLicenseListenerTests.java
index 5a5aacd392f3c..8b5b2b84c3ca8 100644
--- a/x-pack/plugin/geoip-enterprise-downloader/src/test/java/org/elasticsearch/xpack/geoip/EnterpriseGeoIpDownloaderLicenseListenerTests.java
+++ b/x-pack/plugin/geoip-enterprise-downloader/src/test/java/org/elasticsearch/xpack/geoip/EnterpriseGeoIpDownloaderLicenseListenerTests.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -50,7 +51,11 @@ public class EnterpriseGeoIpDownloaderLicenseListenerTests extends ESTestCase {
 
     @Before
     public void setup() {
-        threadPool = new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), MeterRegistry.NOOP);
+        threadPool = new ThreadPool(
+            Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(),
+            MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders()
+        );
     }
 
     @After
diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphInfoTransportAction.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphInfoTransportAction.java
index 7a26b59c3d895..b1029771dccbb 100644
--- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphInfoTransportAction.java
+++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphInfoTransportAction.java
@@ -7,8 +7,8 @@
 package org.elasticsearch.xpack.graph;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphUsageTransportAction.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphUsageTransportAction.java
index f1bd79b2f3b53..e19c5717a7a0a 100644
--- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphUsageTransportAction.java
+++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphUsageTransportAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java
index db825182e4621..36e8eaf94c8be 100644
--- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java
+++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java
@@ -21,12 +21,12 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.graph.Connection;
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java
index fa0c510fde5c0..82c547945cd9f 100644
--- a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportDeleteSamlServiceProviderAction.java
@@ -11,9 +11,9 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument;
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java
index 6cc322e84822d..3413b609815e7 100644
--- a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java
index 68b4759412e70..07d71481326d4 100644
--- a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnAction.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java
index 010467e485ae7..b94eb3d259151 100644
--- a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlMetadataAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider;
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java
index 644225edf8d69..e1c9384298ff9 100644
--- a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/action/TransportSamlValidateAuthnRequestAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.idp.saml.authn.SamlAuthnRequestValidator;
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java
index 6d6f292d112f9..4b59488e3707c 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java
@@ -118,13 +118,20 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception {
             null,
             null
         );
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 2);
 
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -170,7 +177,10 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception {
 
         // data stream was rolled over and has 3 indices, two managed by ILM and the write index will be unmanaged
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -207,7 +217,7 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception {
             // the write index is not managed by DSL either
             ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute(
                 ExplainDataStreamLifecycleAction.INSTANCE,
-                new ExplainDataStreamLifecycleAction.Request(new String[] { writeIndex })
+                new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { writeIndex })
             ).actionGet();
             assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(1));
             ExplainIndexDataStreamLifecycle writeIndexDataStreamLifecycleExplain = dataStreamLifecycleExplainResponse.getIndices().get(0);
@@ -217,7 +227,12 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception {
         // let's make the write index of the data stream managed by DSL
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
-            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, new DataStreamLifecycle())
+            new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName },
+                new DataStreamLifecycle()
+            )
         ).actionGet();
 
         // at this point we should be able to rollover the data stream by indexing only one document (as the data stream lifecycle is
@@ -227,7 +242,10 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception {
         // data stream was rolled over and has 4 indices, 2 managed by ILM, and the latest 2 generations managed by the data stream
         // lifecycle
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -238,7 +256,12 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception {
         // let's migrate this data stream to use the custom data stream lifecycle
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
-            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, customLifecycle.getDataStreamRetention())
+            new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName },
+                customLifecycle.getDataStreamRetention()
+            )
         ).actionGet();
 
         assertBusy(() -> {
@@ -274,7 +297,7 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception {
             // the third generation and the write index are managed based on the custom lifecycle
             ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute(
                 ExplainDataStreamLifecycleAction.INSTANCE,
-                new ExplainDataStreamLifecycleAction.Request(new String[] { thirdGenerationIndex, writeIndex })
+                new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { thirdGenerationIndex, writeIndex })
             ).actionGet();
             assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(2));
             for (ExplainIndexDataStreamLifecycle index : dataStreamLifecycleExplainResponse.getIndices()) {
@@ -300,12 +323,19 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro
             null,
             null
         );
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 2);
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -356,7 +386,10 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro
 
         // data stream was rolled over and has 3 indices, ALL managed by ILM
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -392,7 +425,12 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro
         // let's migrate this data stream to use the data stream lifecycle starting with the next generation
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
-            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, TimeValue.timeValueDays(90))
+            new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName },
+                TimeValue.timeValueDays(90)
+            )
         );
 
         putComposableIndexTemplate(
@@ -410,7 +448,10 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro
 
         // data stream was rolled over and has 4 indices, 3 managed by ILM, and the write index managed by the data stream lifecycle
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -450,7 +491,7 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro
             // the write index is managed by the data stream lifecycle
             ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute(
                 ExplainDataStreamLifecycleAction.INSTANCE,
-                new ExplainDataStreamLifecycleAction.Request(new String[] { writeIndex })
+                new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { writeIndex })
             ).actionGet();
             assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(1));
             ExplainIndexDataStreamLifecycle dataStreamLifecycleExplain = dataStreamLifecycleExplainResponse.getIndices().get(0);
@@ -475,13 +516,20 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
             null,
             null
         );
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 2);
 
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -527,7 +575,10 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
         indexDocs(dataStreamName, 2);
 
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -560,7 +611,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
             // the write index is unmanaged
             ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute(
                 ExplainDataStreamLifecycleAction.INSTANCE,
-                new ExplainDataStreamLifecycleAction.Request(new String[] { writeIndex })
+                new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { writeIndex })
             ).actionGet();
             assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(1));
             ExplainIndexDataStreamLifecycle dataStreamLifecycleExplain = dataStreamLifecycleExplainResponse.getIndices().get(0);
@@ -570,7 +621,12 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
         // let's make the write index of the data stream managed by DSL
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
-            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, new DataStreamLifecycle())
+            new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName },
+                new DataStreamLifecycle()
+            )
         ).actionGet();
 
         // at this point, the write index of the data stream is managed by data stream lifecycle and not by ILM anymore so we can just index
@@ -580,12 +636,20 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
         // let's migrate this data stream to use the custom data stream lifecycle
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
-            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, customLifecycle.getDataStreamRetention())
+            new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName },
+                customLifecycle.getDataStreamRetention()
+            )
         ).actionGet();
 
         // data stream was rolled over and has 4 indices, 2 managed by ILM, and 2 managed by the custom data stream lifecycle
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -627,7 +691,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
             // the third generation and the write index are managed by the data stream lifecycle
             ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute(
                 ExplainDataStreamLifecycleAction.INSTANCE,
-                new ExplainDataStreamLifecycleAction.Request(new String[] { thirdGenerationIndex, writeIndex })
+                new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { thirdGenerationIndex, writeIndex })
             ).actionGet();
             assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(2));
             for (ExplainIndexDataStreamLifecycle index : dataStreamLifecycleExplainResponse.getIndices()) {
@@ -640,7 +704,13 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
         // let's migrate this data stream to disable using the data stream lifecycle
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
-            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, TimeValue.timeValueDays(90), false)
+            new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName },
+                TimeValue.timeValueDays(90),
+                false
+            )
         ).actionGet();
 
         // At this point the data stream is not managed by the data stream lifecycle and all the indices fallback to ILM.
@@ -690,7 +760,10 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw
         indexDocs(dataStreamName, 1);
 
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -706,7 +779,7 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw
             // let's check the indices are managed by the data stream lifecycle
             ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute(
                 ExplainDataStreamLifecycleAction.INSTANCE,
-                new ExplainDataStreamLifecycleAction.Request(new String[] { firstGenerationIndex, writeIndex })
+                new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { firstGenerationIndex, writeIndex })
             ).actionGet();
             assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(2));
             for (ExplainIndexDataStreamLifecycle index : dataStreamLifecycleExplainResponse.getIndices()) {
@@ -743,7 +816,10 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw
         indexDocs(dataStreamName, 1);
 
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -760,7 +836,10 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw
             // let's check the previous indices are managed by the data stream lifecycle
             ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute(
                 ExplainDataStreamLifecycleAction.INSTANCE,
-                new ExplainDataStreamLifecycleAction.Request(new String[] { firstGenerationIndex, secondGenerationIndex })
+                new ExplainDataStreamLifecycleAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    new String[] { firstGenerationIndex, secondGenerationIndex }
+                )
             ).actionGet();
             assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(2));
             for (ExplainIndexDataStreamLifecycle index : dataStreamLifecycleExplainResponse.getIndices()) {
@@ -779,7 +858,10 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw
         // rollover should now happen when indexing 2 documents (as configured in ILM)
         indexDocs(dataStreamName, 2);
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -824,14 +906,21 @@ public void testGetDataStreamResponse() throws Exception {
             null,
             null
         );
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         indexDocs(dataStreamName, 2);
 
         // wait to rollover
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -850,7 +939,12 @@ public void testGetDataStreamResponse() throws Exception {
 
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
-            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, TimeValue.timeValueDays(90))
+            new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName },
+                TimeValue.timeValueDays(90)
+            )
         ).actionGet();
 
         // rollover again - at this point this data stream should have 2 backing indices managed by ILM and the write index managed by
@@ -858,7 +952,10 @@ public void testGetDataStreamResponse() throws Exception {
         indexDocs(dataStreamName, 2);
 
         assertBusy(() -> {
-            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+            GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName }
+            );
             GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
                 .actionGet();
             assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -908,7 +1005,10 @@ public void testGetDataStreamResponse() throws Exception {
 
         // remove ILM policy and prefer_ilm from template
         putComposableIndexTemplate(indexTemplateName, null, List.of(dataStreamName + "*"), Settings.builder().build(), null, null);
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStreamName }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
@@ -919,10 +1019,16 @@ public void testGetDataStreamResponse() throws Exception {
         // disable data stream lifecycle on the data stream. the future generations will be UNMANAGED
         client().execute(
             PutDataStreamLifecycleAction.INSTANCE,
-            new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, TimeValue.timeValueDays(90), false)
+            new PutDataStreamLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                new String[] { dataStreamName },
+                TimeValue.timeValueDays(90),
+                false
+            )
         ).actionGet();
 
-        getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+        getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName });
         getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
         dataStreamInfo = getDataStreamResponse.getDataStreams().get(0);
@@ -976,7 +1082,10 @@ private static DataStreamLifecycle customEnabledLifecycle() {
     }
 
     private List<String> getBackingIndices(String dataStreamName) {
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStreamName }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         return getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().stream().map(Index::getName).toList();
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
index 3530f33704beb..30d1d6f7c914b 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
@@ -496,7 +496,7 @@ public void testPollIntervalUpdate() throws Exception {
             assertThat(indexLifecycleService.getScheduler().jobCount(), equalTo(1));
         });
         {
-            TimeValueSchedule schedule = (TimeValueSchedule) indexLifecycleService.getScheduledJob().getSchedule();
+            TimeValueSchedule schedule = (TimeValueSchedule) indexLifecycleService.getScheduledJob().schedule();
             assertThat(schedule.getInterval(), equalTo(pollInterval));
         }
 
@@ -504,7 +504,7 @@ public void testPollIntervalUpdate() throws Exception {
         TimeValue newPollInterval = TimeValue.timeValueHours(randomLongBetween(6, 1000));
         updateClusterSettings(Settings.builder().put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, newPollInterval.getStringRep()));
         {
-            TimeValueSchedule schedule = (TimeValueSchedule) indexLifecycleService.getScheduledJob().getSchedule();
+            TimeValueSchedule schedule = (TimeValueSchedule) indexLifecycleService.getScheduledJob().schedule();
             assertThat(schedule.getInterval(), equalTo(newPollInterval));
         }
     }
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java
index 4e8df685a25a7..a8f47483a15fc 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java
@@ -11,11 +11,11 @@
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.injection.guice.AbstractModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportAction.java
index de941da93562e..d4d1147c635c0 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.ilm;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java
index c2e2c80998992..9c978ffc25cba 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java
@@ -353,8 +353,8 @@ private void cancelJob() {
 
     @Override
     public void triggered(SchedulerEngine.Event event) {
-        if (event.getJobName().equals(XPackField.INDEX_LIFECYCLE)) {
-            logger.trace("job triggered: " + event.getJobName() + ", " + event.getScheduledTime() + ", " + event.getTriggeredTime());
+        if (event.jobName().equals(XPackField.INDEX_LIFECYCLE)) {
+            logger.trace("job triggered: " + event.jobName() + ", " + event.scheduledTime() + ", " + event.triggeredTime());
             triggerPolicies(clusterService.state(), false);
         }
     }
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleUsageTransportAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleUsageTransportAction.java
index 86f0e3ce5bcb1..bcfb39eb98378 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleUsageTransportAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleUsageTransportAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java
index 05f9fe7820baf..a532392f35080 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java
@@ -20,9 +20,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java
index 9e3399bd220ae..383dc6622f280 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java
@@ -19,9 +19,9 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java
index f58d101a330c8..f4598727d6123 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java
index b18cde4446c74..ea35bcafd9681 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java
index 472b9bdd0b800..48cf84ed7a6a4 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java
@@ -24,10 +24,10 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMoveToStepAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMoveToStepAction.java
index ec905c0e9eb48..ce9f73b065c22 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMoveToStepAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMoveToStepAction.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -33,6 +32,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java
index 26cede5881aa5..92d182ea6d44a 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java
@@ -23,10 +23,10 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRemoveIndexLifecyclePolicyAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRemoveIndexLifecyclePolicyAction.java
index 57cc88d24efbc..4fc8f6b65bbda 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRemoveIndexLifecyclePolicyAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRemoveIndexLifecyclePolicyAction.java
@@ -16,10 +16,10 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java
index ee96fa73838df..83f26d1ec73f3 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportRetryAction.java
@@ -26,12 +26,12 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.LifecycleExecutionState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStartILMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStartILMAction.java
index 5a0e3d1583066..0147f006fe300 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStartILMAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStartILMAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStopILMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStopILMAction.java
index 20f07e6bf074e..bfa6c68bddc95 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStopILMAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportStopILMAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/inference/build.gradle b/x-pack/plugin/inference/build.gradle
index beeec94f21ebf..211b99343340d 100644
--- a/x-pack/plugin/inference/build.gradle
+++ b/x-pack/plugin/inference/build.gradle
@@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 restResources {
   restApi {
-    include '_common', 'bulk', 'indices', 'inference', 'index', 'get', 'update', 'reindex', 'search'
+    include '_common', 'bulk', 'indices', 'inference', 'index', 'get', 'update', 'reindex', 'search', 'field_caps'
   }
 }
 
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CustomElandModelIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CustomElandModelIT.java
new file mode 100644
index 0000000000000..65b7a138e7e1e
--- /dev/null
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CustomElandModelIT.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.inference.TaskType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class CustomElandModelIT extends InferenceBaseRestTest {
+
+    // The model definition is taken from org.elasticsearch.xpack.ml.integration.TextExpansionQueryIT
+
+    static final String BASE_64_ENCODED_MODEL = "UEsDBAAACAgAAAAAAAAAAAAAAAAAA"
+        + "AAAAAAUAA4Ac2ltcGxlbW9kZWwvZGF0YS5wa2xGQgoAWlpaWlpaWlpaWoACY19fdG9yY2hfXwpUaW55VG"
+        + "V4dEV4cGFuc2lvbgpxACmBfShYCAAAAHRyYWluaW5ncQGJWBYAAABfaXNfZnVsbF9iYWNrd2FyZF9ob29"
+        + "rcQJOdWJxAy5QSwcIITmbsFgAAABYAAAAUEsDBBQACAgIAAAAAAAAAAAAAAAAAAAAAAAdAB0Ac2ltcGxl"
+        + "bW9kZWwvY29kZS9fX3RvcmNoX18ucHlGQhkAWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWoWRT4+cMAzF7"
+        + "/spfASJomF3e0Ga3nrrn8vcELIyxAzRhAQlpjvbT19DWDrdquqBA/bvPT87nVUxwsm41xPd+PNtUi4a77"
+        + "KvXs+W8voBAHFSQY3EFCIiHKFp1+p57vs/ShyUccZdoIaz93aBTMR+thbPqru+qKBx8P4q/e8TyxRlmwVc"
+        + "tJp66H1YmCyS7WsZwD50A2L5V7pCBADGTTOj0bGGE7noQyqzv5JDfp0o9fZRCWqP37yjhE4+mqX5X3AdF"
+        + "ZHGM/2TzOHDpy1IvQWR+OWo3KwsRiKdpcqg4pBFDtm+QJ7nqwIPckrlnGfFJG0uNhOl38Sjut3pCqg26Qu"
+        + "Zy8BR9In7ScHHrKkKMW0TIucFrGQXCMpdaDO05O6DpOiy8e4kr0Ed/2YKOIhplW8gPr4ntygrd9ixpx3j9"
+        + "UZZVRagl2c6+imWUzBjuf5m+Ch7afphuvvW+r/0dsfn+2N9MZGb9+/SFtCYdhd83CMYp+mGy0LiKNs8y/e"
+        + "UuEA8B/d2z4dfUEsHCFSE3IaCAQAAIAMAAFBLAwQUAAgICAAAAAAAAAAAAAAAAAAAAAAAJwApAHNpbXBsZ"
+        + "W1vZGVsL2NvZGUvX190b3JjaF9fLnB5LmRlYnVnX3BrbEZCJQBaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlp"
+        + "aWlpaWlpaWlpaWlpahZHLbtNAFIZtp03rSVIuLRKXjdk5ojitKJsiFq24lem0KKSqpRIZt55gE9/GM+lNL"
+        + "Fgx4i1Ys2aHhIBXgAVICNggHgNm6rqJN2BZGv36/v/MOWeea/Z5RVHurLfRUsfZXOnccx522itrd53O0vL"
+        + "qbaKYtsAKUe1pcege7hm9JNtzM8+kOOzNApIX0A3xBXE6YE7g0UWjg2OaZAJXbKvALOnj2GEHKc496ykLkt"
+        + "gNt3Jz17hprCUxFqExe7YIpQkNpO1/kfHhPUdtUAdH2/gfmeYiIFW7IkM6IBP2wrDNbMe3Mjf2ksiK3Hjg"
+        + "hg7F2DN9l/omZZl5Mmez2QRk0q4WUUB0+1oh9nDwxGdUXJdXPMRZQs352eGaRPV9s2lcMeZFGWBfKJJiw0Y"
+        + "gbCMLBaRmXyy4flx6a667Fch55q05QOq2Jg2ANOyZwplhNsjiohVApo7aa21QnNGW5+4GXv8gxK1beBeHSR"
+        + "rhmLXWVh+0aBhErZ7bx1ejxMOhlR6QU4ycNqGyk8/yNGCWkwY7/RCD7UEQek4QszCgDJAzZtfErA0VqHBy9"
+        + "ugQP9pUfUmgCjVYgWNwHFbhBJyEOgSwBuuwARWZmoI6J9PwLfzEocpRpPrT8DP8wqHG0b4UX+E3DiscvRgl"
+        + "XIoi81KKPwioHI5x9EooNKWiy0KOc/T6WF4SssrRuzJ9L2VNRXUhJzj6UKYfS4W/q/5wuh/l4M9R9qsU+y2"
+        + "dpoo2hJzkaEET8r6KRONicnRdK9EbUi6raFVIwNGjsrlbpk6ZPi7TbS3fv3LyNjPiEKzG0aG0tvNb6xw90/"
+        + "whe6ONjnJcUxobHDUqQ8bIOW79BVBLBwhfSmPKdAIAAE4EAABQSwMEAAAICAAAAAAAAAAAAAAAAAAAAAAAA"
+        + "BkABQBzaW1wbGVtb2RlbC9jb25zdGFudHMucGtsRkIBAFqAAikuUEsHCG0vCVcEAAAABAAAAFBLAwQAAAgI"
+        + "AAAAAAAAAAAAAAAAAAAAAAAAEwA7AHNpbXBsZW1vZGVsL3ZlcnNpb25GQjcAWlpaWlpaWlpaWlpaWlpaWlp"
+        + "aWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWjMKUEsHCNGeZ1UCAAAAAgAAAFBLAQIAAA"
+        + "AACAgAAAAAAAAhOZuwWAAAAFgAAAAUAAAAAAAAAAAAAAAAAAAAAABzaW1wbGVtb2RlbC9kYXRhLnBrbFBLA"
+        + "QIAABQACAgIAAAAAABUhNyGggEAACADAAAdAAAAAAAAAAAAAAAAAKgAAABzaW1wbGVtb2RlbC9jb2RlL19f"
+        + "dG9yY2hfXy5weVBLAQIAABQACAgIAAAAAABfSmPKdAIAAE4EAAAnAAAAAAAAAAAAAAAAAJICAABzaW1wbGVt"
+        + "b2RlbC9jb2RlL19fdG9yY2hfXy5weS5kZWJ1Z19wa2xQSwECAAAAAAgIAAAAAAAAbS8JVwQAAAAEAAAAGQAA"
+        + "AAAAAAAAAAAAAACEBQAAc2ltcGxlbW9kZWwvY29uc3RhbnRzLnBrbFBLAQIAAAAACAgAAAAAAADRnmdVAgAA"
+        + "AAIAAAATAAAAAAAAAAAAAAAAANQFAABzaW1wbGVtb2RlbC92ZXJzaW9uUEsGBiwAAAAAAAAAHgMtAAAAAAAA"
+        + "AAAABQAAAAAAAAAFAAAAAAAAAGoBAAAAAAAAUgYAAAAAAABQSwYHAAAAALwHAAAAAAAAAQAAAFBLBQYAAAAABQAFAGoBAABSBgAAAAA=";
+
+    static final long RAW_MODEL_SIZE; // size of the model before base64 encoding
+    static {
+        RAW_MODEL_SIZE = Base64.getDecoder().decode(BASE_64_ENCODED_MODEL).length;
+    }
+
+    // Test a sparse embedding model deployed with the ml trained models APIs
+    public void testSparse() throws IOException {
+        String modelId = "custom-text-expansion-model";
+
+        createTextExpansionModel(modelId);
+        putModelDefinition(modelId, BASE_64_ENCODED_MODEL, RAW_MODEL_SIZE);
+        putVocabulary(
+            List.of("these", "are", "my", "words", "the", "washing", "machine", "is", "leaking", "octopus", "comforter", "smells"),
+            modelId
+        );
+
+        var inferenceConfig = """
+            {
+              "service": "elasticsearch",
+              "service_settings": {
+                "model_id": "custom-text-expansion-model",
+                "num_allocations": 1,
+                "num_threads": 1
+              }
+            }
+            """;
+
+        var inferenceId = "sparse-inf";
+        putModel(inferenceId, inferenceConfig, TaskType.SPARSE_EMBEDDING);
+        var results = inferOnMockService(inferenceId, List.of("washing", "machine"));
+        deleteModel(inferenceId);
+        assertNotNull(results.get("sparse_embedding"));
+    }
+
+    protected void createTextExpansionModel(String modelId) throws IOException {
+        // with_special_tokens: false for this test with limited vocab
+        Request request = new Request("PUT", "/_ml/trained_models/" + modelId);
+        request.setJsonEntity("""
+            {
+               "description": "a text expansion model",
+               "model_type": "pytorch",
+               "inference_config": {
+                 "text_expansion": {
+                   "tokenization": {
+                     "bert": {
+                       "with_special_tokens": false
+                     }
+                   }
+                 }
+               }
+             }""");
+        client().performRequest(request);
+    }
+
+    protected void putVocabulary(List<String> vocabulary, String modelId) throws IOException {
+        List<String> vocabularyWithPad = new ArrayList<>();
+        vocabularyWithPad.add("[PAD]");
+        vocabularyWithPad.add("[UNK]");
+        vocabularyWithPad.addAll(vocabulary);
+        String quotedWords = vocabularyWithPad.stream().map(s -> "\"" + s + "\"").collect(Collectors.joining(","));
+
+        Request request = new Request("PUT", "_ml/trained_models/" + modelId + "/vocabulary");
+        request.setJsonEntity(Strings.format("""
+            { "vocabulary": [%s] }
+            """, quotedWords));
+        client().performRequest(request);
+    }
+
+    protected void putModelDefinition(String modelId, String base64EncodedModel, long unencodedModelSize) throws IOException {
+        Request request = new Request("PUT", "_ml/trained_models/" + modelId + "/definition/0");
+        String body = Strings.format("""
+            {"total_definition_length":%s,"definition": "%s","total_parts": 1}""", unencodedModelSize, base64EncodedModel);
+        request.setJsonEntity(body);
+        client().performRequest(request);
+    }
+}
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
index 242f786e95364..7b897432afa83 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
@@ -49,7 +49,7 @@ public void testGet() throws IOException {
 
         var singleModel = getModels("se_model_1", TaskType.SPARSE_EMBEDDING);
         assertThat(singleModel, hasSize(1));
-        assertEquals("se_model_1", singleModel.get(0).get("model_id"));
+        assertEquals("se_model_1", singleModel.get(0).get("inference_id"));
 
         for (int i = 0; i < 5; i++) {
             deleteModel("se_model_" + i, TaskType.SPARSE_EMBEDDING);
@@ -82,7 +82,7 @@ public void testGetModelWithAnyTaskType() throws IOException {
         String inferenceEntityId = "sparse_embedding_model";
         putModel(inferenceEntityId, mockSparseServiceModelConfig(), TaskType.SPARSE_EMBEDDING);
         var singleModel = getModels(inferenceEntityId, TaskType.ANY);
-        assertEquals(inferenceEntityId, singleModel.get(0).get("model_id"));
+        assertEquals(inferenceEntityId, singleModel.get(0).get("inference_id"));
         assertEquals(TaskType.SPARSE_EMBEDDING.toString(), singleModel.get(0).get("task_type"));
     }
 
@@ -91,7 +91,7 @@ public void testApisWithoutTaskType() throws IOException {
         String modelId = "no_task_type_in_url";
         putModel(modelId, mockSparseServiceModelConfig(TaskType.SPARSE_EMBEDDING));
         var singleModel = getModel(modelId);
-        assertEquals(modelId, singleModel.get("model_id"));
+        assertEquals(modelId, singleModel.get("inference_id"));
         assertEquals(TaskType.SPARSE_EMBEDDING.toString(), singleModel.get("task_type"));
 
         var inference = inferOnMockService(modelId, List.of(randomAlphaOfLength(10)));
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockDenseInferenceServiceIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockDenseInferenceServiceIT.java
index 833b1fd3673fc..5f6bad5687407 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockDenseInferenceServiceIT.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockDenseInferenceServiceIT.java
@@ -22,7 +22,7 @@ public void testMockService() throws IOException {
         var model = getModels(inferenceEntityId, TaskType.TEXT_EMBEDDING).get(0);
 
         for (var modelMap : List.of(putModel, model)) {
-            assertEquals(inferenceEntityId, modelMap.get("model_id"));
+            assertEquals(inferenceEntityId, modelMap.get("inference_id"));
             assertEquals(TaskType.TEXT_EMBEDDING, TaskType.fromString((String) modelMap.get("task_type")));
             assertEquals("text_embedding_test_service", modelMap.get("service"));
         }
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockSparseInferenceServiceIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockSparseInferenceServiceIT.java
index 97e0641f37c33..24ba2708f5de4 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockSparseInferenceServiceIT.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockSparseInferenceServiceIT.java
@@ -24,7 +24,7 @@ public void testMockService() throws IOException {
         var model = getModels(inferenceEntityId, TaskType.SPARSE_EMBEDDING).get(0);
 
         for (var modelMap : List.of(putModel, model)) {
-            assertEquals(inferenceEntityId, modelMap.get("model_id"));
+            assertEquals(inferenceEntityId, modelMap.get("inference_id"));
             assertEquals(TaskType.SPARSE_EMBEDDING, TaskType.fromString((String) modelMap.get("task_type")));
             assertEquals("test_service", modelMap.get("service"));
         }
@@ -77,7 +77,7 @@ public void testMockService_DoesNotReturnHiddenField_InModelResponses() throws I
         var model = getModels(inferenceEntityId, TaskType.SPARSE_EMBEDDING).get(0);
 
         for (var modelMap : List.of(putModel, model)) {
-            assertEquals(inferenceEntityId, modelMap.get("model_id"));
+            assertEquals(inferenceEntityId, modelMap.get("inference_id"));
             assertThat(modelMap.get("service_settings"), is(Map.of("model", "my_model")));
             assertEquals(TaskType.SPARSE_EMBEDDING, TaskType.fromString((String) modelMap.get("task_type")));
             assertEquals("test_service", modelMap.get("service"));
@@ -95,7 +95,7 @@ public void testMockService_DoesReturnHiddenField_InModelResponses() throws IOEx
         var model = getModels(inferenceEntityId, TaskType.SPARSE_EMBEDDING).get(0);
 
         for (var modelMap : List.of(putModel, model)) {
-            assertEquals(inferenceEntityId, modelMap.get("model_id"));
+            assertEquals(inferenceEntityId, modelMap.get("inference_id"));
             assertThat(modelMap.get("service_settings"), is(Map.of("model", "my_model", "hidden_field", "my_hidden_value")));
             assertEquals(TaskType.SPARSE_EMBEDDING, TaskType.fromString((String) modelMap.get("task_type")));
             assertEquals("test_service", modelMap.get("service"));
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/RerankingIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/RerankingIT.java
index 77251ada4c488..893d3fb3e9b80 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/RerankingIT.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/RerankingIT.java
@@ -35,7 +35,7 @@ private String putCohereRerankEndpoint() throws IOException {
                 "api_key": ""
               }
             }
-            """);// TODO remove key
+            """);
         return endpointID;
     }
 
@@ -61,7 +61,7 @@ private String putCohereRerankEndpointWithDocuments() throws IOException {
                 "return_documents": true
               }
             }
-            """);// TODO remove key
+            """);
         return endpointID;
     }
 
@@ -81,13 +81,13 @@ private String putCohereRerankEndpointWithTop2() throws IOException {
               "service": "cohere",
               "service_settings": {
                 "model_id": "rerank-english-v2.0",
-                "api_key": "8TNPBvpBO7oN97009HQHzQbBhNrxmREbcJrZCwkK"
+                "api_key": ""
               },
               "task_settings": {
                 "top_n": 2
               }
             }
-            """);// TODO remove key
+            """);
         return endpointID;
     }
 
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java
index 8da1aaabd517a..73c0f6d4c7685 100644
--- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java
+++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java
@@ -33,6 +33,7 @@
 import java.util.Locale;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticTextInput;
 import static org.hamcrest.Matchers.equalTo;
 
 public class ShardBulkInferenceActionFilterIT extends ESIntegTestCase {
@@ -93,8 +94,8 @@ public void testBulkOperations() throws Exception {
                 String id = Long.toString(totalDocs);
                 boolean isIndexRequest = randomBoolean();
                 Map<String, Object> source = new HashMap<>();
-                source.put("sparse_field", isIndexRequest && rarely() ? null : randomAlphaOfLengthBetween(0, 1000));
-                source.put("dense_field", isIndexRequest && rarely() ? null : randomAlphaOfLengthBetween(0, 1000));
+                source.put("sparse_field", isIndexRequest && rarely() ? null : randomSemanticTextInput());
+                source.put("dense_field", isIndexRequest && rarely() ? null : randomSemanticTextInput());
                 if (isIndexRequest) {
                     bulkReqBuilder.add(new IndexRequestBuilder(client()).setIndex(INDEX_NAME).setId(id).setSource(source));
                     totalDocs++;
diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java
index 17037e56b2db3..d776f3963c2ca 100644
--- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java
+++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsFeatureFlag;
 import org.elasticsearch.xpack.inference.InferencePlugin;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
 import org.elasticsearch.xpack.inference.services.elser.ElserInternalModel;
@@ -101,6 +102,7 @@ public void testStoreModelWithUnknownFields() throws Exception {
     }
 
     public void testGetModel() throws Exception {
+        assumeTrue("Only if 'inference_adaptive_allocations' feature flag is enabled", AdaptiveAllocationsFeatureFlag.isEnabled());
         String inferenceEntityId = "test-get-model";
         Model model = buildElserModelConfig(inferenceEntityId, TaskType.SPARSE_EMBEDDING);
         AtomicReference<Boolean> putModelHolder = new AtomicReference<>();
@@ -410,7 +412,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field("unknown_field", "foo");
-            builder.field(MODEL_ID, getInferenceEntityId());
+            builder.field(INDEX_ONLY_ID_FIELD_NAME, getInferenceEntityId());
             builder.field(TaskType.NAME, getTaskType().toString());
             builder.field(SERVICE, getService());
             builder.field(SERVICE_SETTINGS, getServiceSettings());
@@ -436,7 +438,7 @@ private static class ModelWithUnknownField extends ModelConfigurations {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field("unknown_field", "foo");
-            builder.field(MODEL_ID, getInferenceEntityId());
+            builder.field(INDEX_ONLY_ID_FIELD_NAME, getInferenceEntityId());
             builder.field(TaskType.NAME, getTaskType().toString());
             builder.field(SERVICE, getService());
             builder.field(SERVICE_SETTINGS, getServiceSettings());
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java
index 4cc7f5b502ba9..12a32ecdc6d4f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder;
 import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder;
 
 import java.util.Set;
@@ -20,7 +21,10 @@ public class InferenceFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED);
+        return Set.of(
+            TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED,
+            RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED
+        );
     }
 
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
index 476ab3355a0b8..d4810ba930b44 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference;
 
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.inference.EmptySecretSettings;
 import org.elasticsearch.inference.EmptyTaskSettings;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -24,6 +25,13 @@
 import org.elasticsearch.xpack.core.inference.results.LegacyTextEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettings;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsServiceSettings;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsTaskSettings;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank.AlibabaCloudSearchRerankServiceSettings;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank.AlibabaCloudSearchRerankTaskSettings;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseServiceSettings;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseTaskSettings;
 import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockSecretSettings;
 import org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionServiceSettings;
 import org.elasticsearch.xpack.inference.services.amazonbedrock.completion.AmazonBedrockChatCompletionTaskSettings;
@@ -45,6 +53,7 @@
 import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsTaskSettings;
 import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankServiceSettings;
 import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankTaskSettings;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsServiceSettings;
 import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalServiceSettings;
 import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalTextEmbeddingServiceSettings;
 import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandRerankTaskSettings;
@@ -95,6 +104,9 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         // Empty default task settings
         namedWriteables.add(new NamedWriteableRegistry.Entry(TaskSettings.class, EmptyTaskSettings.NAME, EmptyTaskSettings::new));
 
+        // Empty default secret settings
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SecretSettings.class, EmptySecretSettings.NAME, EmptySecretSettings::new));
+
         // Default secret settings
         namedWriteables.add(new NamedWriteableRegistry.Entry(SecretSettings.class, DefaultSecretSettings.NAME, DefaultSecretSettings::new));
 
@@ -111,6 +123,8 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         addCustomElandWriteables(namedWriteables);
         addAnthropicNamedWritables(namedWriteables);
         addAmazonBedrockNamedWriteables(namedWriteables);
+        addEisNamedWriteables(namedWriteables);
+        addAlibabaCloudSearchNamedWriteables(namedWriteables);
 
         return namedWriteables;
     }
@@ -475,4 +489,67 @@ private static void addAnthropicNamedWritables(List<NamedWriteableRegistry.Entry
             )
         );
     }
+
+    private static void addAlibabaCloudSearchNamedWriteables(List<NamedWriteableRegistry.Entry> namedWriteables) {
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                ServiceSettings.class,
+                AlibabaCloudSearchServiceSettings.NAME,
+                AlibabaCloudSearchServiceSettings::new
+            )
+        );
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                ServiceSettings.class,
+                AlibabaCloudSearchEmbeddingsServiceSettings.NAME,
+                AlibabaCloudSearchEmbeddingsServiceSettings::new
+            )
+        );
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                TaskSettings.class,
+                AlibabaCloudSearchEmbeddingsTaskSettings.NAME,
+                AlibabaCloudSearchEmbeddingsTaskSettings::new
+            )
+        );
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                ServiceSettings.class,
+                AlibabaCloudSearchSparseServiceSettings.NAME,
+                AlibabaCloudSearchSparseServiceSettings::new
+            )
+        );
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                TaskSettings.class,
+                AlibabaCloudSearchSparseTaskSettings.NAME,
+                AlibabaCloudSearchSparseTaskSettings::new
+            )
+        );
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                ServiceSettings.class,
+                AlibabaCloudSearchRerankServiceSettings.NAME,
+                AlibabaCloudSearchRerankServiceSettings::new
+            )
+        );
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                TaskSettings.class,
+                AlibabaCloudSearchRerankTaskSettings.NAME,
+                AlibabaCloudSearchRerankTaskSettings::new
+            )
+        );
+
+    }
+
+    private static void addEisNamedWriteables(List<NamedWriteableRegistry.Entry> namedWriteables) {
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                ServiceSettings.class,
+                ElasticInferenceServiceSparseEmbeddingsServiceSettings.NAME,
+                ElasticInferenceServiceSparseEmbeddingsServiceSettings::new
+            )
+        );
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
index fce2c54c535c9..dff93a63d0647 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.inference.InferenceServiceExtension;
 import org.elasticsearch.inference.InferenceServiceRegistry;
+import org.elasticsearch.node.PluginComponentBinding;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.MapperPlugin;
@@ -62,6 +63,8 @@
 import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
 import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper;
 import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder;
+import org.elasticsearch.xpack.inference.rank.random.RandomRankBuilder;
+import org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder;
 import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankBuilder;
 import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
@@ -71,11 +74,16 @@
 import org.elasticsearch.xpack.inference.rest.RestInferenceAction;
 import org.elasticsearch.xpack.inference.rest.RestPutInferenceModelAction;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchService;
 import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockService;
 import org.elasticsearch.xpack.inference.services.anthropic.AnthropicService;
 import org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioService;
 import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiService;
 import org.elasticsearch.xpack.inference.services.cohere.CohereService;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceComponents;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceFeature;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSettings;
 import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService;
 import org.elasticsearch.xpack.inference.services.elser.ElserInternalService;
 import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioService;
@@ -84,8 +92,8 @@
 import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserService;
 import org.elasticsearch.xpack.inference.services.mistral.MistralService;
 import org.elasticsearch.xpack.inference.services.openai.OpenAiService;
-import org.elasticsearch.xpack.inference.telemetry.InferenceAPMStats;
-import org.elasticsearch.xpack.inference.telemetry.StatsMap;
+import org.elasticsearch.xpack.inference.telemetry.ApmInferenceStats;
+import org.elasticsearch.xpack.inference.telemetry.InferenceStats;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -123,7 +131,7 @@ public class InferencePlugin extends Plugin implements ActionPlugin, ExtensibleP
     private final SetOnce<HttpRequestSender.Factory> httpFactory = new SetOnce<>();
     private final SetOnce<AmazonBedrockRequestSender.Factory> amazonBedrockFactory = new SetOnce<>();
     private final SetOnce<ServiceComponents> serviceComponents = new SetOnce<>();
-
+    private final SetOnce<ElasticInferenceServiceComponents> eisComponents = new SetOnce<>();
     private final SetOnce<InferenceServiceRegistry> inferenceServiceRegistry = new SetOnce<>();
     private final SetOnce<ShardBulkInferenceActionFilter> shardBulkInferenceActionFilter = new SetOnce<>();
     private List<InferenceServiceExtension> inferenceServiceExtensions;
@@ -186,6 +194,15 @@ public Collection<?> createComponents(PluginServices services) {
         var inferenceServices = new ArrayList<>(inferenceServiceExtensions);
         inferenceServices.add(this::getInferenceServiceFactories);
 
+        if (ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled()) {
+            ElasticInferenceServiceSettings eisSettings = new ElasticInferenceServiceSettings(settings);
+            eisComponents.set(new ElasticInferenceServiceComponents(eisSettings.getEisGatewayUrl()));
+
+            inferenceServices.add(
+                () -> List.of(context -> new ElasticInferenceService(httpFactory.get(), serviceComponents.get(), eisComponents.get()))
+            );
+        }
+
         var factoryContext = new InferenceServiceExtension.InferenceServiceFactoryContext(services.client());
         // This must be done after the HttpRequestSenderFactory is created so that the services can get the
         // reference correctly
@@ -196,10 +213,10 @@ public Collection<?> createComponents(PluginServices services) {
         var actionFilter = new ShardBulkInferenceActionFilter(registry, modelRegistry);
         shardBulkInferenceActionFilter.set(actionFilter);
 
-        var statsFactory = new InferenceAPMStats.Factory(services.telemetryProvider().getMeterRegistry());
-        var statsMap = new StatsMap<>(InferenceAPMStats::key, statsFactory::newInferenceRequestAPMCounter);
+        var meterRegistry = services.telemetryProvider().getMeterRegistry();
+        var stats = new PluginComponentBinding<>(InferenceStats.class, ApmInferenceStats.create(meterRegistry));
 
-        return List.of(modelRegistry, registry, httpClientManager, statsMap);
+        return List.of(modelRegistry, registry, httpClientManager, stats);
     }
 
     @Override
@@ -221,6 +238,7 @@ public List<InferenceServiceExtension.Factory> getInferenceServiceFactories() {
             context -> new MistralService(httpFactory.get(), serviceComponents.get()),
             context -> new AnthropicService(httpFactory.get(), serviceComponents.get()),
             context -> new AmazonBedrockService(httpFactory.get(), amazonBedrockFactory.get(), serviceComponents.get()),
+            context -> new AlibabaCloudSearchService(httpFactory.get(), serviceComponents.get()),
             ElasticsearchInternalService::new
         );
     }
@@ -229,6 +247,7 @@ public List<InferenceServiceExtension.Factory> getInferenceServiceFactories() {
     public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         var entries = new ArrayList<>(InferenceNamedWriteablesProvider.getNamedWriteables());
         entries.add(new NamedWriteableRegistry.Entry(RankBuilder.class, TextSimilarityRankBuilder.NAME, TextSimilarityRankBuilder::new));
+        entries.add(new NamedWriteableRegistry.Entry(RankBuilder.class, RandomRankBuilder.NAME, RandomRankBuilder::new));
         return entries;
     }
 
@@ -280,6 +299,7 @@ public List<Setting<?>> getSettings() {
             HttpClientManager.getSettingsDefinitions(),
             ThrottlerManager.getSettingsDefinitions(),
             RetrySettings.getSettingsDefinitions(),
+            ElasticInferenceServiceSettings.getSettingsDefinitions(),
             Truncator.getSettingsDefinitions(),
             RequestExecutorServiceSettings.getSettingsDefinitions(),
             List.of(SKIP_VALIDATE_AND_START)
@@ -321,7 +341,8 @@ public List<QuerySpec<?>> getQueries() {
     @Override
     public List<RetrieverSpec<?>> getRetrievers() {
         return List.of(
-            new RetrieverSpec<>(new ParseField(TextSimilarityRankBuilder.NAME), TextSimilarityRankRetrieverBuilder::fromXContent)
+            new RetrieverSpec<>(new ParseField(TextSimilarityRankBuilder.NAME), TextSimilarityRankRetrieverBuilder::fromXContent),
+            new RetrieverSpec<>(new ParseField(RandomRankBuilder.NAME), RandomRankRetrieverBuilder::fromXContent)
         );
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
index e59ac4e1356f0..3c893f8870627 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
@@ -22,9 +22,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.inference.InferenceServiceRegistry;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceDiagnosticsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceDiagnosticsAction.java
index 8c9ab8f4cdffa..88689035fbd8a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceDiagnosticsAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceDiagnosticsAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java
index ef441693a9a9e..a1f33afa05b5c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java
@@ -13,11 +13,11 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.inference.InferenceServiceRegistry;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java
index 575697b5d0d39..81669a573a5d1 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java
@@ -11,32 +11,36 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.inference.InferenceService;
 import org.elasticsearch.inference.InferenceServiceRegistry;
 import org.elasticsearch.inference.Model;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
+import org.elasticsearch.xpack.inference.telemetry.InferenceStats;
 
 public class TransportInferenceAction extends HandledTransportAction<InferenceAction.Request, InferenceAction.Response> {
 
     private final ModelRegistry modelRegistry;
     private final InferenceServiceRegistry serviceRegistry;
+    private final InferenceStats inferenceStats;
 
     @Inject
     public TransportInferenceAction(
         TransportService transportService,
         ActionFilters actionFilters,
         ModelRegistry modelRegistry,
-        InferenceServiceRegistry serviceRegistry
+        InferenceServiceRegistry serviceRegistry,
+        InferenceStats inferenceStats
     ) {
         super(InferenceAction.NAME, transportService, actionFilters, InferenceAction.Request::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.modelRegistry = modelRegistry;
         this.serviceRegistry = serviceRegistry;
+        this.inferenceStats = inferenceStats;
     }
 
     @Override
@@ -76,6 +80,7 @@ protected void doExecute(Task task, InferenceAction.Request request, ActionListe
                     unparsedModel.settings(),
                     unparsedModel.secrets()
                 );
+            inferenceStats.incrementRequestCount(model);
             inferOnService(model, request, service.get(), delegate);
         });
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageAction.java
index 712cb1ebad781..624afff9f5d11 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceUsageAction.java
@@ -14,9 +14,9 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java
index dd26b07af7f27..b5d57e7afa6e7 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -30,6 +29,7 @@
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java
index f1a590e647dbc..ade0748ef10bf 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java
@@ -492,12 +492,16 @@ private Map<String, List<FieldInferenceRequest>> createFieldInferenceRequests(Bu
      * If {@code valueObj} is not a string or a collection of strings, it throws an ElasticsearchStatusException.
      */
     private static List<String> nodeStringValues(String field, Object valueObj) {
-        if (valueObj instanceof String value) {
+        if (valueObj instanceof Number || valueObj instanceof Boolean) {
+            return List.of(valueObj.toString());
+        } else if (valueObj instanceof String value) {
             return List.of(value);
         } else if (valueObj instanceof Collection<?> values) {
             List<String> valuesString = new ArrayList<>();
             for (var v : values) {
-                if (v instanceof String value) {
+                if (v instanceof Number || v instanceof Boolean) {
+                    valuesString.add(v.toString());
+                } else if (v instanceof String value) {
                     valuesString.add(value);
                 } else {
                     throw new ElasticsearchStatusException(
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchActionCreator.java
new file mode 100644
index 0000000000000..218ca2ef39ed6
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchActionCreator.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.alibabacloudsearch;
+
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank.AlibabaCloudSearchRerankModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseModel;
+
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Provides a way to construct an {@link ExecutableAction} using the visitor pattern based on the alibaba cloud search model type.
+ */
+public class AlibabaCloudSearchActionCreator implements AlibabaCloudSearchActionVisitor {
+    private final Sender sender;
+    private final ServiceComponents serviceComponents;
+
+    public AlibabaCloudSearchActionCreator(Sender sender, ServiceComponents serviceComponents) {
+        this.sender = Objects.requireNonNull(sender);
+        this.serviceComponents = Objects.requireNonNull(serviceComponents);
+    }
+
+    @Override
+    public ExecutableAction create(AlibabaCloudSearchEmbeddingsModel model, Map<String, Object> taskSettings, InputType inputType) {
+        var overriddenModel = AlibabaCloudSearchEmbeddingsModel.of(model, taskSettings, inputType);
+
+        return new AlibabaCloudSearchEmbeddingsAction(sender, overriddenModel, serviceComponents);
+    }
+
+    @Override
+    public ExecutableAction create(AlibabaCloudSearchSparseModel model, Map<String, Object> taskSettings, InputType inputType) {
+        var overriddenModel = AlibabaCloudSearchSparseModel.of(model, taskSettings, inputType);
+
+        return new AlibabaCloudSearchSparseAction(sender, overriddenModel, serviceComponents);
+    }
+
+    @Override
+    public ExecutableAction create(AlibabaCloudSearchRerankModel model, Map<String, Object> taskSettings) {
+        var overriddenModel = AlibabaCloudSearchRerankModel.of(model, taskSettings);
+
+        return new AlibabaCloudSearchRerankAction(sender, overriddenModel, serviceComponents);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchActionVisitor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchActionVisitor.java
new file mode 100644
index 0000000000000..69ae903c7b38f
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchActionVisitor.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.alibabacloudsearch;
+
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank.AlibabaCloudSearchRerankModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseModel;
+
+import java.util.Map;
+
+public interface AlibabaCloudSearchActionVisitor {
+    ExecutableAction create(AlibabaCloudSearchEmbeddingsModel model, Map<String, Object> taskSettings, InputType inputType);
+
+    ExecutableAction create(AlibabaCloudSearchSparseModel model, Map<String, Object> taskSettings, InputType inputType);
+
+    ExecutableAction create(AlibabaCloudSearchRerankModel model, Map<String, Object> taskSettings);
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchEmbeddingsAction.java
new file mode 100644
index 0000000000000..7a22bbf6b4bfd
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchEmbeddingsAction.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.alibabacloudsearch;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount;
+import org.elasticsearch.xpack.inference.external.http.sender.AlibabaCloudSearchEmbeddingsRequestManager;
+import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModel;
+
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage;
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError;
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException;
+
+public class AlibabaCloudSearchEmbeddingsAction implements ExecutableAction {
+    private final AlibabaCloudSearchAccount account;
+    private final AlibabaCloudSearchEmbeddingsModel model;
+    private final String failedToSendRequestErrorMessage;
+    private final Sender sender;
+    private final AlibabaCloudSearchEmbeddingsRequestManager requestCreator;
+
+    public AlibabaCloudSearchEmbeddingsAction(Sender sender, AlibabaCloudSearchEmbeddingsModel model, ServiceComponents serviceComponents) {
+        this.model = Objects.requireNonNull(model);
+        this.sender = Objects.requireNonNull(sender);
+        this.account = new AlibabaCloudSearchAccount(this.model.getSecretSettings().apiKey());
+        this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(null, "AlibabaCloud Search text embeddings");
+        this.requestCreator = AlibabaCloudSearchEmbeddingsRequestManager.of(account, model, serviceComponents.threadPool());
+    }
+
+    @Override
+    public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener<InferenceServiceResults> listener) {
+        try {
+            ActionListener<InferenceServiceResults> wrappedListener = wrapFailuresInElasticsearchException(
+                failedToSendRequestErrorMessage,
+                listener
+            );
+            sender.send(requestCreator, inferenceInputs, timeout, wrappedListener);
+        } catch (ElasticsearchException e) {
+            listener.onFailure(e);
+        } catch (Exception e) {
+            listener.onFailure(createInternalServerError(e, failedToSendRequestErrorMessage));
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchRerankAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchRerankAction.java
new file mode 100644
index 0000000000000..88229ce63463b
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchRerankAction.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.alibabacloudsearch;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount;
+import org.elasticsearch.xpack.inference.external.http.sender.AlibabaCloudSearchRerankRequestManager;
+import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank.AlibabaCloudSearchRerankModel;
+
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage;
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError;
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException;
+
+public class AlibabaCloudSearchRerankAction implements ExecutableAction {
+    private static final Logger logger = LogManager.getLogger(AlibabaCloudSearchRerankAction.class);
+
+    private final AlibabaCloudSearchAccount account;
+    private final AlibabaCloudSearchRerankModel model;
+    private final String failedToSendRequestErrorMessage;
+    private final Sender sender;
+    private final AlibabaCloudSearchRerankRequestManager requestCreator;
+
+    public AlibabaCloudSearchRerankAction(Sender sender, AlibabaCloudSearchRerankModel model, ServiceComponents serviceComponents) {
+        this.model = Objects.requireNonNull(model);
+        this.account = new AlibabaCloudSearchAccount(this.model.getSecretSettings().apiKey());
+        this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(null, "AlibabaCloud Search rerank");
+        this.sender = Objects.requireNonNull(sender);
+        this.requestCreator = AlibabaCloudSearchRerankRequestManager.of(account, model, serviceComponents.threadPool());
+    }
+
+    @Override
+    public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener<InferenceServiceResults> listener) {
+        try {
+            ActionListener<InferenceServiceResults> wrappedListener = wrapFailuresInElasticsearchException(
+                failedToSendRequestErrorMessage,
+                listener
+            );
+            sender.send(requestCreator, inferenceInputs, timeout, wrappedListener);
+        } catch (ElasticsearchException e) {
+            listener.onFailure(e);
+        } catch (Exception e) {
+            listener.onFailure(createInternalServerError(e, failedToSendRequestErrorMessage));
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchSparseAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchSparseAction.java
new file mode 100644
index 0000000000000..2cd31ff83d200
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/alibabacloudsearch/AlibabaCloudSearchSparseAction.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.alibabacloudsearch;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount;
+import org.elasticsearch.xpack.inference.external.http.sender.AlibabaCloudSearchSparseRequestManager;
+import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseModel;
+
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage;
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError;
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException;
+
+public class AlibabaCloudSearchSparseAction implements ExecutableAction {
+    private static final Logger logger = LogManager.getLogger(AlibabaCloudSearchSparseAction.class);
+
+    private final AlibabaCloudSearchAccount account;
+    private final AlibabaCloudSearchSparseModel model;
+    private final String failedToSendRequestErrorMessage;
+    private final Sender sender;
+    private final AlibabaCloudSearchSparseRequestManager requestCreator;
+
+    public AlibabaCloudSearchSparseAction(Sender sender, AlibabaCloudSearchSparseModel model, ServiceComponents serviceComponents) {
+        this.model = Objects.requireNonNull(model);
+        this.account = new AlibabaCloudSearchAccount(this.model.getSecretSettings().apiKey());
+        this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(null, "AlibabaCloud Search sparse embeddings");
+        this.sender = Objects.requireNonNull(sender);
+        requestCreator = AlibabaCloudSearchSparseRequestManager.of(account, model, serviceComponents.threadPool());
+    }
+
+    @Override
+    public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener<InferenceServiceResults> listener) {
+        try {
+            ActionListener<InferenceServiceResults> wrappedListener = wrapFailuresInElasticsearchException(
+                failedToSendRequestErrorMessage,
+                listener
+            );
+            sender.send(requestCreator, inferenceInputs, timeout, wrappedListener);
+        } catch (ElasticsearchException e) {
+            listener.onFailure(e);
+        } catch (Exception e) {
+            listener.onFailure(createInternalServerError(e, failedToSendRequestErrorMessage));
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java
new file mode 100644
index 0000000000000..ea2295979c480
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreator.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.elastic;
+
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction;
+import org.elasticsearch.xpack.inference.external.http.sender.ElasticInferenceServiceSparseEmbeddingsRequestManager;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel;
+
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage;
+
+public class ElasticInferenceServiceActionCreator implements ElasticInferenceServiceActionVisitor {
+
+    private final Sender sender;
+
+    private final ServiceComponents serviceComponents;
+
+    public ElasticInferenceServiceActionCreator(Sender sender, ServiceComponents serviceComponents) {
+        this.sender = Objects.requireNonNull(sender);
+        this.serviceComponents = Objects.requireNonNull(serviceComponents);
+    }
+
+    @Override
+    public ExecutableAction create(ElasticInferenceServiceSparseEmbeddingsModel model) {
+        var requestManager = new ElasticInferenceServiceSparseEmbeddingsRequestManager(model, serviceComponents);
+        var errorMessage = constructFailedToSendRequestMessage(model.uri(), "Elastic Inference Service sparse embeddings");
+        return new SenderExecutableAction(sender, requestManager, errorMessage);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionVisitor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionVisitor.java
new file mode 100644
index 0000000000000..99985e50b2538
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionVisitor.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.elastic;
+
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel;
+
+public interface ElasticInferenceServiceActionVisitor {
+
+    ExecutableAction create(ElasticInferenceServiceSparseEmbeddingsModel model);
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchAccount.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchAccount.java
new file mode 100644
index 0000000000000..6aabbe20cc355
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchAccount.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.alibabacloudsearch;
+
+import org.elasticsearch.common.settings.SecureString;
+
+import java.util.Objects;
+
+public record AlibabaCloudSearchAccount(SecureString apiKey) {
+
+    public AlibabaCloudSearchAccount {
+        Objects.requireNonNull(apiKey);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchResponseHandler.java
new file mode 100644
index 0000000000000..05d51372d9cdc
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchResponseHandler.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.alibabacloudsearch;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser;
+import org.elasticsearch.xpack.inference.external.http.retry.RetryException;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.external.response.alibabacloudsearch.AlibabaCloudSearchErrorResponseEntity;
+import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
+
+import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody;
+
+/**
+ * Defines how to handle various errors returned from the AlibabaCloudSearch integration.
+ */
+public class AlibabaCloudSearchResponseHandler extends BaseResponseHandler {
+
+    public AlibabaCloudSearchResponseHandler(String requestType, ResponseParser parseFunction) {
+        super(requestType, parseFunction, AlibabaCloudSearchErrorResponseEntity::fromResponse);
+    }
+
+    @Override
+    public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result)
+        throws RetryException {
+        checkForFailureStatusCode(request, result);
+        checkForEmptyBody(throttlerManager, logger, request, result);
+    }
+
+    /**
+     * Validates the status code throws an RetryException if not in the range [200, 300).
+     *
+     * @param request The http request
+     * @param result  The http response and body
+     * @throws RetryException Throws if status code is {@code >= 300 or < 200 }
+     */
+    void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException {
+        int statusCode = result.response().getStatusLine().getStatusCode();
+        if (statusCode >= 200 && statusCode < 300) {
+            return;
+        }
+
+        // handle error codes
+        if (statusCode >= 500) {
+            throw new RetryException(false, buildError(SERVER_ERROR, request, result));
+        } else if (statusCode == 429) {
+            throw new RetryException(true, buildError(RATE_LIMIT, request, result));
+        } else if (statusCode == 401) {
+            throw new RetryException(false, buildError(AUTHENTICATION, request, result));
+        } else if (statusCode >= 300 && statusCode < 400) {
+            throw new RetryException(false, buildError(REDIRECTION, request, result));
+        } else {
+            throw new RetryException(false, buildError(UNSUCCESSFUL, request, result));
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecuteOnlyRequestSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecuteOnlyRequestSender.java
index a08acab655936..0826d990a80a5 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecuteOnlyRequestSender.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecuteOnlyRequestSender.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.amazonbedrock;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
@@ -45,7 +44,6 @@ public AmazonBedrockExecuteOnlyRequestSender(AmazonBedrockClientCache clientCach
     public void send(
         Logger logger,
         Request request,
-        HttpClientContext context,
         Supplier<Boolean> hasRequestTimedOutFunction,
         ResponseHandler responseHandler,
         ActionListener<InferenceServiceResults> listener
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java
new file mode 100644
index 0000000000000..15e543fadad71
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.elastic;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler;
+import org.elasticsearch.xpack.inference.external.http.retry.ContentTooLargeException;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser;
+import org.elasticsearch.xpack.inference.external.http.retry.RetryException;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceErrorResponseEntity;
+import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
+
+import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody;
+
+public class ElasticInferenceServiceResponseHandler extends BaseResponseHandler {
+
+    public ElasticInferenceServiceResponseHandler(String requestType, ResponseParser parseFunction) {
+        super(requestType, parseFunction, ElasticInferenceServiceErrorResponseEntity::fromResponse);
+    }
+
+    @Override
+    public void validateResponse(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result)
+        throws RetryException {
+        checkForFailureStatusCode(request, result);
+        checkForEmptyBody(throttlerManager, logger, request, result);
+    }
+
+    void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException {
+        int statusCode = result.response().getStatusLine().getStatusCode();
+        if (statusCode >= 200 && statusCode < 300) {
+            return;
+        }
+
+        if (statusCode == 500) {
+            throw new RetryException(true, buildError(SERVER_ERROR, request, result));
+        } else if (statusCode == 400) {
+            throw new RetryException(false, buildError(BAD_REQUEST, request, result));
+        } else if (statusCode == 405) {
+            throw new RetryException(false, buildError(METHOD_NOT_ALLOWED, request, result));
+        } else if (statusCode == 413) {
+            throw new ContentTooLargeException(buildError(CONTENT_TOO_LARGE, request, result));
+        }
+
+        throw new RetryException(false, buildError(UNSUCCESSFUL, request, result));
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java
index f793cb3586924..c9cbe169ec03d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/BaseResponseHandler.java
@@ -27,6 +27,8 @@ public abstract class BaseResponseHandler implements ResponseHandler {
     public static final String REDIRECTION = "Unhandled redirection";
     public static final String CONTENT_TOO_LARGE = "Received a content too large status code";
     public static final String UNSUCCESSFUL = "Received an unsuccessful status code";
+    public static final String BAD_REQUEST = "Received a bad request status code";
+    public static final String METHOD_NOT_ALLOWED = "Received a method not allowed status code";
 
     protected final String requestType;
     private final ResponseParser parseFunction;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RequestSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RequestSender.java
index 8244e5ad29e95..8e55b0988de6a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RequestSender.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RequestSender.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.retry;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -19,7 +18,6 @@ public interface RequestSender {
     void send(
         Logger logger,
         Request request,
-        HttpClientContext context,
         Supplier<Boolean> hasRequestTimedOutFunction,
         ResponseHandler responseHandler,
         ActionListener<InferenceServiceResults> listener
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java
index dd45501564e4e..263bdea5ce368 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java
@@ -189,12 +189,18 @@ public boolean shouldRetry(Exception e) {
     public void send(
         Logger logger,
         Request request,
-        HttpClientContext context,
         Supplier<Boolean> hasRequestTimedOutFunction,
         ResponseHandler responseHandler,
         ActionListener<InferenceServiceResults> listener
     ) {
-        InternalRetrier retrier = new InternalRetrier(logger, request, context, hasRequestTimedOutFunction, responseHandler, listener);
+        var retrier = new InternalRetrier(
+            logger,
+            request,
+            HttpClientContext.create(),
+            hasRequestTimedOutFunction,
+            responseHandler,
+            listener
+        );
         retrier.run();
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchEmbeddingsRequestManager.java
new file mode 100644
index 0000000000000..55c699bf26e82
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchEmbeddingsRequestManager.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchResponseHandler;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
+import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchEmbeddingsRequest;
+import org.elasticsearch.xpack.inference.external.response.alibabacloudsearch.AlibabaCloudSearchEmbeddingsResponseEntity;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModel;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class AlibabaCloudSearchEmbeddingsRequestManager extends AlibabaCloudSearchRequestManager {
+    private static final Logger logger = LogManager.getLogger(AlibabaCloudSearchEmbeddingsRequestManager.class);
+
+    private static final ResponseHandler HANDLER = createEmbeddingsHandler();
+
+    private static ResponseHandler createEmbeddingsHandler() {
+        return new AlibabaCloudSearchResponseHandler(
+            "alibaba cloud search text embedding",
+            AlibabaCloudSearchEmbeddingsResponseEntity::fromResponse
+        );
+    }
+
+    public static AlibabaCloudSearchEmbeddingsRequestManager of(
+        AlibabaCloudSearchAccount account,
+        AlibabaCloudSearchEmbeddingsModel model,
+        ThreadPool threadPool
+    ) {
+        return new AlibabaCloudSearchEmbeddingsRequestManager(
+            Objects.requireNonNull(account),
+            Objects.requireNonNull(model),
+            Objects.requireNonNull(threadPool)
+        );
+    }
+
+    private final AlibabaCloudSearchEmbeddingsModel model;
+
+    private final AlibabaCloudSearchAccount account;
+
+    private AlibabaCloudSearchEmbeddingsRequestManager(
+        AlibabaCloudSearchAccount account,
+        AlibabaCloudSearchEmbeddingsModel model,
+        ThreadPool threadPool
+    ) {
+        super(threadPool, model);
+        this.account = Objects.requireNonNull(account);
+        this.model = Objects.requireNonNull(model);
+    }
+
+    @Override
+    public void execute(
+        InferenceInputs inferenceInputs,
+        RequestSender requestSender,
+        Supplier<Boolean> hasRequestCompletedFunction,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        List<String> input = DocumentsOnlyInput.of(inferenceInputs).getInputs();
+        AlibabaCloudSearchEmbeddingsRequest request = new AlibabaCloudSearchEmbeddingsRequest(account, input, model);
+
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchRequestManager.java
new file mode 100644
index 0000000000000..c8ade15ac5057
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchRequestManager.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchModel;
+
+import java.util.Objects;
+
+abstract class AlibabaCloudSearchRequestManager extends BaseRequestManager {
+
+    protected AlibabaCloudSearchRequestManager(ThreadPool threadPool, AlibabaCloudSearchModel model) {
+        super(threadPool, model.getInferenceEntityId(), RateLimitGrouping.of(model), model.rateLimitServiceSettings().rateLimitSettings());
+    }
+
+    record RateLimitGrouping(int apiKeyHash) {
+        public static RateLimitGrouping of(AlibabaCloudSearchModel model) {
+            Objects.requireNonNull(model);
+
+            return new RateLimitGrouping(model.rateLimitServiceSettings().hashCode());
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchRerankRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchRerankRequestManager.java
new file mode 100644
index 0000000000000..446db40aa5ae5
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchRerankRequestManager.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchResponseHandler;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
+import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchRerankRequest;
+import org.elasticsearch.xpack.inference.external.response.alibabacloudsearch.AlibabaCloudSearchRerankResponseEntity;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank.AlibabaCloudSearchRerankModel;
+
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class AlibabaCloudSearchRerankRequestManager extends AlibabaCloudSearchRequestManager {
+    private static final Logger logger = LogManager.getLogger(AlibabaCloudSearchRerankRequestManager.class);
+    private static final ResponseHandler HANDLER = createRerankHandler();
+
+    private static ResponseHandler createRerankHandler() {
+        return new AlibabaCloudSearchResponseHandler("alibaba cloud search rerank", AlibabaCloudSearchRerankResponseEntity::fromResponse);
+    }
+
+    public static AlibabaCloudSearchRerankRequestManager of(
+        AlibabaCloudSearchAccount account,
+        AlibabaCloudSearchRerankModel model,
+        ThreadPool threadPool
+    ) {
+        return new AlibabaCloudSearchRerankRequestManager(
+            Objects.requireNonNull(account),
+            Objects.requireNonNull(model),
+            Objects.requireNonNull(threadPool)
+        );
+    }
+
+    private final AlibabaCloudSearchRerankModel model;
+
+    private final AlibabaCloudSearchAccount account;
+
+    private AlibabaCloudSearchRerankRequestManager(
+        AlibabaCloudSearchAccount account,
+        AlibabaCloudSearchRerankModel model,
+        ThreadPool threadPool
+    ) {
+        super(threadPool, model);
+        this.account = account;
+        this.model = model;
+    }
+
+    @Override
+    public void execute(
+        InferenceInputs inferenceInputs,
+        RequestSender requestSender,
+        Supplier<Boolean> hasRequestCompletedFunction,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        var rerankInput = QueryAndDocsInputs.of(inferenceInputs);
+        AlibabaCloudSearchRerankRequest request = new AlibabaCloudSearchRerankRequest(
+            account,
+            rerankInput.getQuery(),
+            rerankInput.getChunks(),
+            model
+        );
+
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchSparseRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchSparseRequestManager.java
new file mode 100644
index 0000000000000..b0cc524bb4cbe
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AlibabaCloudSearchSparseRequestManager.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchResponseHandler;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
+import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchSparseRequest;
+import org.elasticsearch.xpack.inference.external.response.alibabacloudsearch.AlibabaCloudSearchSparseResponseEntity;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseModel;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class AlibabaCloudSearchSparseRequestManager extends AlibabaCloudSearchRequestManager {
+    private static final Logger logger = LogManager.getLogger(AlibabaCloudSearchSparseRequestManager.class);
+
+    private static final ResponseHandler HANDLER = createEmbeddingsHandler();
+
+    private static ResponseHandler createEmbeddingsHandler() {
+        return new AlibabaCloudSearchResponseHandler(
+            "alibaba cloud search sparse embedding",
+            AlibabaCloudSearchSparseResponseEntity::fromResponse
+        );
+    }
+
+    public static AlibabaCloudSearchSparseRequestManager of(
+        AlibabaCloudSearchAccount account,
+        AlibabaCloudSearchSparseModel model,
+        ThreadPool threadPool
+    ) {
+        return new AlibabaCloudSearchSparseRequestManager(
+            Objects.requireNonNull(account),
+            Objects.requireNonNull(model),
+            Objects.requireNonNull(threadPool)
+        );
+    }
+
+    private final AlibabaCloudSearchSparseModel model;
+
+    private final AlibabaCloudSearchAccount account;
+
+    private AlibabaCloudSearchSparseRequestManager(
+        AlibabaCloudSearchAccount account,
+        AlibabaCloudSearchSparseModel model,
+        ThreadPool threadPool
+    ) {
+        super(threadPool, model);
+        this.account = Objects.requireNonNull(account);
+        this.model = Objects.requireNonNull(model);
+    }
+
+    @Override
+    public void execute(
+        InferenceInputs inferenceInputs,
+        RequestSender requestSender,
+        Supplier<Boolean> hasRequestCompletedFunction,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        List<String> input = DocumentsOnlyInput.of(inferenceInputs).getInputs();
+        AlibabaCloudSearchSparseRequest request = new AlibabaCloudSearchSparseRequest(account, input, model);
+
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java
index 8642a19b26a7d..1c6bb58717942 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
@@ -52,7 +51,7 @@ public void execute(
         var responseHandler = new AmazonBedrockChatCompletionResponseHandler();
 
         try {
-            requestSender.send(logger, request, HttpClientContext.create(), hasRequestCompletedFunction, responseHandler, listener);
+            requestSender.send(logger, request, hasRequestCompletedFunction, responseHandler, listener);
         } catch (Exception e) {
             var errorMessage = Strings.format(
                 "Failed to send [completion] request from inference entity id [%s]",
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockEmbeddingsRequestManager.java
index 2f94cdf342938..34aacbf67af6f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockEmbeddingsRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockEmbeddingsRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
@@ -61,7 +60,7 @@ public void execute(
         var responseHandler = new AmazonBedrockEmbeddingsResponseHandler();
         var request = new AmazonBedrockEmbeddingsRequest(truncator, truncatedInput, embeddingsModel, requestEntity, timeout);
         try {
-            requestSender.send(logger, request, HttpClientContext.create(), hasRequestCompletedFunction, responseHandler, listener);
+            requestSender.send(logger, request, hasRequestCompletedFunction, responseHandler, listener);
         } catch (Exception e) {
             var errorMessage = Strings.format(
                 "Failed to send [text_embedding] request from inference entity id [%s]",
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceRequestManager.java
new file mode 100644
index 0000000000000..c857a481f8f04
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceRequestManager.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceModel;
+
+import java.util.Objects;
+
+public abstract class ElasticInferenceServiceRequestManager extends BaseRequestManager {
+
+    protected ElasticInferenceServiceRequestManager(ThreadPool threadPool, ElasticInferenceServiceModel model) {
+        super(threadPool, model.getInferenceEntityId(), RateLimitGrouping.of(model), model.rateLimitServiceSettings().rateLimitSettings());
+    }
+
+    record RateLimitGrouping(int modelIdHash) {
+        public static RateLimitGrouping of(ElasticInferenceServiceModel model) {
+            Objects.requireNonNull(model);
+
+            return new RateLimitGrouping(model.rateLimitServiceSettings().modelId().hashCode());
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java
new file mode 100644
index 0000000000000..b59ac54d5cbb6
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ElasticInferenceServiceSparseEmbeddingsRequestManager.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xpack.inference.common.Truncator;
+import org.elasticsearch.xpack.inference.external.elastic.ElasticInferenceServiceResponseHandler;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
+import org.elasticsearch.xpack.inference.external.request.elastic.ElasticInferenceServiceSparseEmbeddingsRequest;
+import org.elasticsearch.xpack.inference.external.response.elastic.ElasticInferenceServiceSparseEmbeddingsResponseEntity;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel;
+
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.xpack.inference.common.Truncator.truncate;
+
+public class ElasticInferenceServiceSparseEmbeddingsRequestManager extends ElasticInferenceServiceRequestManager {
+
+    private static final Logger logger = LogManager.getLogger(ElasticInferenceServiceSparseEmbeddingsRequestManager.class);
+
+    private static final ResponseHandler HANDLER = createSparseEmbeddingsHandler();
+
+    private final ElasticInferenceServiceSparseEmbeddingsModel model;
+
+    private final Truncator truncator;
+
+    private static ResponseHandler createSparseEmbeddingsHandler() {
+        return new ElasticInferenceServiceResponseHandler(
+            "Elastic Inference Service sparse embeddings",
+            ElasticInferenceServiceSparseEmbeddingsResponseEntity::fromResponse
+        );
+    }
+
+    public ElasticInferenceServiceSparseEmbeddingsRequestManager(
+        ElasticInferenceServiceSparseEmbeddingsModel model,
+        ServiceComponents serviceComponents
+    ) {
+        super(serviceComponents.threadPool(), model);
+        this.model = model;
+        this.truncator = serviceComponents.truncator();
+    }
+
+    @Override
+    public void execute(
+        InferenceInputs inferenceInputs,
+        RequestSender requestSender,
+        Supplier<Boolean> hasRequestCompletedFunction,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        List<String> docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs();
+        var truncatedInput = truncate(docsInput, model.getServiceSettings().maxInputTokens());
+
+        ElasticInferenceServiceSparseEmbeddingsRequest request = new ElasticInferenceServiceSparseEmbeddingsRequest(
+            truncator,
+            truncatedInput,
+            model
+        );
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java
index 214eba4ee3485..241466422e47b 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
@@ -33,7 +32,7 @@ public void run() {
         var inferenceEntityId = request.createHttpRequest().inferenceEntityId();
 
         try {
-            requestSender.send(logger, request, HttpClientContext.create(), hasFinished, responseHandler, listener);
+            requestSender.send(logger, request, hasFinished, responseHandler, listener);
         } catch (Exception e) {
             var errorMessage = Strings.format("Failed to send request from inference entity id [%s]", inferenceEntityId);
             logger.warn(errorMessage, e);
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequest.java
new file mode 100644
index 0000000000000..081854903405e
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequest.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ByteArrayEntity;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount;
+import org.elasticsearch.xpack.inference.external.request.HttpRequest;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsTaskSettings;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.external.request.RequestUtils.buildUri;
+import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader;
+
+public class AlibabaCloudSearchEmbeddingsRequest extends AlibabaCloudSearchRequest {
+
+    private final AlibabaCloudSearchAccount account;
+    private final List<String> input;
+    private final URI uri;
+    private final AlibabaCloudSearchEmbeddingsTaskSettings taskSettings;
+    private final String model;
+    private final String host;
+    private final String workspaceName;
+    private final String httpSchema;
+    private final String inferenceEntityId;
+
+    public AlibabaCloudSearchEmbeddingsRequest(
+        AlibabaCloudSearchAccount account,
+        List<String> input,
+        AlibabaCloudSearchEmbeddingsModel embeddingsModel
+    ) {
+        Objects.requireNonNull(embeddingsModel);
+
+        this.account = Objects.requireNonNull(account);
+        this.input = Objects.requireNonNull(input);
+        taskSettings = embeddingsModel.getTaskSettings();
+        model = embeddingsModel.getServiceSettings().getCommonSettings().modelId();
+        host = embeddingsModel.getServiceSettings().getCommonSettings().getHost();
+        workspaceName = embeddingsModel.getServiceSettings().getCommonSettings().getWorkspaceName();
+        httpSchema = embeddingsModel.getServiceSettings().getCommonSettings().getHttpSchema() != null
+            ? embeddingsModel.getServiceSettings().getCommonSettings().getHttpSchema()
+            : "https";
+        uri = buildUri(null, AlibabaCloudSearchUtils.SERVICE_NAME, this::buildDefaultUri);
+        inferenceEntityId = embeddingsModel.getInferenceEntityId();
+    }
+
+    @Override
+    public HttpRequest createHttpRequest() {
+        HttpPost httpPost = new HttpPost(uri);
+
+        ByteArrayEntity byteEntity = new ByteArrayEntity(
+            Strings.toString(new AlibabaCloudSearchEmbeddingsRequestEntity(input, taskSettings)).getBytes(StandardCharsets.UTF_8)
+        );
+        httpPost.setEntity(byteEntity);
+
+        httpPost.setHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType());
+        httpPost.setHeader(createAuthBearerHeader(account.apiKey()));
+
+        return new HttpRequest(httpPost, getInferenceEntityId());
+    }
+
+    @Override
+    public String getInferenceEntityId() {
+        return inferenceEntityId;
+    }
+
+    @Override
+    public URI getURI() {
+        return uri;
+    }
+
+    @Override
+    public Request truncate() {
+        return this;
+    }
+
+    @Override
+    public boolean[] getTruncationInfo() {
+        return null;
+    }
+
+    URI buildDefaultUri() throws URISyntaxException {
+        return new URIBuilder().setScheme(httpSchema)
+            .setHost(host)
+            .setPathSegments(
+                AlibabaCloudSearchUtils.VERSION_3,
+                AlibabaCloudSearchUtils.OPENAPI_PATH,
+                AlibabaCloudSearchUtils.WORKSPACE_PATH,
+                workspaceName,
+                AlibabaCloudSearchUtils.TEXT_EMBEDDING_PATH,
+                model
+            )
+            .build();
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java
new file mode 100644
index 0000000000000..c2367aeff3070
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsTaskSettings;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsTaskSettings.invalidInputTypeMessage;
+
+public record AlibabaCloudSearchEmbeddingsRequestEntity(List<String> input, AlibabaCloudSearchEmbeddingsTaskSettings taskSettings)
+    implements
+        ToXContentObject {
+
+    private static final String SEARCH_DOCUMENT = "document";
+    private static final String SEARCH_QUERY = "query";
+
+    private static final String TEXTS_FIELD = "input";
+
+    static final String INPUT_TYPE_FIELD = "input_type";
+
+    public AlibabaCloudSearchEmbeddingsRequestEntity {
+        Objects.requireNonNull(input);
+        Objects.requireNonNull(taskSettings);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(TEXTS_FIELD, input);
+
+        String inputType = covertToString(taskSettings.getInputType());
+        if (inputType != null) {
+            builder.field(INPUT_TYPE_FIELD, inputType);
+        }
+
+        builder.endObject();
+        return builder;
+    }
+
+    // default for testing
+    static String covertToString(InputType inputType) {
+        if (inputType == null) {
+            return null;
+        }
+
+        return switch (inputType) {
+            case INGEST -> SEARCH_DOCUMENT;
+            case SEARCH -> SEARCH_QUERY;
+            default -> {
+                assert false : invalidInputTypeMessage(inputType);
+                yield null;
+            }
+        };
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRequest.java
new file mode 100644
index 0000000000000..75fc12e1bad31
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRequest.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.elasticsearch.xpack.inference.external.request.Request;
+
+public abstract class AlibabaCloudSearchRequest implements Request {
+    private final long startTime;
+
+    public AlibabaCloudSearchRequest() {
+        this.startTime = System.currentTimeMillis();
+    }
+
+    public long getStartTime() {
+        return startTime;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRerankRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRerankRequest.java
new file mode 100644
index 0000000000000..878bcc6e6a0db
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRerankRequest.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ByteArrayEntity;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount;
+import org.elasticsearch.xpack.inference.external.request.HttpRequest;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank.AlibabaCloudSearchRerankModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank.AlibabaCloudSearchRerankTaskSettings;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.external.request.RequestUtils.buildUri;
+import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader;
+
+public class AlibabaCloudSearchRerankRequest implements Request {
+    private final AlibabaCloudSearchAccount account;
+    private final String query;
+    private final List<String> input;
+    private final URI uri;
+    private final AlibabaCloudSearchRerankTaskSettings taskSettings;
+    private final String model;
+    private final String host;
+    private final String workspaceName;
+    private final String httpSchema;
+    private final String inferenceEntityId;
+
+    public AlibabaCloudSearchRerankRequest(
+        AlibabaCloudSearchAccount account,
+        String query,
+        List<String> input,
+        AlibabaCloudSearchRerankModel rerankModel
+    ) {
+        Objects.requireNonNull(rerankModel);
+
+        this.account = Objects.requireNonNull(account);
+        this.query = Objects.requireNonNull(query);
+        this.input = Objects.requireNonNull(input);
+        taskSettings = rerankModel.getTaskSettings();
+        model = rerankModel.getServiceSettings().getCommonSettings().modelId();
+        host = rerankModel.getServiceSettings().getCommonSettings().getHost();
+        workspaceName = rerankModel.getServiceSettings().getCommonSettings().getWorkspaceName();
+        httpSchema = rerankModel.getServiceSettings().getCommonSettings().getHttpSchema() != null
+            ? rerankModel.getServiceSettings().getCommonSettings().getHttpSchema()
+            : "https";
+        uri = buildUri(null, AlibabaCloudSearchUtils.SERVICE_NAME, this::buildDefaultUri);
+        inferenceEntityId = rerankModel.getInferenceEntityId();
+    }
+
+    @Override
+    public HttpRequest createHttpRequest() {
+        HttpPost httpPost = new HttpPost(uri);
+
+        ByteArrayEntity byteEntity = new ByteArrayEntity(
+            Strings.toString(new AlibabaCloudSearchRerankRequestEntity(query, input, taskSettings)).getBytes(StandardCharsets.UTF_8)
+        );
+        httpPost.setEntity(byteEntity);
+
+        httpPost.setHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType());
+        httpPost.setHeader(createAuthBearerHeader(account.apiKey()));
+
+        return new HttpRequest(httpPost, getInferenceEntityId());
+    }
+
+    @Override
+    public String getInferenceEntityId() {
+        return inferenceEntityId;
+    }
+
+    @Override
+    public URI getURI() {
+        return uri;
+    }
+
+    @Override
+    public Request truncate() {
+        return this;
+    }
+
+    @Override
+    public boolean[] getTruncationInfo() {
+        return null;
+    }
+
+    URI buildDefaultUri() throws URISyntaxException {
+        return new URIBuilder().setScheme(httpSchema)
+            .setHost(host)
+            .setPathSegments(
+                AlibabaCloudSearchUtils.VERSION_3,
+                AlibabaCloudSearchUtils.OPENAPI_PATH,
+                AlibabaCloudSearchUtils.WORKSPACE_PATH,
+                workspaceName,
+                AlibabaCloudSearchUtils.RERANK_PATH,
+                model
+            )
+            .build();
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRerankRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRerankRequestEntity.java
new file mode 100644
index 0000000000000..054e373e3e525
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRerankRequestEntity.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank.AlibabaCloudSearchRerankTaskSettings;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public record AlibabaCloudSearchRerankRequestEntity(String query, List<String> input, AlibabaCloudSearchRerankTaskSettings taskSettings)
+    implements
+        ToXContentObject {
+
+    private static final String SEARCH_QUERY = "query";
+    private static final String TEXTS_FIELD = "docs";
+
+    public AlibabaCloudSearchRerankRequestEntity {
+        Objects.requireNonNull(query);
+        Objects.requireNonNull(input);
+        Objects.requireNonNull(taskSettings);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.field(SEARCH_QUERY, query);
+            builder.field(TEXTS_FIELD, input);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequest.java
new file mode 100644
index 0000000000000..c7b4c314b07a7
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequest.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ByteArrayEntity;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount;
+import org.elasticsearch.xpack.inference.external.request.HttpRequest;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseTaskSettings;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.external.request.RequestUtils.buildUri;
+import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader;
+
+public class AlibabaCloudSearchSparseRequest extends AlibabaCloudSearchRequest {
+
+    private final AlibabaCloudSearchAccount account;
+    private final List<String> input;
+    private final URI uri;
+    private final AlibabaCloudSearchSparseTaskSettings taskSettings;
+    private final String model;
+    private final String host;
+    private final String workspaceName;
+    private final String httpSchema;
+    private final String inferenceEntityId;
+
+    public AlibabaCloudSearchSparseRequest(
+        AlibabaCloudSearchAccount account,
+        List<String> input,
+        AlibabaCloudSearchSparseModel sparseEmbeddingsModel
+    ) {
+        Objects.requireNonNull(sparseEmbeddingsModel);
+
+        this.account = Objects.requireNonNull(account);
+        this.input = Objects.requireNonNull(input);
+        taskSettings = sparseEmbeddingsModel.getTaskSettings();
+        model = sparseEmbeddingsModel.getServiceSettings().getCommonSettings().modelId();
+        host = sparseEmbeddingsModel.getServiceSettings().getCommonSettings().getHost();
+        workspaceName = sparseEmbeddingsModel.getServiceSettings().getCommonSettings().getWorkspaceName();
+        httpSchema = sparseEmbeddingsModel.getServiceSettings().getCommonSettings().getHttpSchema() != null
+            ? sparseEmbeddingsModel.getServiceSettings().getCommonSettings().getHttpSchema()
+            : "https";
+        uri = buildUri(null, AlibabaCloudSearchUtils.SERVICE_NAME, this::buildDefaultUri);
+        inferenceEntityId = sparseEmbeddingsModel.getInferenceEntityId();
+    }
+
+    @Override
+    public HttpRequest createHttpRequest() {
+        HttpPost httpPost = new HttpPost(uri);
+
+        ByteArrayEntity byteEntity = new ByteArrayEntity(
+            Strings.toString(new AlibabaCloudSearchSparseRequestEntity(input, taskSettings)).getBytes(StandardCharsets.UTF_8)
+        );
+        httpPost.setEntity(byteEntity);
+
+        httpPost.setHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType());
+        httpPost.setHeader(createAuthBearerHeader(account.apiKey()));
+
+        return new HttpRequest(httpPost, getInferenceEntityId());
+    }
+
+    @Override
+    public String getInferenceEntityId() {
+        return inferenceEntityId;
+    }
+
+    @Override
+    public URI getURI() {
+        return uri;
+    }
+
+    @Override
+    public Request truncate() {
+        return this;
+    }
+
+    @Override
+    public boolean[] getTruncationInfo() {
+        return null;
+    }
+
+    URI buildDefaultUri() throws URISyntaxException {
+        return new URIBuilder().setScheme(httpSchema)
+            .setHost(host)
+            .setPathSegments(
+                AlibabaCloudSearchUtils.VERSION_3,
+                AlibabaCloudSearchUtils.OPENAPI_PATH,
+                AlibabaCloudSearchUtils.WORKSPACE_PATH,
+                workspaceName,
+                AlibabaCloudSearchUtils.SPARSE_EMBEDDING_PATH,
+                model
+            )
+            .build();
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java
new file mode 100644
index 0000000000000..3aec226bfc277
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseTaskSettings;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public record AlibabaCloudSearchSparseRequestEntity(List<String> input, AlibabaCloudSearchSparseTaskSettings taskSettings)
+    implements
+        ToXContentObject {
+
+    private static final String TEXTS_FIELD = "input";
+
+    static final String INPUT_TYPE_FIELD = "input_type";
+
+    static final String RETURN_TOKEN_FIELD = "return_token";
+
+    public AlibabaCloudSearchSparseRequestEntity {
+        Objects.requireNonNull(input);
+        Objects.requireNonNull(taskSettings);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(TEXTS_FIELD, input);
+        String inputType = AlibabaCloudSearchEmbeddingsRequestEntity.covertToString(taskSettings.getInputType());
+        if (inputType != null) {
+            builder.field(INPUT_TYPE_FIELD, inputType);
+        }
+        if (taskSettings.isReturnToken() != null) {
+            builder.field(RETURN_TOKEN_FIELD, taskSettings.isReturnToken());
+        }
+        builder.endObject();
+        return builder;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchUtils.java
new file mode 100644
index 0000000000000..7d671471976f5
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchUtils.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+public class AlibabaCloudSearchUtils {
+    public static final String SERVICE_NAME = "alibabacloud-ai-search";
+    public static final String VERSION_3 = "v3";
+    public static final String OPENAPI_PATH = "openapi";
+    public static final String WORKSPACE_PATH = "workspaces";
+    public static final String TEXT_EMBEDDING_PATH = "text-embedding";
+    public static final String SPARSE_EMBEDDING_PATH = "text-sparse-embedding";
+    public static final String RERANK_PATH = "ranker";
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceRequest.java
new file mode 100644
index 0000000000000..03eec913a265f
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceRequest.java
@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.elastic;
+
+import org.elasticsearch.xpack.inference.external.request.Request;
+
+public interface ElasticInferenceServiceRequest extends Request {}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java
new file mode 100644
index 0000000000000..41a2ef1c3ccda
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequest.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.elastic;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.message.BasicHeader;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.common.Truncator;
+import org.elasticsearch.xpack.inference.external.request.HttpRequest;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModel;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+public class ElasticInferenceServiceSparseEmbeddingsRequest implements ElasticInferenceServiceRequest {
+
+    private final URI uri;
+
+    private final ElasticInferenceServiceSparseEmbeddingsModel model;
+
+    private final Truncator.TruncationResult truncationResult;
+    private final Truncator truncator;
+
+    public ElasticInferenceServiceSparseEmbeddingsRequest(
+        Truncator truncator,
+        Truncator.TruncationResult truncationResult,
+        ElasticInferenceServiceSparseEmbeddingsModel model
+    ) {
+        this.truncator = truncator;
+        this.truncationResult = truncationResult;
+        this.model = Objects.requireNonNull(model);
+        this.uri = model.uri();
+    }
+
+    @Override
+    public HttpRequest createHttpRequest() {
+        var httpPost = new HttpPost(uri);
+        var requestEntity = Strings.toString(new ElasticInferenceServiceSparseEmbeddingsRequestEntity(truncationResult.input()));
+
+        ByteArrayEntity byteEntity = new ByteArrayEntity(requestEntity.getBytes(StandardCharsets.UTF_8));
+        httpPost.setEntity(byteEntity);
+
+        httpPost.setHeader(new BasicHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType()));
+
+        return new HttpRequest(httpPost, getInferenceEntityId());
+    }
+
+    @Override
+    public String getInferenceEntityId() {
+        return model.getInferenceEntityId();
+    }
+
+    @Override
+    public URI getURI() {
+        return this.uri;
+    }
+
+    @Override
+    public Request truncate() {
+        var truncatedInput = truncator.truncate(truncationResult.input());
+
+        return new ElasticInferenceServiceSparseEmbeddingsRequest(truncator, truncatedInput, model);
+    }
+
+    @Override
+    public boolean[] getTruncationInfo() {
+        return truncationResult.truncated().clone();
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java
new file mode 100644
index 0000000000000..301bbf0146c14
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntity.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.elastic;
+
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public record ElasticInferenceServiceSparseEmbeddingsRequestEntity(List<String> inputs) implements ToXContentObject {
+
+    private static final String INPUT_FIELD = "input";
+
+    public ElasticInferenceServiceSparseEmbeddingsRequestEntity {
+        Objects.requireNonNull(inputs);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.startArray(INPUT_FIELD);
+
+        {
+            for (String input : inputs) {
+                builder.value(input);
+            }
+        }
+
+        builder.endArray();
+        builder.endObject();
+
+        return builder;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntity.java
new file mode 100644
index 0000000000000..33fa645b107bc
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntity.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.alibabacloudsearch;
+
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
+
+public class AlibabaCloudSearchEmbeddingsResponseEntity extends AlibabaCloudSearchResponseEntity {
+    private static final String FAILED_TO_FIND_FIELD_TEMPLATE =
+        "Failed to find required field [%s] in AlibabaCloud Search embeddings response";
+
+    /**
+     * Parses the AlibabaCloud Search embedding json response.
+     * For a request like:
+     *
+     * <pre>
+     * <code>
+     * {
+     *  "texts": ["hello this is my name", "I wish I was there!"]
+     * }
+     * </code>
+     * </pre>
+     *
+     * The response would look like:
+     *
+     * <pre>
+     * <code>
+     * {
+     *     "request_id": "B4AB89C8-B135-xxxx-A6F8-2BAB801A2CE4",
+     *     "latency": 38,
+     *     "usage": {
+     *         "token_count": 3072
+     *     },
+     *     "result": {
+     *         "embeddings": [
+     *             {
+     *                 "index": 0,
+     *                 "embedding": [
+     *                     -0.02868066355586052,
+     *                     0.022033605724573135,
+     *                     -0.0417383536696434,
+     *                     -0.044081952422857285,
+     *                     0.02141784131526947,
+     *                     -8.240503375418484E-4,
+     *                     -0.01309406291693449,
+     *                     -0.02169642224907875,
+     *                     -0.03996409475803375,
+     *                     0.008053945377469063,
+     *                     ...
+     *                     -0.05131729692220688,
+     *                     -0.016595875844359398
+     *                 ]
+     *             }
+     *         ]
+     *     }
+     * }
+     * </code>
+     * </pre>
+     */
+    public static InferenceTextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException {
+        return fromResponse(request, response, parser -> {
+            positionParserAtTokenAfterField(parser, "embeddings", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+            List<InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding> embeddingList = XContentParserUtils.parseList(
+                parser,
+                AlibabaCloudSearchEmbeddingsResponseEntity::parseEmbeddingObject
+            );
+
+            return new InferenceTextEmbeddingFloatResults(embeddingList);
+        });
+    }
+
+    private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseEmbeddingObject(XContentParser parser)
+        throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+
+        positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+        List<Float> embeddingValues = XContentParserUtils.parseList(parser, AlibabaCloudSearchEmbeddingsResponseEntity::parseEmbeddingList);
+
+        // the parser is currently sitting at an ARRAY_END so go to the next token
+        parser.nextToken();
+        // if there are additional fields within this object, lets skip them, so we can begin parsing the next embedding array
+        parser.skipChildren();
+
+        return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValues);
+    }
+
+    private static float parseEmbeddingList(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
+        return parser.floatValue();
+    }
+
+    private AlibabaCloudSearchEmbeddingsResponseEntity() {}
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntity.java
new file mode 100644
index 0000000000000..77a0c6ecc7cc8
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntity.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.alibabacloudsearch;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage;
+
+public class AlibabaCloudSearchErrorResponseEntity implements ErrorMessage {
+    private static final Logger logger = LogManager.getLogger(AlibabaCloudSearchErrorResponseEntity.class);
+
+    private final String errorMessage;
+
+    private AlibabaCloudSearchErrorResponseEntity(String errorMessage) {
+        this.errorMessage = errorMessage;
+    }
+
+    @Override
+    public String getErrorMessage() {
+        return errorMessage;
+    }
+
+    /**
+     * An example error response for invalid auth would look like
+     * <code>
+     *     {
+     *      "request_id": "651B3087-8A07-xxxx-xxxx-9C4E7B60F52D",
+     *      "latency": 0,
+     *      "code": "InvalidParameter",
+     *      "message": "JSON parse error: Cannot deserialize value of type `InputType` from String \"xxx\""
+     *      }
+     * </code>
+     *
+     *
+     * @param response The error response
+     * @return An error entity if the response is JSON with the above structure
+     * or null if the response does not contain the message field
+     */
+    public static AlibabaCloudSearchErrorResponseEntity fromResponse(HttpResult response) {
+        try (
+            XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON)
+                .createParser(XContentParserConfiguration.EMPTY, response.body())
+        ) {
+            var responseMap = jsonParser.map();
+            if (logger.isDebugEnabled()) {
+                logger.debug("Received error response: {}", responseMap);
+            }
+
+            var message = (String) responseMap.get("message");
+            if (message != null) {
+                return new AlibabaCloudSearchErrorResponseEntity(message);
+            }
+        } catch (Exception e) {
+            // swallow the error
+        }
+
+        return null;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchRerankResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchRerankResponseEntity.java
new file mode 100644
index 0000000000000..9a18c56e86475
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchRerankResponseEntity.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.alibabacloudsearch;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
+
+public class AlibabaCloudSearchRerankResponseEntity {
+    private static final String FAILED_TO_FIND_FIELD_TEMPLATE = "Failed to find required field [%s] in AlibabaCloud Search rerank response";
+
+    private static final Logger logger = LogManager.getLogger(AlibabaCloudSearchRerankResponseEntity.class);
+
+    /**
+     * Parses the AlibabaCloud Search rerank json response.
+     * For a request like:
+     *
+     * <pre>
+     * <code>
+     * {
+     *  "query": "上海有什么好玩的",
+     *  "docs" : ["上海有许多好玩的地方",
+     *             "北京有许多好玩的地方"]
+     * }
+     * </code>
+     * </pre>
+     *
+     * The response would look like:
+     *
+     * <pre>
+     * <code>
+     *     {
+     *   "request_id": "450fcb80-f796-xxxx-xxxx-e1e86d29aa9f",
+     *   "latency": 564.903929,
+     *   "usage": {
+     *     "doc_count": 2
+     *   }
+     *   "result": {
+     *    "scores":[
+     *      {
+     *        "index":1,
+     *        "score": 1.37
+     *      },
+     *      {
+     *        "index":0,
+     *        "score": -0.3
+     *      }
+     *    ]
+     *   }
+     * }
+     * </code>
+     * </pre>
+     */
+    public static InferenceServiceResults fromResponse(Request request, HttpResult response) throws IOException {
+        var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE);
+
+        try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) {
+            moveToFirstToken(jsonParser);
+
+            XContentParser.Token token = jsonParser.currentToken();
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+
+            positionParserAtTokenAfterField(jsonParser, "result", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+            positionParserAtTokenAfterField(jsonParser, "scores", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+            token = jsonParser.currentToken();
+            if (token == XContentParser.Token.START_ARRAY) {
+                return new RankedDocsResults(parseList(jsonParser, AlibabaCloudSearchRerankResponseEntity::parseRankedDocObject));
+            } else {
+                throwUnknownToken(token, jsonParser);
+            }
+
+            // This should never be reached. The above code should either return successfully or hit the throwUnknownToken
+            // or throw a parsing exception
+            throw new IllegalStateException("Reached an invalid state while parsing the AlibabaCloudSearch response");
+        }
+    }
+
+    private static RankedDocsResults.RankedDoc parseRankedDocObject(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+        int index = -1;
+        float score = -1;
+        String documentText = null;
+        parser.nextToken();
+        while (parser.currentToken() != XContentParser.Token.END_OBJECT) {
+            if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                switch (parser.currentName()) {
+                    case "index":
+                        parser.nextToken(); // move to VALUE_NUMBER
+                        index = parser.intValue();
+                        parser.nextToken(); // move to next FIELD_NAME or END_OBJECT
+                        break;
+                    case "score":
+                        parser.nextToken(); // move to VALUE_NUMBER
+                        score = parser.floatValue();
+                        parser.nextToken(); // move to next FIELD_NAME or END_OBJECT
+                        break;
+                    default:
+                        throwUnknownField(parser.currentName(), parser);
+                }
+            } else {
+                parser.nextToken();
+            }
+        }
+
+        if (index == -1) {
+            logger.warn("Failed to find required field [index] in AlibabaCloudSearch rerank response");
+        }
+        if (score == -1) {
+            logger.warn("Failed to find required field [relevance_score] in AlibabaCloudSearch rerank response");
+        }
+        // documentText may or may not be present depending on the request parameter
+
+        return new RankedDocsResults.RankedDoc(index, score, documentText);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchResponseEntity.java
new file mode 100644
index 0000000000000..156f2c1d4078c
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchResponseEntity.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.alibabacloudsearch;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchRequest;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
+
+public abstract class AlibabaCloudSearchResponseEntity {
+    private static final Logger logger = LogManager.getLogger(AlibabaCloudSearchResponseEntity.class);
+
+    public static <R> R fromResponse(Request request, HttpResult response, CheckedFunction<XContentParser, R, IOException> function)
+        throws IOException {
+        var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE);
+        AlibabaCloudSearchRequest alibabaCloudSearchRequest = (AlibabaCloudSearchRequest) request;
+
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) {
+            moveToFirstToken(parser);
+
+            XContentParser.Token token = parser.currentToken();
+            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
+
+            R result = null;
+            String requestID = null;
+            float latency = 0;
+            Map<String, Object> usage = null;
+            while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                String currentFieldName = parser.currentName();
+                parser.nextToken();
+                switch (currentFieldName) {
+                    case "result":
+                        result = function.apply(parser);
+                        break;
+                    case "request_id":
+                        requestID = parser.text();
+                        break;
+                    case "latency":
+                        latency = parser.floatValue();
+                        break;
+                    case "usage":
+                        usage = parser.map();
+                        break;
+                    default:
+                        parser.skipChildren();
+                }
+            }
+
+            logger.debug(
+                "AlibabaCloud Search uri [{}] response: request_id [{}], latency [{}ms], client cost [{}ms], usage [{}]",
+                request.getURI().getPath(),
+                requestID,
+                latency,
+                System.currentTimeMillis() - alibabaCloudSearchRequest.getStartTime(),
+                usage
+            );
+            return result;
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchSparseResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchSparseResponseEntity.java
new file mode 100644
index 0000000000000..c903bfd188116
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchSparseResponseEntity.java
@@ -0,0 +1,199 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.alibabacloudsearch;
+
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
+import org.elasticsearch.xpack.core.ml.search.WeightedToken;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
+
+public class AlibabaCloudSearchSparseResponseEntity extends AlibabaCloudSearchResponseEntity {
+    private static final String FAILED_TO_FIND_FIELD_TEMPLATE =
+        "Failed to find required field [%s] in AlibabaCloud Search sparse embeddings response";
+
+    /**
+     * Parses the AlibabaCloud Search sparse embedding json response.
+     * For a request like:
+     *
+     * <pre>
+     * <code>
+     * {
+     *  "texts": ["hello this is my name", "I wish I was there!"]
+     * }
+     * </code>
+     * </pre>
+     *
+     * The response would look like:
+     *
+     * <pre>
+     * <code>
+     *     {
+     *   "request_id": "DDC4306F-xxxx-xxxx-xxxx-92C5CEA756A0",
+     *   "latency": 25,
+     *   "usage": {
+     *     "token_count": 11
+     *   },
+     *   "result": {
+     *     "sparse_embeddings": [
+     *       {
+     *         "index": 0,
+     *         "embedding": [
+     *           {
+     *             "token_id": 6,
+     *             "weight": 0.1014404296875,
+     *             "token": ""
+     *           },
+     *           {
+     *             "token_id": 163040,
+     *             "weight": 0.2841796875,
+     *             "token": "科学技术"
+     *           },
+     *           {
+     *             "token_id": 354,
+     *             "weight": 0.1431884765625,
+     *             "token": "是"
+     *           },
+     *           {
+     *             "token_id": 5998,
+     *             "weight": 0.1614990234375,
+     *             "token": "第一"
+     *           },
+     *           {
+     *             "token_id": 8550,
+     *             "weight": 0.239013671875,
+     *             "token": "生产"
+     *           },
+     *           {
+     *             "token_id": 2017,
+     *             "weight": 0.161376953125,
+     *             "token": "力"
+     *           }
+     *         ]
+     *       },
+     *       {
+     *         "index": 1,
+     *         "embedding": [
+     *           {
+     *             "token_id": 9803,
+     *             "weight": 0.1949462890625,
+     *             "token": "open"
+     *           },
+     *           {
+     *             "token_id": 86250,
+     *             "weight": 0.317138671875,
+     *             "token": "search"
+     *           },
+     *           {
+     *             "token_id": 5889,
+     *             "weight": 0.175048828125,
+     *             "token": "产品"
+     *           },
+     *           {
+     *             "token_id": 2564,
+     *             "weight": 0.1163330078125,
+     *             "token": "文"
+     *           },
+     *           {
+     *             "token_id": 59529,
+     *             "weight": 0.16650390625,
+     *             "token": "档"
+     *           }
+     *         ]
+     *       }
+     *     ]
+     *   }
+     * }
+     * </code>
+     * </pre>
+     */
+    public static SparseEmbeddingResults fromResponse(Request request, HttpResult response) throws IOException {
+        return fromResponse(request, response, jsonParser -> {
+            positionParserAtTokenAfterField(jsonParser, "sparse_embeddings", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+            List<SparseEmbeddingResults.Embedding> embeddingList = XContentParserUtils.parseList(
+                jsonParser,
+                AlibabaCloudSearchSparseResponseEntity::parseEmbeddingObject
+            );
+
+            return new SparseEmbeddingResults(embeddingList);
+        });
+    }
+
+    private static SparseEmbeddingResults.Embedding parseEmbeddingObject(XContentParser parser) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+
+        positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+        List<WeightedToken> tokens = parseWeightedTokenList(parser);
+
+        // the parser is currently sitting at an ARRAY_END so go to the next token
+        parser.nextToken();
+        // if there are additional fields within this object, lets skip them, so we can begin parsing the next embedding array
+        parser.skipChildren();
+
+        return new SparseEmbeddingResults.Embedding(tokens, false);
+    }
+
+    private static List<WeightedToken> parseWeightedTokenList(XContentParser parser) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
+        if (parser.nextToken() == XContentParser.Token.END_ARRAY) {
+            return List.of();
+        }
+        final ArrayList<WeightedToken> list = new ArrayList<>();
+        do {
+            WeightedToken token = parseEmbeddingList(parser);
+            if (token != null) {
+                list.add(token);
+            }
+        } while (parser.nextToken() != XContentParser.Token.END_ARRAY);
+        return list;
+    }
+
+    private static WeightedToken parseEmbeddingList(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
+        Map<String, Object> values = parser.map();
+        Object tokenName;
+        if (values.containsKey("token")) {
+            tokenName = values.get("token");
+        } else {
+            tokenName = values.get("token_id");
+        }
+        float weight = Float.parseFloat(values.get("weight").toString());
+
+        if (invalidToken(tokenName) || weight <= 0.0f) {
+            return null;
+        }
+
+        return new WeightedToken(tokenName.toString(), weight);
+    }
+
+    private static boolean invalidToken(Object tokenName) {
+        if (tokenName == null) {
+            return true;
+        }
+
+        String token = tokenName.toString();
+        if (token.isEmpty() || token.contains(".")) {
+            return true;
+        }
+
+        return false;
+    }
+
+    private AlibabaCloudSearchSparseResponseEntity() {}
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java
new file mode 100644
index 0000000000000..c860821c81bbf
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntity.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.elastic;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.http.retry.ErrorMessage;
+
+public class ElasticInferenceServiceErrorResponseEntity implements ErrorMessage {
+
+    private final String errorMessage;
+
+    private static final Logger logger = LogManager.getLogger(ElasticInferenceServiceErrorResponseEntity.class);
+
+    private ElasticInferenceServiceErrorResponseEntity(String errorMessage) {
+        this.errorMessage = errorMessage;
+    }
+
+    @Override
+    public String getErrorMessage() {
+        return errorMessage;
+    }
+
+    /**
+     * An example error response would look like
+     *
+     * <code>
+     *     {
+     *         "error": "some error"
+     *     }
+     * </code>
+     *
+     * @param response The error response
+     * @return An error entity if the response is JSON with the above structure
+     * or null if the response does not contain the error field
+     */
+    public static @Nullable ElasticInferenceServiceErrorResponseEntity fromResponse(HttpResult response) {
+        try (
+            XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON)
+                .createParser(XContentParserConfiguration.EMPTY, response.body())
+        ) {
+            var responseMap = jsonParser.map();
+            var error = (String) responseMap.get("error");
+            if (error != null) {
+                return new ElasticInferenceServiceErrorResponseEntity(error);
+            }
+        } catch (Exception e) {
+            logger.debug("Failed to parse error response", e);
+        }
+
+        return null;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntity.java
new file mode 100644
index 0000000000000..2b36cc5d22cd4
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntity.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.elastic;
+
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
+import org.elasticsearch.xpack.core.ml.search.WeightedToken;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList;
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
+
+public class ElasticInferenceServiceSparseEmbeddingsResponseEntity {
+
+    private static final String FAILED_TO_FIND_FIELD_TEMPLATE =
+        "Failed to find required field [%s] in Elastic Inference Service embeddings response";
+
+    /**
+     * Parses the EIS json response.
+     *
+     * For a request like:
+     *
+     * <pre>
+     *     <code>
+     *         {
+     *             "inputs": ["Embed this text", "Embed this text, too"]
+     *         }
+     *     </code>
+     * </pre>
+     *
+     * The response would look like:
+     *
+     * <pre>
+     *     <code>
+     *         {
+     *           "data": [
+     *                     {
+     *                       "Embed": 2.1259406,
+     *                       "this": 1.7073475,
+     *                       "text": 0.9020516
+     *                     },
+     *                    (...)
+     *                  ],
+     *           "meta": {
+     *               "processing_latency": ...,
+     *               "request_time": ...
+     *           }
+     *     </code>
+     * </pre>
+     */
+
+    public static SparseEmbeddingResults fromResponse(Request request, HttpResult response) throws IOException {
+        var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE);
+
+        try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) {
+            moveToFirstToken(jsonParser);
+
+            XContentParser.Token token = jsonParser.currentToken();
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+
+            positionParserAtTokenAfterField(jsonParser, "data", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+            var truncationResults = request.getTruncationInfo();
+            List<SparseEmbeddingResults.Embedding> parsedEmbeddings = parseList(
+                jsonParser,
+                (parser, index) -> ElasticInferenceServiceSparseEmbeddingsResponseEntity.parseExpansionResult(
+                    truncationResults,
+                    parser,
+                    index
+                )
+            );
+
+            if (parsedEmbeddings.isEmpty()) {
+                return new SparseEmbeddingResults(Collections.emptyList());
+            }
+
+            return new SparseEmbeddingResults(parsedEmbeddings);
+        }
+    }
+
+    private static SparseEmbeddingResults.Embedding parseExpansionResult(boolean[] truncationResults, XContentParser parser, int index)
+        throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
+
+        List<WeightedToken> weightedTokens = new ArrayList<>();
+        token = parser.nextToken();
+        while (token != null && token != XContentParser.Token.END_OBJECT) {
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
+            var floatToken = parser.nextToken();
+            ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, floatToken, parser);
+
+            weightedTokens.add(new WeightedToken(parser.currentName(), parser.floatValue()));
+
+            token = parser.nextToken();
+        }
+
+        // prevent an out of bounds if for some reason the truncation list is smaller than the results
+        var isTruncated = truncationResults != null && index < truncationResults.length && truncationResults[index];
+        return new SparseEmbeddingResults.Embedding(weightedTokens, isTruncated);
+    }
+
+    private ElasticInferenceServiceSparseEmbeddingsResponseEntity() {}
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
index b9b95afbf6dc6..71906a720e969 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
@@ -7,12 +7,12 @@
 
 package org.elasticsearch.xpack.inference.mapper;
 
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
-import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -60,6 +60,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
 
@@ -320,7 +321,7 @@ public SemanticTextFieldType(
             IndexVersion indexVersionCreated,
             Map<String, String> meta
         ) {
-            super(name, false, false, false, TextSearchInfo.NONE, meta);
+            super(name, true, false, false, TextSearchInfo.NONE, meta);
             this.inferenceId = inferenceId;
             this.modelSettings = modelSettings;
             this.inferenceField = inferenceField;
@@ -383,6 +384,11 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
             throw new IllegalArgumentException("[semantic_text] fields do not support sorting, scripting or aggregating");
         }
 
+        @Override
+        public boolean fieldHasValue(FieldInfos fieldInfos) {
+            return fieldInfos.fieldInfo(getEmbeddingsFieldName(name())) != null;
+        }
+
         public QueryBuilder semanticQuery(InferenceResults inferenceResults, float boost, String queryName) {
             String nestedFieldPath = getChunksFieldName(name());
             String inferenceResultsFieldName = getEmbeddingsFieldName(name());
@@ -469,7 +475,7 @@ private static ObjectMapper createInferenceField(
         @Nullable SemanticTextField.ModelSettings modelSettings,
         Function<Query, BitSetProducer> bitSetProducer
     ) {
-        return new ObjectMapper.Builder(INFERENCE_FIELD, Explicit.EXPLICIT_TRUE).dynamic(ObjectMapper.Dynamic.FALSE)
+        return new ObjectMapper.Builder(INFERENCE_FIELD, Optional.of(ObjectMapper.Subobjects.ENABLED)).dynamic(ObjectMapper.Dynamic.FALSE)
             .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer))
             .build(context);
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankBuilder.java
new file mode 100644
index 0000000000000..fdb5503e491eb
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankBuilder.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.rank.random;
+
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.rank.RankBuilder;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext;
+import org.elasticsearch.search.rank.feature.RankFeatureDoc;
+import org.elasticsearch.search.rank.rerank.RerankingQueryPhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.rerank.RerankingQueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.rerank.RerankingRankFeaturePhaseRankShardContext;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+import static org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder.FIELD_FIELD;
+import static org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder.SEED_FIELD;
+
+/**
+ * A {@code RankBuilder} that performs reranking with random scores, used for testing.
+ */
+public class RandomRankBuilder extends RankBuilder {
+
+    public static final String NAME = "random_reranker";
+
+    static final ConstructingObjectParser<RandomRankBuilder, Void> PARSER = new ConstructingObjectParser<>(NAME, args -> {
+        Integer rankWindowSize = args[0] == null ? DEFAULT_RANK_WINDOW_SIZE : (Integer) args[0];
+        String field = (String) args[1];
+        Integer seed = (Integer) args[2];
+
+        return new RandomRankBuilder(rankWindowSize, field, seed);
+    });
+
+    static {
+        PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD);
+        PARSER.declareString(constructorArg(), FIELD_FIELD);
+        PARSER.declareInt(optionalConstructorArg(), SEED_FIELD);
+    }
+
+    private final String field;
+    private final Integer seed;
+
+    public RandomRankBuilder(int rankWindowSize, String field, Integer seed) {
+        super(rankWindowSize);
+
+        if (field == null || field.isEmpty()) {
+            throw new IllegalArgumentException("field is required");
+        }
+
+        this.field = field;
+        this.seed = seed;
+    }
+
+    public RandomRankBuilder(StreamInput in) throws IOException {
+        super(in);
+        // rankWindowSize deserialization is handled by the parent class RankBuilder
+        this.field = in.readString();
+        this.seed = in.readOptionalInt();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.RANDOM_RERANKER_RETRIEVER;
+    }
+
+    @Override
+    public void doWriteTo(StreamOutput out) throws IOException {
+        // rankWindowSize serialization is handled by the parent class RankBuilder
+        out.writeString(field);
+        out.writeOptionalInt(seed);
+    }
+
+    @Override
+    public void doXContent(XContentBuilder builder, Params params) throws IOException {
+        // rankWindowSize serialization is handled by the parent class RankBuilder
+        builder.field(FIELD_FIELD.getPreferredName(), field);
+        if (seed != null) {
+            builder.field(SEED_FIELD.getPreferredName(), seed);
+        }
+    }
+
+    @Override
+    public boolean isCompoundBuilder() {
+        return false;
+    }
+
+    @Override
+    public Explanation explainHit(Explanation baseExplanation, RankDoc scoreDoc, List<String> queryNames) {
+        if (scoreDoc == null) {
+            return baseExplanation;
+        }
+        if (false == baseExplanation.isMatch()) {
+            return baseExplanation;
+        }
+
+        assert scoreDoc instanceof RankFeatureDoc : "ScoreDoc is not an instance of RankFeatureDoc";
+        RankFeatureDoc rankFeatureDoc = (RankFeatureDoc) scoreDoc;
+
+        return Explanation.match(
+            rankFeatureDoc.score,
+            "rank after reranking: [" + rankFeatureDoc.rank + "] using seed [" + seed + "] with score: [" + rankFeatureDoc.score + "]",
+            baseExplanation
+        );
+    }
+
+    @Override
+    public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
+        return new RerankingQueryPhaseRankShardContext(queries, rankWindowSize());
+    }
+
+    @Override
+    public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) {
+        return new RerankingQueryPhaseRankCoordinatorContext(rankWindowSize());
+    }
+
+    @Override
+    public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+        return new RerankingRankFeaturePhaseRankShardContext(field);
+    }
+
+    @Override
+    public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from, Client client) {
+        return new RandomRankFeaturePhaseRankCoordinatorContext(size, from, rankWindowSize(), seed);
+    }
+
+    public String field() {
+        return field;
+    }
+
+    @Override
+    protected boolean doEquals(RankBuilder other) {
+        RandomRankBuilder that = (RandomRankBuilder) other;
+        return Objects.equals(field, that.field) && Objects.equals(seed, that.seed);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(field, seed);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankFeaturePhaseRankCoordinatorContext.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankFeaturePhaseRankCoordinatorContext.java
new file mode 100644
index 0000000000000..446d8e5862dd2
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankFeaturePhaseRankCoordinatorContext.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.rank.random;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.feature.RankFeatureDoc;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Random;
+
+/**
+ * A {@code RankFeaturePhaseRankCoordinatorContext} that performs a rerank inference call to determine relevance scores for documents within
+ * the provided rank window.
+ */
+public class RandomRankFeaturePhaseRankCoordinatorContext extends RankFeaturePhaseRankCoordinatorContext {
+
+    private final Integer seed;
+
+    public RandomRankFeaturePhaseRankCoordinatorContext(int size, int from, int rankWindowSize, Integer seed) {
+        super(size, from, rankWindowSize);
+        this.seed = seed;
+    }
+
+    @Override
+    protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
+        // Generate random scores seeded by doc
+        float[] scores = new float[featureDocs.length];
+        for (int i = 0; i < featureDocs.length; i++) {
+            RankFeatureDoc featureDoc = featureDocs[i];
+            int doc = featureDoc.doc;
+            long docSeed = seed != null ? seed + doc : doc;
+            scores[i] = new Random(docSeed).nextFloat();
+        }
+        scoreListener.onResponse(scores);
+    }
+
+    /**
+     * Sorts documents by score descending.
+     * @param originalDocs documents to process
+     */
+    @Override
+    protected RankFeatureDoc[] preprocess(RankFeatureDoc[] originalDocs) {
+        return Arrays.stream(originalDocs)
+            .sorted(Comparator.comparing((RankFeatureDoc doc) -> doc.score).reversed())
+            .toArray(RankFeatureDoc[]::new);
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java
new file mode 100644
index 0000000000000..eb36c445506a7
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.rank.random;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.retriever.RetrieverBuilder;
+import org.elasticsearch.search.retriever.RetrieverParserContext;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * A {@code RetrieverBuilder} for randomly scoring a set of documents using the {@code RandomRankBuilder}
+ */
+public class RandomRankRetrieverBuilder extends RetrieverBuilder {
+
+    public static final NodeFeature RANDOM_RERANKER_RETRIEVER_SUPPORTED = new NodeFeature("random_reranker_retriever_supported");
+
+    public static final ParseField RETRIEVER_FIELD = new ParseField("retriever");
+    public static final ParseField FIELD_FIELD = new ParseField("field");
+    public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size");
+    public static final ParseField SEED_FIELD = new ParseField("seed");
+
+    public static final ConstructingObjectParser<RandomRankRetrieverBuilder, RetrieverParserContext> PARSER =
+        new ConstructingObjectParser<>(RandomRankBuilder.NAME, args -> {
+            RetrieverBuilder retrieverBuilder = (RetrieverBuilder) args[0];
+            String field = (String) args[1];
+            int rankWindowSize = args[2] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[2];
+            Integer seed = (Integer) args[3];
+
+            return new RandomRankRetrieverBuilder(retrieverBuilder, field, rankWindowSize, seed);
+        });
+
+    static {
+        PARSER.declareNamedObject(constructorArg(), (p, c, n) -> p.namedObject(RetrieverBuilder.class, n, c), RETRIEVER_FIELD);
+        PARSER.declareString(optionalConstructorArg(), FIELD_FIELD);
+        PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD);
+        PARSER.declareInt(optionalConstructorArg(), SEED_FIELD);
+
+        RetrieverBuilder.declareBaseParserFields(RandomRankBuilder.NAME, PARSER);
+    }
+
+    public static RandomRankRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException {
+        if (context.clusterSupportsFeature(RANDOM_RERANKER_RETRIEVER_SUPPORTED) == false) {
+            throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + RandomRankBuilder.NAME + "]");
+        }
+        return PARSER.apply(parser, context);
+    }
+
+    private final RetrieverBuilder retrieverBuilder;
+    private final String field;
+    private final int rankWindowSize;
+    private final Integer seed;
+
+    public RandomRankRetrieverBuilder(RetrieverBuilder retrieverBuilder, String field, int rankWindowSize, Integer seed) {
+        this.retrieverBuilder = retrieverBuilder;
+        this.field = field;
+        this.rankWindowSize = rankWindowSize;
+        this.seed = seed;
+    }
+
+    @Override
+    public QueryBuilder topDocsQuery() {
+        return retrieverBuilder.topDocsQuery();
+    }
+
+    @Override
+    public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
+        retrieverBuilder.extractToSearchSourceBuilder(searchSourceBuilder, compoundUsed);
+
+        // Combining with other rank builder (such as RRF) is not supported
+        if (searchSourceBuilder.rankBuilder() != null) {
+            throw new IllegalArgumentException("random rank builder cannot be combined with other rank builders");
+        }
+
+        searchSourceBuilder.rankBuilder(new RandomRankBuilder(this.rankWindowSize, this.field, this.seed));
+    }
+
+    @Override
+    public String getName() {
+        return RandomRankBuilder.NAME;
+    }
+
+    public int rankWindowSize() {
+        return rankWindowSize;
+    }
+
+    @Override
+    protected void doToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(RETRIEVER_FIELD.getPreferredName());
+        builder.startObject();
+        builder.field(retrieverBuilder.getName(), retrieverBuilder);
+        builder.endObject();
+        builder.field(FIELD_FIELD.getPreferredName(), field);
+        builder.field(RANK_WINDOW_SIZE_FIELD.getPreferredName(), rankWindowSize);
+        if (seed != null) {
+            builder.field(SEED_FIELD.getPreferredName(), seed);
+        }
+    }
+
+    @Override
+    protected boolean doEquals(Object other) {
+        RandomRankRetrieverBuilder that = (RandomRankRetrieverBuilder) other;
+        return Objects.equals(retrieverBuilder, that.retrieverBuilder)
+            && Objects.equals(field, that.field)
+            && Objects.equals(rankWindowSize, that.rankWindowSize)
+            && Objects.equals(seed, that.seed);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(retrieverBuilder, field, rankWindowSize, seed);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankFeaturePhaseRankCoordinatorContext.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankFeaturePhaseRankCoordinatorContext.java
index a22126439e9e2..cad11cbdc9d5b 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankFeaturePhaseRankCoordinatorContext.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankFeaturePhaseRankCoordinatorContext.java
@@ -14,8 +14,11 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
 import org.elasticsearch.search.rank.feature.RankFeatureDoc;
+import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
+import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankTaskSettings;
+import org.elasticsearch.xpack.inference.services.googlevertexai.rerank.GoogleVertexAiRerankTaskSettings;
 
 import java.util.Arrays;
 import java.util.Comparator;
@@ -53,24 +56,84 @@ public TextSimilarityRankFeaturePhaseRankCoordinatorContext(
     protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
         // Wrap the provided rankListener to an ActionListener that would handle the response from the inference service
         // and then pass the results
-        final ActionListener<InferenceAction.Response> actionListener = scoreListener.delegateFailureAndWrap((l, r) -> {
-            float[] scores = extractScoresFromResponse(r);
-            if (scores.length != featureDocs.length) {
+        final ActionListener<InferenceAction.Response> inferenceListener = scoreListener.delegateFailureAndWrap((l, r) -> {
+            InferenceServiceResults results = r.getResults();
+            assert results instanceof RankedDocsResults;
+
+            // Ensure we get exactly as many scores as the number of docs we passed, otherwise we may return incorrect results
+            List<RankedDocsResults.RankedDoc> rankedDocs = ((RankedDocsResults) results).getRankedDocs();
+
+            if (rankedDocs.size() != featureDocs.length) {
                 l.onFailure(
-                    new IllegalStateException("Document and score count mismatch: [" + featureDocs.length + "] vs [" + scores.length + "]")
+                    new IllegalStateException(
+                        "Reranker input document count and returned score count mismatch: ["
+                            + featureDocs.length
+                            + "] vs ["
+                            + rankedDocs.size()
+                            + "]"
+                    )
                 );
             } else {
+                float[] scores = extractScoresFromRankedDocs(rankedDocs);
                 l.onResponse(scores);
             }
         });
 
-        List<String> featureData = Arrays.stream(featureDocs).map(x -> x.featureData).toList();
-        InferenceAction.Request request = generateRequest(featureData);
-        try {
-            client.execute(InferenceAction.INSTANCE, request, actionListener);
-        } finally {
-            request.decRef();
-        }
+        // top N listener
+        ActionListener<GetInferenceModelAction.Response> topNListener = scoreListener.delegateFailureAndWrap((l, r) -> {
+            // The rerank inference endpoint may have an override to return top N documents only, in that case let's fail fast to avoid
+            // assigning scores to the wrong input
+            Integer configuredTopN = null;
+            if (r.getEndpoints().isEmpty() == false
+                && r.getEndpoints().get(0).getTaskSettings() instanceof CohereRerankTaskSettings cohereTaskSettings) {
+                configuredTopN = cohereTaskSettings.getTopNDocumentsOnly();
+            } else if (r.getEndpoints().isEmpty() == false
+                && r.getEndpoints().get(0).getTaskSettings() instanceof GoogleVertexAiRerankTaskSettings googleVertexAiTaskSettings) {
+                    configuredTopN = googleVertexAiTaskSettings.topN();
+                }
+            if (configuredTopN != null && configuredTopN < rankWindowSize) {
+                l.onFailure(
+                    new IllegalArgumentException(
+                        "Inference endpoint ["
+                            + inferenceId
+                            + "] is configured to return the top ["
+                            + configuredTopN
+                            + "] results, but rank_window_size is ["
+                            + rankWindowSize
+                            + "]. Reduce rank_window_size to be less than or equal to the configured top N value."
+                    )
+                );
+                return;
+            }
+
+            // Short circuit on empty results after request validation
+            if (featureDocs.length == 0) {
+                inferenceListener.onResponse(new InferenceAction.Response(new RankedDocsResults(List.of())));
+            } else {
+                List<String> featureData = Arrays.stream(featureDocs).map(x -> x.featureData).toList();
+                InferenceAction.Request inferenceRequest = generateRequest(featureData);
+                try {
+                    client.execute(InferenceAction.INSTANCE, inferenceRequest, inferenceListener);
+                } finally {
+                    inferenceRequest.decRef();
+                }
+            }
+        });
+
+        GetInferenceModelAction.Request getModelRequest = new GetInferenceModelAction.Request(inferenceId, TaskType.RERANK);
+        client.execute(GetInferenceModelAction.INSTANCE, getModelRequest, topNListener);
+    }
+
+    /**
+     * Sorts documents by score descending and discards those with a score less than minScore.
+     * @param originalDocs documents to process
+     */
+    @Override
+    protected RankFeatureDoc[] preprocess(RankFeatureDoc[] originalDocs) {
+        return Arrays.stream(originalDocs)
+            .filter(doc -> minScore == null || doc.score >= minScore)
+            .sorted(Comparator.comparing((RankFeatureDoc doc) -> doc.score).reversed())
+            .toArray(RankFeatureDoc[]::new);
     }
 
     protected InferenceAction.Request generateRequest(List<String> docFeatures) {
@@ -85,11 +148,7 @@ protected InferenceAction.Request generateRequest(List<String> docFeatures) {
         );
     }
 
-    private float[] extractScoresFromResponse(InferenceAction.Response response) {
-        InferenceServiceResults results = response.getResults();
-        assert results instanceof RankedDocsResults;
-
-        List<RankedDocsResults.RankedDoc> rankedDocs = ((RankedDocsResults) results).getRankedDocs();
+    private float[] extractScoresFromRankedDocs(List<RankedDocsResults.RankedDoc> rankedDocs) {
         float[] scores = new float[rankedDocs.size()];
         for (RankedDocsResults.RankedDoc rankedDoc : rankedDocs) {
             scores[rankedDoc.index()] = rankedDoc.relevanceScore();
@@ -97,16 +156,4 @@ private float[] extractScoresFromResponse(InferenceAction.Response response) {
 
         return scores;
     }
-
-    /**
-     * Sorts documents by score descending and discards those with a score less than minScore.
-     * @param originalDocs documents to process
-     */
-    @Override
-    protected RankFeatureDoc[] preprocess(RankFeatureDoc[] originalDocs) {
-        return Arrays.stream(originalDocs)
-            .filter(doc -> minScore == null || doc.score >= minScore)
-            .sorted(Comparator.comparing((RankFeatureDoc doc) -> doc.score).reversed())
-            .toArray(RankFeatureDoc[]::new);
-    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java
index a81fbb51f678d..927c708268a49 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java
@@ -9,6 +9,8 @@
 
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.retriever.RetrieverBuilder;
@@ -20,6 +22,7 @@
 import org.elasticsearch.xpack.core.XPackPlugin;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Objects;
 
 import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE;
@@ -99,10 +102,61 @@ public TextSimilarityRankRetrieverBuilder(
         this.minScore = minScore;
     }
 
+    public TextSimilarityRankRetrieverBuilder(
+        RetrieverBuilder retrieverBuilder,
+        String inferenceId,
+        String inferenceText,
+        String field,
+        int rankWindowSize,
+        Float minScore,
+        String retrieverName,
+        List<QueryBuilder> preFilterQueryBuilders
+    ) {
+        this.retrieverBuilder = retrieverBuilder;
+        this.inferenceId = inferenceId;
+        this.inferenceText = inferenceText;
+        this.field = field;
+        this.rankWindowSize = rankWindowSize;
+        this.minScore = minScore;
+        this.retrieverName = retrieverName;
+        this.preFilterQueryBuilders = preFilterQueryBuilders;
+    }
+
+    @Override
+    public QueryBuilder topDocsQuery() {
+        // the original matching set of the TextSimilarityRank retriever is specified by its nested retriever
+        return retrieverBuilder.topDocsQuery();
+    }
+
+    @Override
+    public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException {
+        // rewrite prefilters
+        boolean hasChanged = false;
+        var newPreFilters = rewritePreFilters(ctx);
+        hasChanged |= newPreFilters != preFilterQueryBuilders;
+
+        // rewrite nested retriever
+        RetrieverBuilder newRetriever = retrieverBuilder.rewrite(ctx);
+        hasChanged |= newRetriever != retrieverBuilder;
+        if (hasChanged) {
+            return new TextSimilarityRankRetrieverBuilder(
+                newRetriever,
+                field,
+                inferenceText,
+                inferenceId,
+                rankWindowSize,
+                minScore,
+                this.retrieverName,
+                newPreFilters
+            );
+        }
+        return this;
+    }
+
     @Override
     public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
+        retrieverBuilder.getPreFilterQueryBuilders().addAll(preFilterQueryBuilders);
         retrieverBuilder.extractToSearchSourceBuilder(searchSourceBuilder, compoundUsed);
-
         // Combining with other rank builder (such as RRF) is not supported yet
         if (searchSourceBuilder.rankBuilder() != null) {
             throw new IllegalArgumentException("text similarity rank builder cannot be combined with other rank builders");
@@ -113,6 +167,13 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder
         );
     }
 
+    /**
+     * Determines if this retriever contains sub-retrievers that need to be executed prior to search.
+     */
+    public boolean isCompound() {
+        return retrieverBuilder.isCompound();
+    }
+
     @Override
     public String getName() {
         return TextSimilarityRankBuilder.NAME;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java
index ae82264a77a0d..a6e4fcae7169f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java
@@ -74,7 +74,10 @@ public static UnparsedModel unparsedModelFromMap(ModelConfigMap modelConfigMap)
             if (modelConfigMap.config() == null) {
                 throw new ElasticsearchStatusException("Missing config map", RestStatus.BAD_REQUEST);
             }
-            String inferenceEntityId = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), ModelConfigurations.MODEL_ID);
+            String inferenceEntityId = ServiceUtils.removeStringOrThrowIfNull(
+                modelConfigMap.config(),
+                ModelConfigurations.INDEX_ONLY_ID_FIELD_NAME
+            );
             String service = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), ModelConfigurations.SERVICE);
             String taskTypeStr = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), TaskType.NAME);
             TaskType taskType = TaskType.fromString(taskTypeStr);
@@ -375,7 +378,10 @@ public void deleteModel(String inferenceEntityId, ActionListener<Boolean> listen
     private static IndexRequest createIndexRequest(String docId, String indexName, ToXContentObject body, boolean allowOverwriting) {
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
             var request = new IndexRequest(indexName);
-            XContentBuilder source = body.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            XContentBuilder source = body.toXContent(
+                builder,
+                new ToXContent.MapParams(Map.of(ModelConfigurations.USE_ID_FOR_INDEX, Boolean.TRUE.toString()))
+            );
             var operation = allowOverwriting ? DocWriteRequest.OpType.INDEX : DocWriteRequest.OpType.CREATE;
 
             return request.opType(operation).id(docId).source(source);
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java
index f4bbcbebf0340..f5c30d0a94c54 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
-import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 
 import java.io.IOException;
@@ -59,6 +59,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
             InferenceAction.Request.DEFAULT_TIMEOUT
         );
         requestBuilder.setInferenceTimeout(inferTimeout);
-        return channel -> client.execute(InferenceAction.INSTANCE, requestBuilder.build(), new RestToXContentListener<>(channel));
+        return channel -> client.execute(InferenceAction.INSTANCE, requestBuilder.build(), new RestChunkedToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchModel.java
new file mode 100644
index 0000000000000..e26953754b1d6
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchModel.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch;
+
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionVisitor;
+
+import java.util.Map;
+import java.util.Objects;
+
+public abstract class AlibabaCloudSearchModel extends Model {
+    private final AlibabaCloudSearchRateLimitServiceSettings rateLimitServiceSettings;
+
+    public AlibabaCloudSearchModel(
+        ModelConfigurations configurations,
+        ModelSecrets secrets,
+        AlibabaCloudSearchRateLimitServiceSettings rateLimitServiceSettings
+    ) {
+        super(configurations, secrets);
+        this.rateLimitServiceSettings = Objects.requireNonNull(rateLimitServiceSettings);
+    }
+
+    protected AlibabaCloudSearchModel(AlibabaCloudSearchModel model, TaskSettings taskSettings) {
+        super(model, taskSettings);
+        rateLimitServiceSettings = model.rateLimitServiceSettings();
+    }
+
+    protected AlibabaCloudSearchModel(AlibabaCloudSearchModel model, ServiceSettings serviceSettings) {
+        super(model, serviceSettings);
+        rateLimitServiceSettings = model.rateLimitServiceSettings();
+    }
+
+    public abstract ExecutableAction accept(AlibabaCloudSearchActionVisitor creator, Map<String, Object> taskSettings, InputType inputType);
+
+    public AlibabaCloudSearchRateLimitServiceSettings rateLimitServiceSettings() {
+        return rateLimitServiceSettings;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchRateLimitServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchRateLimitServiceSettings.java
new file mode 100644
index 0000000000000..db09f1d66e77b
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchRateLimitServiceSettings.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch;
+
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+public interface AlibabaCloudSearchRateLimitServiceSettings {
+    RateLimitSettings rateLimitSettings();
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java
new file mode 100644
index 0000000000000..0c48c99b4b81e
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java
@@ -0,0 +1,318 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.ChunkedInferenceServiceResults;
+import org.elasticsearch.inference.ChunkingOptions;
+import org.elasticsearch.inference.InferenceService;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionCreator;
+import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.external.http.sender.QueryAndDocsInputs;
+import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchUtils;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.SenderService;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.ServiceUtils;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsServiceSettings;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank.AlibabaCloudSearchRerankModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseModel;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.core.inference.action.InferenceAction.Request.DEFAULT_TIMEOUT;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap;
+
+public class AlibabaCloudSearchService extends SenderService {
+    public static final String NAME = AlibabaCloudSearchUtils.SERVICE_NAME;
+
+    public AlibabaCloudSearchService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
+        super(factory, serviceComponents);
+    }
+
+    @Override
+    public String name() {
+        return NAME;
+    }
+
+    @Override
+    public void parseRequestConfig(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> config,
+        Set<String> platformArchitectures,
+        ActionListener<Model> parsedModelListener
+    ) {
+        try {
+            Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+            Map<String, Object> taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS);
+
+            AlibabaCloudSearchModel model = createModel(
+                inferenceEntityId,
+                taskType,
+                serviceSettingsMap,
+                taskSettingsMap,
+                serviceSettingsMap,
+                TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME),
+                ConfigurationParseContext.REQUEST
+            );
+
+            throwIfNotEmptyMap(config, NAME);
+            throwIfNotEmptyMap(serviceSettingsMap, NAME);
+            throwIfNotEmptyMap(taskSettingsMap, NAME);
+
+            parsedModelListener.onResponse(model);
+        } catch (Exception e) {
+            parsedModelListener.onFailure(e);
+        }
+    }
+
+    private static AlibabaCloudSearchModel createModelWithoutLoggingDeprecations(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secretSettings,
+        String failureMessage
+    ) {
+        return createModel(
+            inferenceEntityId,
+            taskType,
+            serviceSettings,
+            taskSettings,
+            secretSettings,
+            failureMessage,
+            ConfigurationParseContext.PERSISTENT
+        );
+    }
+
+    private static AlibabaCloudSearchModel createModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secretSettings,
+        String failureMessage,
+        ConfigurationParseContext context
+    ) {
+        return switch (taskType) {
+            case TEXT_EMBEDDING -> new AlibabaCloudSearchEmbeddingsModel(
+                inferenceEntityId,
+                taskType,
+                NAME,
+                serviceSettings,
+                taskSettings,
+                secretSettings,
+                context
+            );
+            case SPARSE_EMBEDDING -> new AlibabaCloudSearchSparseModel(
+                inferenceEntityId,
+                taskType,
+                NAME,
+                serviceSettings,
+                taskSettings,
+                secretSettings,
+                context
+            );
+            case RERANK -> new AlibabaCloudSearchRerankModel(
+                inferenceEntityId,
+                taskType,
+                NAME,
+                serviceSettings,
+                taskSettings,
+                secretSettings,
+                context
+            );
+            default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
+        };
+    }
+
+    @Override
+    public AlibabaCloudSearchModel parsePersistedConfigWithSecrets(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> config,
+        Map<String, Object> secrets
+    ) {
+        Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+        Map<String, Object> taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS);
+        Map<String, Object> secretSettingsMap = removeFromMapOrThrowIfNull(secrets, ModelSecrets.SECRET_SETTINGS);
+
+        return createModelWithoutLoggingDeprecations(
+            inferenceEntityId,
+            taskType,
+            serviceSettingsMap,
+            taskSettingsMap,
+            secretSettingsMap,
+            parsePersistedConfigErrorMsg(inferenceEntityId, NAME)
+        );
+    }
+
+    @Override
+    public AlibabaCloudSearchModel parsePersistedConfig(String inferenceEntityId, TaskType taskType, Map<String, Object> config) {
+        Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+        Map<String, Object> taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS);
+
+        return createModelWithoutLoggingDeprecations(
+            inferenceEntityId,
+            taskType,
+            serviceSettingsMap,
+            taskSettingsMap,
+            null,
+            parsePersistedConfigErrorMsg(inferenceEntityId, NAME)
+        );
+    }
+
+    @Override
+    public void doInfer(
+        Model model,
+        String query,
+        List<String> input,
+        Map<String, Object> taskSettings,
+        InputType inputType,
+        TimeValue timeout,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        if (model instanceof AlibabaCloudSearchModel == false) {
+            listener.onFailure(createInvalidModelException(model));
+            return;
+        }
+
+        AlibabaCloudSearchModel alibabaCloudSearchModel = (AlibabaCloudSearchModel) model;
+        var actionCreator = new AlibabaCloudSearchActionCreator(getSender(), getServiceComponents());
+
+        var action = alibabaCloudSearchModel.accept(actionCreator, taskSettings, inputType);
+        action.execute(new QueryAndDocsInputs(query, input), timeout, listener);
+    }
+
+    @Override
+    public void doInfer(
+        Model model,
+        List<String> input,
+        Map<String, Object> taskSettings,
+        InputType inputType,
+        TimeValue timeout,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        if (model instanceof AlibabaCloudSearchModel == false) {
+            listener.onFailure(createInvalidModelException(model));
+            return;
+        }
+
+        AlibabaCloudSearchModel alibabaCloudSearchModel = (AlibabaCloudSearchModel) model;
+        var actionCreator = new AlibabaCloudSearchActionCreator(getSender(), getServiceComponents());
+
+        var action = alibabaCloudSearchModel.accept(actionCreator, taskSettings, inputType);
+        action.execute(new DocumentsOnlyInput(input), timeout, listener);
+    }
+
+    @Override
+    protected void doChunkedInfer(
+        Model model,
+        @Nullable String query,
+        List<String> input,
+        Map<String, Object> taskSettings,
+        InputType inputType,
+        ChunkingOptions chunkingOptions,
+        TimeValue timeout,
+        ActionListener<List<ChunkedInferenceServiceResults>> listener
+    ) {
+        listener.onFailure(new ElasticsearchStatusException("Chunking not supported by the {} service", RestStatus.BAD_REQUEST, NAME));
+    }
+
+    /**
+     * For text embedding models get the embedding size and
+     * update the service settings.
+     *
+     * @param model The new model
+     * @param listener The listener
+     */
+    @Override
+    public void checkModelConfig(Model model, ActionListener<Model> listener) {
+        if (model instanceof AlibabaCloudSearchEmbeddingsModel embeddingsModel) {
+            ServiceUtils.getEmbeddingSize(
+                model,
+                this,
+                listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size)))
+            );
+        } else {
+            checkAlibabaCloudSearchServiceConfig(model, this, listener);
+        }
+    }
+
+    private AlibabaCloudSearchEmbeddingsModel updateModelWithEmbeddingDetails(AlibabaCloudSearchEmbeddingsModel model, int embeddingSize) {
+        AlibabaCloudSearchEmbeddingsServiceSettings serviceSettings = new AlibabaCloudSearchEmbeddingsServiceSettings(
+            new AlibabaCloudSearchServiceSettings(
+                model.getServiceSettings().getCommonSettings().modelId(),
+                model.getServiceSettings().getCommonSettings().getHost(),
+                model.getServiceSettings().getCommonSettings().getWorkspaceName(),
+                model.getServiceSettings().getCommonSettings().getHttpSchema(),
+                model.getServiceSettings().getCommonSettings().rateLimitSettings()
+            ),
+            SimilarityMeasure.DOT_PRODUCT,
+            embeddingSize,
+            model.getServiceSettings().getMaxInputTokens()
+        );
+
+        return new AlibabaCloudSearchEmbeddingsModel(model, serviceSettings);
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED;
+    }
+
+    /**
+     * For other models except of text embedding
+     * check the model's service settings and task settings
+     *
+     * @param model The new model
+     * @param service The inferenceService
+     * @param listener The listener
+     */
+    private void checkAlibabaCloudSearchServiceConfig(Model model, InferenceService service, ActionListener<Model> listener) {
+        String input = ALIBABA_CLOUD_SEARCH_SERVICE_CONFIG_INPUT;
+        String query = model.getTaskType().equals(TaskType.RERANK) ? ALIBABA_CLOUD_SEARCH_SERVICE_CONFIG_QUERY : null;
+
+        service.infer(
+            model,
+            query,
+            List.of(input),
+            Map.of(),
+            InputType.INGEST,
+            DEFAULT_TIMEOUT,
+            listener.delegateFailureAndWrap((delegate, r) -> {
+                listener.onResponse(model);
+            })
+        );
+    }
+
+    private static final String ALIBABA_CLOUD_SEARCH_SERVICE_CONFIG_INPUT = "input";
+    private static final String ALIBABA_CLOUD_SEARCH_SERVICE_CONFIG_QUERY = "query";
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceSettings.java
new file mode 100644
index 0000000000000..3500bdf814e16
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceSettings.java
@@ -0,0 +1,193 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalString;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString;
+
+public class AlibabaCloudSearchServiceSettings extends FilteredXContentObject
+    implements
+        ServiceSettings,
+        AlibabaCloudSearchRateLimitServiceSettings {
+
+    public static final String NAME = "alibabacloud_search_service_settings";
+    public static final String SERVICE_ID = "service_id";
+    public static final String HOST = "host";
+    public static final String WORKSPACE_NAME = "workspace";
+    public static final String HTTP_SCHEMA_NAME = "http_schema";
+
+    private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(1_000);
+
+    public static AlibabaCloudSearchServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
+        ValidationException validationException = new ValidationException();
+
+        String modelId = extractRequiredString(map, SERVICE_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        String host = extractRequiredString(map, HOST, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        var workspaceName = extractRequiredString(map, WORKSPACE_NAME, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        var httpSchema = extractOptionalString(map, HTTP_SCHEMA_NAME, ModelConfigurations.SERVICE_SETTINGS, validationException);
+
+        if (httpSchema != null) {
+            var validSchemas = Set.of("https", "http");
+            if (validSchemas.contains(httpSchema) == false) {
+                validationException.addValidationError("Invalid value for [http_schema]. Must be one of [https, http]");
+            }
+        }
+
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            AlibabaCloudSearchService.NAME,
+            context
+        );
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new AlibabaCloudSearchServiceSettings(modelId, host, workspaceName, httpSchema, rateLimitSettings);
+    }
+
+    private final String serviceId;
+    private final String host;
+    private final String workspaceName;
+    private final String httpSchema;
+    private final RateLimitSettings rateLimitSettings;
+
+    public AlibabaCloudSearchServiceSettings(
+        String serviceId,
+        String host,
+        String workspaceName,
+        @Nullable String httpSchema,
+        @Nullable RateLimitSettings rateLimitSettings
+    ) {
+        this.serviceId = serviceId;
+        this.host = host;
+        this.workspaceName = workspaceName;
+        this.httpSchema = httpSchema;
+        this.rateLimitSettings = Objects.requireNonNullElse(rateLimitSettings, DEFAULT_RATE_LIMIT_SETTINGS);
+    }
+
+    public AlibabaCloudSearchServiceSettings(StreamInput in) throws IOException {
+        serviceId = in.readString();
+        host = in.readString();
+        workspaceName = in.readString();
+        httpSchema = in.readOptionalString();
+        rateLimitSettings = new RateLimitSettings(in);
+    }
+
+    @Override
+    public String modelId() {
+        return serviceId;
+    }
+
+    public String getHost() {
+        return host;
+    }
+
+    public String getWorkspaceName() {
+        return workspaceName;
+    }
+
+    public String getHttpSchema() {
+        return httpSchema;
+    }
+
+    @Override
+    public RateLimitSettings rateLimitSettings() {
+        return rateLimitSettings;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        toXContentFragment(builder, params);
+
+        builder.endObject();
+        return builder;
+    }
+
+    public XContentBuilder toXContentFragment(XContentBuilder builder, Params params) throws IOException {
+        return toXContentFragmentOfExposedFields(builder, params);
+    }
+
+    @Override
+    public XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException {
+        if (serviceId != null) {
+            builder.field(SERVICE_ID, serviceId);
+        }
+        builder.field(HOST, host);
+        builder.field(WORKSPACE_NAME, workspaceName);
+        if (httpSchema != null) {
+            builder.field(HTTP_SCHEMA_NAME, httpSchema);
+        }
+        rateLimitSettings.toXContent(builder, params);
+
+        return builder;
+    }
+
+    @Override
+    public ToXContentObject getFilteredXContentObject() {
+        return this;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(serviceId);
+        out.writeString(host);
+        out.writeString(workspaceName);
+        out.writeOptionalString(httpSchema);
+        rateLimitSettings.writeTo(out);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AlibabaCloudSearchServiceSettings that = (AlibabaCloudSearchServiceSettings) o;
+        return Objects.equals(serviceId, that.serviceId)
+            && Objects.equals(host, that.host)
+            && Objects.equals(workspaceName, that.workspaceName)
+            && Objects.equals(httpSchema, that.httpSchema);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(serviceId, host, workspaceName, httpSchema);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModel.java
new file mode 100644
index 0000000000000..87e5e59ae3434
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModel.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionVisitor;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchModel;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+
+import java.util.Map;
+
+public class AlibabaCloudSearchEmbeddingsModel extends AlibabaCloudSearchModel {
+    public static AlibabaCloudSearchEmbeddingsModel of(
+        AlibabaCloudSearchEmbeddingsModel model,
+        Map<String, Object> taskSettings,
+        InputType inputType
+    ) {
+        var requestTaskSettings = AlibabaCloudSearchEmbeddingsTaskSettings.fromMap(taskSettings);
+        return new AlibabaCloudSearchEmbeddingsModel(
+            model,
+            AlibabaCloudSearchEmbeddingsTaskSettings.of(model.getTaskSettings(), requestTaskSettings, inputType)
+        );
+    }
+
+    public AlibabaCloudSearchEmbeddingsModel(
+        String modelId,
+        TaskType taskType,
+        String service,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secrets,
+        ConfigurationParseContext context
+    ) {
+        this(
+            modelId,
+            taskType,
+            service,
+            AlibabaCloudSearchEmbeddingsServiceSettings.fromMap(serviceSettings, context),
+            AlibabaCloudSearchEmbeddingsTaskSettings.fromMap(taskSettings),
+            DefaultSecretSettings.fromMap(secrets)
+        );
+    }
+
+    // should only be used for testing
+    AlibabaCloudSearchEmbeddingsModel(
+        String modelId,
+        TaskType taskType,
+        String service,
+        AlibabaCloudSearchEmbeddingsServiceSettings serviceSettings,
+        AlibabaCloudSearchEmbeddingsTaskSettings taskSettings,
+        @Nullable DefaultSecretSettings secretSettings
+    ) {
+        super(
+            new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings),
+            new ModelSecrets(secretSettings),
+            serviceSettings.getCommonSettings()
+        );
+    }
+
+    private AlibabaCloudSearchEmbeddingsModel(
+        AlibabaCloudSearchEmbeddingsModel model,
+        AlibabaCloudSearchEmbeddingsTaskSettings taskSettings
+    ) {
+        super(model, taskSettings);
+    }
+
+    public AlibabaCloudSearchEmbeddingsModel(
+        AlibabaCloudSearchEmbeddingsModel model,
+        AlibabaCloudSearchEmbeddingsServiceSettings serviceSettings
+    ) {
+        super(model, serviceSettings);
+    }
+
+    @Override
+    public AlibabaCloudSearchEmbeddingsServiceSettings getServiceSettings() {
+        return (AlibabaCloudSearchEmbeddingsServiceSettings) super.getServiceSettings();
+    }
+
+    @Override
+    public AlibabaCloudSearchEmbeddingsTaskSettings getTaskSettings() {
+        return (AlibabaCloudSearchEmbeddingsTaskSettings) super.getTaskSettings();
+    }
+
+    @Override
+    public DefaultSecretSettings getSecretSettings() {
+        return (DefaultSecretSettings) super.getSecretSettings();
+    }
+
+    @Override
+    public ExecutableAction accept(AlibabaCloudSearchActionVisitor visitor, Map<String, Object> taskSettings, InputType inputType) {
+        return visitor.create(this, taskSettings, inputType);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsServiceSettings.java
new file mode 100644
index 0000000000000..76dfd01f333da
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsServiceSettings.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettings;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS;
+import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS;
+import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeAsType;
+
+public class AlibabaCloudSearchEmbeddingsServiceSettings implements ServiceSettings {
+    public static final String NAME = "alibabacloud_search_embeddings_service_settings";
+
+    public static AlibabaCloudSearchEmbeddingsServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
+        ValidationException validationException = new ValidationException();
+        var commonServiceSettings = AlibabaCloudSearchServiceSettings.fromMap(map, context);
+
+        SimilarityMeasure similarity = extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        Integer dims = removeAsType(map, DIMENSIONS, Integer.class);
+        Integer maxInputTokens = removeAsType(map, MAX_INPUT_TOKENS, Integer.class);
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new AlibabaCloudSearchEmbeddingsServiceSettings(commonServiceSettings, similarity, dims, maxInputTokens);
+    }
+
+    private final AlibabaCloudSearchServiceSettings commonSettings;
+    private final SimilarityMeasure similarity;
+    private final Integer dimensions;
+    private final Integer maxInputTokens;
+
+    public AlibabaCloudSearchEmbeddingsServiceSettings(
+        AlibabaCloudSearchServiceSettings commonSettings,
+        @Nullable SimilarityMeasure similarity,
+        @Nullable Integer dimensions,
+        @Nullable Integer maxInputTokens
+    ) {
+        this.commonSettings = commonSettings;
+        this.similarity = similarity;
+        this.dimensions = dimensions;
+        this.maxInputTokens = maxInputTokens;
+    }
+
+    public AlibabaCloudSearchEmbeddingsServiceSettings(StreamInput in) throws IOException {
+        commonSettings = new AlibabaCloudSearchServiceSettings(in);
+        similarity = in.readOptionalEnum(SimilarityMeasure.class);
+        dimensions = in.readOptionalVInt();
+        maxInputTokens = in.readOptionalVInt();
+    }
+
+    public AlibabaCloudSearchServiceSettings getCommonSettings() {
+        return commonSettings;
+    }
+
+    public SimilarityMeasure getSimilarity() {
+        return similarity;
+    }
+
+    public Integer getDimensions() {
+        return dimensions;
+    }
+
+    public Integer getMaxInputTokens() {
+        return maxInputTokens;
+    }
+
+    @Override
+    public String modelId() {
+        return commonSettings.modelId();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        commonSettings.toXContentFragment(builder, params);
+        if (similarity != null) {
+            builder.field(SIMILARITY, similarity);
+        }
+        if (dimensions != null) {
+            builder.field(DIMENSIONS, dimensions);
+        }
+        if (maxInputTokens != null) {
+            builder.field(MAX_INPUT_TOKENS, maxInputTokens);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public ToXContentObject getFilteredXContentObject() {
+        return this;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        commonSettings.writeTo(out);
+        out.writeOptionalEnum(similarity);
+        out.writeOptionalVInt(dimensions);
+        out.writeOptionalVInt(maxInputTokens);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AlibabaCloudSearchEmbeddingsServiceSettings that = (AlibabaCloudSearchEmbeddingsServiceSettings) o;
+        return Objects.equals(commonSettings, that.commonSettings)
+            && Objects.equals(similarity, that.similarity)
+            && Objects.equals(dimensions, that.dimensions)
+            && Objects.equals(maxInputTokens, that.maxInputTokens);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(commonSettings, similarity, dimensions, maxInputTokens);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java
new file mode 100644
index 0000000000000..abfd49940b67b
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettings.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalEnum;
+
+/**
+ * Defines the task settings for the alibaba cloud search text embeddings service.
+ *
+ * <p>
+ * <a href="https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details">
+ * See api docs for details.</a>
+ * </p>
+ */
+public class AlibabaCloudSearchEmbeddingsTaskSettings implements TaskSettings {
+
+    public static final String NAME = "alibabacloud_search_embeddings_task_settings";
+    public static final AlibabaCloudSearchEmbeddingsTaskSettings EMPTY_SETTINGS = new AlibabaCloudSearchEmbeddingsTaskSettings(
+        (InputType) null
+    );
+    static final String INPUT_TYPE = "input_type";
+    static final EnumSet<InputType> VALID_REQUEST_VALUES = EnumSet.of(InputType.INGEST, InputType.SEARCH);
+
+    public static AlibabaCloudSearchEmbeddingsTaskSettings fromMap(Map<String, Object> map) {
+        if (map == null || map.isEmpty()) {
+            return EMPTY_SETTINGS;
+        }
+
+        ValidationException validationException = new ValidationException();
+
+        InputType inputType = extractOptionalEnum(
+            map,
+            INPUT_TYPE,
+            ModelConfigurations.TASK_SETTINGS,
+            InputType::fromString,
+            VALID_REQUEST_VALUES,
+            validationException
+        );
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new AlibabaCloudSearchEmbeddingsTaskSettings(inputType);
+    }
+
+    /**
+     * Creates a new {@link AlibabaCloudSearchEmbeddingsTaskSettings} by preferring non-null fields from the provided parameters.
+     * For the input type, preference is given to requestInputType if it is not null and not UNSPECIFIED.
+     * Then preference is given to the requestTaskSettings and finally to originalSettings even if the value is null.
+     * <p>
+     * Similarly, for the truncation field preference is given to requestTaskSettings if it is not null and then to
+     * originalSettings.
+     *
+     * @param originalSettings    the settings stored as part of the inference entity configuration
+     * @param requestTaskSettings the settings passed in within the task_settings field of the request
+     * @param requestInputType    the input type passed in the request parameters
+     * @return a constructed {@link AlibabaCloudSearchEmbeddingsTaskSettings}
+     */
+    public static AlibabaCloudSearchEmbeddingsTaskSettings of(
+        AlibabaCloudSearchEmbeddingsTaskSettings originalSettings,
+        AlibabaCloudSearchEmbeddingsTaskSettings requestTaskSettings,
+        InputType requestInputType
+    ) {
+        var inputTypeToUse = getValidInputType(originalSettings, requestTaskSettings, requestInputType);
+
+        return new AlibabaCloudSearchEmbeddingsTaskSettings(inputTypeToUse);
+    }
+
+    private static InputType getValidInputType(
+        AlibabaCloudSearchEmbeddingsTaskSettings originalSettings,
+        AlibabaCloudSearchEmbeddingsTaskSettings requestTaskSettings,
+        InputType requestInputType
+    ) {
+        InputType inputTypeToUse = originalSettings.inputType;
+
+        if (VALID_REQUEST_VALUES.contains(requestInputType)) {
+            inputTypeToUse = requestInputType;
+        } else if (requestTaskSettings.inputType != null) {
+            inputTypeToUse = requestTaskSettings.inputType;
+        }
+
+        return inputTypeToUse;
+    }
+
+    private final InputType inputType;
+
+    public AlibabaCloudSearchEmbeddingsTaskSettings(StreamInput in) throws IOException {
+        this(in.readOptionalEnum(InputType.class));
+    }
+
+    public AlibabaCloudSearchEmbeddingsTaskSettings(@Nullable InputType inputType) {
+        validateInputType(inputType);
+        this.inputType = inputType;
+    }
+
+    private static void validateInputType(InputType inputType) {
+        if (inputType == null) {
+            return;
+        }
+
+        assert VALID_REQUEST_VALUES.contains(inputType) : invalidInputTypeMessage(inputType);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (inputType != null) {
+            builder.field(INPUT_TYPE, inputType);
+        }
+
+        builder.endObject();
+        return builder;
+    }
+
+    public InputType getInputType() {
+        return inputType;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalEnum(inputType);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AlibabaCloudSearchEmbeddingsTaskSettings that = (AlibabaCloudSearchEmbeddingsTaskSettings) o;
+        return Objects.equals(inputType, that.inputType);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(inputType);
+    }
+
+    public static String invalidInputTypeMessage(InputType inputType) {
+        return Strings.format("received invalid input type value [%s]", inputType.toString());
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankModel.java
new file mode 100644
index 0000000000000..a9152b6edd4c5
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankModel.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionVisitor;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchModel;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+
+import java.util.Map;
+
+public class AlibabaCloudSearchRerankModel extends AlibabaCloudSearchModel {
+    public static AlibabaCloudSearchRerankModel of(AlibabaCloudSearchRerankModel model, Map<String, Object> taskSettings) {
+        var requestTaskSettings = AlibabaCloudSearchRerankTaskSettings.fromMap(taskSettings);
+        return new AlibabaCloudSearchRerankModel(
+            model,
+            AlibabaCloudSearchRerankTaskSettings.of(model.getTaskSettings(), requestTaskSettings)
+        );
+    }
+
+    public AlibabaCloudSearchRerankModel(
+        String modelId,
+        TaskType taskType,
+        String service,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secrets,
+        ConfigurationParseContext context
+    ) {
+        this(
+            modelId,
+            taskType,
+            service,
+            AlibabaCloudSearchRerankServiceSettings.fromMap(serviceSettings, context),
+            AlibabaCloudSearchRerankTaskSettings.fromMap(taskSettings),
+            DefaultSecretSettings.fromMap(secrets)
+        );
+    }
+
+    // should only be used for testing
+    AlibabaCloudSearchRerankModel(
+        String modelId,
+        TaskType taskType,
+        String service,
+        AlibabaCloudSearchRerankServiceSettings serviceSettings,
+        AlibabaCloudSearchRerankTaskSettings taskSettings,
+        @Nullable DefaultSecretSettings secretSettings
+    ) {
+        super(
+            new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings),
+            new ModelSecrets(secretSettings),
+            serviceSettings.getCommonSettings()
+        );
+    }
+
+    private AlibabaCloudSearchRerankModel(AlibabaCloudSearchRerankModel model, AlibabaCloudSearchRerankTaskSettings taskSettings) {
+        super(model, taskSettings);
+    }
+
+    public AlibabaCloudSearchRerankModel(AlibabaCloudSearchRerankModel model, AlibabaCloudSearchRerankServiceSettings serviceSettings) {
+        super(model, serviceSettings);
+    }
+
+    @Override
+    public AlibabaCloudSearchRerankServiceSettings getServiceSettings() {
+        return (AlibabaCloudSearchRerankServiceSettings) super.getServiceSettings();
+    }
+
+    @Override
+    public AlibabaCloudSearchRerankTaskSettings getTaskSettings() {
+        return (AlibabaCloudSearchRerankTaskSettings) super.getTaskSettings();
+    }
+
+    @Override
+    public DefaultSecretSettings getSecretSettings() {
+        return (DefaultSecretSettings) super.getSecretSettings();
+    }
+
+    @Override
+    public ExecutableAction accept(AlibabaCloudSearchActionVisitor visitor, Map<String, Object> taskSettings, InputType inputType) {
+        return visitor.create(this, taskSettings);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankServiceSettings.java
new file mode 100644
index 0000000000000..42c7238aefa7f
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankServiceSettings.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettings;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class AlibabaCloudSearchRerankServiceSettings implements ServiceSettings {
+    public static final String NAME = "alibabacloud_search_rerank_service_settings";
+
+    public static AlibabaCloudSearchRerankServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
+        ValidationException validationException = new ValidationException();
+        var commonServiceSettings = AlibabaCloudSearchServiceSettings.fromMap(map, context);
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new AlibabaCloudSearchRerankServiceSettings(commonServiceSettings);
+    }
+
+    private final AlibabaCloudSearchServiceSettings commonSettings;
+
+    public AlibabaCloudSearchRerankServiceSettings(AlibabaCloudSearchServiceSettings commonSettings) {
+        this.commonSettings = commonSettings;
+    }
+
+    public AlibabaCloudSearchRerankServiceSettings(StreamInput in) throws IOException {
+        commonSettings = new AlibabaCloudSearchServiceSettings(in);
+    }
+
+    public AlibabaCloudSearchServiceSettings getCommonSettings() {
+        return commonSettings;
+    }
+
+    @Override
+    public String modelId() {
+        return commonSettings.modelId();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        commonSettings.toXContentFragment(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public ToXContentObject getFilteredXContentObject() {
+        return this;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        commonSettings.writeTo(out);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AlibabaCloudSearchRerankServiceSettings that = (AlibabaCloudSearchRerankServiceSettings) o;
+        return Objects.equals(commonSettings, that.commonSettings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(commonSettings);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java
new file mode 100644
index 0000000000000..e9fb468eab7fb
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/rerank/AlibabaCloudSearchRerankTaskSettings.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Defines the task settings for the AlibabaCloudSearch rerank service.
+ *
+ * <p>
+ * <a href="https://help.aliyun.com/zh/open-search/search-platform/developer-reference/ranker-api-details">See api docs for details.</a>
+ * </p>
+ */
+public class AlibabaCloudSearchRerankTaskSettings implements TaskSettings {
+    public static final String NAME = "alibabacloud_search_rerank_task_settings";
+
+    static final AlibabaCloudSearchRerankTaskSettings EMPTY_SETTINGS = new AlibabaCloudSearchRerankTaskSettings();
+
+    public static AlibabaCloudSearchRerankTaskSettings fromMap(Map<String, Object> map) {
+        ValidationException validationException = new ValidationException();
+
+        if (map == null || map.isEmpty()) {
+            return EMPTY_SETTINGS;
+        }
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return of();
+    }
+
+    /**
+     * Creates a new {@link AlibabaCloudSearchRerankTaskSettings}
+     * by preferring non-null fields from the request settings over the original settings.
+     * @return a constructed {@link AlibabaCloudSearchRerankTaskSettings}
+     */
+    public static AlibabaCloudSearchRerankTaskSettings of(
+        AlibabaCloudSearchRerankTaskSettings originalSettings,
+        AlibabaCloudSearchRerankTaskSettings requestTaskSettings
+    ) {
+        return new AlibabaCloudSearchRerankTaskSettings();
+    }
+
+    public static AlibabaCloudSearchRerankTaskSettings of() {
+        return new AlibabaCloudSearchRerankTaskSettings();
+    }
+
+    public AlibabaCloudSearchRerankTaskSettings(StreamInput in) {
+        this();
+    }
+
+    public AlibabaCloudSearchRerankTaskSettings() {}
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {}
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash();
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModel.java
new file mode 100644
index 0000000000000..b551ba389136b
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModel.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionVisitor;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchModel;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+
+import java.util.Map;
+
+public class AlibabaCloudSearchSparseModel extends AlibabaCloudSearchModel {
+    public static AlibabaCloudSearchSparseModel of(
+        AlibabaCloudSearchSparseModel model,
+        Map<String, Object> taskSettings,
+        InputType inputType
+    ) {
+        var requestTaskSettings = AlibabaCloudSearchSparseTaskSettings.fromMap(taskSettings);
+        return new AlibabaCloudSearchSparseModel(
+            model,
+            AlibabaCloudSearchSparseTaskSettings.of(model.getTaskSettings(), requestTaskSettings, inputType)
+        );
+    }
+
+    public AlibabaCloudSearchSparseModel(
+        String modelId,
+        TaskType taskType,
+        String service,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secrets,
+        ConfigurationParseContext context
+    ) {
+        this(
+            modelId,
+            taskType,
+            service,
+            AlibabaCloudSearchSparseServiceSettings.fromMap(serviceSettings, context),
+            AlibabaCloudSearchSparseTaskSettings.fromMap(taskSettings),
+            DefaultSecretSettings.fromMap(secrets)
+        );
+    }
+
+    // should only be used for testing
+    AlibabaCloudSearchSparseModel(
+        String modelId,
+        TaskType taskType,
+        String service,
+        AlibabaCloudSearchSparseServiceSettings serviceSettings,
+        AlibabaCloudSearchSparseTaskSettings taskSettings,
+        @Nullable DefaultSecretSettings secretSettings
+    ) {
+        super(
+            new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings),
+            new ModelSecrets(secretSettings),
+            serviceSettings.getCommonSettings()
+        );
+    }
+
+    private AlibabaCloudSearchSparseModel(AlibabaCloudSearchSparseModel model, AlibabaCloudSearchSparseTaskSettings taskSettings) {
+        super(model, taskSettings);
+    }
+
+    public AlibabaCloudSearchSparseModel(AlibabaCloudSearchSparseModel model, AlibabaCloudSearchSparseServiceSettings serviceSettings) {
+        super(model, serviceSettings);
+    }
+
+    @Override
+    public AlibabaCloudSearchSparseServiceSettings getServiceSettings() {
+        return (AlibabaCloudSearchSparseServiceSettings) super.getServiceSettings();
+    }
+
+    @Override
+    public AlibabaCloudSearchSparseTaskSettings getTaskSettings() {
+        return (AlibabaCloudSearchSparseTaskSettings) super.getTaskSettings();
+    }
+
+    @Override
+    public DefaultSecretSettings getSecretSettings() {
+        return (DefaultSecretSettings) super.getSecretSettings();
+    }
+
+    @Override
+    public ExecutableAction accept(AlibabaCloudSearchActionVisitor visitor, Map<String, Object> taskSettings, InputType inputType) {
+        return visitor.create(this, taskSettings, inputType);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseServiceSettings.java
new file mode 100644
index 0000000000000..fe44c936c4e61
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseServiceSettings.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettings;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class AlibabaCloudSearchSparseServiceSettings implements ServiceSettings {
+    public static final String NAME = "alibabacloud_search_sparse_embeddings_service_settings";
+
+    public static AlibabaCloudSearchSparseServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
+        ValidationException validationException = new ValidationException();
+        var commonServiceSettings = AlibabaCloudSearchServiceSettings.fromMap(map, context);
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new AlibabaCloudSearchSparseServiceSettings(commonServiceSettings);
+    }
+
+    private final AlibabaCloudSearchServiceSettings commonSettings;
+
+    public AlibabaCloudSearchSparseServiceSettings(AlibabaCloudSearchServiceSettings commonSettings) {
+        this.commonSettings = commonSettings;
+    }
+
+    public AlibabaCloudSearchSparseServiceSettings(StreamInput in) throws IOException {
+        commonSettings = new AlibabaCloudSearchServiceSettings(in);
+    }
+
+    public AlibabaCloudSearchServiceSettings getCommonSettings() {
+        return commonSettings;
+    }
+
+    @Override
+    public String modelId() {
+        return commonSettings.modelId();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        commonSettings.toXContentFragment(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public ToXContentObject getFilteredXContentObject() {
+        return this;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        commonSettings.writeTo(out);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AlibabaCloudSearchSparseServiceSettings that = (AlibabaCloudSearchSparseServiceSettings) o;
+        return Objects.equals(commonSettings, that.commonSettings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(commonSettings);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java
new file mode 100644
index 0000000000000..2b1e9ace1b24c
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettings.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalBoolean;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalEnum;
+
+/**
+ * Defines the task settings for the alibabacloud search text sparse embeddings service.
+ *
+ * <p>
+ * <a href="https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-sparse-embedding-api-details">
+ * See api docs for details.</a>
+ * </p>
+ */
+public class AlibabaCloudSearchSparseTaskSettings implements TaskSettings {
+
+    public static final String NAME = "alibabacloud_search_sparse_embeddings_task_settings";
+    public static final AlibabaCloudSearchSparseTaskSettings EMPTY_SETTINGS = new AlibabaCloudSearchSparseTaskSettings(null, null);
+    static final String INPUT_TYPE = "input_type";
+    static final String RETURN_TOKEN = "return_token";
+    static final EnumSet<InputType> VALID_REQUEST_VALUES = EnumSet.of(InputType.INGEST, InputType.SEARCH);
+
+    public static AlibabaCloudSearchSparseTaskSettings fromMap(Map<String, Object> map) {
+        if (map == null || map.isEmpty()) {
+            return EMPTY_SETTINGS;
+        }
+
+        ValidationException validationException = new ValidationException();
+
+        InputType inputType = extractOptionalEnum(
+            map,
+            INPUT_TYPE,
+            ModelConfigurations.TASK_SETTINGS,
+            InputType::fromString,
+            VALID_REQUEST_VALUES,
+            validationException
+        );
+
+        Boolean returnToken = extractOptionalBoolean(map, RETURN_TOKEN, validationException);
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new AlibabaCloudSearchSparseTaskSettings(inputType, returnToken);
+    }
+
+    /**
+     * Creates a new {@link AlibabaCloudSearchSparseTaskSettings} by preferring non-null fields from the provided parameters.
+     * For the input type, preference is given to requestInputType if it is not null and not UNSPECIFIED.
+     * Then preference is given to the requestTaskSettings and finally to originalSettings even if the value is null.
+     * <p>
+     * Similarly, for the truncation field preference is given to requestTaskSettings if it is not null and then to
+     * originalSettings.
+     *
+     * @param originalSettings    the settings stored as part of the inference entity configuration
+     * @param requestTaskSettings the settings passed in within the task_settings field of the request
+     * @param requestInputType    the input type passed in the request parameters
+     * @return a constructed {@link AlibabaCloudSearchSparseTaskSettings}
+     */
+    public static AlibabaCloudSearchSparseTaskSettings of(
+        AlibabaCloudSearchSparseTaskSettings originalSettings,
+        AlibabaCloudSearchSparseTaskSettings requestTaskSettings,
+        InputType requestInputType
+    ) {
+        var inputTypeToUse = getValidInputType(originalSettings, requestTaskSettings, requestInputType);
+        var returnToken = requestTaskSettings.isReturnToken() != null
+            ? requestTaskSettings.isReturnToken()
+            : originalSettings.isReturnToken();
+        return new AlibabaCloudSearchSparseTaskSettings(inputTypeToUse, returnToken);
+    }
+
+    private static InputType getValidInputType(
+        AlibabaCloudSearchSparseTaskSettings originalSettings,
+        AlibabaCloudSearchSparseTaskSettings requestTaskSettings,
+        InputType requestInputType
+    ) {
+        InputType inputTypeToUse = originalSettings.inputType;
+
+        if (VALID_REQUEST_VALUES.contains(requestInputType)) {
+            inputTypeToUse = requestInputType;
+        } else if (requestTaskSettings.inputType != null) {
+            inputTypeToUse = requestTaskSettings.inputType;
+        }
+
+        return inputTypeToUse;
+    }
+
+    private final InputType inputType;
+    private final Boolean returnToken;
+
+    public AlibabaCloudSearchSparseTaskSettings(StreamInput in) throws IOException {
+        this(in.readOptionalEnum(InputType.class), in.readOptionalBoolean());
+    }
+
+    public AlibabaCloudSearchSparseTaskSettings(@Nullable InputType inputType, Boolean returnToken) {
+        validateInputType(inputType);
+        this.inputType = inputType;
+        this.returnToken = returnToken;
+    }
+
+    private static void validateInputType(InputType inputType) {
+        if (inputType == null) {
+            return;
+        }
+
+        assert VALID_REQUEST_VALUES.contains(inputType) : invalidInputTypeMessage(inputType);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (inputType != null) {
+            builder.field(INPUT_TYPE, inputType);
+        }
+        if (returnToken != null) {
+            builder.field(RETURN_TOKEN, returnToken);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public InputType getInputType() {
+        return inputType;
+    }
+
+    public Boolean isReturnToken() {
+        return returnToken;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_ALIBABACLOUD_SEARCH_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalEnum(inputType);
+        out.writeOptionalBoolean(returnToken);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AlibabaCloudSearchSparseTaskSettings that = (AlibabaCloudSearchSparseTaskSettings) o;
+        return Objects.equals(inputType, that.inputType) && Objects.equals(returnToken, that.returnToken);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(inputType, returnToken);
+    }
+
+    public static String invalidInputTypeMessage(InputType inputType) {
+        return Strings.format("received invalid input type value [%s]", inputType.toString());
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionServiceSettings.java
index 3a70a26a82387..7976407b6fb67 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionServiceSettings.java
@@ -113,7 +113,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_COMPLETION_INFERENCE_SERVICE_ADDED;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java
index fc11d96269b68..49684d6749756 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionTaskSettings.java
@@ -141,7 +141,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java
index dc001993b366f..f5d70b89cee50 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsTaskSettings.java
@@ -76,7 +76,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java
index 48e45f368bfe2..06217e8079b06 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java
@@ -104,7 +104,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java
index d736948afb5a2..941a4bdeeb41a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java
@@ -290,7 +290,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java
index 49329a55a18ef..36cf89ea2d846 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsTaskSettings.java
@@ -91,7 +91,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingType.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingType.java
index 8dbbbf7011e86..11e405df3cde9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingType.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingType.java
@@ -112,7 +112,7 @@ public DenseVectorFieldMapper.ElementType toElementType() {
      * @return the embedding type that is known to the version passed in
      */
     public static CohereEmbeddingType translateToVersion(CohereEmbeddingType embeddingType, TransportVersion version) {
-        if (version.before(TransportVersions.ML_INFERENCE_EMBEDDING_BYTE_ADDED) && embeddingType == BYTE) {
+        if (version.before(TransportVersions.V_8_14_0) && embeddingType == BYTE) {
             return INT8;
         }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java
index 538d88a59ca76..fea5226bf9c6f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java
@@ -28,7 +28,7 @@ public static CohereEmbeddingsModel of(CohereEmbeddingsModel model, Map<String,
     }
 
     public CohereEmbeddingsModel(
-        String modelId,
+        String inferenceId,
         TaskType taskType,
         String service,
         Map<String, Object> serviceSettings,
@@ -37,7 +37,7 @@ public CohereEmbeddingsModel(
         ConfigurationParseContext context
     ) {
         this(
-            modelId,
+            inferenceId,
             taskType,
             service,
             CohereEmbeddingsServiceSettings.fromMap(serviceSettings, context),
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java
index b5ff2e628f58e..1132dff34ed6e 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java
@@ -154,7 +154,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_COHERE_RERANK;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java
index 82f2d0e6f7ada..a01f6a4e65b8f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankTaskSettings.java
@@ -136,7 +136,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_COHERE_RERANK;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java
new file mode 100644
index 0000000000000..f77217f9c02f9
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java
@@ -0,0 +1,274 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.ChunkedInferenceServiceResults;
+import org.elasticsearch.inference.ChunkingOptions;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
+import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults;
+import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
+import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
+import org.elasticsearch.xpack.inference.external.action.elastic.ElasticInferenceServiceActionCreator;
+import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.SenderService;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap;
+
+public class ElasticInferenceService extends SenderService {
+
+    public static final String NAME = "elastic";
+
+    private final ElasticInferenceServiceComponents elasticInferenceServiceComponents;
+
+    public ElasticInferenceService(
+        HttpRequestSender.Factory factory,
+        ServiceComponents serviceComponents,
+        ElasticInferenceServiceComponents eisComponents
+    ) {
+        super(factory, serviceComponents);
+        this.elasticInferenceServiceComponents = eisComponents;
+    }
+
+    @Override
+    protected void doInfer(
+        Model model,
+        List<String> input,
+        Map<String, Object> taskSettings,
+        InputType inputType,
+        TimeValue timeout,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        if (model instanceof ElasticInferenceServiceModel == false) {
+            listener.onFailure(createInvalidModelException(model));
+            return;
+        }
+
+        ElasticInferenceServiceModel elasticInferenceServiceModel = (ElasticInferenceServiceModel) model;
+        var actionCreator = new ElasticInferenceServiceActionCreator(getSender(), getServiceComponents());
+
+        var action = elasticInferenceServiceModel.accept(actionCreator, taskSettings);
+        action.execute(new DocumentsOnlyInput(input), timeout, listener);
+    }
+
+    @Override
+    protected void doInfer(
+        Model model,
+        String query,
+        List<String> input,
+        Map<String, Object> taskSettings,
+        InputType inputType,
+        TimeValue timeout,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        throw new UnsupportedOperationException("Query input not supported for Elastic Inference Service");
+    }
+
+    @Override
+    protected void doChunkedInfer(
+        Model model,
+        String query,
+        List<String> input,
+        Map<String, Object> taskSettings,
+        InputType inputType,
+        ChunkingOptions chunkingOptions,
+        TimeValue timeout,
+        ActionListener<List<ChunkedInferenceServiceResults>> listener
+    ) {
+        // Pass-through without actually performing chunking (result will have a single chunk per input)
+        ActionListener<InferenceServiceResults> inferListener = listener.delegateFailureAndWrap(
+            (delegate, response) -> delegate.onResponse(translateToChunkedResults(input, response))
+        );
+
+        doInfer(model, input, taskSettings, inputType, timeout, inferListener);
+    }
+
+    @Override
+    public String name() {
+        return NAME;
+    }
+
+    @Override
+    public void parseRequestConfig(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> config,
+        Set<String> platformArchitectures,
+        ActionListener<Model> parsedModelListener
+    ) {
+        try {
+            Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+            Map<String, Object> taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS);
+
+            ElasticInferenceServiceModel model = createModel(
+                inferenceEntityId,
+                taskType,
+                serviceSettingsMap,
+                taskSettingsMap,
+                serviceSettingsMap,
+                elasticInferenceServiceComponents,
+                TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME),
+                ConfigurationParseContext.REQUEST
+            );
+
+            throwIfNotEmptyMap(config, NAME);
+            throwIfNotEmptyMap(serviceSettingsMap, NAME);
+            throwIfNotEmptyMap(taskSettingsMap, NAME);
+
+            parsedModelListener.onResponse(model);
+        } catch (Exception e) {
+            parsedModelListener.onFailure(e);
+        }
+    }
+
+    private static ElasticInferenceServiceModel createModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secretSettings,
+        ElasticInferenceServiceComponents eisServiceComponents,
+        String failureMessage,
+        ConfigurationParseContext context
+    ) {
+        return switch (taskType) {
+            case SPARSE_EMBEDDING -> new ElasticInferenceServiceSparseEmbeddingsModel(
+                inferenceEntityId,
+                taskType,
+                NAME,
+                serviceSettings,
+                taskSettings,
+                secretSettings,
+                eisServiceComponents,
+                context
+            );
+            default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
+        };
+    }
+
+    @Override
+    public Model parsePersistedConfigWithSecrets(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> config,
+        Map<String, Object> secrets
+    ) {
+        Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+        Map<String, Object> taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS);
+        Map<String, Object> secretSettingsMap = removeFromMapOrDefaultEmpty(secrets, ModelSecrets.SECRET_SETTINGS);
+
+        return createModelFromPersistent(
+            inferenceEntityId,
+            taskType,
+            serviceSettingsMap,
+            taskSettingsMap,
+            secretSettingsMap,
+            parsePersistedConfigErrorMsg(inferenceEntityId, NAME)
+        );
+    }
+
+    @Override
+    public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, Map<String, Object> config) {
+        Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+        Map<String, Object> taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS);
+
+        return createModelFromPersistent(
+            inferenceEntityId,
+            taskType,
+            serviceSettingsMap,
+            taskSettingsMap,
+            null,
+            parsePersistedConfigErrorMsg(inferenceEntityId, NAME)
+        );
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_EIS_INTEGRATION_ADDED;
+    }
+
+    private ElasticInferenceServiceModel createModelFromPersistent(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secretSettings,
+        String failureMessage
+    ) {
+        return createModel(
+            inferenceEntityId,
+            taskType,
+            serviceSettings,
+            taskSettings,
+            secretSettings,
+            elasticInferenceServiceComponents,
+            failureMessage,
+            ConfigurationParseContext.PERSISTENT
+        );
+    }
+
+    @Override
+    public void checkModelConfig(Model model, ActionListener<Model> listener) {
+        if (model instanceof ElasticInferenceServiceSparseEmbeddingsModel embeddingsModel) {
+            listener.onResponse(updateModelWithEmbeddingDetails(embeddingsModel));
+        } else {
+            listener.onResponse(model);
+        }
+    }
+
+    private static List<ChunkedInferenceServiceResults> translateToChunkedResults(
+        List<String> inputs,
+        InferenceServiceResults inferenceResults
+    ) {
+        if (inferenceResults instanceof SparseEmbeddingResults sparseEmbeddingResults) {
+            return InferenceChunkedSparseEmbeddingResults.listOf(inputs, sparseEmbeddingResults);
+        } else if (inferenceResults instanceof ErrorInferenceResults error) {
+            return List.of(new ErrorChunkedInferenceResults(error.getException()));
+        } else {
+            String expectedClass = Strings.format("%s", SparseEmbeddingResults.class.getSimpleName());
+            throw createInvalidChunkedResultException(expectedClass, inferenceResults.getWriteableName());
+        }
+    }
+
+    private ElasticInferenceServiceSparseEmbeddingsModel updateModelWithEmbeddingDetails(
+        ElasticInferenceServiceSparseEmbeddingsModel model
+    ) {
+        ElasticInferenceServiceSparseEmbeddingsServiceSettings serviceSettings = new ElasticInferenceServiceSparseEmbeddingsServiceSettings(
+            model.getServiceSettings().modelId(),
+            model.getServiceSettings().maxInputTokens(),
+            model.getServiceSettings().rateLimitSettings()
+        );
+
+        return new ElasticInferenceServiceSparseEmbeddingsModel(model, serviceSettings);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceComponents.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceComponents.java
new file mode 100644
index 0000000000000..4386964e927d2
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceComponents.java
@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+public record ElasticInferenceServiceComponents(String eisGatewayUrl) {}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceFeature.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceFeature.java
new file mode 100644
index 0000000000000..b0fb6d14ee6f7
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceFeature.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+import org.elasticsearch.common.util.FeatureFlag;
+
+/**
+ * Elastic Inference Service (EIS) feature flag. When the feature is complete, this flag will be removed.
+ * Enable feature via JVM option: `-Des.eis_feature_flag_enabled=true`.
+ */
+public class ElasticInferenceServiceFeature {
+
+    public static final FeatureFlag ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG = new FeatureFlag("eis");
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceModel.java
new file mode 100644
index 0000000000000..e7809d869fec4
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceModel.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.elastic.ElasticInferenceServiceActionVisitor;
+
+import java.util.Map;
+import java.util.Objects;
+
+public abstract class ElasticInferenceServiceModel extends Model {
+
+    private final ElasticInferenceServiceRateLimitServiceSettings rateLimitServiceSettings;
+
+    private final ElasticInferenceServiceComponents elasticInferenceServiceComponents;
+
+    public ElasticInferenceServiceModel(
+        ModelConfigurations configurations,
+        ModelSecrets secrets,
+        ElasticInferenceServiceRateLimitServiceSettings rateLimitServiceSettings,
+        ElasticInferenceServiceComponents elasticInferenceServiceComponents
+    ) {
+        super(configurations, secrets);
+
+        this.rateLimitServiceSettings = Objects.requireNonNull(rateLimitServiceSettings);
+        this.elasticInferenceServiceComponents = Objects.requireNonNull(elasticInferenceServiceComponents);
+    }
+
+    public ElasticInferenceServiceModel(ElasticInferenceServiceModel model, ServiceSettings serviceSettings) {
+        super(model, serviceSettings);
+
+        this.rateLimitServiceSettings = model.rateLimitServiceSettings();
+        this.elasticInferenceServiceComponents = model.elasticInferenceServiceComponents();
+    }
+
+    public ElasticInferenceServiceRateLimitServiceSettings rateLimitServiceSettings() {
+        return rateLimitServiceSettings;
+    }
+
+    public ElasticInferenceServiceComponents elasticInferenceServiceComponents() {
+        return elasticInferenceServiceComponents;
+    }
+
+    public abstract ExecutableAction accept(ElasticInferenceServiceActionVisitor visitor, Map<String, Object> taskSettings);
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceRateLimitServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceRateLimitServiceSettings.java
new file mode 100644
index 0000000000000..2ec562b61fa01
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceRateLimitServiceSettings.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+public interface ElasticInferenceServiceRateLimitServiceSettings {
+
+    String modelId();
+
+    RateLimitSettings rateLimitSettings();
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java
new file mode 100644
index 0000000000000..8525710c6cf23
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSettings.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.List;
+
+public class ElasticInferenceServiceSettings {
+
+    static final Setting<String> EIS_GATEWAY_URL = Setting.simpleString("xpack.inference.eis.gateway.url", Setting.Property.NodeScope);
+
+    // Adjust this variable to be volatile, if the setting can be updated at some point in time
+    private final String eisGatewayUrl;
+
+    public ElasticInferenceServiceSettings(Settings settings) {
+        eisGatewayUrl = EIS_GATEWAY_URL.get(settings);
+    }
+
+    public static List<Setting<?>> getSettingsDefinitions() {
+        return List.of(EIS_GATEWAY_URL);
+    }
+
+    public String getEisGatewayUrl() {
+        return eisGatewayUrl;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java
new file mode 100644
index 0000000000000..163e3dd654150
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModel.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.EmptySecretSettings;
+import org.elasticsearch.inference.EmptyTaskSettings;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.SecretSettings;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.elastic.ElasticInferenceServiceActionVisitor;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.elser.ElserModels;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+
+public class ElasticInferenceServiceSparseEmbeddingsModel extends ElasticInferenceServiceModel {
+
+    private final URI uri;
+
+    public ElasticInferenceServiceSparseEmbeddingsModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        String service,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        Map<String, Object> secrets,
+        ElasticInferenceServiceComponents elasticInferenceServiceComponents,
+        ConfigurationParseContext context
+    ) {
+        this(
+            inferenceEntityId,
+            taskType,
+            service,
+            ElasticInferenceServiceSparseEmbeddingsServiceSettings.fromMap(serviceSettings, context),
+            EmptyTaskSettings.INSTANCE,
+            EmptySecretSettings.INSTANCE,
+            elasticInferenceServiceComponents
+        );
+    }
+
+    public ElasticInferenceServiceSparseEmbeddingsModel(
+        ElasticInferenceServiceSparseEmbeddingsModel model,
+        ElasticInferenceServiceSparseEmbeddingsServiceSettings serviceSettings
+    ) {
+        super(model, serviceSettings);
+
+        try {
+            this.uri = createUri();
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    ElasticInferenceServiceSparseEmbeddingsModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        String service,
+        ElasticInferenceServiceSparseEmbeddingsServiceSettings serviceSettings,
+        @Nullable TaskSettings taskSettings,
+        @Nullable SecretSettings secretSettings,
+        ElasticInferenceServiceComponents elasticInferenceServiceComponents
+    ) {
+        super(
+            new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings),
+            new ModelSecrets(secretSettings),
+            serviceSettings,
+            elasticInferenceServiceComponents
+        );
+
+        try {
+            this.uri = createUri();
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public ExecutableAction accept(ElasticInferenceServiceActionVisitor visitor, Map<String, Object> taskSettings) {
+        return visitor.create(this);
+    }
+
+    @Override
+    public ElasticInferenceServiceSparseEmbeddingsServiceSettings getServiceSettings() {
+        return (ElasticInferenceServiceSparseEmbeddingsServiceSettings) super.getServiceSettings();
+    }
+
+    public URI uri() {
+        return uri;
+    }
+
+    private URI createUri() throws URISyntaxException {
+        String modelId = getServiceSettings().modelId();
+        String modelIdUriPath;
+
+        switch (modelId) {
+            case ElserModels.ELSER_V2_MODEL -> modelIdUriPath = "ELSERv2";
+            default -> throw new IllegalArgumentException("Unsupported model for EIS [" + modelId + "]");
+        }
+
+        return new URI(elasticInferenceServiceComponents().eisGatewayUrl() + "/sparse-text-embedding/" + modelIdUriPath);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java
new file mode 100644
index 0000000000000..15b89525f7915
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettings.java
@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.elser.ElserModels;
+import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS;
+import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString;
+
+public class ElasticInferenceServiceSparseEmbeddingsServiceSettings extends FilteredXContentObject
+    implements
+        ServiceSettings,
+        ElasticInferenceServiceRateLimitServiceSettings {
+
+    public static final String NAME = "elastic_inference_service_sparse_embeddings_service_settings";
+
+    private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(1_000);
+
+    public static ElasticInferenceServiceSparseEmbeddingsServiceSettings fromMap(
+        Map<String, Object> map,
+        ConfigurationParseContext context
+    ) {
+        ValidationException validationException = new ValidationException();
+
+        String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        Integer maxInputTokens = extractOptionalPositiveInteger(
+            map,
+            MAX_INPUT_TOKENS,
+            ModelConfigurations.SERVICE_SETTINGS,
+            validationException
+        );
+
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            ElasticInferenceService.NAME,
+            context
+        );
+
+        if (modelId != null && ElserModels.isValidEisModel(modelId) == false) {
+            validationException.addValidationError("unknown ELSER model id [" + modelId + "]");
+        }
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new ElasticInferenceServiceSparseEmbeddingsServiceSettings(modelId, maxInputTokens, rateLimitSettings);
+    }
+
+    private final String modelId;
+
+    private final Integer maxInputTokens;
+    private final RateLimitSettings rateLimitSettings;
+
+    public ElasticInferenceServiceSparseEmbeddingsServiceSettings(
+        String modelId,
+        @Nullable Integer maxInputTokens,
+        RateLimitSettings rateLimitSettings
+    ) {
+        this.modelId = Objects.requireNonNull(modelId);
+        this.maxInputTokens = maxInputTokens;
+        this.rateLimitSettings = Objects.requireNonNullElse(rateLimitSettings, DEFAULT_RATE_LIMIT_SETTINGS);
+    }
+
+    public ElasticInferenceServiceSparseEmbeddingsServiceSettings(StreamInput in) throws IOException {
+        this.modelId = in.readString();
+        this.maxInputTokens = in.readOptionalVInt();
+        this.rateLimitSettings = new RateLimitSettings(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    public String modelId() {
+        return modelId;
+    }
+
+    public Integer maxInputTokens() {
+        return maxInputTokens;
+    }
+
+    @Override
+    public RateLimitSettings rateLimitSettings() {
+        return rateLimitSettings;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_EIS_INTEGRATION_ADDED;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        toXContentFragmentOfExposedFields(builder, params);
+
+        builder.endObject();
+
+        return builder;
+    }
+
+    @Override
+    protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException {
+        builder.field(MODEL_ID, modelId);
+        if (maxInputTokens != null) {
+            builder.field(MAX_INPUT_TOKENS, maxInputTokens);
+        }
+        rateLimitSettings.toXContent(builder, params);
+
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(modelId);
+        out.writeOptionalVInt(maxInputTokens);
+        rateLimitSettings.writeTo(out);
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) return true;
+        if (object == null || getClass() != object.getClass()) return false;
+        ElasticInferenceServiceSparseEmbeddingsServiceSettings that = (ElasticInferenceServiceSparseEmbeddingsServiceSettings) object;
+        return Objects.equals(modelId, that.modelId)
+            && Objects.equals(maxInputTokens, that.maxInputTokens)
+            && Objects.equals(rateLimitSettings, that.rateLimitSettings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(modelId, maxInputTokens, rateLimitSettings);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java
index 574ca77d4587e..457416370e559 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java
@@ -154,10 +154,10 @@ public void isModelDownloaded(Model model, ActionListener<Boolean> listener) {
             executeAsyncWithOrigin(client, INFERENCE_ORIGIN, GetTrainedModelsAction.INSTANCE, getRequest, getModelsResponseListener);
         } else {
             listener.onFailure(
-                new IllegalArgumentException(
-                    "Unable to determine supported model for ["
+                new IllegalStateException(
+                    "Can not check the download status of the model used by ["
                         + model.getConfigurations().getInferenceEntityId()
-                        + "] please verify the request and submit a bug report if necessary."
+                        + "] as the model_id cannot be found."
                 )
             );
         }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java
index 0b586af5005fb..523aff20b8e05 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankTaskSettings.java
@@ -107,7 +107,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_COHERE_RERANK;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
index ef72b022e929e..cca8ae63e974c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
@@ -7,8 +7,6 @@
 
 package org.elasticsearch.xpack.inference.services.elasticsearch;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
@@ -27,19 +25,18 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
+import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults;
 import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults;
 import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
+import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
 import org.elasticsearch.xpack.core.ml.action.InferModelAction;
-import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction;
-import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
-import org.elasticsearch.xpack.core.ml.action.StopTrainedModelDeploymentAction;
-import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
-import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput;
 import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResults;
+import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
@@ -53,8 +50,6 @@
 import java.util.Set;
 import java.util.function.Function;
 
-import static org.elasticsearch.xpack.core.ClientHelper.INFERENCE_ORIGIN;
-import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull;
@@ -71,15 +66,13 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi
         MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86
     );
 
-    private static final Logger logger = LogManager.getLogger(ElasticsearchInternalService.class);
-
     public ElasticsearchInternalService(InferenceServiceExtension.InferenceServiceFactoryContext context) {
         super(context);
     }
 
     @Override
     protected EnumSet<TaskType> supportedTaskTypes() {
-        return EnumSet.of(TaskType.RERANK, TaskType.TEXT_EMBEDDING);
+        return EnumSet.of(TaskType.RERANK, TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING);
     }
 
     @Override
@@ -161,6 +154,12 @@ private static CustomElandModel createCustomElandModel(
                 NAME,
                 CustomElandInternalTextEmbeddingServiceSettings.fromMap(serviceSettings, context)
             );
+            case SPARSE_EMBEDDING -> new CustomElandModel(
+                inferenceEntityId,
+                taskType,
+                NAME,
+                elandServiceSettings(serviceSettings, context)
+            );
             case RERANK -> new CustomElandRerankModel(
                 inferenceEntityId,
                 taskType,
@@ -334,6 +333,8 @@ public void infer(
             inferTextEmbedding(model, input, inputType, timeout, listener);
         } else if (TaskType.RERANK.equals(taskType)) {
             inferRerank(model, query, input, inputType, timeout, taskSettings, listener);
+        } else if (TaskType.SPARSE_EMBEDDING.equals(taskType)) {
+            inferSparseEmbedding(model, input, inputType, timeout, listener);
         } else {
             throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST);
         }
@@ -364,6 +365,31 @@ public void inferTextEmbedding(
         );
     }
 
+    public void inferSparseEmbedding(
+        Model model,
+        List<String> inputs,
+        InputType inputType,
+        TimeValue timeout,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        var request = buildInferenceRequest(
+            model.getConfigurations().getInferenceEntityId(),
+            TextExpansionConfigUpdate.EMPTY_UPDATE,
+            inputs,
+            inputType,
+            timeout,
+            false
+        );
+
+        client.execute(
+            InferModelAction.INSTANCE,
+            request,
+            listener.delegateFailureAndWrap(
+                (l, inferenceResult) -> l.onResponse(SparseEmbeddingResults.of(inferenceResult.getInferenceResults()))
+            )
+        );
+    }
+
     public void inferRerank(
         Model model,
         String query,
@@ -422,7 +448,7 @@ public void chunkedInfer(
         TimeValue timeout,
         ActionListener<List<ChunkedInferenceServiceResults>> listener
     ) {
-        if (TaskType.TEXT_EMBEDDING.isAnyOrSame(model.getTaskType()) == false) {
+        if ((TaskType.TEXT_EMBEDDING.equals(model.getTaskType()) || TaskType.SPARSE_EMBEDDING.equals(model.getTaskType())) == false) {
             listener.onFailure(
                 new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(model.getTaskType(), NAME), RestStatus.BAD_REQUEST)
             );
@@ -464,6 +490,8 @@ private static List<ChunkedInferenceServiceResults> translateToChunkedResults(Li
     private static ChunkedInferenceServiceResults translateToChunkedResult(InferenceResults inferenceResult) {
         if (inferenceResult instanceof MlChunkedTextEmbeddingFloatResults mlChunkedResult) {
             return InferenceChunkedTextEmbeddingFloatResults.ofMlResults(mlChunkedResult);
+        } else if (inferenceResult instanceof MlChunkedTextExpansionResults mlChunkedResult) {
+            return InferenceChunkedSparseEmbeddingResults.ofMlResult(mlChunkedResult);
         } else if (inferenceResult instanceof ErrorInferenceResults error) {
             return new ErrorChunkedInferenceResults(error.getException());
         } else {
@@ -471,106 +499,9 @@ private static ChunkedInferenceServiceResults translateToChunkedResult(Inference
         }
     }
 
-    @Override
-    public void start(Model model, ActionListener<Boolean> listener) {
-        if (model instanceof ElasticsearchInternalModel == false) {
-            listener.onFailure(notElasticsearchModelException(model));
-            return;
-        }
-
-        if (model.getTaskType() != TaskType.TEXT_EMBEDDING && model.getTaskType() != TaskType.RERANK) {
-            listener.onFailure(
-                new IllegalStateException(TaskType.unsupportedTaskTypeErrorMsg(model.getConfigurations().getTaskType(), NAME))
-            );
-            return;
-        }
-
-        var startRequest = ((ElasticsearchInternalModel) model).getStartTrainedModelDeploymentActionRequest();
-        var responseListener = ((ElasticsearchInternalModel) model).getCreateTrainedModelAssignmentActionListener(model, listener);
-
-        client.execute(StartTrainedModelDeploymentAction.INSTANCE, startRequest, responseListener);
-    }
-
-    @Override
-    public void stop(String inferenceEntityId, ActionListener<Boolean> listener) {
-        var request = new StopTrainedModelDeploymentAction.Request(inferenceEntityId);
-        request.setForce(true);
-        client.execute(
-            StopTrainedModelDeploymentAction.INSTANCE,
-            request,
-            listener.delegateFailureAndWrap((delegatedResponseListener, response) -> delegatedResponseListener.onResponse(Boolean.TRUE))
-        );
-    }
-
-    @Override
-    public void putModel(Model model, ActionListener<Boolean> listener) {
-        if (model instanceof ElasticsearchInternalModel == false) {
-            listener.onFailure(notElasticsearchModelException(model));
-            return;
-        } else if (model instanceof MultilingualE5SmallModel e5Model) {
-            String modelId = e5Model.getServiceSettings().modelId();
-            var input = new TrainedModelInput(List.<String>of("text_field")); // by convention text_field is used
-            var config = TrainedModelConfig.builder().setInput(input).setModelId(modelId).validate(true).build();
-            PutTrainedModelAction.Request putRequest = new PutTrainedModelAction.Request(config, false, true);
-            executeAsyncWithOrigin(
-                client,
-                INFERENCE_ORIGIN,
-                PutTrainedModelAction.INSTANCE,
-                putRequest,
-                ActionListener.wrap(response -> listener.onResponse(Boolean.TRUE), e -> {
-                    if (e instanceof ElasticsearchStatusException esException
-                        && esException.getMessage().contains(PutTrainedModelAction.MODEL_ALREADY_EXISTS_ERROR_MESSAGE_FRAGMENT)) {
-                        listener.onResponse(Boolean.TRUE);
-                    } else {
-                        listener.onFailure(e);
-                    }
-                })
-            );
-        } else if (model instanceof CustomElandModel) {
-            logger.info("Custom eland model detected, model must have been already loaded into the cluster with eland.");
-            listener.onResponse(Boolean.TRUE);
-        } else {
-            listener.onFailure(
-                new IllegalArgumentException(
-                    "Can not download model automatically for ["
-                        + model.getConfigurations().getInferenceEntityId()
-                        + "] you may need to download it through the trained models API or with eland."
-                )
-            );
-            return;
-        }
-    }
-
-    @Override
-    public void isModelDownloaded(Model model, ActionListener<Boolean> listener) {
-        ActionListener<GetTrainedModelsAction.Response> getModelsResponseListener = listener.delegateFailure((delegate, response) -> {
-            if (response.getResources().count() < 1) {
-                delegate.onResponse(Boolean.FALSE);
-            } else {
-                delegate.onResponse(Boolean.TRUE);
-            }
-        });
-
-        if (model.getServiceSettings() instanceof ElasticsearchInternalServiceSettings internalServiceSettings) {
-            String modelId = internalServiceSettings.modelId();
-            GetTrainedModelsAction.Request getRequest = new GetTrainedModelsAction.Request(modelId);
-            executeAsyncWithOrigin(client, INFERENCE_ORIGIN, GetTrainedModelsAction.INSTANCE, getRequest, getModelsResponseListener);
-        } else if (model instanceof ElasticsearchInternalModel == false) {
-            listener.onFailure(notElasticsearchModelException(model));
-        } else {
-            listener.onFailure(
-                new IllegalArgumentException(
-                    "Unable to determine supported model for ["
-                        + model.getConfigurations().getInferenceEntityId()
-                        + "] please verify the request and submit a bug report if necessary."
-                )
-            );
-        }
-    }
-
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_L2_NORM_SIMILARITY_ADDED;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java
index 1acf19c5373b7..8de791325a6df 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsFeatureFlag;
 import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings;
 import org.elasticsearch.xpack.inference.services.ServiceUtils;
 
@@ -87,12 +88,18 @@ protected static ElasticsearchInternalServiceSettings.Builder fromMap(
         String modelId = extractOptionalString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
 
         if (numAllocations == null && adaptiveAllocationsSettings == null) {
-            validationException.addValidationError(
-                ServiceUtils.missingOneOfSettingsErrorMsg(
-                    List.of(NUM_ALLOCATIONS, ADAPTIVE_ALLOCATIONS),
-                    ModelConfigurations.SERVICE_SETTINGS
-                )
-            );
+            if (AdaptiveAllocationsFeatureFlag.isEnabled()) {
+                validationException.addValidationError(
+                    ServiceUtils.missingOneOfSettingsErrorMsg(
+                        List.of(NUM_ALLOCATIONS, ADAPTIVE_ALLOCATIONS),
+                        ModelConfigurations.SERVICE_SETTINGS
+                    )
+                );
+            } else {
+                validationException.addValidationError(
+                    ServiceUtils.missingSettingErrorMsg(NUM_ALLOCATIONS, ModelConfigurations.SERVICE_SETTINGS)
+                );
+            }
         }
 
         // if an error occurred while parsing, we'll set these to an invalid value, so we don't accidentally get a
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java
index 03d7682600e7c..948117954a63f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
 import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
-import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
 import org.elasticsearch.xpack.core.ml.action.InferModelAction;
 import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults;
@@ -43,26 +42,15 @@
 import java.util.Map;
 import java.util.Set;
 
-import static org.elasticsearch.xpack.core.ClientHelper.INFERENCE_ORIGIN;
-import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap;
+import static org.elasticsearch.xpack.inference.services.elser.ElserModels.ELSER_V2_MODEL;
+import static org.elasticsearch.xpack.inference.services.elser.ElserModels.ELSER_V2_MODEL_LINUX_X86;
 
 public class ElserInternalService extends BaseElasticsearchInternalService {
 
     public static final String NAME = "elser";
 
-    static final String ELSER_V1_MODEL = ".elser_model_1";
-    // Default non platform specific v2 model
-    static final String ELSER_V2_MODEL = ".elser_model_2";
-    static final String ELSER_V2_MODEL_LINUX_X86 = ".elser_model_2_linux-x86_64";
-
-    public static Set<String> VALID_ELSER_MODEL_IDS = Set.of(
-        ElserInternalService.ELSER_V1_MODEL,
-        ElserInternalService.ELSER_V2_MODEL,
-        ElserInternalService.ELSER_V2_MODEL_LINUX_X86
-    );
-
     private static final String OLD_MODEL_ID_FIELD_NAME = "model_version";
 
     public ElserInternalService(InferenceServiceExtension.InferenceServiceFactoryContext context) {
@@ -251,31 +239,6 @@ private void checkCompatibleTaskType(TaskType taskType) {
         }
     }
 
-    @Override
-    public void isModelDownloaded(Model model, ActionListener<Boolean> listener) {
-        ActionListener<GetTrainedModelsAction.Response> getModelsResponseListener = listener.delegateFailure((delegate, response) -> {
-            if (response.getResources().count() < 1) {
-                delegate.onResponse(Boolean.FALSE);
-            } else {
-                delegate.onResponse(Boolean.TRUE);
-            }
-        });
-
-        if (model instanceof ElserInternalModel elserModel) {
-            String modelId = elserModel.getServiceSettings().modelId();
-            GetTrainedModelsAction.Request getRequest = new GetTrainedModelsAction.Request(modelId);
-            executeAsyncWithOrigin(client, INFERENCE_ORIGIN, GetTrainedModelsAction.INSTANCE, getRequest, getModelsResponseListener);
-        } else {
-            listener.onFailure(
-                new IllegalArgumentException(
-                    "Can not download model automatically for ["
-                        + model.getConfigurations().getInferenceEntityId()
-                        + "] you may need to download it through the trained models API or with eland."
-                )
-            );
-        }
-    }
-
     private static ElserMlNodeTaskSettings taskSettingsFromMap(TaskType taskType, Map<String, Object> config) {
         if (taskType != TaskType.SPARSE_EMBEDDING) {
             throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST);
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java
index 75797919b3616..fcbabd5a88fc6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java
@@ -18,8 +18,6 @@
 import java.util.Arrays;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.inference.services.elser.ElserInternalService.VALID_ELSER_MODEL_IDS;
-
 public class ElserInternalServiceSettings extends ElasticsearchInternalServiceSettings {
 
     public static final String NAME = "elser_mlnode_service_settings";
@@ -29,10 +27,10 @@ public static ElasticsearchInternalServiceSettings.Builder fromRequestMap(Map<St
         var baseSettings = ElasticsearchInternalServiceSettings.fromMap(map, validationException);
 
         String modelId = baseSettings.getModelId();
-        if (modelId != null && VALID_ELSER_MODEL_IDS.contains(modelId) == false) {
+        if (modelId != null && ElserModels.isValidModel(modelId) == false) {
             var ve = new ValidationException();
             ve.addValidationError(
-                "Unknown ELSER model ID [" + modelId + "]. Valid models are " + Arrays.toString(VALID_ELSER_MODEL_IDS.toArray())
+                "Unknown ELSER model ID [" + modelId + "]. Valid models are " + Arrays.toString(ElserModels.VALID_ELSER_MODEL_IDS.toArray())
             );
             throw ve;
         }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserModels.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserModels.java
new file mode 100644
index 0000000000000..af94d2813dd2c
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserModels.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elser;
+
+import java.util.Set;
+
+public class ElserModels {
+
+    public static final String ELSER_V1_MODEL = ".elser_model_1";
+    // Default non platform specific v2 model
+    public static final String ELSER_V2_MODEL = ".elser_model_2";
+    public static final String ELSER_V2_MODEL_LINUX_X86 = ".elser_model_2_linux-x86_64";
+
+    public static Set<String> VALID_ELSER_MODEL_IDS = Set.of(
+        ElserModels.ELSER_V1_MODEL,
+        ElserModels.ELSER_V2_MODEL,
+        ElserModels.ELSER_V2_MODEL_LINUX_X86
+    );
+
+    public static boolean isValidModel(String model) {
+        return VALID_ELSER_MODEL_IDS.contains(model);
+    }
+
+    public static boolean isValidEisModel(String model) {
+        return ELSER_V2_MODEL.equals(model);
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java
index c4ab8bd99b8b0..96436996098f8 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java
@@ -187,7 +187,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_COMPLETION_INFERENCE_SERVICE_ADDED;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java
index 2d5a407f3c1a6..922bbcda2c746 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionTaskSettings.java
@@ -82,7 +82,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_COMPLETION_INFERENCE_SERVICE_ADDED;
+        return TransportVersions.V_8_14_0;
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
index d474e935fbda7..6ef1f6f0feefe 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
@@ -150,7 +150,7 @@ public OpenAiEmbeddingsServiceSettings(
         @Nullable RateLimitSettings rateLimitSettings
     ) {
         this.uri = uri;
-        this.modelId = modelId;
+        this.modelId = Objects.requireNonNull(modelId);
         this.organizationId = organizationId;
         this.similarity = similarity;
         this.dimensions = dimensions;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/ApmInferenceStats.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/ApmInferenceStats.java
new file mode 100644
index 0000000000000..ae14a0792dead
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/ApmInferenceStats.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.telemetry;
+
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.telemetry.metric.LongCounter;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+import java.util.HashMap;
+import java.util.Objects;
+
+public class ApmInferenceStats implements InferenceStats {
+    private final LongCounter inferenceAPMRequestCounter;
+
+    public ApmInferenceStats(LongCounter inferenceAPMRequestCounter) {
+        this.inferenceAPMRequestCounter = Objects.requireNonNull(inferenceAPMRequestCounter);
+    }
+
+    @Override
+    public void incrementRequestCount(Model model) {
+        var service = model.getConfigurations().getService();
+        var taskType = model.getTaskType();
+        var modelId = model.getServiceSettings().modelId();
+
+        var attributes = new HashMap<String, Object>(5);
+        attributes.put("service", service);
+        attributes.put("task_type", taskType.toString());
+        if (modelId != null) {
+            attributes.put("model_id", modelId);
+        }
+
+        inferenceAPMRequestCounter.incrementBy(1, attributes);
+    }
+
+    public static ApmInferenceStats create(MeterRegistry meterRegistry) {
+        return new ApmInferenceStats(
+            meterRegistry.registerLongCounter(
+                "es.inference.requests.count.total",
+                "Inference API request counts for a particular service, task type, model ID",
+                "operations"
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceAPMStats.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceAPMStats.java
deleted file mode 100644
index 76977fef76045..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceAPMStats.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.telemetry;
-
-import org.elasticsearch.inference.Model;
-import org.elasticsearch.telemetry.metric.LongCounter;
-import org.elasticsearch.telemetry.metric.MeterRegistry;
-
-import java.util.Map;
-import java.util.Objects;
-
-public class InferenceAPMStats extends InferenceStats {
-
-    private final LongCounter inferenceAPMRequestCounter;
-
-    public InferenceAPMStats(Model model, MeterRegistry meterRegistry) {
-        super(model);
-        this.inferenceAPMRequestCounter = meterRegistry.registerLongCounter(
-            "es.inference.requests.count",
-            "Inference API request counts for a particular service, task type, model ID",
-            "operations"
-        );
-    }
-
-    @Override
-    public void increment() {
-        super.increment();
-        inferenceAPMRequestCounter.incrementBy(1, Map.of("service", service, "task_type", taskType.toString(), "model_id", modelId));
-    }
-
-    public static final class Factory {
-        private final MeterRegistry meterRegistry;
-
-        public Factory(MeterRegistry meterRegistry) {
-            this.meterRegistry = Objects.requireNonNull(meterRegistry);
-        }
-
-        public InferenceAPMStats newInferenceRequestAPMCounter(Model model) {
-            return new InferenceAPMStats(model, meterRegistry);
-        }
-    }
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceStats.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceStats.java
index d639f9da71f56..d080e818e45fc 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceStats.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceStats.java
@@ -8,52 +8,14 @@
 package org.elasticsearch.xpack.inference.telemetry;
 
 import org.elasticsearch.inference.Model;
-import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.xpack.core.inference.InferenceRequestStats;
 
-import java.util.Objects;
-import java.util.concurrent.atomic.LongAdder;
+public interface InferenceStats {
 
-public class InferenceStats implements Stats {
-    protected final String service;
-    protected final TaskType taskType;
-    protected final String modelId;
-    protected final LongAdder counter = new LongAdder();
+    /**
+     * Increment the counter for a particular value in a thread safe manner.
+     * @param model the model to increment request count for
+     */
+    void incrementRequestCount(Model model);
 
-    public static String key(Model model) {
-        StringBuilder builder = new StringBuilder();
-        builder.append(model.getConfigurations().getService());
-        builder.append(":");
-        builder.append(model.getTaskType());
-
-        if (model.getServiceSettings().modelId() != null) {
-            builder.append(":");
-            builder.append(model.getServiceSettings().modelId());
-        }
-
-        return builder.toString();
-    }
-
-    public InferenceStats(Model model) {
-        Objects.requireNonNull(model);
-
-        service = model.getConfigurations().getService();
-        taskType = model.getTaskType();
-        modelId = model.getServiceSettings().modelId();
-    }
-
-    @Override
-    public void increment() {
-        counter.increment();
-    }
-
-    @Override
-    public long getCount() {
-        return counter.sum();
-    }
-
-    @Override
-    public InferenceRequestStats toSerializableForm() {
-        return new InferenceRequestStats(service, taskType, modelId, getCount());
-    }
+    InferenceStats NOOP = model -> {};
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/Stats.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/Stats.java
deleted file mode 100644
index bb1e9c98fc2cb..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/Stats.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.telemetry;
-
-import org.elasticsearch.xpack.core.inference.SerializableStats;
-
-public interface Stats {
-
-    /**
-     * Increase the counter by one.
-     */
-    void increment();
-
-    /**
-     * Return the current value of the counter.
-     * @return the current value of the counter
-     */
-    long getCount();
-
-    /**
-     * Convert the object into a serializable form that can be written across nodes and returned in xcontent format.
-     * @return the serializable format of the object
-     */
-    SerializableStats toSerializableForm();
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/StatsMap.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/StatsMap.java
deleted file mode 100644
index 1cfecfb4507d6..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/StatsMap.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.telemetry;
-
-import org.elasticsearch.xpack.core.inference.SerializableStats;
-
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-/**
- * A map to provide tracking incrementing statistics.
- *
- * @param <Input> The input to derive the keys and values for the map
- * @param <Value> The type of the values stored in the map
- */
-public class StatsMap<Input, Value extends Stats> {
-
-    private final ConcurrentMap<String, Value> stats = new ConcurrentHashMap<>();
-    private final Function<Input, String> keyCreator;
-    private final Function<Input, Value> valueCreator;
-
-    /**
-     * @param keyCreator a function for creating a key in the map based on the input provided
-     * @param valueCreator a function for creating a value in the map based on the input provided
-     */
-    public StatsMap(Function<Input, String> keyCreator, Function<Input, Value> valueCreator) {
-        this.keyCreator = Objects.requireNonNull(keyCreator);
-        this.valueCreator = Objects.requireNonNull(valueCreator);
-    }
-
-    /**
-     * Increment the counter for a particular value in a thread safe manner.
-     * @param input the input to derive the appropriate key in the map
-     */
-    public void increment(Input input) {
-        var value = stats.computeIfAbsent(keyCreator.apply(input), key -> valueCreator.apply(input));
-        value.increment();
-    }
-
-    /**
-     * Build a map that can be serialized. This takes a snapshot of the current state. Any concurrent calls to increment may or may not
-     * be represented in the resulting serializable map.
-     * @return a map that is more easily serializable
-     */
-    public Map<String, SerializableStats> toSerializableMap() {
-        return stats.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().toSerializableForm()));
-    }
-}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java
new file mode 100644
index 0000000000000..b50ea9e5ee224
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/EmptySecretSettingsTests.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.inference.EmptySecretSettings;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+public class EmptySecretSettingsTests extends AbstractWireSerializingTestCase<EmptySecretSettings> {
+
+    public static EmptySecretSettings createRandom() {
+        return EmptySecretSettings.INSTANCE; // no options to randomise
+    }
+
+    @Override
+    protected Writeable.Reader<EmptySecretSettings> instanceReader() {
+        return EmptySecretSettings::new;
+    }
+
+    @Override
+    protected EmptySecretSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected EmptySecretSettings mutateInstance(EmptySecretSettings instance) {
+        // All instances are the same and have no fields, nothing to mutate
+        return null;
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/InputTypeTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/InputTypeTests.java
index d275d00373cbe..055b4581e067b 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/InputTypeTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/InputTypeTests.java
@@ -14,4 +14,8 @@ public class InputTypeTests extends ESTestCase {
     public static InputType randomWithoutUnspecified() {
         return randomFrom(InputType.INGEST, InputType.SEARCH, InputType.CLUSTERING, InputType.CLASSIFICATION);
     }
+
+    public static InputType randomWithIngestAndSearch() {
+        return randomFrom(InputType.INGEST, InputType.SEARCH);
+    }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java
index f63a6369b21a6..d78ea7933e836 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
 import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults;
+import org.elasticsearch.xpack.inference.mapper.SemanticTextField;
 import org.elasticsearch.xpack.inference.model.TestModel;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
 import org.junit.After;
@@ -56,7 +57,9 @@
 import static org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilter.DEFAULT_BATCH_SIZE;
 import static org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilter.getIndexRequestOrNull;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticText;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticTextInput;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSparseEmbeddings;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.semanticTextFieldFromChunkedInferenceResults;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.toChunkedResult;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -331,16 +334,34 @@ private static BulkItemRequest[] randomBulkItemRequest(
         for (var entry : fieldInferenceMap.values()) {
             String field = entry.getName();
             var model = modelMap.get(entry.getInferenceId());
-            String text = randomAlphaOfLengthBetween(10, 20);
-            docMap.put(field, text);
-            expectedDocMap.put(field, text);
+            Object inputObject = randomSemanticTextInput();
+            String inputText = inputObject.toString();
+            docMap.put(field, inputObject);
+            expectedDocMap.put(field, inputText);
             if (model == null) {
                 // ignore results, the doc should fail with a resource not found exception
                 continue;
             }
-            var result = randomSemanticText(field, model, List.of(text), requestContentType);
-            model.putResult(text, toChunkedResult(result));
-            expectedDocMap.put(field, result);
+
+            SemanticTextField semanticTextField;
+            // The model is not field aware and that is why we are skipping the embedding generation process for existing values.
+            // This prevents a situation where embeddings in the expected docMap do not match those in the model, which could happen if
+            // embeddings were overwritten.
+            if (model.hasResult(inputText)) {
+                ChunkedInferenceServiceResults results = model.getResults(inputText);
+                semanticTextField = semanticTextFieldFromChunkedInferenceResults(
+                    field,
+                    model,
+                    List.of(inputText),
+                    results,
+                    requestContentType
+                );
+            } else {
+                semanticTextField = randomSemanticText(field, model, List.of(inputText), requestContentType);
+                model.putResult(inputText, toChunkedResult(semanticTextField));
+            }
+
+            expectedDocMap.put(field, semanticTextField);
         }
 
         int requestId = randomIntBetween(0, Integer.MAX_VALUE);
@@ -383,5 +404,9 @@ ChunkedInferenceServiceResults getResults(String text) {
         void putResult(String text, ChunkedInferenceServiceResults result) {
             resultMap.put(text, result);
         }
+
+        boolean hasResult(String text) {
+            return resultMap.containsKey(text);
+        }
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java
new file mode 100644
index 0000000000000..1081a60ba6866
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/elastic/ElasticInferenceServiceActionCreatorTests.java
@@ -0,0 +1,289 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.elastic;
+
+import org.apache.http.HttpHeaders;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.http.MockResponse;
+import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.inference.action.InferenceAction;
+import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
+import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests;
+import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
+import org.elasticsearch.xpack.inference.results.SparseEmbeddingResultsTests;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModelTests;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool;
+import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty;
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
+import static org.elasticsearch.xpack.inference.external.http.retry.RetrySettingsTests.buildSettingsWithRetryFields;
+import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender;
+import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+
+public class ElasticInferenceServiceActionCreatorTests extends ESTestCase {
+
+    private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS);
+    private final MockWebServer webServer = new MockWebServer();
+    private ThreadPool threadPool;
+    private HttpClientManager clientManager;
+
+    @Before
+    public void init() throws Exception {
+        webServer.start();
+        threadPool = createThreadPool(inferenceUtilityPool());
+        clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class));
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        clientManager.close();
+        terminate(threadPool);
+        webServer.close();
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testExecute_ReturnsSuccessfulResponse_ForElserAction() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var sender = createSender(senderFactory)) {
+            sender.start();
+
+            String responseJson = """
+                {
+                    "data": [
+                        {
+                            "hello": 2.1259406,
+                            "greet": 1.7073475
+                        }
+                    ]
+                }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer));
+            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+            var action = actionCreator.create(model);
+
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            action.execute(new DocumentsOnlyInput(List.of("hello world")), InferenceAction.Request.DEFAULT_TIMEOUT, listener);
+
+            var result = listener.actionGet(TIMEOUT);
+
+            assertThat(
+                result.asMap(),
+                is(
+                    SparseEmbeddingResultsTests.buildExpectationSparseEmbeddings(
+                        List.of(
+                            new SparseEmbeddingResultsTests.EmbeddingExpectation(Map.of("hello", 2.1259406f, "greet", 1.7073475f), false)
+                        )
+                    )
+                )
+            );
+
+            assertThat(webServer.requests(), hasSize(1));
+            assertNull(webServer.requests().get(0).getUri().getQuery());
+            assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType()));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            assertThat(requestMap.size(), is(1));
+            assertThat(requestMap.get("input"), instanceOf(List.class));
+            var inputList = (List<String>) requestMap.get("input");
+            assertThat(inputList, contains("hello world"));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testSend_FailsFromInvalidResponseFormat_ForElserAction() throws IOException {
+        // timeout as zero for no retries
+        var settings = buildSettingsWithRetryFields(
+            TimeValue.timeValueMillis(1),
+            TimeValue.timeValueMinutes(1),
+            TimeValue.timeValueSeconds(0)
+        );
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager, settings);
+
+        try (var sender = createSender(senderFactory)) {
+            sender.start();
+
+            // This will fail because the expected output is {"data": [{...}]}
+            String responseJson = """
+                {
+                    "data": {
+                        "hello": 2.1259406
+                    }
+                }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer));
+            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+            var action = actionCreator.create(model);
+
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            action.execute(new DocumentsOnlyInput(List.of("hello world")), InferenceAction.Request.DEFAULT_TIMEOUT, listener);
+
+            var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT));
+            assertThat(
+                thrownException.getMessage(),
+                is("Failed to parse object: expecting token of type [START_ARRAY] but found [START_OBJECT]")
+            );
+
+            assertThat(webServer.requests(), hasSize(1));
+            assertNull(webServer.requests().get(0).getUri().getQuery());
+            assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType()));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            assertThat(requestMap.size(), is(1));
+            assertThat(requestMap.get("input"), instanceOf(List.class));
+            var inputList = (List<String>) requestMap.get("input");
+            assertThat(inputList, contains("hello world"));
+        }
+    }
+
+    public void testExecute_ReturnsSuccessfulResponse_AfterTruncating() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var sender = createSender(senderFactory)) {
+            sender.start();
+
+            String responseJsonContentTooLarge = """
+                {
+                    "error": "Input validation error: `input` must have less than 512 tokens. Given: 571",
+                    "error_type": "Validation"
+                }
+                """;
+
+            String responseJson = """
+                {
+                    "data": [
+                        {
+                            "hello": 2.1259406,
+                            "greet": 1.7073475
+                        }
+                    ]
+                }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(413).setBody(responseJsonContentTooLarge));
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer));
+            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+            var action = actionCreator.create(model);
+
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            action.execute(new DocumentsOnlyInput(List.of("hello world")), InferenceAction.Request.DEFAULT_TIMEOUT, listener);
+
+            var result = listener.actionGet(TIMEOUT);
+
+            assertThat(
+                result.asMap(),
+                is(
+                    SparseEmbeddingResultsTests.buildExpectationSparseEmbeddings(
+                        List.of(
+                            new SparseEmbeddingResultsTests.EmbeddingExpectation(Map.of("hello", 2.1259406f, "greet", 1.7073475f), true)
+                        )
+                    )
+                )
+            );
+
+            assertThat(webServer.requests(), hasSize(2));
+            {
+                assertNull(webServer.requests().get(0).getUri().getQuery());
+                assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType()));
+
+                var initialRequestAsMap = entityAsMap(webServer.requests().get(0).getBody());
+                var initialInputs = initialRequestAsMap.get("input");
+                assertThat(initialInputs, is(List.of("hello world")));
+            }
+            {
+                assertNull(webServer.requests().get(1).getUri().getQuery());
+                assertThat(webServer.requests().get(1).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType()));
+
+                var truncatedRequest = entityAsMap(webServer.requests().get(1).getBody());
+                var truncatedInputs = truncatedRequest.get("input");
+                assertThat(truncatedInputs, is(List.of("hello")));
+            }
+        }
+    }
+
+    public void testExecute_TruncatesInputBeforeSending() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var sender = createSender(senderFactory)) {
+            sender.start();
+
+            String responseJson = """
+                {
+                    "data": [
+                        {
+                            "hello": 2.1259406,
+                            "greet": 1.7073475
+                        }
+                    ]
+                }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            // truncated to 1 token = 3 characters
+            var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer), 1);
+            var actionCreator = new ElasticInferenceServiceActionCreator(sender, createWithEmptySettings(threadPool));
+            var action = actionCreator.create(model);
+
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            action.execute(new DocumentsOnlyInput(List.of("hello world")), InferenceAction.Request.DEFAULT_TIMEOUT, listener);
+
+            var result = listener.actionGet(TIMEOUT);
+
+            assertThat(
+                result.asMap(),
+                is(
+                    SparseEmbeddingResultsTests.buildExpectationSparseEmbeddings(
+                        List.of(
+                            new SparseEmbeddingResultsTests.EmbeddingExpectation(Map.of("hello", 2.1259406f, "greet", 1.7073475f), true)
+                        )
+                    )
+                )
+            );
+
+            assertThat(webServer.requests(), hasSize(1));
+            assertNull(webServer.requests().get(0).getUri().getQuery());
+            assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType()));
+
+            var initialRequestAsMap = entityAsMap(webServer.requests().get(0).getBody());
+            var initialInputs = initialRequestAsMap.get("input");
+            assertThat(initialInputs, is(List.of("hel")));
+        }
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandlerTests.java
new file mode 100644
index 0000000000000..ea30ee29ff5a8
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandlerTests.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.elastic;
+
+import org.apache.http.Header;
+import org.apache.http.HeaderElement;
+import org.apache.http.HttpResponse;
+import org.apache.http.StatusLine;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.http.retry.ContentTooLargeException;
+import org.elasticsearch.xpack.inference.external.http.retry.RetryException;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.hamcrest.MatcherAssert;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.core.Is.is;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ElasticInferenceServiceResponseHandlerTests extends ESTestCase {
+
+    public void testCheckForFailureStatusCode_DoesNotThrowFor200() {
+        callCheckForFailureStatusCode(200, "id");
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor400() {
+        var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(400, "id"));
+        assertFalse(exception.shouldRetry());
+        MatcherAssert.assertThat(
+            exception.getCause().getMessage(),
+            containsString("Received a bad request status code for request from inference entity id [id] status [400]")
+        );
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST));
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor405() {
+        var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(405, "id"));
+        assertFalse(exception.shouldRetry());
+        MatcherAssert.assertThat(
+            exception.getCause().getMessage(),
+            containsString("Received a method not allowed status code for request from inference entity id [id] status [405]")
+        );
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.METHOD_NOT_ALLOWED));
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor413() {
+        var exception = expectThrows(ContentTooLargeException.class, () -> callCheckForFailureStatusCode(413, "id"));
+        assertTrue(exception.shouldRetry());
+        MatcherAssert.assertThat(
+            exception.getCause().getMessage(),
+            containsString("Received a content too large status code for request from inference entity id [id] status [413]")
+        );
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.REQUEST_ENTITY_TOO_LARGE));
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor500_WithShouldRetryTrue() {
+        var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(500, "id"));
+        assertTrue(exception.shouldRetry());
+        MatcherAssert.assertThat(
+            exception.getCause().getMessage(),
+            containsString("Received a server error status code for request from inference entity id [id] status [500]")
+        );
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST));
+    }
+
+    public void testCheckForFailureStatusCode_ThrowsFor402() {
+        var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(402, "id"));
+        assertFalse(exception.shouldRetry());
+        MatcherAssert.assertThat(
+            exception.getCause().getMessage(),
+            containsString("Received an unsuccessful status code for request from inference entity id [id] status [402]")
+        );
+        MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.PAYMENT_REQUIRED));
+    }
+
+    private static void callCheckForFailureStatusCode(int statusCode, String modelId) {
+        callCheckForFailureStatusCode(statusCode, null, modelId);
+    }
+
+    private static void callCheckForFailureStatusCode(int statusCode, @Nullable String errorMessage, String modelId) {
+        var statusLine = mock(StatusLine.class);
+        when(statusLine.getStatusCode()).thenReturn(statusCode);
+
+        var httpResponse = mock(HttpResponse.class);
+        when(httpResponse.getStatusLine()).thenReturn(statusLine);
+        var header = mock(Header.class);
+        when(header.getElements()).thenReturn(new HeaderElement[] {});
+        when(httpResponse.getFirstHeader(anyString())).thenReturn(header);
+
+        String responseJson = Strings.format("""
+                {
+                    "message": "%s"
+                }
+            """, errorMessage);
+
+        var mockRequest = mock(Request.class);
+        when(mockRequest.getInferenceEntityId()).thenReturn(modelId);
+        var httpResult = new HttpResult(httpResponse, errorMessage == null ? new byte[] {} : responseJson.getBytes(StandardCharsets.UTF_8));
+        var handler = new ElasticInferenceServiceResponseHandler("", (request, result) -> null);
+
+        handler.checkForFailureStatusCode(mockRequest, httpResult);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java
index c2842a1278a49..f70ab43908827 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java
@@ -10,7 +10,6 @@
 import org.apache.http.ConnectionClosedException;
 import org.apache.http.HttpResponse;
 import org.apache.http.StatusLine;
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
@@ -80,7 +79,7 @@ public void testSend_CallsSenderAgain_AfterValidateResponseThrowsAnException() t
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1);
 
         assertThat(listener.actionGet(TIMEOUT), is(inferenceResults));
         verify(httpClient, times(2)).send(any(), any(), any());
@@ -111,7 +110,7 @@ public void testSend_CallsSenderAgain_WhenAFailureStatusCodeIsReturned() throws
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1);
 
         assertThat(listener.actionGet(TIMEOUT), is(inferenceResults));
         verify(httpClient, times(2)).send(any(), any(), any());
@@ -139,7 +138,7 @@ public void testSend_CallsSenderAgain_WhenParsingFailsOnce() throws IOException
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1);
 
         assertThat(listener.actionGet(TIMEOUT), is(inferenceResults));
         verify(httpClient, times(2)).send(any(), any(), any());
@@ -167,7 +166,7 @@ public void testSend_DoesNotCallSenderAgain_WhenParsingFailsWithNonRetryableExce
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 0);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 0);
 
         var thrownException = expectThrows(IllegalStateException.class, () -> listener.actionGet(TIMEOUT));
         assertThat(thrownException.getMessage(), is("failed"));
@@ -202,7 +201,7 @@ public void testSend_CallsSenderAgain_WhenHttpResultListenerCallsOnFailureOnce()
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1);
 
         assertThat(listener.actionGet(TIMEOUT), is(inferenceResults));
         verify(httpClient, times(2)).send(any(), any(), any());
@@ -235,7 +234,7 @@ public void testSend_CallsSenderAgain_WhenHttpResultListenerCallsOnFailureOnce_W
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1);
 
         assertThat(listener.actionGet(TIMEOUT), is(inferenceResults));
         verify(httpClient, times(2)).send(any(), any(), any());
@@ -268,7 +267,7 @@ public void testSend_CallsSenderAgain_WhenHttpResultListenerCallsOnFailureOnceWi
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1);
 
         assertThat(listener.actionGet(TIMEOUT), is(inferenceResults));
         verify(httpClient, times(2)).send(any(), any(), any());
@@ -295,7 +294,7 @@ public void testSend_ReturnsFailure_WhenHttpResultListenerCallsOnFailureOnceWith
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 0);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 0);
 
         var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT));
         assertThat(thrownException.getMessage(), is("Invalid host [null], please check that the URL is correct."));
@@ -317,10 +316,7 @@ public void testSend_ReturnsElasticsearchExceptionFailure_WhenTheHttpClientThrow
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(
-            () -> retrier.send(mock(Logger.class), mockRequest("id"), HttpClientContext.create(), () -> false, handler, listener),
-            0
-        );
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest("id"), () -> false, handler, listener), 0);
 
         var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT));
         assertThat(thrownException.getMessage(), is("Http client failed to send request from inference entity id [id]"));
@@ -354,7 +350,7 @@ public void testSend_ReturnsFailure_WhenValidateResponseThrowsAnException_AfterO
         var retrier = createRetrier(sender);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1);
 
         var thrownException = expectThrows(IllegalStateException.class, () -> listener.actionGet(TIMEOUT));
         assertThat(thrownException.getMessage(), is("failed again"));
@@ -391,7 +387,7 @@ public void testSend_ReturnsFailure_WhenValidateResponseThrowsAnElasticsearchExc
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1);
 
         var thrownException = expectThrows(RetryException.class, () -> listener.actionGet(TIMEOUT));
         assertThat(thrownException.getMessage(), is("failed again"));
@@ -423,7 +419,7 @@ public void testSend_ReturnsFailure_WhenHttpResultsListenerCallsOnFailure_AfterO
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1);
 
         var thrownException = expectThrows(RetryException.class, () -> listener.actionGet(TIMEOUT));
         assertThat(thrownException.getMessage(), is("failed again"));
@@ -449,7 +445,7 @@ public void testSend_ReturnsFailure_WhenHttpResultsListenerCallsOnFailure_WithNo
         var retrier = createRetrier(httpClient);
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 0);
+        executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 0);
 
         var thrownException = expectThrows(IllegalStateException.class, () -> listener.actionGet(TIMEOUT));
         assertThat(thrownException.getMessage(), is("failed"));
@@ -484,7 +480,7 @@ public void testSend_DoesNotRetryIndefinitely() throws IOException {
             );
 
             var listener = new PlainActionFuture<InferenceServiceResults>();
-            retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener);
+            retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener);
 
             // Assert that the retrying sender stopped after max retires even though the exception is retryable
             var thrownException = expectThrows(UncategorizedExecutionException.class, () -> listener.actionGet(TIMEOUT));
@@ -524,7 +520,7 @@ public void testSend_DoesNotRetryIndefinitely_WithAlwaysRetryingResponseHandler(
             );
 
             var listener = new PlainActionFuture<InferenceServiceResults>();
-            retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener);
+            retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener);
 
             // Assert that the retrying sender stopped after max retires
             var thrownException = expectThrows(UncategorizedExecutionException.class, () -> listener.actionGet(TIMEOUT));
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java
index 762a3a74184a4..e09e4968571e5 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java
@@ -123,7 +123,7 @@ public void testIsTerminated_AfterStopFromSeparateThread() {
             waitToShutdown.countDown();
             waitToReturnFromSend.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS);
             return Void.TYPE;
-        }).when(requestSender).send(any(), any(), any(), any(), any(), any());
+        }).when(requestSender).send(any(), any(), any(), any(), any());
 
         var service = createRequestExecutorService(null, requestSender);
 
@@ -203,7 +203,7 @@ public void testTaskThrowsError_CallsOnFailure() {
         doAnswer(invocation -> {
             service.shutdown();
             throw new IllegalArgumentException("failed");
-        }).when(requestSender).send(any(), any(), any(), any(), any(), any());
+        }).when(requestSender).send(any(), any(), any(), any(), any());
 
         PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
 
@@ -270,13 +270,13 @@ public void testExecute_PreservesThreadContext() throws InterruptedException, Ex
             assertNull(serviceThreadContext.getHeader(headerKey));
 
             @SuppressWarnings("unchecked")
-            ActionListener<InferenceServiceResults> listener = (ActionListener<InferenceServiceResults>) invocation.getArguments()[5];
+            ActionListener<InferenceServiceResults> listener = invocation.getArgument(4, ActionListener.class);
             listener.onResponse(null);
 
             waitToShutdown.countDown();
             waitToReturnFromSend.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS);
             return Void.TYPE;
-        }).when(requestSender).send(any(), any(), any(), any(), any(), any());
+        }).when(requestSender).send(any(), any(), any(), any(), any());
 
         var finishedOnResponse = new CountDownLatch(1);
         ActionListener<InferenceServiceResults> listener = new ActionListener<>() {
@@ -422,7 +422,7 @@ public void testChangingCapacity_SetsCapacityToTwo() throws ExecutionException,
             waitToShutdown.countDown();
             waitToReturnFromSend.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS);
             return Void.TYPE;
-        }).when(requestSender).send(any(), any(), any(), any(), any(), any());
+        }).when(requestSender).send(any(), any(), any(), any(), any());
 
         Future<?> executorTermination = submitShutdownRequest(waitToShutdown, waitToReturnFromSend, service);
 
@@ -467,7 +467,7 @@ public void testChangingCapacity_DoesNotRejectsOverflowTasks_BecauseOfQueueFull(
             waitToShutdown.countDown();
             waitToReturnFromSend.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS);
             return Void.TYPE;
-        }).when(requestSender).send(any(), any(), any(), any(), any(), any());
+        }).when(requestSender).send(any(), any(), any(), any(), any());
 
         Future<?> executorTermination = submitShutdownRequest(waitToShutdown, waitToReturnFromSend, service);
 
@@ -528,7 +528,7 @@ public void testChangingCapacity_ToZero_SetsQueueCapacityToUnbounded() throws IO
             waitToShutdown.countDown();
             waitToReturnFromSend.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS);
             return Void.TYPE;
-        }).when(requestSender).send(any(), any(), any(), any(), any(), any());
+        }).when(requestSender).send(any(), any(), any(), any(), any());
 
         Future<?> executorTermination = submitShutdownRequest(waitToShutdown, waitToReturnFromSend, service);
 
@@ -598,11 +598,11 @@ public void testDoesNotExecuteTask_WhenCannotReserveTokens_AndThenCanReserve_And
         doAnswer(invocation -> {
             service.shutdown();
             return Void.TYPE;
-        }).when(requestSender).send(any(), any(), any(), any(), any(), any());
+        }).when(requestSender).send(any(), any(), any(), any(), any());
 
         service.start();
 
-        verify(requestSender, times(1)).send(any(), any(), any(), any(), any(), any());
+        verify(requestSender, times(1)).send(any(), any(), any(), any(), any());
     }
 
     public void testRemovesRateLimitGroup_AfterStaleDuration() {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManagerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManagerTests.java
index 8b7c01ae133cf..d8a1f2c4227e4 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManagerTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManagerTests.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -47,7 +46,6 @@ public static RequestManager createMock(RequestSender requestSender, String infe
             requestSender.send(
                 mock(Logger.class),
                 RequestTests.mockRequest(inferenceEntityId),
-                HttpClientContext.create(),
                 () -> false,
                 mock(ResponseHandler.class),
                 listener
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java
new file mode 100644
index 0000000000000..6aaab219c331d
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsTaskSettings;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+
+public class AlibabaCloudSearchEmbeddingsRequestEntityTests extends ESTestCase {
+    public void testXContent_WritesAllFields_WhenTheyAreDefined() throws IOException {
+        var entity = new AlibabaCloudSearchEmbeddingsRequestEntity(
+            List.of("abc"),
+            new AlibabaCloudSearchEmbeddingsTaskSettings(InputType.INGEST)
+        );
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        MatcherAssert.assertThat(xContentResult, is("""
+            {"input":["abc"],"input_type":"document"}"""));
+    }
+
+    public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws IOException {
+        var entity = new AlibabaCloudSearchEmbeddingsRequestEntity(List.of("abc"), AlibabaCloudSearchEmbeddingsTaskSettings.EMPTY_SETTINGS);
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        MatcherAssert.assertThat(xContentResult, is("""
+            {"input":["abc"]}"""));
+    }
+
+    public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() {
+        var thrownException = expectThrows(
+            AssertionError.class,
+            () -> AlibabaCloudSearchEmbeddingsRequestEntity.covertToString(InputType.UNSPECIFIED)
+        );
+        MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]"));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestTests.java
new file mode 100644
index 0000000000000..378401f589b19
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestTests.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModelTests;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsServiceSettingsTests;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsTaskSettingsTests;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests.getSecretSettingsMap;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
+public class AlibabaCloudSearchEmbeddingsRequestTests extends ESTestCase {
+    public void testCreateRequest() throws IOException {
+        var request = createRequest(
+            List.of("abc"),
+            AlibabaCloudSearchEmbeddingsModelTests.createModel(
+                "embedding_test",
+                TaskType.TEXT_EMBEDDING,
+                AlibabaCloudSearchEmbeddingsServiceSettingsTests.getServiceSettingsMap("embeddings_test", "host", "default"),
+                AlibabaCloudSearchEmbeddingsTaskSettingsTests.getTaskSettingsMap(null),
+                getSecretSettingsMap("secret")
+            )
+        );
+
+        var httpRequest = request.createHttpRequest();
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+        MatcherAssert.assertThat(
+            httpPost.getURI().toString(),
+            is("https://host/v3/openapi/workspaces/default/text-embedding/embeddings_test")
+        );
+        MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+        MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret"));
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"))));
+    }
+
+    public static AlibabaCloudSearchEmbeddingsRequest createRequest(List<String> input, AlibabaCloudSearchEmbeddingsModel model) {
+        var account = new AlibabaCloudSearchAccount(model.getSecretSettings().apiKey());
+        return new AlibabaCloudSearchEmbeddingsRequest(account, input, model);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRerankRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRerankRequestEntityTests.java
new file mode 100644
index 0000000000000..8f981d64d36eb
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchRerankRequestEntityTests.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.rerank.AlibabaCloudSearchRerankTaskSettings;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+
+public class AlibabaCloudSearchRerankRequestEntityTests extends ESTestCase {
+    public void testXContent_WritesAllFields_WhenTheyAreDefined() throws IOException {
+        var entity = new AlibabaCloudSearchRerankRequestEntity("query", List.of("abc"), new AlibabaCloudSearchRerankTaskSettings());
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        MatcherAssert.assertThat(xContentResult, is("""
+            {"query":"query","docs":["abc"]}"""));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntityTests.java
new file mode 100644
index 0000000000000..6ae209bc3c6f1
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntityTests.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseTaskSettings;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+
+public class AlibabaCloudSearchSparseRequestEntityTests extends ESTestCase {
+    public void testXContent_WritesAllFields_WhenTheyAreDefined() throws IOException {
+        var entity = new AlibabaCloudSearchSparseRequestEntity(
+            List.of("abc"),
+            new AlibabaCloudSearchSparseTaskSettings(InputType.INGEST, true)
+        );
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        MatcherAssert.assertThat(xContentResult, is("""
+            {"input":["abc"],"input_type":"document","return_token":true}"""));
+    }
+
+    public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws IOException {
+        var entity = new AlibabaCloudSearchSparseRequestEntity(List.of("abc"), AlibabaCloudSearchSparseTaskSettings.EMPTY_SETTINGS);
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        MatcherAssert.assertThat(xContentResult, is("""
+            {"input":["abc"]}"""));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestTests.java
new file mode 100644
index 0000000000000..74fc225820641
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestTests.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.alibabacloudsearch;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.alibabacloudsearch.AlibabaCloudSearchAccount;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseModelTests;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseServiceSettingsTests;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse.AlibabaCloudSearchSparseTaskSettingsTests;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests.getSecretSettingsMap;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
+public class AlibabaCloudSearchSparseRequestTests extends ESTestCase {
+    public void testCreateRequest() throws IOException {
+        var request = createRequest(
+            List.of("abc"),
+            AlibabaCloudSearchSparseModelTests.createModel(
+                "embedding_test",
+                TaskType.TEXT_EMBEDDING,
+                AlibabaCloudSearchSparseServiceSettingsTests.getServiceSettingsMap("embeddings_test", "host", "default"),
+                AlibabaCloudSearchSparseTaskSettingsTests.getTaskSettingsMap(null, null),
+                getSecretSettingsMap("secret")
+            )
+        );
+
+        var httpRequest = request.createHttpRequest();
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+        MatcherAssert.assertThat(
+            httpPost.getURI().toString(),
+            is("https://host/v3/openapi/workspaces/default/text-sparse-embedding/embeddings_test")
+        );
+        MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+        MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret"));
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"))));
+    }
+
+    public static AlibabaCloudSearchSparseRequest createRequest(List<String> input, AlibabaCloudSearchSparseModel model) {
+        var account = new AlibabaCloudSearchAccount(model.getSecretSettings().apiKey());
+        return new AlibabaCloudSearchSparseRequest(account, input, model);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java
new file mode 100644
index 0000000000000..7b10cf600275c
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestEntityTests.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.elastic;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString;
+
+public class ElasticInferenceServiceSparseEmbeddingsRequestEntityTests extends ESTestCase {
+
+    public void testToXContent_SingleInput() throws IOException {
+        var entity = new ElasticInferenceServiceSparseEmbeddingsRequestEntity(List.of("abc"));
+        String xContentString = xContentEntityToString(entity);
+        assertThat(xContentString, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "input": ["abc"]
+            }"""));
+    }
+
+    public void testToXContent_MultipleInputs() throws IOException {
+        var entity = new ElasticInferenceServiceSparseEmbeddingsRequestEntity(List.of("abc", "def"));
+        String xContentString = xContentEntityToString(entity);
+        assertThat(xContentString, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "input": [
+                    "abc",
+                    "def"
+                ]
+            }
+            """));
+    }
+
+    private String xContentEntityToString(ElasticInferenceServiceSparseEmbeddingsRequestEntity entity) throws IOException {
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        return Strings.toString(builder);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java
new file mode 100644
index 0000000000000..0f2c859fb62d5
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/elastic/ElasticInferenceServiceSparseEmbeddingsRequestTests.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.elastic;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.common.Truncator;
+import org.elasticsearch.xpack.inference.common.TruncatorTests;
+import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsModelTests;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
+public class ElasticInferenceServiceSparseEmbeddingsRequestTests extends ESTestCase {
+
+    public void testCreateHttpRequest() throws IOException {
+        var url = "http://eis-gateway.com";
+        var input = "input";
+
+        var request = createRequest(url, input);
+        var httpRequest = request.createHttpRequest();
+
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+        assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        assertThat(requestMap.size(), equalTo(1));
+        assertThat(requestMap.get("input"), is(List.of(input)));
+    }
+
+    public void testTruncate_ReducesInputTextSizeByHalf() throws IOException {
+        var url = "http://eis-gateway.com";
+        var input = "abcd";
+
+        var request = createRequest(url, input);
+        var truncatedRequest = request.truncate();
+
+        var httpRequest = truncatedRequest.createHttpRequest();
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        assertThat(requestMap, aMapWithSize(1));
+        assertThat(requestMap.get("input"), is(List.of("ab")));
+    }
+
+    public void testIsTruncated_ReturnsTrue() {
+        var url = "http://eis-gateway.com";
+        var input = "abcd";
+
+        var request = createRequest(url, input);
+        assertFalse(request.getTruncationInfo()[0]);
+
+        var truncatedRequest = request.truncate();
+        assertTrue(truncatedRequest.getTruncationInfo()[0]);
+    }
+
+    public ElasticInferenceServiceSparseEmbeddingsRequest createRequest(String url, String input) {
+        var embeddingsModel = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(url);
+
+        return new ElasticInferenceServiceSparseEmbeddingsRequest(
+            TruncatorTests.createTruncator(),
+            new Truncator.TruncationResult(List.of(input), new boolean[] { false }),
+            embeddingsModel
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntityTests.java
new file mode 100644
index 0000000000000..33fa6a2a542cb
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntityTests.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.alibabacloudsearch;
+
+import org.apache.http.HttpResponse;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchRequest;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class AlibabaCloudSearchEmbeddingsResponseEntityTests extends ESTestCase {
+    public void testFromResponse_CreatesResultsForASingleItem() throws IOException, URISyntaxException {
+        String responseJson = """
+            {
+                "request_id": "B4AB89C8-B135-xxxx-A6F8-2BAB801A2CE4",
+                "latency": 38,
+                "usage": {
+                    "token_count": 3072
+                },
+                "result": {
+                    "embeddings": [
+                        {
+                            "index": 0,
+                            "embedding": [
+                                -0.02868066355586052,
+                                0.022033605724573135
+                            ]
+                        }
+                    ]
+                }
+            }
+            """;
+
+        AlibabaCloudSearchRequest request = mock(AlibabaCloudSearchRequest.class);
+        URI uri = new URI("mock_uri");
+        when(request.getURI()).thenReturn(uri);
+
+        InferenceTextEmbeddingFloatResults parsedResults = AlibabaCloudSearchEmbeddingsResponseEntity.fromResponse(
+            request,
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(
+                List.of(
+                    new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(
+                        new float[] { -0.02868066355586052f, 0.022033605724573135f }
+                    )
+                )
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntityTests.java
new file mode 100644
index 0000000000000..a03349c66b6d5
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchErrorResponseEntityTests.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.alibabacloudsearch;
+
+import org.apache.http.HttpResponse;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.mockito.Mockito.mock;
+
+public class AlibabaCloudSearchErrorResponseEntityTests extends ESTestCase {
+    public void testFromResponse() {
+        String responseJson = """
+            {
+                "request_id": "651B3087-8A07-4BF3-B931-9C4E7B60F52D",
+                "latency": 0,
+                "code": "InvalidParameter",
+                "message": "JSON parse error: Cannot deserialize value of type `InputType` from String \\"xxx\\""
+            }
+            """;
+
+        AlibabaCloudSearchErrorResponseEntity errorMessage = AlibabaCloudSearchErrorResponseEntity.fromResponse(
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+        assertNotNull(errorMessage);
+        assertEquals("JSON parse error: Cannot deserialize value of type `InputType` from String \"xxx\"", errorMessage.getErrorMessage());
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchRerankResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchRerankResponseEntityTests.java
new file mode 100644
index 0000000000000..bebc8bb66f207
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchRerankResponseEntityTests.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.alibabacloudsearch;
+
+import org.apache.http.HttpResponse;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+
+public class AlibabaCloudSearchRerankResponseEntityTests extends ESTestCase {
+
+    public void testFromResponse_CreatesResultsForASingleItem() throws IOException {
+        InferenceServiceResults parsedResults = AlibabaCloudSearchRerankResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), responseLiteral.getBytes(StandardCharsets.UTF_8))
+        );
+
+        MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class));
+        List<RankedDocsResults.RankedDoc> expected = responseLiteralDocs();
+        for (int i = 0; i < ((RankedDocsResults) parsedResults).getRankedDocs().size(); i++) {
+            assertThat(((RankedDocsResults) parsedResults).getRankedDocs().get(i).index(), is(expected.get(i).index()));
+        }
+    }
+
+    private final String responseLiteral = """
+        {
+          "request_id": "450fcb80-f796-46c1-8d69-e1e86d29aa9f",
+          "latency": 564.903929,
+          "usage": {
+            "doc_count": 2
+          },
+          "result": {
+           "scores":[
+             {
+               "index":1,
+               "score": 1.37
+             },
+             {
+               "index":0,
+               "score": -0.3
+             }
+           ]
+          }
+        }
+        """;
+
+    private ArrayList<RankedDocsResults.RankedDoc> responseLiteralDocs() {
+        var list = new ArrayList<RankedDocsResults.RankedDoc>();
+
+        list.add(new RankedDocsResults.RankedDoc(1, 1.37F, null));
+        list.add(new RankedDocsResults.RankedDoc(0, -0.3F, null));
+        return list;
+    };
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchSparseResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchSparseResponseEntityTests.java
new file mode 100644
index 0000000000000..a6d3a4b77d74f
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchSparseResponseEntityTests.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.alibabacloudsearch;
+
+import org.apache.http.HttpResponse;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
+import org.elasticsearch.xpack.core.ml.search.WeightedToken;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchRequest;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class AlibabaCloudSearchSparseResponseEntityTests extends ESTestCase {
+    public void testFromResponse_CreatesResultsForASingleItem() throws IOException, URISyntaxException {
+        String responseJson = """
+            {
+              "request_id": "DDC4306F-xxxx-xxxx-xxxx-92C5CEA756A0",
+              "latency": 25,
+              "usage": {
+                "token_count": 11
+              },
+              "result": {
+                "sparse_embeddings": [
+                  {
+                    "index": 0,
+                    "embedding": [
+                      {
+                        "token_id": 6,
+                        "weight": 0.1014404296875
+                      },
+                      {
+                        "token_id": 163040,
+                        "weight": 0.2841796875
+                      },
+                      {
+                        "token_id": 354,
+                        "weight": 0.1431884765625
+                      }
+                    ]
+                  }
+                ]
+              }
+            }
+            """;
+
+        AlibabaCloudSearchRequest request = mock(AlibabaCloudSearchRequest.class);
+        URI uri = new URI("mock_uri");
+        when(request.getURI()).thenReturn(uri);
+
+        SparseEmbeddingResults parsedResults = AlibabaCloudSearchSparseResponseEntity.fromResponse(
+            request,
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(
+                List.of(
+                    new SparseEmbeddingResults.Embedding(
+                        List.of(
+                            new WeightedToken("6", 0.1014404296875f),
+                            new WeightedToken("163040", 0.2841796875f),
+                            new WeightedToken("354", 0.1431884765625f)
+                        ),
+                        false
+                    )
+                )
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntityTests.java
new file mode 100644
index 0000000000000..4da0518084828
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceErrorResponseEntityTests.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.elastic;
+
+import org.apache.http.HttpResponse;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+
+public class ElasticInferenceServiceErrorResponseEntityTests extends ESTestCase {
+
+    public void testFromResponse() {
+        String responseJson = """
+            {
+                "error": "error"
+            }
+            """;
+
+        ElasticInferenceServiceErrorResponseEntity errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse(
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertNotNull(errorResponseEntity);
+        assertThat(errorResponseEntity.getErrorMessage(), is("error"));
+    }
+
+    public void testFromResponse_NoErrorMessagePresent() {
+        String responseJson = """
+            {
+                "not_error": "error"
+            }
+            """;
+
+        ElasticInferenceServiceErrorResponseEntity errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse(
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertNull(errorResponseEntity);
+    }
+
+    public void testFromResponse_InvalidJson() {
+        String invalidResponseJson = """
+            {
+            """;
+
+        ElasticInferenceServiceErrorResponseEntity errorResponseEntity = ElasticInferenceServiceErrorResponseEntity.fromResponse(
+            new HttpResult(mock(HttpResponse.class), invalidResponseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertNull(errorResponseEntity);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntityTests.java
new file mode 100644
index 0000000000000..6e1994260ca0c
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/elastic/ElasticInferenceServiceSparseEmbeddingsResponseEntityTests.java
@@ -0,0 +1,241 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.elastic;
+
+import org.apache.http.HttpResponse;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
+import org.elasticsearch.xpack.core.ml.search.WeightedToken;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ElasticInferenceServiceSparseEmbeddingsResponseEntityTests extends ESTestCase {
+
+    public void testSparseEmbeddingsResponse_SingleEmbeddingInData_NoMeta_NoTruncation() throws Exception {
+        String responseJson = """
+            {
+                "data": [
+                     {
+                       "a": 1.23,
+                       "is": 4.56,
+                       "it": 7.89
+                    }
+                ]
+            }
+            """;
+
+        SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(
+                List.of(
+                    SparseEmbeddingResults.Embedding.create(
+                        List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)),
+                        false
+                    )
+                )
+            )
+        );
+    }
+
+    public void testSparseEmbeddingsResponse_MultipleEmbeddingsInData_NoMeta_NoTruncation() throws Exception {
+        String responseJson = """
+            {
+                "data": [
+                     {
+                       "a": 1.23,
+                       "is": 4.56,
+                       "it": 7.89
+                    },
+                    {
+                       "b": 1.23,
+                       "it": 4.56,
+                       "is": 7.89
+                    }
+                ]
+            }
+            """;
+
+        SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(
+                List.of(
+                    SparseEmbeddingResults.Embedding.create(
+                        List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)),
+                        false
+                    ),
+                    SparseEmbeddingResults.Embedding.create(
+                        List.of(new WeightedToken("b", 1.23F), new WeightedToken("it", 4.56F), new WeightedToken("is", 7.89F)),
+                        false
+                    )
+                )
+            )
+        );
+    }
+
+    public void testSparseEmbeddingsResponse_SingleEmbeddingInData_NoMeta_Truncated() throws Exception {
+        String responseJson = """
+            {
+                "data": [
+                     {
+                       "a": 1.23,
+                       "is": 4.56,
+                       "it": 7.89
+                    }
+                ]
+            }
+            """;
+
+        var request = mock(Request.class);
+        when(request.getTruncationInfo()).thenReturn(new boolean[] { true });
+
+        SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse(
+            request,
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(
+                List.of(
+                    SparseEmbeddingResults.Embedding.create(
+                        List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)),
+                        true
+                    )
+                )
+            )
+        );
+    }
+
+    public void testSparseEmbeddingsResponse_MultipleEmbeddingsInData_NoMeta_Truncated() throws Exception {
+        String responseJson = """
+            {
+                "data": [
+                     {
+                       "a": 1.23,
+                       "is": 4.56,
+                       "it": 7.89
+                    },
+                    {
+                       "b": 1.23,
+                       "it": 4.56,
+                       "is": 7.89
+                    }
+                ]
+            }
+            """;
+
+        var request = mock(Request.class);
+        when(request.getTruncationInfo()).thenReturn(new boolean[] { true, false });
+
+        SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse(
+            request,
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(
+                List.of(
+                    SparseEmbeddingResults.Embedding.create(
+                        List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)),
+                        true
+                    ),
+                    SparseEmbeddingResults.Embedding.create(
+                        List.of(new WeightedToken("b", 1.23F), new WeightedToken("it", 4.56F), new WeightedToken("is", 7.89F)),
+                        false
+                    )
+                )
+            )
+        );
+    }
+
+    public void testSparseEmbeddingsResponse_SingleEmbeddingInData_IgnoresMetaBeforeData_NoTruncation() throws Exception {
+        String responseJson = """
+            {
+                "meta": {
+                    "processing_latency": 1.23
+                },
+                "data": [
+                     {
+                       "a": 1.23,
+                       "is": 4.56,
+                       "it": 7.89
+                    }
+                ]
+            }
+            """;
+
+        SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(
+                List.of(
+                    SparseEmbeddingResults.Embedding.create(
+                        List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)),
+                        false
+                    )
+                )
+            )
+        );
+    }
+
+    public void testSparseEmbeddingsResponse_SingleEmbeddingInData_IgnoresMetaAfterData_NoTruncation() throws Exception {
+        String responseJson = """
+            {
+                "data": [
+                     {
+                       "a": 1.23,
+                       "is": 4.56,
+                       "it": 7.89
+                    }
+                ],
+                "meta": {
+                    "processing_latency": 1.23
+                }
+            }
+            """;
+
+        SparseEmbeddingResults parsedResults = ElasticInferenceServiceSparseEmbeddingsResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(
+                List.of(
+                    SparseEmbeddingResults.Embedding.create(
+                        List.of(new WeightedToken("a", 1.23F), new WeightedToken("is", 4.56F), new WeightedToken("it", 7.89F)),
+                        false
+                    )
+                )
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java
index c3c416d8fe65e..e350a539ba928 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java
@@ -310,7 +310,7 @@ public void testFails_ResponseIsInvalidJson_MissingSquareBracket() {
             )
         );
 
-        assertThat(thrownException.getMessage(), containsString("expected close marker for Array (start marker at [Source: (byte[])"));
+        assertThat(thrownException.getMessage(), containsString("expected close marker for Array (start marker at"));
     }
 
     public void testFails_ResponseIsInvalidJson_MissingField() {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java
index 1cae8d981313f..bb0691c691176 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java
@@ -8,6 +8,8 @@
 package org.elasticsearch.xpack.inference.mapper;
 
 import org.apache.lucene.document.FeatureField;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
@@ -63,6 +65,7 @@
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.BiConsumer;
 
@@ -130,6 +133,25 @@ protected IngestScriptSupport ingestScriptSupport() {
         throw new AssumptionViolatedException("not supported");
     }
 
+    @Override
+    public MappedFieldType getMappedFieldType() {
+        return new SemanticTextFieldMapper.SemanticTextFieldType(
+            "field",
+            "fake-inference-id",
+            null,
+            null,
+            IndexVersion.current(),
+            Map.of()
+        );
+    }
+
+    @Override
+    protected void assertSearchable(MappedFieldType fieldType) {
+        assertThat(fieldType, instanceOf(SemanticTextFieldMapper.SemanticTextFieldType.class));
+        assertTrue(fieldType.isIndexed());
+        assertTrue(fieldType.isSearchable());
+    }
+
     public void testDefaults() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
         assertEquals(Strings.toString(fieldMapping(this::minimalMapping)), mapper.mappingSource().toString());
@@ -141,6 +163,13 @@ public void testDefaults() throws Exception {
         assertTrue(fields.isEmpty());
     }
 
+    @Override
+    public void testFieldHasValue() {
+        MappedFieldType fieldType = getMappedFieldType();
+        FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { getFieldInfoWithName(getEmbeddingsFieldName("field")) });
+        assertTrue(fieldType.fieldHasValue(fieldInfos));
+    }
+
     public void testInferenceIdNotPresent() {
         Exception e = expectThrows(
             MapperParsingException.class,
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java
index 2a64f77e28756..563093930c358 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java
@@ -86,7 +86,7 @@ protected void assertEqualInstances(SemanticTextField expectedInstance, Semantic
 
     @Override
     protected SemanticTextField createTestInstance() {
-        List<String> rawValues = randomList(1, 5, () -> randomAlphaOfLengthBetween(10, 20));
+        List<String> rawValues = randomList(1, 5, () -> randomSemanticTextInput().toString());
         try { // try catch required for override
             return randomSemanticText(NAME, TestModel.createRandomInstance(), rawValues, randomFrom(XContentType.values()));
         } catch (IOException e) {
@@ -192,6 +192,16 @@ public static SemanticTextField randomSemanticText(String fieldName, Model model
             case SPARSE_EMBEDDING -> randomSparseEmbeddings(inputs);
             default -> throw new AssertionError("invalid task type: " + model.getTaskType().name());
         };
+        return semanticTextFieldFromChunkedInferenceResults(fieldName, model, inputs, results, contentType);
+    }
+
+    public static SemanticTextField semanticTextFieldFromChunkedInferenceResults(
+        String fieldName,
+        Model model,
+        List<String> inputs,
+        ChunkedInferenceServiceResults results,
+        XContentType contentType
+    ) {
         return new SemanticTextField(
             fieldName,
             inputs,
@@ -204,6 +214,24 @@ public static SemanticTextField randomSemanticText(String fieldName, Model model
         );
     }
 
+    /**
+     * Returns a randomly generated object for Semantic Text tests purpose.
+     */
+    public static Object randomSemanticTextInput() {
+        if (rarely()) {
+            return switch (randomIntBetween(0, 4)) {
+                case 0 -> randomInt();
+                case 1 -> randomLong();
+                case 2 -> randomFloat();
+                case 3 -> randomBoolean();
+                case 4 -> randomDouble();
+                default -> throw new IllegalStateException("Illegal state while generating random semantic text input");
+            };
+        } else {
+            return randomAlphaOfLengthBetween(10, 20);
+        }
+    }
+
     public static ChunkedInferenceServiceResults toChunkedResult(SemanticTextField field) throws IOException {
         switch (field.inference().modelSettings().taskType()) {
             case SPARSE_EMBEDDING -> {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankBuilderTests.java
new file mode 100644
index 0000000000000..c464dbaea47cd
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankBuilderTests.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.rank.random;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+
+import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE;
+
+public class RandomRankBuilderTests extends AbstractXContentSerializingTestCase<RandomRankBuilder> {
+
+    @Override
+    protected RandomRankBuilder createTestInstance() {
+        return new RandomRankBuilder(randomIntBetween(1, 1000), "my-field", randomBoolean() ? randomIntBetween(1, 1000) : null);
+    }
+
+    @Override
+    protected RandomRankBuilder mutateInstance(RandomRankBuilder instance) throws IOException {
+        String field = instance.field() + randomAlphaOfLength(2);
+        int rankWindowSize = randomValueOtherThan(instance.rankWindowSize(), this::randomRankWindowSize);
+        Integer seed = randomBoolean() ? randomIntBetween(1, 1000) : null;
+        return new RandomRankBuilder(rankWindowSize, field, seed);
+    }
+
+    @Override
+    protected Writeable.Reader<RandomRankBuilder> instanceReader() {
+        return RandomRankBuilder::new;
+    }
+
+    @Override
+    protected RandomRankBuilder doParseInstance(XContentParser parser) throws IOException {
+        parser.nextToken();
+        assertEquals(parser.currentToken(), XContentParser.Token.START_OBJECT);
+        parser.nextToken();
+        assertEquals(parser.currentToken(), XContentParser.Token.FIELD_NAME);
+        assertEquals(parser.currentName(), RandomRankBuilder.NAME);
+        RandomRankBuilder builder = RandomRankBuilder.PARSER.parse(parser, null);
+        parser.nextToken();
+        assertEquals(parser.currentToken(), XContentParser.Token.END_OBJECT);
+        parser.nextToken();
+        assertNull(parser.currentToken());
+        return builder;
+    }
+
+    private int randomRankWindowSize() {
+        return randomIntBetween(0, 1000);
+    }
+
+    public void testParserDefaults() throws IOException {
+        String json = """
+            {
+              "field": "my-field"
+            }""";
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) {
+            RandomRankBuilder parsed = RandomRankBuilder.PARSER.parse(parser, null);
+            assertEquals(DEFAULT_RANK_WINDOW_SIZE, parsed.rankWindowSize());
+        }
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java
new file mode 100644
index 0000000000000..c33f30d461350
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.rank.random;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.search.retriever.RetrieverBuilder;
+import org.elasticsearch.search.retriever.RetrieverParserContext;
+import org.elasticsearch.search.retriever.TestRetrieverBuilder;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.usage.SearchUsage;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankBuilder;
+import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE;
+
+public class RandomRankRetrieverBuilderTests extends AbstractXContentTestCase<RandomRankRetrieverBuilder> {
+
+    /**
+     * Creates a random {@link RandomRankRetrieverBuilder}. The created instance
+     * is not guaranteed to pass {@link SearchRequest} validation. This is purely
+     * for x-content testing.
+     */
+    public static RandomRankRetrieverBuilder createRandomRankRetrieverBuilder() {
+        return new RandomRankRetrieverBuilder(
+            TestRetrieverBuilder.createRandomTestRetrieverBuilder(),
+            randomAlphaOfLength(10),
+            randomIntBetween(1, 10000),
+            randomBoolean() ? randomIntBetween(1, 1000) : null
+        );
+    }
+
+    @Override
+    protected RandomRankRetrieverBuilder createTestInstance() {
+        return createRandomRankRetrieverBuilder();
+    }
+
+    @Override
+    protected RandomRankRetrieverBuilder doParseInstance(XContentParser parser) {
+        return RandomRankRetrieverBuilder.PARSER.apply(
+            parser,
+            new RetrieverParserContext(
+                new SearchUsage(),
+                nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED
+            )
+        );
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>();
+        entries.add(
+            new NamedXContentRegistry.Entry(
+                RetrieverBuilder.class,
+                TestRetrieverBuilder.TEST_SPEC.getName(),
+                (p, c) -> TestRetrieverBuilder.TEST_SPEC.getParser().fromXContent(p, (RetrieverParserContext) c),
+                TestRetrieverBuilder.TEST_SPEC.getName().getForRestApiVersion()
+            )
+        );
+        entries.add(
+            new NamedXContentRegistry.Entry(
+                RetrieverBuilder.class,
+                new ParseField(TextSimilarityRankBuilder.NAME),
+                (p, c) -> TextSimilarityRankRetrieverBuilder.PARSER.apply(p, (RetrieverParserContext) c)
+            )
+        );
+        return new NamedXContentRegistry(entries);
+    }
+
+    public void testParserDefaults() throws IOException {
+        String json = """
+            {
+              "retriever": {
+                "test": {
+                  "value": "my-test-retriever"
+                }
+              },
+              "field": "my-field"
+            }""";
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) {
+            RandomRankRetrieverBuilder parsed = RandomRankRetrieverBuilder.PARSER.parse(parser, null);
+            assertEquals(DEFAULT_RANK_WINDOW_SIZE, parsed.rankWindowSize());
+        }
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankFeaturePhaseRankCoordinatorContextTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankFeaturePhaseRankCoordinatorContextTests.java
index 50d91a2271de6..d6c476cdc15d6 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankFeaturePhaseRankCoordinatorContextTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankFeaturePhaseRankCoordinatorContextTests.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.search.rank.feature.RankFeatureDoc;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.inference.action.InferenceAction;
+import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.argThat;
@@ -54,10 +54,28 @@ public void onFailure(Exception e) {
                 fail();
             }
         });
+        verify(mockClient).execute(
+            eq(GetInferenceModelAction.INSTANCE),
+            argThat(actionRequest -> ((GetInferenceModelAction.Request) actionRequest).getTaskType().equals(TaskType.RERANK)),
+            any()
+        );
+    }
 
+    public void testComputeScoresForEmpty() {
+        subject.computeScores(new RankFeatureDoc[0], new ActionListener<>() {
+            @Override
+            public void onResponse(float[] floats) {
+                assertArrayEquals(new float[0], floats, 0.0f);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                fail();
+            }
+        });
         verify(mockClient).execute(
-            eq(InferenceAction.INSTANCE),
-            argThat(actionRequest -> ((InferenceAction.Request) actionRequest).getTaskType().equals(TaskType.RERANK)),
+            eq(GetInferenceModelAction.INSTANCE),
+            argThat(actionRequest -> ((GetInferenceModelAction.Request) actionRequest).getTaskType().equals(TaskType.RERANK)),
             any()
         );
     }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java
index 51f240be6fbeb..332e567cae796 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java
@@ -8,10 +8,22 @@
 package org.elasticsearch.xpack.inference.rank.textsimilarity;
 
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.RandomQueryBuilder;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.index.query.Rewriteable;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.builder.SubSearchSourceBuilder;
 import org.elasticsearch.search.retriever.RetrieverBuilder;
 import org.elasticsearch.search.retriever.RetrieverParserContext;
 import org.elasticsearch.search.retriever.TestRetrieverBuilder;
 import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.usage.SearchUsage;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
@@ -23,6 +35,10 @@
 import java.util.List;
 
 import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.mockito.Mockito.mock;
 
 public class TextSimilarityRankRetrieverBuilderTests extends AbstractXContentTestCase<TextSimilarityRankRetrieverBuilder> {
 
@@ -32,8 +48,17 @@ public class TextSimilarityRankRetrieverBuilderTests extends AbstractXContentTes
      * for x-content testing.
      */
     public static TextSimilarityRankRetrieverBuilder createRandomTextSimilarityRankRetrieverBuilder() {
+        return createRandomTextSimilarityRankRetrieverBuilder(TestRetrieverBuilder.createRandomTestRetrieverBuilder());
+    }
+
+    /**
+     * Creates a random {@link TextSimilarityRankRetrieverBuilder}. The created instance
+     * is not guaranteed to pass {@link SearchRequest} validation. This is purely
+     * for x-content testing.
+     */
+    public static TextSimilarityRankRetrieverBuilder createRandomTextSimilarityRankRetrieverBuilder(RetrieverBuilder innerRetriever) {
         return new TextSimilarityRankRetrieverBuilder(
-            TestRetrieverBuilder.createRandomTestRetrieverBuilder(),
+            innerRetriever,
             randomAlphaOfLength(10),
             randomAlphaOfLength(20),
             randomAlphaOfLength(50),
@@ -104,4 +129,122 @@ public void testParserDefaults() throws IOException {
         }
     }
 
+    public void testRewriteInnerRetriever() throws IOException {
+        final boolean[] rewritten = { false };
+        List<QueryBuilder> preFilterQueryBuilders = new ArrayList<>();
+        if (randomBoolean()) {
+            for (int i = 0; i < randomIntBetween(1, 5); i++) {
+                preFilterQueryBuilders.add(RandomQueryBuilder.createQuery(random()));
+            }
+        }
+        RetrieverBuilder innerRetriever = new TestRetrieverBuilder("top-level-retriever") {
+            @Override
+            public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException {
+                if (randomBoolean()) {
+                    return this;
+                }
+                rewritten[0] = true;
+                return new TestRetrieverBuilder("nested-rewritten-retriever") {
+                    @Override
+                    public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
+                        if (preFilterQueryBuilders.isEmpty() == false) {
+                            BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+
+                            for (QueryBuilder preFilterQueryBuilder : preFilterQueryBuilders) {
+                                boolQueryBuilder.filter(preFilterQueryBuilder);
+                            }
+                            boolQueryBuilder.must(new RangeQueryBuilder("some_field"));
+                            searchSourceBuilder.subSearches().add(new SubSearchSourceBuilder(boolQueryBuilder));
+                        } else {
+                            searchSourceBuilder.subSearches().add(new SubSearchSourceBuilder(new RangeQueryBuilder("some_field")));
+                        }
+                    }
+                };
+            }
+
+            @Override
+            public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
+                if (preFilterQueryBuilders.isEmpty() == false) {
+                    BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+
+                    for (QueryBuilder preFilterQueryBuilder : preFilterQueryBuilders) {
+                        boolQueryBuilder.filter(preFilterQueryBuilder);
+                    }
+                    boolQueryBuilder.must(new TermQueryBuilder("field", "value"));
+                    searchSourceBuilder.subSearches().add(new SubSearchSourceBuilder(boolQueryBuilder));
+                } else {
+                    searchSourceBuilder.subSearches().add(new SubSearchSourceBuilder(new TermQueryBuilder("field", "value")));
+                }
+            }
+        };
+        TextSimilarityRankRetrieverBuilder textSimilarityRankRetrieverBuilder = createRandomTextSimilarityRankRetrieverBuilder(
+            innerRetriever
+        );
+        textSimilarityRankRetrieverBuilder.getPreFilterQueryBuilders().addAll(preFilterQueryBuilders);
+        SearchSourceBuilder source = new SearchSourceBuilder().retriever(textSimilarityRankRetrieverBuilder);
+        QueryRewriteContext queryRewriteContext = mock(QueryRewriteContext.class);
+        source = Rewriteable.rewrite(source, queryRewriteContext);
+        assertNull(source.retriever());
+        if (false == preFilterQueryBuilders.isEmpty()) {
+            if (source.query() instanceof MatchAllQueryBuilder == false && source.query() instanceof MatchNoneQueryBuilder == false) {
+                assertThat(source.query(), instanceOf(BoolQueryBuilder.class));
+                BoolQueryBuilder bq = (BoolQueryBuilder) source.query();
+                assertFalse(bq.must().isEmpty());
+                assertThat(bq.must().size(), equalTo(1));
+                if (rewritten[0]) {
+                    assertThat(bq.must().get(0), instanceOf(RangeQueryBuilder.class));
+                } else {
+                    assertThat(bq.must().get(0), instanceOf(TermQueryBuilder.class));
+                }
+                for (int j = 0; j < bq.filter().size(); j++) {
+                    assertEqualQueryOrMatchAllNone(bq.filter().get(j), preFilterQueryBuilders.get(j));
+                }
+            }
+        } else {
+            if (rewritten[0]) {
+                assertThat(source.query(), instanceOf(RangeQueryBuilder.class));
+            } else {
+                assertThat(source.query(), instanceOf(TermQueryBuilder.class));
+            }
+        }
+    }
+
+    public void testIsCompound() {
+        RetrieverBuilder compoundInnerRetriever = new TestRetrieverBuilder(ESTestCase.randomAlphaOfLengthBetween(5, 10)) {
+            @Override
+            public boolean isCompound() {
+                return true;
+            }
+        };
+        RetrieverBuilder nonCompoundInnerRetriever = new TestRetrieverBuilder(ESTestCase.randomAlphaOfLengthBetween(5, 10)) {
+            @Override
+            public boolean isCompound() {
+                return false;
+            }
+        };
+        TextSimilarityRankRetrieverBuilder compoundTextSimilarityRankRetrieverBuilder = createRandomTextSimilarityRankRetrieverBuilder(
+            compoundInnerRetriever
+        );
+        assertTrue(compoundTextSimilarityRankRetrieverBuilder.isCompound());
+        TextSimilarityRankRetrieverBuilder nonCompoundTextSimilarityRankRetrieverBuilder = createRandomTextSimilarityRankRetrieverBuilder(
+            nonCompoundInnerRetriever
+        );
+        assertFalse(nonCompoundTextSimilarityRankRetrieverBuilder.isCompound());
+    }
+
+    public void testTopDocsQuery() {
+        RetrieverBuilder innerRetriever = new TestRetrieverBuilder(ESTestCase.randomAlphaOfLengthBetween(5, 10)) {
+            @Override
+            public QueryBuilder topDocsQuery() {
+                return new TermQueryBuilder("field", "value");
+            }
+        };
+        TextSimilarityRankRetrieverBuilder retriever = createRandomTextSimilarityRankRetrieverBuilder(innerRetriever);
+        assertThat(retriever.topDocsQuery(), instanceOf(TermQueryBuilder.class));
+    }
+
+    private static void assertEqualQueryOrMatchAllNone(QueryBuilder actual, QueryBuilder expected) {
+        assertThat(actual, anyOf(instanceOf(MatchAllQueryBuilder.class), instanceOf(MatchNoneQueryBuilder.class), equalTo(expected)));
+    }
+
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java
index 7fbfe70dbcfe7..a26dc50097cf5 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.inference.rank.textsimilarity;
 
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.inference.InputType;
@@ -29,22 +30,46 @@
 import java.util.Objects;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 public class TextSimilarityRankTests extends ESSingleNodeTestCase {
 
     /**
-     * {@code TextSimilarityRankBuilder} that simulates an inference call that returns a different number of results as the input.
+     * {@code TextSimilarityRankBuilder} that sets top_n in the inference endpoint's task settings.
+     * See {@code TextSimilarityTestPlugin -> TestFilter -> handleGetInferenceModelActionRequest} for the logic that extracts the top_n
+     * value.
      */
-    public static class InvalidInferenceResultCountProvidingTextSimilarityRankBuilder extends TextSimilarityRankBuilder {
+    public static class TopNConfigurationAcceptingTextSimilarityRankBuilder extends TextSimilarityRankBuilder {
 
-        public InvalidInferenceResultCountProvidingTextSimilarityRankBuilder(
+        public TopNConfigurationAcceptingTextSimilarityRankBuilder(
             String field,
             String inferenceId,
             String inferenceText,
             int rankWindowSize,
-            Float minScore
+            Float minScore,
+            int topN
+        ) {
+            super(field, inferenceId + "-task-settings-top-" + topN, inferenceText, rankWindowSize, minScore);
+        }
+    }
+
+    /**
+     * {@code TextSimilarityRankBuilder} that simulates an inference call returning N results.
+     */
+    public static class InferenceResultCountAcceptingTextSimilarityRankBuilder extends TextSimilarityRankBuilder {
+
+        private final int inferenceResultCount;
+
+        public InferenceResultCountAcceptingTextSimilarityRankBuilder(
+            String field,
+            String inferenceId,
+            String inferenceText,
+            int rankWindowSize,
+            Float minScore,
+            int inferenceResultCount
         ) {
             super(field, inferenceId, inferenceText, rankWindowSize, minScore);
+            this.inferenceResultCount = inferenceResultCount;
         }
 
         @Override
@@ -62,10 +87,10 @@ public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorCo
                 protected InferenceAction.Request generateRequest(List<String> docFeatures) {
                     return new InferenceAction.Request(
                         TaskType.RERANK,
-                        inferenceId,
+                        this.inferenceId,
                         inferenceText,
                         docFeatures,
-                        Map.of("invalidInferenceResultCount", true),
+                        Map.of("inferenceResultCount", inferenceResultCount),
                         InputType.SEARCH,
                         InferenceAction.Request.DEFAULT_TIMEOUT
                     );
@@ -151,17 +176,38 @@ public void testRerankInferenceFailure() {
         );
     }
 
-    public void testRerankInferenceResultMismatch() {
-        ElasticsearchAssertions.assertFailures(
+    public void testRerankTopNConfigurationAndRankWindowSizeMismatch() {
+        SearchPhaseExecutionException ex = expectThrows(
+            SearchPhaseExecutionException.class,
             // Execute search with text similarity reranking
             client.prepareSearch()
                 .setRankBuilder(
-                    new InvalidInferenceResultCountProvidingTextSimilarityRankBuilder("text", "my-rerank-model", "my query", 100, 1.5f)
+                    // Simulate reranker configuration with top_n=3 in task_settings, which is different from rank_window_size=10
+                    // (Note: top_n comes from inferenceId, there's no other easy way of passing this to the mocked get model request)
+                    new TopNConfigurationAcceptingTextSimilarityRankBuilder("text", "my-rerank-model", "my query", 100, 1.5f, 3)
                 )
-                .setQuery(QueryBuilders.matchAllQuery()),
-            RestStatus.INTERNAL_SERVER_ERROR,
-            containsString("Failed to execute phase [rank-feature], Computing updated ranks for results failed")
+                .setQuery(QueryBuilders.matchAllQuery())
+        );
+        assertThat(ex.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(
+            ex.getDetailedMessage(),
+            containsString("Reduce rank_window_size to be less than or equal to the configured top N value")
+        );
+    }
+
+    public void testRerankInputSizeAndInferenceResultsMismatch() {
+        SearchPhaseExecutionException ex = expectThrows(
+            SearchPhaseExecutionException.class,
+            // Execute search with text similarity reranking
+            client.prepareSearch()
+                .setRankBuilder(
+                    // Simulate reranker returning different number of results from input
+                    new InferenceResultCountAcceptingTextSimilarityRankBuilder("text", "my-rerank-model", "my query", 100, 1.5f, 4)
+                )
+                .setQuery(QueryBuilders.matchAllQuery())
         );
+        assertThat(ex.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
+        assertThat(ex.getDetailedMessage(), containsString("Reranker input document count and returned score count mismatch"));
     }
 
     private static void assertHitHasRankScoreAndText(SearchHit hit, int expectedRank, float expectedScore, String expectedText) {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityTestPlugin.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityTestPlugin.java
index 1e457a1a27c92..6d0c15d5c0bfe 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityTestPlugin.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityTestPlugin.java
@@ -21,7 +21,9 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.inference.EmptyTaskSettings;
 import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -39,8 +41,12 @@
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
+import org.elasticsearch.xpack.inference.services.cohere.CohereService;
+import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankServiceSettings;
+import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankTaskSettings;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -48,6 +54,8 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
@@ -100,7 +108,6 @@ public int order() {
         }
 
         @Override
-        @SuppressWarnings("unchecked")
         public <Request extends ActionRequest, Response extends ActionResponse> void apply(
             Task task,
             String action,
@@ -108,23 +115,59 @@ public <Request extends ActionRequest, Response extends ActionResponse> void app
             ActionListener<Response> listener,
             ActionFilterChain<Request, Response> chain
         ) {
-            // For any other action than inference, execute normally
-            if (action.equals(InferenceAction.INSTANCE.name()) == false) {
+            if (action.equals(GetInferenceModelAction.INSTANCE.name())) {
+                assert request instanceof GetInferenceModelAction.Request;
+                handleGetInferenceModelActionRequest((GetInferenceModelAction.Request) request, listener);
+            } else if (action.equals(InferenceAction.INSTANCE.name())) {
+                assert request instanceof InferenceAction.Request;
+                handleInferenceActionRequest((InferenceAction.Request) request, listener);
+            } else {
+                // For any other action than get model and inference, execute normally
                 chain.proceed(task, action, request, listener);
-                return;
             }
+        }
 
-            assert request instanceof InferenceAction.Request;
-            boolean shouldThrow = (boolean) ((InferenceAction.Request) request).getTaskSettings().getOrDefault("throwing", false);
-            boolean hasInvalidInferenceResultCount = (boolean) ((InferenceAction.Request) request).getTaskSettings()
-                .getOrDefault("invalidInferenceResultCount", false);
+        @SuppressWarnings("unchecked")
+        private <Response extends ActionResponse> void handleGetInferenceModelActionRequest(
+            GetInferenceModelAction.Request request,
+            ActionListener<Response> listener
+        ) {
+            String inferenceEntityId = request.getInferenceEntityId();
+            Integer topN = null;
+            Matcher extractTopN = Pattern.compile(".*(task-settings-top-\\d+).*").matcher(inferenceEntityId);
+            if (extractTopN.find()) {
+                topN = Integer.parseInt(extractTopN.group(1).replaceAll("\\D", ""));
+            }
+
+            ActionResponse response = new GetInferenceModelAction.Response(
+                List.of(
+                    new ModelConfigurations(
+                        request.getInferenceEntityId(),
+                        request.getTaskType(),
+                        CohereService.NAME,
+                        new CohereRerankServiceSettings("uri", "model", null),
+                        topN == null ? new EmptyTaskSettings() : new CohereRerankTaskSettings(topN, null, null)
+                    )
+                )
+            );
+            listener.onResponse((Response) response);
+        }
+
+        @SuppressWarnings("unchecked")
+        private <Response extends ActionResponse> void handleInferenceActionRequest(
+            InferenceAction.Request request,
+            ActionListener<Response> listener
+        ) {
+            Map<String, Object> taskSettings = request.getTaskSettings();
+            boolean shouldThrow = (boolean) taskSettings.getOrDefault("throwing", false);
+            Integer inferenceResultCount = (Integer) taskSettings.get("inferenceResultCount");
 
             if (shouldThrow) {
                 listener.onFailure(new UnsupportedOperationException("simulated failure"));
             } else {
                 List<RankedDocsResults.RankedDoc> rankedDocsResults = new ArrayList<>();
-                List<String> inputs = ((InferenceAction.Request) request).getInput();
-                int resultCount = hasInvalidInferenceResultCount ? inputs.size() - 1 : inputs.size();
+                List<String> inputs = request.getInput();
+                int resultCount = inferenceResultCount == null ? inputs.size() : inferenceResultCount;
                 for (int i = 0; i < resultCount; i++) {
                     rankedDocsResults.add(new RankedDocsResults.RankedDoc(i, Float.parseFloat(inputs.get(i)), inputs.get(i)));
                 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceSettingsTests.java
new file mode 100644
index 0000000000000..d7965a38c845b
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceSettingsTests.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+
+public class AlibabaCloudSearchServiceSettingsTests extends AbstractWireSerializingTestCase<AlibabaCloudSearchServiceSettings> {
+    /**
+     * The created settings can have a url set to null.
+     */
+    public static AlibabaCloudSearchServiceSettings createRandom() {
+        var model = randomAlphaOfLength(15);
+        String host = randomAlphaOfLength(15);
+        String workspaceName = randomAlphaOfLength(10);
+        String httpSchema = "https";
+        return new AlibabaCloudSearchServiceSettings(model, host, workspaceName, httpSchema, RateLimitSettingsTests.createRandom());
+    }
+
+    public void testFromMap() throws URISyntaxException {
+        var model = "model";
+        var host = "host";
+        var workspaceName = "default";
+        var httpSchema = "https";
+        var serviceSettings = AlibabaCloudSearchServiceSettings.fromMap(
+            new HashMap<>(
+                Map.of(
+                    AlibabaCloudSearchServiceSettings.SERVICE_ID,
+                    model,
+                    AlibabaCloudSearchServiceSettings.HOST,
+                    host,
+                    AlibabaCloudSearchServiceSettings.WORKSPACE_NAME,
+                    workspaceName,
+                    AlibabaCloudSearchServiceSettings.HTTP_SCHEMA_NAME,
+                    httpSchema
+                )
+            ),
+            null
+        );
+
+        MatcherAssert.assertThat(serviceSettings, is(new AlibabaCloudSearchServiceSettings(model, host, workspaceName, httpSchema, null)));
+    }
+
+    public void testFromMap_WithRateLimit() {
+        var model = "model";
+        var host = "host";
+        var workspaceName = "default";
+        var httpSchema = "https";
+        var serviceSettings = AlibabaCloudSearchServiceSettings.fromMap(
+            new HashMap<>(
+                Map.of(
+                    AlibabaCloudSearchServiceSettings.SERVICE_ID,
+                    model,
+                    AlibabaCloudSearchServiceSettings.HOST,
+                    host,
+                    AlibabaCloudSearchServiceSettings.WORKSPACE_NAME,
+                    workspaceName,
+                    AlibabaCloudSearchServiceSettings.HTTP_SCHEMA_NAME,
+                    httpSchema,
+                    RateLimitSettings.FIELD_NAME,
+                    new HashMap<>(Map.of(RateLimitSettings.REQUESTS_PER_MINUTE_FIELD, 3))
+                )
+            ),
+            null
+        );
+
+        MatcherAssert.assertThat(
+            serviceSettings,
+            is(new AlibabaCloudSearchServiceSettings(model, host, workspaceName, httpSchema, new RateLimitSettings(3)))
+        );
+    }
+
+    public void testXContent() throws IOException {
+        var entity = new AlibabaCloudSearchServiceSettings("model_id_name", "host_name", "workspace_name", null, null);
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, is("""
+            {"service_id":"model_id_name","host":"host_name","workspace":"workspace_name","rate_limit":{"requests_per_minute":1000}}"""));
+    }
+
+    @Override
+    protected Writeable.Reader<AlibabaCloudSearchServiceSettings> instanceReader() {
+        return AlibabaCloudSearchServiceSettings::new;
+    }
+
+    @Override
+    protected AlibabaCloudSearchServiceSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected AlibabaCloudSearchServiceSettings mutateInstance(AlibabaCloudSearchServiceSettings instance) throws IOException {
+        return null;
+    }
+
+    public static Map<String, Object> getServiceSettingsMap(String serviceId, String host, String workspaceName) {
+        var map = new HashMap<String, Object>();
+        map.put(AlibabaCloudSearchServiceSettings.SERVICE_ID, serviceId);
+        map.put(AlibabaCloudSearchServiceSettings.HOST, host);
+        map.put(AlibabaCloudSearchServiceSettings.WORKSPACE_NAME, workspaceName);
+        return map;
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java
new file mode 100644
index 0000000000000..cc70b61226fe3
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java
@@ -0,0 +1,172 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults;
+import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests;
+import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
+import org.elasticsearch.xpack.inference.services.ServiceFields;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModelTests;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsServiceSettingsTests;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsTaskSettingsTests;
+import org.hamcrest.MatcherAssert;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool;
+import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty;
+import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
+import static org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests.getSecretSettingsMap;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.mockito.Mockito.mock;
+
+public class AlibabaCloudSearchServiceTests extends ESTestCase {
+    private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS);
+    private ThreadPool threadPool;
+    private HttpClientManager clientManager;
+
+    @Before
+    public void init() throws Exception {
+        threadPool = createThreadPool(inferenceUtilityPool());
+        clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class));
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        clientManager.close();
+        terminate(threadPool);
+    }
+
+    public void testParseRequestConfig_CreatesAnEmbeddingsModel() throws IOException {
+        try (var service = new AlibabaCloudSearchService(mock(HttpRequestSender.Factory.class), createWithEmptySettings(threadPool))) {
+            ActionListener<Model> modelVerificationListener = ActionListener.wrap(model -> {
+                assertThat(model, instanceOf(AlibabaCloudSearchEmbeddingsModel.class));
+
+                var embeddingsModel = (AlibabaCloudSearchEmbeddingsModel) model;
+                assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("service_id"));
+                assertThat(embeddingsModel.getServiceSettings().getCommonSettings().getHost(), is("host"));
+                assertThat(embeddingsModel.getServiceSettings().getCommonSettings().getWorkspaceName(), is("default"));
+                assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+            }, e -> fail("Model parsing should have succeeded " + e.getMessage()));
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                getRequestConfigMap(
+                    AlibabaCloudSearchEmbeddingsServiceSettingsTests.getServiceSettingsMap("service_id", "host", "default"),
+                    AlibabaCloudSearchEmbeddingsTaskSettingsTests.getTaskSettingsMap(null),
+                    getSecretSettingsMap("secret")
+                ),
+                Set.of(),
+                modelVerificationListener
+            );
+        }
+    }
+
+    public void testCheckModelConfig() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new AlibabaCloudSearchService(senderFactory, createWithEmptySettings(threadPool)) {
+            @Override
+            public void doInfer(
+                Model model,
+                List<String> input,
+                Map<String, Object> taskSettings,
+                InputType inputType,
+                TimeValue timeout,
+                ActionListener<InferenceServiceResults> listener
+            ) {
+                InferenceTextEmbeddingFloatResults results = new InferenceTextEmbeddingFloatResults(
+                    List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.028680f, 0.022033f }))
+                );
+
+                listener.onResponse(results);
+            }
+        }) {
+            Map<String, Object> serviceSettingsMap = new HashMap<>();
+            serviceSettingsMap.put(AlibabaCloudSearchServiceSettings.SERVICE_ID, "service_id");
+            serviceSettingsMap.put(AlibabaCloudSearchServiceSettings.HOST, "host");
+            serviceSettingsMap.put(AlibabaCloudSearchServiceSettings.WORKSPACE_NAME, "default");
+            serviceSettingsMap.put(ServiceFields.DIMENSIONS, 1536);
+
+            Map<String, Object> taskSettingsMap = new HashMap<>();
+
+            Map<String, Object> secretSettingsMap = new HashMap<>();
+            secretSettingsMap.put("api_key", "secret");
+
+            var model = AlibabaCloudSearchEmbeddingsModelTests.createModel(
+                "service",
+                TaskType.TEXT_EMBEDDING,
+                serviceSettingsMap,
+                taskSettingsMap,
+                secretSettingsMap
+            );
+            PlainActionFuture<Model> listener = new PlainActionFuture<>();
+            service.checkModelConfig(model, listener);
+            var result = listener.actionGet(TIMEOUT);
+
+            Map<String, Object> expectedServiceSettingsMap = new HashMap<>();
+            expectedServiceSettingsMap.put(AlibabaCloudSearchServiceSettings.SERVICE_ID, "service_id");
+            expectedServiceSettingsMap.put(AlibabaCloudSearchServiceSettings.HOST, "host");
+            expectedServiceSettingsMap.put(AlibabaCloudSearchServiceSettings.WORKSPACE_NAME, "default");
+            expectedServiceSettingsMap.put(ServiceFields.SIMILARITY, "DOT_PRODUCT");
+            expectedServiceSettingsMap.put(ServiceFields.DIMENSIONS, 2);
+
+            Map<String, Object> expectedTaskSettingsMap = new HashMap<>();
+
+            Map<String, Object> expectedSecretSettingsMap = new HashMap<>();
+            expectedSecretSettingsMap.put("api_key", "secret");
+
+            var expectedModel = AlibabaCloudSearchEmbeddingsModelTests.createModel(
+                "service",
+                TaskType.TEXT_EMBEDDING,
+                expectedServiceSettingsMap,
+                expectedTaskSettingsMap,
+                expectedSecretSettingsMap
+            );
+
+            MatcherAssert.assertThat(result, is(expectedModel));
+        }
+    }
+
+    private Map<String, Object> getRequestConfigMap(
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        Map<String, Object> secretSettings
+    ) {
+        var builtServiceSettings = new HashMap<>();
+        builtServiceSettings.putAll(serviceSettings);
+        builtServiceSettings.putAll(secretSettings);
+
+        return new HashMap<>(
+            Map.of(ModelConfigurations.SERVICE_SETTINGS, builtServiceSettings, ModelConfigurations.TASK_SETTINGS, taskSettings)
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModelTests.java
new file mode 100644
index 0000000000000..fca0ee11e5c78
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModelTests.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchUtils;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests;
+import org.hamcrest.MatcherAssert;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+
+public class AlibabaCloudSearchEmbeddingsModelTests extends ESTestCase {
+    public void testOverride() {
+        AlibabaCloudSearchEmbeddingsTaskSettings taskSettings = AlibabaCloudSearchEmbeddingsTaskSettingsTests.createRandom();
+        var model = createModel(
+            "service",
+            TaskType.TEXT_EMBEDDING,
+            AlibabaCloudSearchEmbeddingsServiceSettingsTests.createRandom(),
+            taskSettings,
+            DefaultSecretSettingsTests.createRandom()
+        );
+
+        var overriddenModel = AlibabaCloudSearchEmbeddingsModel.of(model, Map.of(), taskSettings.getInputType());
+        MatcherAssert.assertThat(overriddenModel, is(model));
+    }
+
+    public static AlibabaCloudSearchEmbeddingsModel createModel(
+        String modelId,
+        TaskType taskType,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secrets
+    ) {
+        return new AlibabaCloudSearchEmbeddingsModel(
+            modelId,
+            taskType,
+            AlibabaCloudSearchUtils.SERVICE_NAME,
+            serviceSettings,
+            taskSettings,
+            secrets,
+            null
+        );
+    }
+
+    public static AlibabaCloudSearchEmbeddingsModel createModel(
+        String modelId,
+        TaskType taskType,
+        AlibabaCloudSearchEmbeddingsServiceSettings serviceSettings,
+        AlibabaCloudSearchEmbeddingsTaskSettings taskSettings,
+        @Nullable DefaultSecretSettings secretSettings
+    ) {
+        return new AlibabaCloudSearchEmbeddingsModel(
+            modelId,
+            taskType,
+            AlibabaCloudSearchUtils.SERVICE_NAME,
+            serviceSettings,
+            taskSettings,
+            secretSettings
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsServiceSettingsTests.java
new file mode 100644
index 0000000000000..815e6d0311195
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsServiceSettingsTests.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.inference.services.ServiceFields;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettings;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettingsTests;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+
+public class AlibabaCloudSearchEmbeddingsServiceSettingsTests extends AbstractWireSerializingTestCase<
+    AlibabaCloudSearchEmbeddingsServiceSettings> {
+    public static AlibabaCloudSearchEmbeddingsServiceSettings createRandom() {
+        var commonSettings = AlibabaCloudSearchServiceSettingsTests.createRandom();
+        var similarity = SimilarityMeasure.DOT_PRODUCT;
+        var dims = 1536;
+        var maxInputTokens = 512;
+        return new AlibabaCloudSearchEmbeddingsServiceSettings(commonSettings, similarity, dims, maxInputTokens);
+    }
+
+    public void testFromMap() {
+        var similarity = SimilarityMeasure.DOT_PRODUCT.toString();
+        var dims = 1536;
+        var maxInputTokens = 512;
+        var model = "model";
+        var host = "host";
+        var workspaceName = "default";
+        var httpSchema = "https";
+        var serviceSettings = AlibabaCloudSearchEmbeddingsServiceSettings.fromMap(
+            new HashMap<>(
+                Map.of(
+                    ServiceFields.SIMILARITY,
+                    similarity,
+                    ServiceFields.DIMENSIONS,
+                    dims,
+                    ServiceFields.MAX_INPUT_TOKENS,
+                    maxInputTokens,
+                    AlibabaCloudSearchServiceSettings.HOST,
+                    host,
+                    AlibabaCloudSearchServiceSettings.SERVICE_ID,
+                    model,
+                    AlibabaCloudSearchServiceSettings.WORKSPACE_NAME,
+                    workspaceName,
+                    AlibabaCloudSearchServiceSettings.HTTP_SCHEMA_NAME,
+                    httpSchema
+                )
+            ),
+            null
+        );
+
+        MatcherAssert.assertThat(
+            serviceSettings,
+            is(
+                new AlibabaCloudSearchEmbeddingsServiceSettings(
+                    new AlibabaCloudSearchServiceSettings(model, host, workspaceName, httpSchema, null),
+                    SimilarityMeasure.DOT_PRODUCT,
+                    dims,
+                    maxInputTokens
+                )
+            )
+        );
+    }
+
+    @Override
+    protected Writeable.Reader<AlibabaCloudSearchEmbeddingsServiceSettings> instanceReader() {
+        return AlibabaCloudSearchEmbeddingsServiceSettings::new;
+    }
+
+    @Override
+    protected AlibabaCloudSearchEmbeddingsServiceSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected AlibabaCloudSearchEmbeddingsServiceSettings mutateInstance(AlibabaCloudSearchEmbeddingsServiceSettings instance)
+        throws IOException {
+        return null;
+    }
+
+    public static Map<String, Object> getServiceSettingsMap(String serviceId, String host, String workspaceName) {
+        return AlibabaCloudSearchServiceSettingsTests.getServiceSettingsMap(serviceId, host, workspaceName);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettingsTests.java
new file mode 100644
index 0000000000000..244685d8e9833
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsTaskSettingsTests.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.InputTypeTests.randomWithIngestAndSearch;
+import static org.hamcrest.Matchers.is;
+
+public class AlibabaCloudSearchEmbeddingsTaskSettingsTests extends AbstractWireSerializingTestCase<
+    AlibabaCloudSearchEmbeddingsTaskSettings> {
+    public static AlibabaCloudSearchEmbeddingsTaskSettings createRandom() {
+        var inputType = randomBoolean() ? randomWithIngestAndSearch() : null;
+
+        return new AlibabaCloudSearchEmbeddingsTaskSettings(inputType);
+    }
+
+    public void testFromMap() {
+        MatcherAssert.assertThat(
+            AlibabaCloudSearchEmbeddingsTaskSettings.fromMap(
+                new HashMap<>(Map.of(AlibabaCloudSearchEmbeddingsTaskSettings.INPUT_TYPE, "ingest"))
+            ),
+            is(new AlibabaCloudSearchEmbeddingsTaskSettings(InputType.INGEST))
+        );
+    }
+
+    public void testFromMap_WhenInputTypeIsNull() {
+        InputType inputType = null;
+        MatcherAssert.assertThat(
+            AlibabaCloudSearchEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of())),
+            is(new AlibabaCloudSearchEmbeddingsTaskSettings(inputType))
+        );
+    }
+
+    @Override
+    protected Writeable.Reader<AlibabaCloudSearchEmbeddingsTaskSettings> instanceReader() {
+        return AlibabaCloudSearchEmbeddingsTaskSettings::new;
+    }
+
+    @Override
+    protected AlibabaCloudSearchEmbeddingsTaskSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected AlibabaCloudSearchEmbeddingsTaskSettings mutateInstance(AlibabaCloudSearchEmbeddingsTaskSettings instance)
+        throws IOException {
+        return null;
+    }
+
+    public static Map<String, Object> getTaskSettingsMap(@Nullable InputType inputType) {
+        var map = new HashMap<String, Object>();
+
+        if (inputType != null) {
+            map.put(AlibabaCloudSearchEmbeddingsTaskSettings.INPUT_TYPE, inputType.toString());
+        }
+
+        return map;
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModelTests.java
new file mode 100644
index 0000000000000..4e9179b66c36f
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModelTests.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchUtils;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests;
+import org.hamcrest.MatcherAssert;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+
+public class AlibabaCloudSearchSparseModelTests extends ESTestCase {
+    public void testOverride() {
+        AlibabaCloudSearchSparseTaskSettings taskSettings = AlibabaCloudSearchSparseTaskSettingsTests.createRandom();
+        var model = createModel(
+            "service",
+            TaskType.TEXT_EMBEDDING,
+            AlibabaCloudSearchSparseServiceSettingsTests.createRandom(),
+            taskSettings,
+            DefaultSecretSettingsTests.createRandom()
+        );
+
+        var overriddenModel = AlibabaCloudSearchSparseModel.of(model, Map.of(), taskSettings.getInputType());
+        MatcherAssert.assertThat(overriddenModel, is(model));
+    }
+
+    public static AlibabaCloudSearchSparseModel createModel(
+        String modelId,
+        TaskType taskType,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secrets
+    ) {
+        return new AlibabaCloudSearchSparseModel(
+            modelId,
+            taskType,
+            AlibabaCloudSearchUtils.SERVICE_NAME,
+            serviceSettings,
+            taskSettings,
+            secrets,
+            null
+        );
+    }
+
+    public static AlibabaCloudSearchSparseModel createModel(
+        String modelId,
+        TaskType taskType,
+        AlibabaCloudSearchSparseServiceSettings serviceSettings,
+        AlibabaCloudSearchSparseTaskSettings taskSettings,
+        @Nullable DefaultSecretSettings secretSettings
+    ) {
+        return new AlibabaCloudSearchSparseModel(
+            modelId,
+            taskType,
+            AlibabaCloudSearchUtils.SERVICE_NAME,
+            serviceSettings,
+            taskSettings,
+            secretSettings
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseServiceSettingsTests.java
new file mode 100644
index 0000000000000..8dc635a52f06f
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseServiceSettingsTests.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettings;
+import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettingsTests;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+
+public class AlibabaCloudSearchSparseServiceSettingsTests extends AbstractWireSerializingTestCase<AlibabaCloudSearchSparseServiceSettings> {
+    public static AlibabaCloudSearchSparseServiceSettings createRandom() {
+        var commonSettings = AlibabaCloudSearchServiceSettingsTests.createRandom();
+        return new AlibabaCloudSearchSparseServiceSettings(commonSettings);
+    }
+
+    public void testFromMap() {
+        var model = "model";
+        var host = "host";
+        var workspaceName = "default";
+        var httpSchema = "https";
+        var serviceSettings = AlibabaCloudSearchSparseServiceSettings.fromMap(
+            new HashMap<>(
+                Map.of(
+                    AlibabaCloudSearchServiceSettings.HOST,
+                    host,
+                    AlibabaCloudSearchServiceSettings.SERVICE_ID,
+                    model,
+                    AlibabaCloudSearchServiceSettings.WORKSPACE_NAME,
+                    workspaceName,
+                    AlibabaCloudSearchServiceSettings.HTTP_SCHEMA_NAME,
+                    httpSchema
+                )
+            ),
+            null
+        );
+
+        MatcherAssert.assertThat(
+            serviceSettings,
+            is(
+                new AlibabaCloudSearchSparseServiceSettings(
+                    new AlibabaCloudSearchServiceSettings(model, host, workspaceName, httpSchema, null)
+                )
+            )
+        );
+    }
+
+    @Override
+    protected Writeable.Reader<AlibabaCloudSearchSparseServiceSettings> instanceReader() {
+        return AlibabaCloudSearchSparseServiceSettings::new;
+    }
+
+    @Override
+    protected AlibabaCloudSearchSparseServiceSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected AlibabaCloudSearchSparseServiceSettings mutateInstance(AlibabaCloudSearchSparseServiceSettings instance) throws IOException {
+        return null;
+    }
+
+    public static Map<String, Object> getServiceSettingsMap(String serviceId, String host, String workspaceName) {
+        return AlibabaCloudSearchServiceSettingsTests.getServiceSettingsMap(serviceId, host, workspaceName);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettingsTests.java
new file mode 100644
index 0000000000000..b16d96f9a081b
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseTaskSettingsTests.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.hamcrest.MatcherAssert;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.InputTypeTests.randomWithIngestAndSearch;
+import static org.hamcrest.Matchers.is;
+
+public class AlibabaCloudSearchSparseTaskSettingsTests extends AbstractWireSerializingTestCase<AlibabaCloudSearchSparseTaskSettings> {
+    public static AlibabaCloudSearchSparseTaskSettings createRandom() {
+        var inputType = randomBoolean() ? randomWithIngestAndSearch() : null;
+        var returnToken = randomBoolean();
+
+        return new AlibabaCloudSearchSparseTaskSettings(inputType, returnToken);
+    }
+
+    public void testFromMap() {
+        MatcherAssert.assertThat(
+            AlibabaCloudSearchSparseTaskSettings.fromMap(new HashMap<>(Map.of(AlibabaCloudSearchSparseTaskSettings.INPUT_TYPE, "ingest"))),
+            is(new AlibabaCloudSearchSparseTaskSettings(InputType.INGEST, null))
+        );
+    }
+
+    public void testFromMap_WhenInputTypeIsNull() {
+        InputType inputType = null;
+        MatcherAssert.assertThat(
+            AlibabaCloudSearchSparseTaskSettings.fromMap(new HashMap<>(Map.of())),
+            is(new AlibabaCloudSearchSparseTaskSettings(inputType, null))
+        );
+    }
+
+    @Override
+    protected Writeable.Reader<AlibabaCloudSearchSparseTaskSettings> instanceReader() {
+        return AlibabaCloudSearchSparseTaskSettings::new;
+    }
+
+    @Override
+    protected AlibabaCloudSearchSparseTaskSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected AlibabaCloudSearchSparseTaskSettings mutateInstance(AlibabaCloudSearchSparseTaskSettings instance) throws IOException {
+        return null;
+    }
+
+    public static Map<String, Object> getTaskSettingsMap(@Nullable InputType inputType, @Nullable Boolean returnToken) {
+        var map = new HashMap<String, Object>();
+
+        if (inputType != null) {
+            map.put(AlibabaCloudSearchSparseTaskSettings.INPUT_TYPE, inputType.toString());
+        }
+
+        if (returnToken != null) {
+            map.put(AlibabaCloudSearchSparseTaskSettings.RETURN_TOKEN, returnToken);
+        }
+
+        return map;
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingTypeTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingTypeTests.java
index ed13e5a87e71b..3aa423d5bbafd 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingTypeTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingTypeTests.java
@@ -38,14 +38,14 @@ public void testTranslateToVersion_ReturnsFloat_WhenVersionIsBeforeByteEnumAddit
 
     public void testTranslateToVersion_ReturnsByte_WhenVersionOnByteEnumAddition_WhenSpecifyingByte() {
         assertThat(
-            CohereEmbeddingType.translateToVersion(CohereEmbeddingType.BYTE, TransportVersions.ML_INFERENCE_EMBEDDING_BYTE_ADDED),
+            CohereEmbeddingType.translateToVersion(CohereEmbeddingType.BYTE, TransportVersions.V_8_14_0),
             is(CohereEmbeddingType.BYTE)
         );
     }
 
     public void testTranslateToVersion_ReturnsFloat_WhenVersionOnByteEnumAddition_WhenSpecifyingFloat() {
         assertThat(
-            CohereEmbeddingType.translateToVersion(CohereEmbeddingType.FLOAT, TransportVersions.ML_INFERENCE_EMBEDDING_BYTE_ADDED),
+            CohereEmbeddingType.translateToVersion(CohereEmbeddingType.FLOAT, TransportVersions.V_8_14_0),
             is(CohereEmbeddingType.FLOAT)
         );
     }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java
new file mode 100644
index 0000000000000..af13ce7944685
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+import org.elasticsearch.inference.EmptySecretSettings;
+import org.elasticsearch.inference.EmptyTaskSettings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.services.elser.ElserModels;
+
+public class ElasticInferenceServiceSparseEmbeddingsModelTests extends ESTestCase {
+
+    public static ElasticInferenceServiceSparseEmbeddingsModel createModel(String url) {
+        return createModel(url, null);
+    }
+
+    public static ElasticInferenceServiceSparseEmbeddingsModel createModel(String url, Integer maxInputTokens) {
+        return new ElasticInferenceServiceSparseEmbeddingsModel(
+            "id",
+            TaskType.SPARSE_EMBEDDING,
+            "service",
+            new ElasticInferenceServiceSparseEmbeddingsServiceSettings(ElserModels.ELSER_V2_MODEL, maxInputTokens, null),
+            EmptyTaskSettings.INSTANCE,
+            EmptySecretSettings.INSTANCE,
+            new ElasticInferenceServiceComponents(url)
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java
new file mode 100644
index 0000000000000..a2b36cf9abdd5
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.ServiceFields;
+import org.elasticsearch.xpack.inference.services.elser.ElserModels;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.services.elser.ElserModelsTests.randomElserModel;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+
+public class ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests extends AbstractWireSerializingTestCase<
+    ElasticInferenceServiceSparseEmbeddingsServiceSettings> {
+
+    @Override
+    protected Writeable.Reader<ElasticInferenceServiceSparseEmbeddingsServiceSettings> instanceReader() {
+        return ElasticInferenceServiceSparseEmbeddingsServiceSettings::new;
+    }
+
+    @Override
+    protected ElasticInferenceServiceSparseEmbeddingsServiceSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected ElasticInferenceServiceSparseEmbeddingsServiceSettings mutateInstance(
+        ElasticInferenceServiceSparseEmbeddingsServiceSettings instance
+    ) throws IOException {
+        return randomValueOtherThan(instance, ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests::createRandom);
+    }
+
+    public void testFromMap() {
+        var modelId = ElserModels.ELSER_V2_MODEL;
+
+        var serviceSettings = ElasticInferenceServiceSparseEmbeddingsServiceSettings.fromMap(
+            new HashMap<>(Map.of(ServiceFields.MODEL_ID, modelId)),
+            ConfigurationParseContext.REQUEST
+        );
+
+        assertThat(serviceSettings, is(new ElasticInferenceServiceSparseEmbeddingsServiceSettings(modelId, null, null)));
+    }
+
+    public void testFromMap_InvalidElserModelId() {
+        var invalidModelId = "invalid";
+
+        ValidationException validationException = expectThrows(
+            ValidationException.class,
+            () -> ElasticInferenceServiceSparseEmbeddingsServiceSettings.fromMap(
+                new HashMap<>(Map.of(ServiceFields.MODEL_ID, invalidModelId)),
+                ConfigurationParseContext.REQUEST
+            )
+        );
+
+        assertThat(validationException.getMessage(), containsString(Strings.format("unknown ELSER model id [%s]", invalidModelId)));
+    }
+
+    public void testToXContent_WritesAlLFields() throws IOException {
+        var modelId = ElserModels.ELSER_V1_MODEL;
+        var maxInputTokens = 10;
+        var serviceSettings = new ElasticInferenceServiceSparseEmbeddingsServiceSettings(modelId, maxInputTokens, null);
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        serviceSettings.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, is(Strings.format("""
+            {"model_id":"%s","max_input_tokens":%d,"rate_limit":{"requests_per_minute":1000}}""", modelId, maxInputTokens)));
+    }
+
+    public static ElasticInferenceServiceSparseEmbeddingsServiceSettings createRandom() {
+        return new ElasticInferenceServiceSparseEmbeddingsServiceSettings(randomElserModel(), randomNonNegativeInt(), null);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java
new file mode 100644
index 0000000000000..62416f05800c6
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java
@@ -0,0 +1,523 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elastic;
+
+import org.apache.http.HttpHeaders;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.ChunkedInferenceServiceResults;
+import org.elasticsearch.inference.ChunkingOptions;
+import org.elasticsearch.inference.EmptySecretSettings;
+import org.elasticsearch.inference.EmptyTaskSettings;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.http.MockResponse;
+import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.inference.action.InferenceAction;
+import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults;
+import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults;
+import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
+import org.elasticsearch.xpack.inference.results.SparseEmbeddingResultsTests;
+import org.elasticsearch.xpack.inference.services.ServiceFields;
+import org.elasticsearch.xpack.inference.services.elser.ElserModels;
+import org.elasticsearch.xpack.inference.services.openai.OpenAiService;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.xpack.inference.Utils.getInvalidModel;
+import static org.elasticsearch.xpack.inference.Utils.getModelListenerForException;
+import static org.elasticsearch.xpack.inference.Utils.getPersistedConfigMap;
+import static org.elasticsearch.xpack.inference.Utils.getRequestConfigMap;
+import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool;
+import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty;
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
+import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class ElasticInferenceServiceTests extends ESTestCase {
+
+    private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS);
+    private final MockWebServer webServer = new MockWebServer();
+    private ThreadPool threadPool;
+
+    private HttpClientManager clientManager;
+
+    @Before
+    public void init() throws Exception {
+        webServer.start();
+        threadPool = createThreadPool(inferenceUtilityPool());
+        clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class));
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        clientManager.close();
+        terminate(threadPool);
+        webServer.close();
+    }
+
+    public void testParseRequestConfig_CreatesASparseEmbeddingsModel() throws IOException {
+        try (var service = createServiceWithMockSender()) {
+            ActionListener<Model> modelListener = ActionListener.wrap(model -> {
+                assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class));
+
+                var completionModel = (ElasticInferenceServiceSparseEmbeddingsModel) model;
+                assertThat(completionModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL));
+
+            }, e -> fail("Model parsing should have succeeded, but failed: " + e.getMessage()));
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.SPARSE_EMBEDDING,
+                getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), Map.of(), Map.of()),
+                Set.of(),
+                modelListener
+            );
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsUnsupportedModelType() throws IOException {
+        try (var service = createServiceWithMockSender()) {
+            var failureListener = getModelListenerForException(
+                ElasticsearchStatusException.class,
+                "The [elastic] service does not support task type [completion]"
+            );
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.COMPLETION,
+                getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), Map.of(), Map.of()),
+                Set.of(),
+                failureListener
+            );
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInConfig() throws IOException {
+        try (var service = createServiceWithMockSender()) {
+            var config = getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), Map.of(), Map.of());
+            config.put("extra_key", "value");
+
+            var failureListener = getModelListenerForException(
+                ElasticsearchStatusException.class,
+                "Model configuration contains settings [{extra_key=value}] unknown to the [elastic] service"
+            );
+            service.parseRequestConfig("id", TaskType.SPARSE_EMBEDDING, config, Set.of(), failureListener);
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInServiceSettingsMap() throws IOException {
+        try (var service = createServiceWithMockSender()) {
+            Map<String, Object> serviceSettings = new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL));
+            serviceSettings.put("extra_key", "value");
+
+            var config = getRequestConfigMap(serviceSettings, Map.of(), Map.of());
+
+            var failureListener = getModelListenerForException(
+                ElasticsearchStatusException.class,
+                "Model configuration contains settings [{extra_key=value}] unknown to the [elastic] service"
+            );
+            service.parseRequestConfig("id", TaskType.SPARSE_EMBEDDING, config, Set.of(), failureListener);
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInTaskSettingsMap() throws IOException {
+        try (var service = createServiceWithMockSender()) {
+            var taskSettings = Map.of("extra_key", (Object) "value");
+
+            var config = getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), taskSettings, Map.of());
+
+            var failureListener = getModelListenerForException(
+                ElasticsearchStatusException.class,
+                "Model configuration contains settings [{extra_key=value}] unknown to the [elastic] service"
+            );
+            service.parseRequestConfig("id", TaskType.SPARSE_EMBEDDING, config, Set.of(), failureListener);
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInSecretSettingsMap() throws IOException {
+        try (var service = createServiceWithMockSender()) {
+            var secretSettings = Map.of("extra_key", (Object) "value");
+
+            var config = getRequestConfigMap(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL), Map.of(), secretSettings);
+
+            var failureListener = getModelListenerForException(
+                ElasticsearchStatusException.class,
+                "Model configuration contains settings [{extra_key=value}] unknown to the [elastic] service"
+            );
+            service.parseRequestConfig("id", TaskType.SPARSE_EMBEDDING, config, Set.of(), failureListener);
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_CreatesASparseEmbeddingModel() throws IOException {
+        try (var service = createServiceWithMockSender()) {
+            var persistedConfig = getPersistedConfigMap(
+                new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL)),
+                Map.of(),
+                Map.of()
+            );
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.SPARSE_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class));
+
+            var sparseEmbeddingsModel = (ElasticInferenceServiceSparseEmbeddingsModel) model;
+            assertThat(sparseEmbeddingsModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL));
+            assertThat(sparseEmbeddingsModel.getTaskSettings(), is(EmptyTaskSettings.INSTANCE));
+            assertThat(sparseEmbeddingsModel.getSecretSettings(), is(EmptySecretSettings.INSTANCE));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInConfig() throws IOException {
+        try (var service = createServiceWithMockSender()) {
+            var persistedConfig = getPersistedConfigMap(
+                new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL)),
+                Map.of(),
+                Map.of()
+            );
+            persistedConfig.config().put("extra_key", "value");
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.SPARSE_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class));
+
+            var completionModel = (ElasticInferenceServiceSparseEmbeddingsModel) model;
+            assertThat(completionModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL));
+            assertThat(completionModel.getTaskSettings(), is(EmptyTaskSettings.INSTANCE));
+            assertThat(completionModel.getSecretSettings(), is(EmptySecretSettings.INSTANCE));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInServiceSettings() throws IOException {
+        try (var service = createServiceWithMockSender()) {
+            Map<String, Object> serviceSettingsMap = new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL));
+            serviceSettingsMap.put("extra_key", "value");
+
+            var persistedConfig = getPersistedConfigMap(serviceSettingsMap, Map.of(), Map.of());
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.SPARSE_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class));
+
+            var completionModel = (ElasticInferenceServiceSparseEmbeddingsModel) model;
+            assertThat(completionModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL));
+            assertThat(completionModel.getTaskSettings(), is(EmptyTaskSettings.INSTANCE));
+            assertThat(completionModel.getSecretSettings(), is(EmptySecretSettings.INSTANCE));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInTaskSettings() throws IOException {
+        try (var service = createServiceWithMockSender()) {
+            var taskSettings = Map.of("extra_key", (Object) "value");
+
+            var persistedConfig = getPersistedConfigMap(
+                new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL)),
+                taskSettings,
+                Map.of()
+            );
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.SPARSE_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class));
+
+            var completionModel = (ElasticInferenceServiceSparseEmbeddingsModel) model;
+            assertThat(completionModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL));
+            assertThat(completionModel.getTaskSettings(), is(EmptyTaskSettings.INSTANCE));
+            assertThat(completionModel.getSecretSettings(), is(EmptySecretSettings.INSTANCE));
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInSecretsSettings() throws IOException {
+        try (var service = createServiceWithMockSender()) {
+            var secretSettingsMap = Map.of("extra_key", (Object) "value");
+
+            var persistedConfig = getPersistedConfigMap(
+                new HashMap<>(Map.of(ServiceFields.MODEL_ID, ElserModels.ELSER_V2_MODEL)),
+                Map.of(),
+                secretSettingsMap
+            );
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.SPARSE_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            assertThat(model, instanceOf(ElasticInferenceServiceSparseEmbeddingsModel.class));
+
+            var completionModel = (ElasticInferenceServiceSparseEmbeddingsModel) model;
+            assertThat(completionModel.getServiceSettings().modelId(), is(ElserModels.ELSER_V2_MODEL));
+            assertThat(completionModel.getTaskSettings(), is(EmptyTaskSettings.INSTANCE));
+            assertThat(completionModel.getSecretSettings(), is(EmptySecretSettings.INSTANCE));
+        }
+    }
+
+    public void testCheckModelConfig_ReturnsNewModelReference() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new OpenAiService(senderFactory, createWithEmptySettings(threadPool))) {
+            var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer));
+            PlainActionFuture<Model> listener = new PlainActionFuture<>();
+            service.checkModelConfig(model, listener);
+
+            var returnedModel = listener.actionGet(TIMEOUT);
+            assertThat(returnedModel, is(ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer))));
+        }
+    }
+
+    public void testInfer_ThrowsErrorWhenModelIsNotAValidModel() throws IOException {
+        var sender = mock(Sender.class);
+
+        var factory = mock(HttpRequestSender.Factory.class);
+        when(factory.createSender()).thenReturn(sender);
+
+        var mockModel = getInvalidModel("model_id", "service_name");
+
+        try (
+            var service = new ElasticInferenceService(
+                factory,
+                createWithEmptySettings(threadPool),
+                new ElasticInferenceServiceComponents(null)
+            )
+        ) {
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                mockModel,
+                null,
+                List.of(""),
+                new HashMap<>(),
+                InputType.INGEST,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT));
+            MatcherAssert.assertThat(
+                thrownException.getMessage(),
+                is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.")
+            );
+
+            verify(factory, times(1)).createSender();
+            verify(sender, times(1)).start();
+        }
+
+        verify(sender, times(1)).close();
+        verifyNoMoreInteractions(factory);
+        verifyNoMoreInteractions(sender);
+    }
+
+    public void testInfer_ThrowsWhenQueryIsPresent() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (
+            var service = new ElasticInferenceService(
+                senderFactory,
+                createWithEmptySettings(threadPool),
+                new ElasticInferenceServiceComponents(getUrl(webServer))
+            )
+        ) {
+            var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(getUrl(webServer));
+
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            UnsupportedOperationException exception = expectThrows(
+                UnsupportedOperationException.class,
+                () -> service.infer(
+                    model,
+                    "should throw",
+                    List.of("abc"),
+                    new HashMap<>(),
+                    InputType.INGEST,
+                    InferenceAction.Request.DEFAULT_TIMEOUT,
+                    listener
+                )
+            );
+
+            assertThat(exception.getMessage(), is("Query input not supported for Elastic Inference Service"));
+        }
+    }
+
+    public void testInfer_SendsEmbeddingsRequest() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+        var eisGatewayUrl = getUrl(webServer);
+
+        try (
+            var service = new ElasticInferenceService(
+                senderFactory,
+                createWithEmptySettings(threadPool),
+                new ElasticInferenceServiceComponents(eisGatewayUrl)
+            )
+        ) {
+            String responseJson = """
+                {
+                    "data": [
+                        {
+                            "hello": 2.1259406,
+                            "greet": 1.7073475
+                        }
+                    ]
+                }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(eisGatewayUrl);
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                null,
+                List.of("input text"),
+                new HashMap<>(),
+                InputType.INGEST,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+            var result = listener.actionGet(TIMEOUT);
+
+            assertThat(
+                result.asMap(),
+                Matchers.is(
+                    SparseEmbeddingResultsTests.buildExpectationSparseEmbeddings(
+                        List.of(
+                            new SparseEmbeddingResultsTests.EmbeddingExpectation(Map.of("hello", 2.1259406f, "greet", 1.7073475f), false)
+                        )
+                    )
+                )
+            );
+            var request = webServer.requests().get(0);
+            assertNull(request.getUri().getQuery());
+            assertThat(request.getHeader(HttpHeaders.CONTENT_TYPE), Matchers.equalTo(XContentType.JSON.mediaType()));
+
+            var requestMap = entityAsMap(request.getBody());
+            assertThat(requestMap, is(Map.of("input", List.of("input text"))));
+        }
+    }
+
+    public void testChunkedInfer_PassesThrough() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+        var eisGatewayUrl = getUrl(webServer);
+
+        try (
+            var service = new ElasticInferenceService(
+                senderFactory,
+                createWithEmptySettings(threadPool),
+                new ElasticInferenceServiceComponents(eisGatewayUrl)
+            )
+        ) {
+            String responseJson = """
+                {
+                    "data": [
+                        {
+                            "hello": 2.1259406,
+                            "greet": 1.7073475
+                        }
+                    ]
+                }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = ElasticInferenceServiceSparseEmbeddingsModelTests.createModel(eisGatewayUrl);
+            PlainActionFuture<List<ChunkedInferenceServiceResults>> listener = new PlainActionFuture<>();
+            service.chunkedInfer(
+                model,
+                List.of("input text"),
+                new HashMap<>(),
+                InputType.INGEST,
+                new ChunkingOptions(null, null),
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var results = listener.actionGet(TIMEOUT);
+            MatcherAssert.assertThat(
+                results.get(0).asMap(),
+                Matchers.is(
+                    Map.of(
+                        InferenceChunkedSparseEmbeddingResults.FIELD_NAME,
+                        List.of(
+                            Map.of(
+                                ChunkedNlpInferenceResults.TEXT,
+                                "input text",
+                                ChunkedNlpInferenceResults.INFERENCE,
+                                Map.of("hello", 2.1259406f, "greet", 1.7073475f)
+                            )
+                        )
+                    )
+                )
+            );
+            MatcherAssert.assertThat(webServer.requests(), hasSize(1));
+            assertNull(webServer.requests().get(0).getUri().getQuery());
+            MatcherAssert.assertThat(
+                webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE),
+                equalTo(XContentType.JSON.mediaType())
+            );
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            assertThat(requestMap, is(Map.of("input", List.of("input text"))));
+        }
+    }
+
+    private ElasticInferenceService createServiceWithMockSender() {
+        return new ElasticInferenceService(
+            mock(HttpRequestSender.Factory.class),
+            createWithEmptySettings(threadPool),
+            new ElasticInferenceServiceComponents(null)
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
index e6fd725a50198..257616033f080 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.inference.ChunkingOptions;
+import org.elasticsearch.inference.EmptyTaskSettings;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceExtension;
 import org.elasticsearch.inference.InputType;
@@ -31,6 +32,7 @@
 import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
+import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults;
 import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
 import org.elasticsearch.xpack.core.ml.action.InferModelAction;
@@ -39,8 +41,10 @@
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings;
 import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
+import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResultsTests;
 import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResults;
 import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResultsTests;
+import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults;
 import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate;
@@ -52,12 +56,10 @@
 import org.mockito.Mockito;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -76,7 +78,6 @@
 
 public class ElasticsearchInternalServiceTests extends ESTestCase {
 
-    TaskType taskType = TaskType.TEXT_EMBEDDING;
     String randomInferenceEntityId = randomAlphaOfLength(10);
 
     private static ThreadPool threadPool;
@@ -92,7 +93,25 @@ public void shutdownThreadPool() {
     }
 
     public void testParseRequestConfig() {
+        var service = createService(mock(Client.class));
+        var settings = new HashMap<String, Object>();
+        settings.put(
+            ModelConfigurations.SERVICE_SETTINGS,
+            new HashMap<>(
+                Map.of(ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4)
+            )
+        );
 
+        ActionListener<Model> modelListener = ActionListener.<Model>wrap(
+            model -> fail("Model parsing should have failed"),
+            e -> assertThat(e, instanceOf(IllegalArgumentException.class))
+        );
+
+        var taskType = randomFrom(TaskType.TEXT_EMBEDDING, TaskType.RERANK, TaskType.SPARSE_EMBEDDING);
+        service.parseRequestConfig(randomInferenceEntityId, taskType, settings, Set.of(), modelListener);
+    }
+
+    public void testParseRequestConfig_Misconfigured() {
         // Null model variant
         {
             var service = createService(mock(Client.class));
@@ -109,43 +128,10 @@ public void testParseRequestConfig() {
                 e -> assertThat(e, instanceOf(IllegalArgumentException.class))
             );
 
+            var taskType = randomFrom(TaskType.TEXT_EMBEDDING, TaskType.RERANK, TaskType.SPARSE_EMBEDDING);
             service.parseRequestConfig(randomInferenceEntityId, taskType, settings, Set.of(), modelListener);
         }
 
-        // Valid model variant
-        {
-            var service = createService(mock(Client.class));
-            var settings = new HashMap<String, Object>();
-            settings.put(
-                ModelConfigurations.SERVICE_SETTINGS,
-                new HashMap<>(
-                    Map.of(
-                        ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS,
-                        1,
-                        ElasticsearchInternalServiceSettings.NUM_THREADS,
-                        4,
-                        ElasticsearchInternalServiceSettings.MODEL_ID,
-                        ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID
-                    )
-                )
-            );
-
-            var e5ServiceSettings = new MultilingualE5SmallInternalServiceSettings(
-                1,
-                4,
-                ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID,
-                null
-            );
-
-            service.parseRequestConfig(
-                randomInferenceEntityId,
-                taskType,
-                settings,
-                Set.of(),
-                getModelVerificationActionListener(e5ServiceSettings)
-            );
-        }
-
         // Invalid config map
         {
             var service = createService(mock(Client.class));
@@ -163,10 +149,12 @@ public void testParseRequestConfig() {
                 e -> assertThat(e, instanceOf(ElasticsearchStatusException.class))
             );
 
+            var taskType = randomFrom(TaskType.TEXT_EMBEDDING, TaskType.RERANK, TaskType.SPARSE_EMBEDDING);
             service.parseRequestConfig(randomInferenceEntityId, taskType, settings, Set.of(), modelListener);
         }
+    }
 
-        // Invalid service settings
+    public void testParseRequestConfig_E5() {
         {
             var service = createService(mock(Client.class));
             var settings = new HashMap<String, Object>();
@@ -179,52 +167,28 @@ public void testParseRequestConfig() {
                         ElasticsearchInternalServiceSettings.NUM_THREADS,
                         4,
                         ElasticsearchInternalServiceSettings.MODEL_ID,
-                        ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID, // we can't directly test the eland case until we mock
-                                                                                     // the threadpool within the client
-                        "not_a_valid_service_setting",
-                        randomAlphaOfLength(10)
+                        ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID
                     )
                 )
             );
 
-            ActionListener<Model> modelListener = ActionListener.<Model>wrap(
-                model -> fail("Model parsing should have failed"),
-                e -> assertThat(e, instanceOf(ElasticsearchStatusException.class))
-            );
-
-            service.parseRequestConfig(randomInferenceEntityId, taskType, settings, Set.of(), modelListener);
-        }
-
-        // Extra service settings
-        {
-            var service = createService(mock(Client.class));
-            var settings = new HashMap<String, Object>();
-            settings.put(
-                ModelConfigurations.SERVICE_SETTINGS,
-                new HashMap<>(
-                    Map.of(
-                        ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS,
-                        1,
-                        ElasticsearchInternalServiceSettings.NUM_THREADS,
-                        4,
-                        ElasticsearchInternalServiceSettings.MODEL_ID,
-                        ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID, // we can't directly test the eland case until we mock
-                                                                                     // the threadpool within the client
-                        "extra_setting_that_should_not_be_here",
-                        randomAlphaOfLength(10)
-                    )
-                )
+            var e5ServiceSettings = new MultilingualE5SmallInternalServiceSettings(
+                1,
+                4,
+                ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID,
+                null
             );
 
-            ActionListener<Model> modelListener = ActionListener.<Model>wrap(
-                model -> fail("Model parsing should have failed"),
-                e -> assertThat(e, instanceOf(ElasticsearchStatusException.class))
+            service.parseRequestConfig(
+                randomInferenceEntityId,
+                TaskType.TEXT_EMBEDDING,
+                settings,
+                Set.of(),
+                getModelVerificationActionListener(e5ServiceSettings)
             );
-
-            service.parseRequestConfig(randomInferenceEntityId, taskType, settings, Set.of(), modelListener);
         }
 
-        // Extra settings
+        // Invalid service settings
         {
             var service = createService(mock(Client.class));
             var settings = new HashMap<String, Object>();
@@ -237,19 +201,19 @@ public void testParseRequestConfig() {
                         ElasticsearchInternalServiceSettings.NUM_THREADS,
                         4,
                         ElasticsearchInternalServiceSettings.MODEL_ID,
-                        ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID // we can't directly test the eland case until we mock
-                        // the threadpool within the client
+                        ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID,
+                        "not_a_valid_service_setting",
+                        randomAlphaOfLength(10)
                     )
                 )
             );
-            settings.put("extra_setting_that_should_not_be_here", randomAlphaOfLength(10));
 
             ActionListener<Model> modelListener = ActionListener.<Model>wrap(
                 model -> fail("Model parsing should have failed"),
                 e -> assertThat(e, instanceOf(ElasticsearchStatusException.class))
             );
 
-            service.parseRequestConfig(randomInferenceEntityId, taskType, settings, Set.of(), modelListener);
+            service.parseRequestConfig(randomInferenceEntityId, TaskType.TEXT_EMBEDDING, settings, Set.of(), modelListener);
         }
     }
 
@@ -342,10 +306,53 @@ public void testParseRequestConfig_Rerank_DefaultTaskSettings() {
         }
     }
 
+    @SuppressWarnings("unchecked")
+    public void testParseRequestConfig_SparseEmbedding() {
+        var client = mock(Client.class);
+        doAnswer(invocation -> {
+            var listener = (ActionListener<GetTrainedModelsAction.Response>) invocation.getArguments()[2];
+            listener.onResponse(
+                new GetTrainedModelsAction.Response(new QueryPage<>(List.of(mock(TrainedModelConfig.class)), 1, mock(ParseField.class)))
+            );
+            return null;
+        }).when(client).execute(Mockito.same(GetTrainedModelsAction.INSTANCE), any(), any());
+
+        when(client.threadPool()).thenReturn(threadPool);
+
+        var service = createService(client);
+        var settings = new HashMap<String, Object>();
+        settings.put(
+            ModelConfigurations.SERVICE_SETTINGS,
+            new HashMap<>(
+                Map.of(
+                    ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS,
+                    1,
+                    ElasticsearchInternalServiceSettings.NUM_THREADS,
+                    4,
+                    ElasticsearchInternalServiceSettings.MODEL_ID,
+                    "foo"
+                )
+            )
+        );
+
+        ActionListener<Model> modelListener = ActionListener.<Model>wrap(model -> {
+            assertThat(model, instanceOf(CustomElandModel.class));
+            assertThat(model.getTaskSettings(), instanceOf(EmptyTaskSettings.class));
+            assertThat(model.getServiceSettings(), instanceOf(CustomElandInternalServiceSettings.class));
+        }, e -> { fail("Model parsing failed " + e.getMessage()); });
+
+        service.parseRequestConfig(randomInferenceEntityId, TaskType.SPARSE_EMBEDDING, settings, Set.of(), modelListener);
+    }
+
     private ActionListener<Model> getModelVerificationActionListener(MultilingualE5SmallInternalServiceSettings e5ServiceSettings) {
         return ActionListener.<Model>wrap(model -> {
             assertEquals(
-                new MultilingualE5SmallModel(randomInferenceEntityId, taskType, ElasticsearchInternalService.NAME, e5ServiceSettings),
+                new MultilingualE5SmallModel(
+                    randomInferenceEntityId,
+                    TaskType.TEXT_EMBEDDING,
+                    ElasticsearchInternalService.NAME,
+                    e5ServiceSettings
+                ),
                 model
             );
         }, e -> { fail("Model parsing failed " + e.getMessage()); });
@@ -371,7 +378,10 @@ public void testParsePersistedConfig() {
                 )
             );
 
-            expectThrows(IllegalArgumentException.class, () -> service.parsePersistedConfig(randomInferenceEntityId, taskType, settings));
+            expectThrows(
+                IllegalArgumentException.class,
+                () -> service.parsePersistedConfig(randomInferenceEntityId, TaskType.TEXT_EMBEDDING, settings)
+            );
 
         }
 
@@ -397,12 +407,17 @@ public void testParsePersistedConfig() {
 
             CustomElandEmbeddingModel parsedModel = (CustomElandEmbeddingModel) service.parsePersistedConfig(
                 randomInferenceEntityId,
-                taskType,
+                TaskType.TEXT_EMBEDDING,
                 settings
             );
             var elandServiceSettings = new CustomElandInternalTextEmbeddingServiceSettings(1, 4, "invalid", null);
             assertEquals(
-                new CustomElandEmbeddingModel(randomInferenceEntityId, taskType, ElasticsearchInternalService.NAME, elandServiceSettings),
+                new CustomElandEmbeddingModel(
+                    randomInferenceEntityId,
+                    TaskType.TEXT_EMBEDDING,
+                    ElasticsearchInternalService.NAME,
+                    elandServiceSettings
+                ),
                 parsedModel
             );
         }
@@ -436,11 +451,16 @@ public void testParsePersistedConfig() {
 
             MultilingualE5SmallModel parsedModel = (MultilingualE5SmallModel) service.parsePersistedConfig(
                 randomInferenceEntityId,
-                taskType,
+                TaskType.TEXT_EMBEDDING,
                 settings
             );
             assertEquals(
-                new MultilingualE5SmallModel(randomInferenceEntityId, taskType, ElasticsearchInternalService.NAME, e5ServiceSettings),
+                new MultilingualE5SmallModel(
+                    randomInferenceEntityId,
+                    TaskType.TEXT_EMBEDDING,
+                    ElasticsearchInternalService.NAME,
+                    e5ServiceSettings
+                ),
                 parsedModel
             );
         }
@@ -456,6 +476,8 @@ public void testParsePersistedConfig() {
                 )
             );
             settings.put("not_a_valid_config_setting", randomAlphaOfLength(10));
+
+            var taskType = randomFrom(TaskType.TEXT_EMBEDDING, TaskType.RERANK, TaskType.SPARSE_EMBEDDING);
             expectThrows(IllegalArgumentException.class, () -> service.parsePersistedConfig(randomInferenceEntityId, taskType, settings));
         }
 
@@ -476,12 +498,13 @@ public void testParsePersistedConfig() {
                     )
                 )
             );
+            var taskType = randomFrom(TaskType.TEXT_EMBEDDING, TaskType.RERANK, TaskType.SPARSE_EMBEDDING);
             expectThrows(IllegalArgumentException.class, () -> service.parsePersistedConfig(randomInferenceEntityId, taskType, settings));
         }
     }
 
     @SuppressWarnings("unchecked")
-    public void testChunkInfer() {
+    public void testChunkInfer_e5() {
         var mlTrainedModelResults = new ArrayList<InferenceResults>();
         mlTrainedModelResults.add(MlChunkedTextEmbeddingFloatResultsTests.createRandomResults());
         mlTrainedModelResults.add(MlChunkedTextEmbeddingFloatResultsTests.createRandomResults());
@@ -568,6 +591,63 @@ public void testChunkInfer() {
         assertTrue("Listener not called", gotResults.get());
     }
 
+    @SuppressWarnings("unchecked")
+    public void testChunkInfer_Sparse() {
+        var mlTrainedModelResults = new ArrayList<InferenceResults>();
+        mlTrainedModelResults.add(InferenceChunkedTextExpansionResultsTests.createRandomResults());
+        mlTrainedModelResults.add(InferenceChunkedTextExpansionResultsTests.createRandomResults());
+        mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom")));
+        var response = new InferModelAction.Response(mlTrainedModelResults, "foo", true);
+
+        ThreadPool threadpool = new TestThreadPool("test");
+        Client client = mock(Client.class);
+        when(client.threadPool()).thenReturn(threadpool);
+        doAnswer(invocationOnMock -> {
+            var listener = (ActionListener<InferModelAction.Response>) invocationOnMock.getArguments()[2];
+            listener.onResponse(response);
+            return null;
+        }).when(client).execute(same(InferModelAction.INSTANCE), any(InferModelAction.Request.class), any(ActionListener.class));
+
+        var model = new CustomElandModel(
+            "foo",
+            TaskType.SPARSE_EMBEDDING,
+            "elasticsearch",
+            new ElasticsearchInternalServiceSettings(1, 1, "model-id", null)
+        );
+        var service = createService(client);
+
+        var gotResults = new AtomicBoolean();
+        var resultsListener = ActionListener.<List<ChunkedInferenceServiceResults>>wrap(chunkedResponse -> {
+            assertThat(chunkedResponse, hasSize(3));
+            assertThat(chunkedResponse.get(0), instanceOf(InferenceChunkedSparseEmbeddingResults.class));
+            var result1 = (InferenceChunkedSparseEmbeddingResults) chunkedResponse.get(0);
+            assertEquals(((MlChunkedTextExpansionResults) mlTrainedModelResults.get(0)).getChunks(), result1.getChunkedResults());
+            assertThat(chunkedResponse.get(1), instanceOf(InferenceChunkedSparseEmbeddingResults.class));
+            var result2 = (InferenceChunkedSparseEmbeddingResults) chunkedResponse.get(1);
+            assertEquals(((MlChunkedTextExpansionResults) mlTrainedModelResults.get(1)).getChunks(), result2.getChunkedResults());
+            var result3 = (ErrorChunkedInferenceResults) chunkedResponse.get(2);
+            assertThat(result3.getException(), instanceOf(RuntimeException.class));
+            assertThat(result3.getException().getMessage(), containsString("boom"));
+            gotResults.set(true);
+        }, ESTestCase::fail);
+
+        service.chunkedInfer(
+            model,
+            null,
+            List.of("foo", "bar"),
+            Map.of(),
+            InputType.SEARCH,
+            new ChunkingOptions(null, null),
+            InferenceAction.Request.DEFAULT_TIMEOUT,
+            ActionListener.runAfter(resultsListener, () -> terminate(threadpool))
+        );
+
+        if (gotResults.get() == false) {
+            terminate(threadpool);
+        }
+        assertTrue("Listener not called", gotResults.get());
+    }
+
     @SuppressWarnings("unchecked")
     public void testChunkInferSetsTokenization() {
         var expectedSpan = new AtomicInteger();
@@ -711,7 +791,7 @@ public void testParseRequestConfigEland_PreservesTaskType() {
             )
         );
 
-        var taskType = randomFrom(EnumSet.of(TaskType.RERANK, TaskType.TEXT_EMBEDDING));
+        var taskType = randomFrom(EnumSet.of(TaskType.RERANK, TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING));
         CustomElandModel expectedModel = getCustomElandModel(taskType);
 
         PlainActionFuture<Model> listener = new PlainActionFuture<>();
@@ -739,6 +819,13 @@ private CustomElandModel getCustomElandModel(TaskType taskType) {
                 ElasticsearchInternalService.NAME,
                 serviceSettings
             );
+        } else if (taskType == TaskType.SPARSE_EMBEDDING) {
+            expectedModel = new CustomElandModel(
+                randomInferenceEntityId,
+                taskType,
+                ElasticsearchInternalService.NAME,
+                new CustomElandInternalServiceSettings(1, 4, "custom-model", null)
+            );
         }
         return expectedModel;
     }
@@ -867,21 +954,4 @@ private ElasticsearchInternalService createService(Client client) {
         var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client);
         return new ElasticsearchInternalService(context);
     }
-
-    public static Model randomModelConfig(String inferenceEntityId) {
-        List<String> givenList = Arrays.asList("MultilingualE5SmallModel");
-        Random rand = org.elasticsearch.common.Randomness.get();
-        String model = givenList.get(rand.nextInt(givenList.size()));
-
-        return switch (model) {
-            case "MultilingualE5SmallModel" -> new MultilingualE5SmallModel(
-                inferenceEntityId,
-                TaskType.TEXT_EMBEDDING,
-                ElasticsearchInternalService.NAME,
-                MultilingualE5SmallInternalServiceSettingsTests.createRandom()
-            );
-            default -> throw new IllegalArgumentException("model " + model + " is not supported for testing");
-        };
-    }
-
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java
index ec753b9bec887..ffbdf1a5a6178 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java
@@ -16,12 +16,12 @@
 import java.io.IOException;
 import java.util.HashSet;
 
+import static org.elasticsearch.xpack.inference.services.elser.ElserModelsTests.randomElserModel;
+
 public class ElserInternalServiceSettingsTests extends AbstractWireSerializingTestCase<ElserInternalServiceSettings> {
 
     public static ElserInternalServiceSettings createRandom() {
-        return new ElserInternalServiceSettings(
-            ElasticsearchInternalServiceSettingsTests.validInstance(randomFrom(ElserInternalService.VALID_ELSER_MODEL_IDS))
-        );
+        return new ElserInternalServiceSettings(ElasticsearchInternalServiceSettingsTests.validInstance(randomElserModel()));
     }
 
     public void testBwcWrite() throws IOException {
@@ -67,7 +67,7 @@ protected ElserInternalServiceSettings mutateInstance(ElserInternalServiceSettin
                 )
             );
             case 2 -> {
-                var versions = new HashSet<>(ElserInternalService.VALID_ELSER_MODEL_IDS);
+                var versions = new HashSet<>(ElserModels.VALID_ELSER_MODEL_IDS);
                 versions.remove(instance.modelId());
                 yield new ElserInternalServiceSettings(
                     new ElasticsearchInternalServiceSettings(
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java
index f950e515a5336..85add1a0090c8 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java
@@ -171,7 +171,7 @@ public void testParseConfigStrictWithNoTaskSettings() {
             "foo",
             TaskType.SPARSE_EMBEDDING,
             ElserInternalService.NAME,
-            new ElserInternalServiceSettings(1, 4, ElserInternalService.ELSER_V2_MODEL, null),
+            new ElserInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null),
             ElserMlNodeTaskSettings.DEFAULT
         );
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserModelsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserModelsTests.java
new file mode 100644
index 0000000000000..f56e941dcc8c0
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserModelsTests.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.elser;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class ElserModelsTests extends ESTestCase {
+
+    public static String randomElserModel() {
+        return randomFrom(ElserModels.VALID_ELSER_MODEL_IDS);
+    }
+
+    public void testIsValidModel() {
+        assertTrue(ElserModels.isValidModel(randomElserModel()));
+    }
+
+    public void testIsValidEisModel() {
+        assertTrue(ElserModels.isValidEisModel(ElserModels.ELSER_V2_MODEL));
+    }
+
+    public void testIsInvalidModel() {
+        assertFalse(ElserModels.isValidModel("invalid"));
+    }
+
+    public void testIsInvalidEisModel() {
+        assertFalse(ElserModels.isValidEisModel(ElserModels.ELSER_V2_MODEL_LINUX_X86));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/telemetry/ApmInferenceStatsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/telemetry/ApmInferenceStatsTests.java
new file mode 100644
index 0000000000000..1a5aba5f89ad2
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/telemetry/ApmInferenceStatsTests.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.telemetry;
+
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.telemetry.metric.LongCounter;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Map;
+
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class ApmInferenceStatsTests extends ESTestCase {
+
+    public void testRecordWithModel() {
+        var longCounter = mock(LongCounter.class);
+
+        var stats = new ApmInferenceStats(longCounter);
+
+        stats.incrementRequestCount(model("service", TaskType.ANY, "modelId"));
+
+        verify(longCounter).incrementBy(
+            eq(1L),
+            eq(Map.of("service", "service", "task_type", TaskType.ANY.toString(), "model_id", "modelId"))
+        );
+    }
+
+    public void testRecordWithoutModel() {
+        var longCounter = mock(LongCounter.class);
+
+        var stats = new ApmInferenceStats(longCounter);
+
+        stats.incrementRequestCount(model("service", TaskType.ANY, null));
+
+        verify(longCounter).incrementBy(eq(1L), eq(Map.of("service", "service", "task_type", TaskType.ANY.toString())));
+    }
+
+    public void testCreation() {
+        assertNotNull(ApmInferenceStats.create(MeterRegistry.NOOP));
+    }
+
+    private Model model(String service, TaskType taskType, String modelId) {
+        var configuration = mock(ModelConfigurations.class);
+        when(configuration.getService()).thenReturn(service);
+        var settings = mock(ServiceSettings.class);
+        if (modelId != null) {
+            when(settings.modelId()).thenReturn(modelId);
+        }
+
+        var model = mock(Model.class);
+        when(model.getTaskType()).thenReturn(taskType);
+        when(model.getConfigurations()).thenReturn(configuration);
+        when(model.getServiceSettings()).thenReturn(settings);
+
+        return model;
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/telemetry/StatsMapTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/telemetry/StatsMapTests.java
deleted file mode 100644
index fcd8d3d7cefbc..0000000000000
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/telemetry/StatsMapTests.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.telemetry;
-
-import org.elasticsearch.inference.TaskType;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
-import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsModel;
-import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsServiceSettingsTests;
-import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsTaskSettingsTests;
-import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModel;
-import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsServiceSettingsTests;
-import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsTaskSettingsTests;
-
-import java.util.Map;
-
-import static org.hamcrest.Matchers.is;
-
-public class StatsMapTests extends ESTestCase {
-    public void testAddingEntry_InitializesTheCountToOne() {
-        var stats = new StatsMap<>(InferenceStats::key, InferenceStats::new);
-
-        stats.increment(
-            new OpenAiEmbeddingsModel(
-                "inference_id",
-                TaskType.TEXT_EMBEDDING,
-                "openai",
-                OpenAiEmbeddingsServiceSettingsTests.getServiceSettingsMap("modelId", null, null),
-                OpenAiEmbeddingsTaskSettingsTests.getTaskSettingsMap(null),
-                null,
-                ConfigurationParseContext.REQUEST
-            )
-        );
-
-        var converted = stats.toSerializableMap();
-
-        assertThat(
-            converted,
-            is(
-                Map.of(
-                    "openai:text_embedding:modelId",
-                    new org.elasticsearch.xpack.core.inference.InferenceRequestStats("openai", TaskType.TEXT_EMBEDDING, "modelId", 1)
-                )
-            )
-        );
-    }
-
-    public void testIncrementingWithSeparateModels_IncrementsTheCounterToTwo() {
-        var stats = new StatsMap<>(InferenceStats::key, InferenceStats::new);
-
-        var model1 = new OpenAiEmbeddingsModel(
-            "inference_id",
-            TaskType.TEXT_EMBEDDING,
-            "openai",
-            OpenAiEmbeddingsServiceSettingsTests.getServiceSettingsMap("modelId", null, null),
-            OpenAiEmbeddingsTaskSettingsTests.getTaskSettingsMap(null),
-            null,
-            ConfigurationParseContext.REQUEST
-        );
-
-        var model2 = new OpenAiEmbeddingsModel(
-            "inference_id",
-            TaskType.TEXT_EMBEDDING,
-            "openai",
-            OpenAiEmbeddingsServiceSettingsTests.getServiceSettingsMap("modelId", null, null),
-            OpenAiEmbeddingsTaskSettingsTests.getTaskSettingsMap(null),
-            null,
-            ConfigurationParseContext.REQUEST
-        );
-
-        stats.increment(model1);
-        stats.increment(model2);
-
-        var converted = stats.toSerializableMap();
-
-        assertThat(
-            converted,
-            is(
-                Map.of(
-                    "openai:text_embedding:modelId",
-                    new org.elasticsearch.xpack.core.inference.InferenceRequestStats("openai", TaskType.TEXT_EMBEDDING, "modelId", 2)
-                )
-            )
-        );
-    }
-
-    public void testNullModelId_ResultsInKeyWithout() {
-        var stats = new StatsMap<>(InferenceStats::key, InferenceStats::new);
-
-        stats.increment(
-            new CohereEmbeddingsModel(
-                "inference_id",
-                TaskType.TEXT_EMBEDDING,
-                "cohere",
-                CohereEmbeddingsServiceSettingsTests.getServiceSettingsMap(null, null, null),
-                CohereEmbeddingsTaskSettingsTests.getTaskSettingsMap(null, null),
-                null,
-                ConfigurationParseContext.REQUEST
-            )
-        );
-
-        var converted = stats.toSerializableMap();
-
-        assertThat(
-            converted,
-            is(
-                Map.of(
-                    "cohere:text_embedding",
-                    new org.elasticsearch.xpack.core.inference.InferenceRequestStats("cohere", TaskType.TEXT_EMBEDDING, null, 1)
-                )
-            )
-        );
-    }
-}
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml
index d7f7e21e6f428..3f907ae1de6cd 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml
@@ -63,6 +63,82 @@ setup:
   - match: { "test-index.mappings.properties.sparse_field.model_settings.task_type": sparse_embedding }
   - length: { "test-index.mappings.properties.sparse_field": 3 }
 
+---
+"Field caps with sparse embedding":
+
+  - requires:
+      cluster_features: "gte_v8.16.0"
+      reason: field_caps support for semantic_text added in 8.16.0
+
+  - do:
+      field_caps:
+        include_empty_fields: true
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - exists: fields.dense_field
+
+  - do:
+      field_caps:
+        include_empty_fields: false
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - not_exists: fields.sparse_field
+  - not_exists: fields.dense_field
+
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: sparse-inference-id
+              model_settings:
+                task_type: sparse_embedding
+              chunks:
+                - text: "these are not the droids you're looking for"
+                  embeddings:
+                    feature_0: 1.0
+                    feature_1: 2.0
+                    feature_2: 3.0
+                    feature_3: 4.0
+                - text: "He's free to go around"
+                  embeddings:
+                    feature_4: 0.1
+                    feature_5: 0.2
+                    feature_6: 0.3
+                    feature_7: 0.4
+        refresh: true
+
+  - do:
+      field_caps:
+        include_empty_fields: true
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - exists: fields.dense_field
+  - match: { fields.sparse_field.semantic_text.searchable: true }
+  - match: { fields.dense_field.semantic_text.searchable: true }
+
+  - do:
+      field_caps:
+        include_empty_fields: false
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - not_exists: fields.dense_field
+  - match: { fields.sparse_field.semantic_text.searchable: true }
+
 ---
 "Indexes dense vector document":
 
@@ -105,6 +181,77 @@ setup:
   - match: { "test-index.mappings.properties.dense_field.model_settings.task_type": text_embedding }
   - length: { "test-index.mappings.properties.dense_field": 3 }
 
+---
+"Field caps with text embedding":
+
+  - requires:
+      cluster_features: "gte_v8.16.0"
+      reason: field_caps support for semantic_text added in 8.16.0
+
+  - do:
+      field_caps:
+        include_empty_fields: true
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - exists: fields.dense_field
+
+  - do:
+      field_caps:
+        include_empty_fields: false
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - not_exists: fields.sparse_field
+  - not_exists: fields.dense_field
+
+  - do:
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          dense_field:
+            text: "these are not the droids you're looking for. He's free to go around"
+            inference:
+              inference_id: dense-inference-id
+              model_settings:
+                task_type: text_embedding
+                dimensions: 4
+                similarity: cosine
+                element_type: float
+              chunks:
+                - text: "these are not the droids you're looking for"
+                  embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ]
+                - text: "He's free to go around"
+                  embeddings: [ 0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896 ]
+        refresh: true
+
+  - do:
+      field_caps:
+        include_empty_fields: true
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - exists: fields.dense_field
+  - match: { fields.sparse_field.semantic_text.searchable: true }
+  - match: { fields.dense_field.semantic_text.searchable: true }
+
+  - do:
+      field_caps:
+        include_empty_fields: false
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - not_exists: fields.sparse_field
+  - exists: fields.dense_field
+  - match: { fields.dense_field.semantic_text.searchable: true }
+
 ---
 "Can't be used as a multifield":
 
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
index f467691600766..f58a5c33fd85d 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
@@ -51,7 +51,7 @@ setup:
                 type: text
 
 ---
-"Calculates text expansion and embedding results for new documents":
+"Calculates sparse embedding and text embedding results for new documents":
   - do:
       index:
         index: test-index
@@ -74,6 +74,93 @@ setup:
   - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
   - match: { _source.non_inference_field: "non inference test" }
 
+---
+"Calculates sparse embedding and text embedding results for new documents with integer value":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: 75
+          dense_field: 100
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "75" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "75" }
+  - match: { _source.dense_field.text: "100" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "100" }
+
+---
+"Calculates sparse embedding and text embedding results for new documents with boolean value":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: true
+          dense_field: false
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "true" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "true" }
+  - match: { _source.dense_field.text: "false" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "false" }
+
+---
+"Calculates sparse embedding and text embedding results for new documents with a collection of mixed data types":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: [false, 75, "inference test", 13.49]
+          dense_field: [true, 49.99, "another inference test", 5654]
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - length: { _source.sparse_field.text: 4 }
+  - match: { _source.sparse_field.text.0: "false" }
+  - match: { _source.sparse_field.text.1: "75" }
+  - match: { _source.sparse_field.text.2: "inference test" }
+  - match: { _source.sparse_field.text.3: "13.49" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - exists: _source.sparse_field.inference.chunks.1.embeddings
+  - exists: _source.sparse_field.inference.chunks.2.embeddings
+  - exists: _source.sparse_field.inference.chunks.3.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "false" }
+  - match: { _source.sparse_field.inference.chunks.1.text: "75" }
+  - match: { _source.sparse_field.inference.chunks.2.text: "inference test" }
+  - match: { _source.sparse_field.inference.chunks.3.text: "13.49" }
+
+  - length: { _source.dense_field.text: 4 }
+  - match: { _source.dense_field.text.0: "true" }
+  - match: { _source.dense_field.text.1: "49.99" }
+  - match: { _source.dense_field.text.2: "another inference test" }
+  - match: { _source.dense_field.text.3: "5654" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - exists: _source.dense_field.inference.chunks.1.embeddings
+  - exists: _source.dense_field.inference.chunks.2.embeddings
+  - exists: _source.dense_field.inference.chunks.3.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "true" }
+  - match: { _source.dense_field.inference.chunks.1.text: "49.99" }
+  - match: { _source.dense_field.inference.chunks.2.text: "another inference test" }
+  - match: { _source.dense_field.inference.chunks.3.text: "5654" }
+
 ---
 "Inference fields do not create new mappings":
   - do:
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml
index 5ee7a943c4d35..932ee4854f445 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml
@@ -90,6 +90,58 @@ setup:
   - close_to: { hits.hits.0._score: { value: 3.7837332e17, error: 1e10 } }
   - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
 
+---
+"Numeric query using a sparse embedding model":
+  - skip:
+      features: [ "headers", "close_to" ]
+
+  - do:
+      index:
+        index: test-sparse-index
+        id: doc_1
+        body:
+          inference_field: [40, 49.678]
+        refresh: true
+
+  - do:
+      search:
+        index: test-sparse-index
+        body:
+          query:
+            semantic:
+              field: "inference_field"
+              query: "40"
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
+
+---
+"Boolean query using a sparse embedding model":
+  - skip:
+      features: [ "headers", "close_to" ]
+
+  - do:
+      index:
+        index: test-sparse-index
+        id: doc_1
+        body:
+          inference_field: true
+        refresh: true
+
+  - do:
+      search:
+        index: test-sparse-index
+        body:
+          query:
+            semantic:
+              field: "inference_field"
+              query: "true"
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0._source.inference_field.inference.chunks: 1 }
+
 ---
 "Query using a dense embedding model":
   - skip:
@@ -121,6 +173,58 @@ setup:
   - close_to: { hits.hits.0._score: { value: 1.0, error: 0.0001 } }
   - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
 
+---
+"Numeric query using a dense embedding model":
+  - skip:
+      features: [ "headers", "close_to" ]
+
+  - do:
+      index:
+        index: test-dense-index
+        id: doc_1
+        body:
+          inference_field: [45.1, 100]
+        refresh: true
+
+  - do:
+      search:
+        index: test-dense-index
+        body:
+          query:
+            semantic:
+              field: "inference_field"
+              query: "45.1"
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0._source.inference_field.inference.chunks: 2 }
+
+---
+"Boolean query using a dense embedding model":
+  - skip:
+      features: [ "headers", "close_to" ]
+
+  - do:
+      index:
+        index: test-dense-index
+        id: doc_1
+        body:
+          inference_field: false
+        refresh: true
+
+  - do:
+      search:
+        index: test-dense-index
+        body:
+          query:
+            semantic:
+              field: "inference_field"
+              query: "false"
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "doc_1" }
+  - length: { hits.hits.0._source.inference_field.inference.chunks: 1 }
+
 ---
 "Query using a dense embedding model that uses byte embeddings":
   - skip:
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml
index 2e01e2b9c8d04..e2c1417057578 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml
@@ -29,6 +29,8 @@ setup:
                 type: text
               topic:
                 type: keyword
+              subtopic:
+                type: keyword
 
   - do:
       index:
@@ -36,7 +38,8 @@ setup:
         id: doc_1
         body:
           text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun."
-          topic: ["science"]
+          topic: [ "science" ]
+          subtopic: [ "technology" ]
         refresh: true
 
   - do:
@@ -45,7 +48,8 @@ setup:
         id: doc_2
         body:
           text: "The phases of the Moon come from the position of the Moon relative to the Earth and Sun."
-          topic: ["science"]
+          topic: [ "science" ]
+          subtopic: [ "astronomy" ]
         refresh: true
 
   - do:
@@ -54,8 +58,9 @@ setup:
         id: doc_3
         body:
           text: "Sun Moon Lake is a lake in Nantou County, Taiwan. It is the largest lake in Taiwan."
-          topic: ["geography"]
+          topic: [ "geography" ]
         refresh: true
+
 ---
 "Simple text similarity rank retriever":
 
@@ -78,7 +83,7 @@ setup:
               field: text
           size: 10
 
-  - match: { hits.total.value : 2 }
+  - match: { hits.total.value: 2 }
   - length: { hits.hits: 2 }
 
   - match: { hits.hits.0._id: "doc_2" }
@@ -88,3 +93,88 @@ setup:
   - match: { hits.hits.1._id: "doc_1" }
   - match: { hits.hits.1._rank: 2 }
   - close_to: { hits.hits.1._score: { value: 0.2, error: 0.001 } }
+
+---
+"Simple text similarity rank retriever and filtering":
+
+  - do:
+      search:
+        index: test-index
+        body:
+          track_total_hits: true
+          fields: [ "text", "topic" ]
+          retriever:
+            text_similarity_reranker:
+              retriever:
+                standard:
+                  query:
+                    term:
+                      topic: "science"
+              filter:
+                term:
+                  subtopic: "technology"
+              rank_window_size: 10
+              inference_id: my-rerank-model
+              inference_text: "How often does the moon hide the sun?"
+              field: text
+          size: 10
+
+  - match: { hits.total.value: 1 }
+  - length: { hits.hits: 1 }
+
+  - match: { hits.hits.0._id: "doc_1" }
+  - match: { hits.hits.0._rank: 1 }
+  - close_to: { hits.hits.0._score: { value: 0.2, error: 0.001 } }
+
+
+---
+"Text similarity reranking fails if the inference ID does not exist":
+  - do:
+      catch: /Inference endpoint not found/
+      search:
+        index: test-index
+        body:
+          track_total_hits: true
+          fields: [ "text", "topic" ]
+          retriever:
+            text_similarity_reranker:
+              retriever:
+                standard:
+                  query:
+                    term:
+                      topic: "science"
+              filter:
+                term:
+                  subtopic: "technology"
+              rank_window_size: 10
+              inference_id: i-dont-exist
+              inference_text: "How often does the moon hide the sun?"
+              field: text
+          size: 10
+
+---
+"Text similarity reranking fails if the inference ID does not exist and result set is empty":
+  - requires:
+      cluster_features: "gte_v8.15.1"
+      reason: bug fixed in 8.15.1
+
+  - do:
+      catch: /Inference endpoint not found/
+      search:
+        index: test-index
+        body:
+          track_total_hits: true
+          fields: [ "text", "topic" ]
+          retriever:
+            text_similarity_reranker:
+              retriever:
+                standard:
+                  query:
+                    term:
+                      topic: "asdfasdf"
+              rank_window_size: 10
+              inference_id: i-dont-exist
+              inference_text: "asdfasdf"
+              field: text
+          size: 10
+
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/80_random_rerank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/80_random_rerank_retriever.yml
new file mode 100644
index 0000000000000..d33f57f763db8
--- /dev/null
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/80_random_rerank_retriever.yml
@@ -0,0 +1,94 @@
+setup:
+  - requires:
+      cluster_features: "gte_v8.16.0"
+      reason: random rerank retriever introduced in 8.16.0
+      test_runner_features: "close_to"
+
+  - do:
+      indices.create:
+        index: test-index
+        body:
+          mappings:
+            properties:
+              text:
+                type: text
+              topic:
+                type: keyword
+              subtopic:
+                type: keyword
+
+  - do:
+      bulk:
+        refresh: true
+        index: test-index
+        body: |
+          {"index": { "_id": "doc_1" } }
+          { "text": "Pugs are proof that even nature has a sense of humor." }
+          {"index": { "_id": "doc_2" } }
+          { "text": "A pugs snore can rival a chainsaw, but it's somehow adorable." }
+          {"index": { "_id": "doc_3" } }
+          { "text": "Pugs are like potato chips; you can't have just one wrinkle." }
+          {"index": { "_id": "doc_4" } }
+          { "text": "Pugs don't walk; pugs waddle majestically." }
+          {"index": { "_id": "doc_5" } }
+          { "text": "A pugs life goal: be the ultimate couch potato, and they're crushing it." }
+---
+"Random rerank retriever predictably shuffles results":
+
+  - do:
+      search:
+        index: test-index
+        body:
+          query:
+            query_string:
+              query: "pugs"
+          size: 10
+
+  - match: { hits.total.value: 5 }
+  - length: { hits.hits: 5 }
+
+  - match: { hits.hits.0._id: "doc_4" }
+  - close_to: { hits.hits.0._score: { value: 0.136, error: 0.001 } }
+
+  - do:
+      search:
+        index: test-index
+        body:
+          retriever:
+            random_reranker:
+              retriever:
+                standard:
+                  query:
+                    query_string:
+                      query: "pugs"
+              field: text
+              seed: 42
+              rank_window_size: 10
+          size: 10
+
+  - match: { hits.total.value: 5 }
+  - length: { hits.hits: 5 }
+
+  - match: { hits.hits.0._id: "doc_1" }
+  - close_to: { hits.hits.0._score: { value: 0.727, error: 0.001 } }
+
+  - do:
+      search:
+        index: test-index
+        body:
+          retriever:
+            random_reranker:
+              retriever:
+                standard:
+                  query:
+                    query_string:
+                      query: "pugs"
+              field: text
+              rank_window_size: 10
+          size: 10
+
+  - match: { hits.total.value: 5 }
+  - length: { hits.hits: 5 }
+
+  - match: { hits.hits.0._id: "doc_3" }
+  - close_to: { hits.hits.0._score: { value: 0.731, error: 0.001 } }
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
index a2970112b4964..86e797d369e6b 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.logstash;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
index a6e112949cbd1..f7504c0227b89 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineAction.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineAction.java
index 4c0e90ce97b1b..4a3c3fa520ea5 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineAction.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineAction.java
@@ -15,9 +15,9 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.logstash.Logstash;
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineAction.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineAction.java
index 6d9a244c13dce..081a170aac9f1 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineAction.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineAction.java
@@ -21,12 +21,12 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportPutPipelineAction.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportPutPipelineAction.java
index 7125cc12f6cfd..6ffdd50e8eb83 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportPutPipelineAction.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/TransportPutPipelineAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.logstash.Logstash;
diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricInfoTransportAction.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricInfoTransportAction.java
index 248c256941d98..9d05fe2ecb315 100644
--- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricInfoTransportAction.java
+++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricInfoTransportAction.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.aggregatemetric;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricUsageTransportAction.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricUsageTransportAction.java
index 45069a58bec19..ca107bccc75d0 100644
--- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricUsageTransportAction.java
+++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricUsageTransportAction.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java
index 2602c2947420b..a547e56b43949 100644
--- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java
+++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java
@@ -720,21 +720,19 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
         return new CompositeSyntheticFieldLoader(
             leafName(),
             fullPath(),
-            new AggregateMetricSyntheticFieldLoader(fullPath(), leafName(), metrics),
+            new AggregateMetricSyntheticFieldLoader(fullPath(), metrics),
             new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath())
         );
     }
 
-    public static class AggregateMetricSyntheticFieldLoader implements CompositeSyntheticFieldLoader.SyntheticFieldLoaderLayer {
+    public static class AggregateMetricSyntheticFieldLoader implements CompositeSyntheticFieldLoader.Layer {
         private final String name;
-        private final String simpleName;
         private final EnumSet<Metric> metrics;
         private final Map<Metric, SortedNumericDocValues> metricDocValues = new EnumMap<>(Metric.class);
         private final Set<Metric> metricHasValue = EnumSet.noneOf(Metric.class);
 
-        protected AggregateMetricSyntheticFieldLoader(String name, String simpleName, EnumSet<Metric> metrics) {
+        protected AggregateMetricSyntheticFieldLoader(String name, EnumSet<Metric> metrics) {
             this.name = name;
-            this.simpleName = simpleName;
             this.metrics = metrics;
         }
 
diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java
index 4b7b27bf2cec3..89c2799d8327d 100644
--- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java
+++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java
@@ -136,6 +136,11 @@ public boolean needs_score() {
                         return false;
                     }
 
+                    @Override
+                    public boolean needs_termStats() {
+                        return false;
+                    }
+
                     @Override
                     public ScoreScript newInstance(DocReader docReader) {
                         return new ScoreScript(Map.of(), searchExecutionContext.lookup(), docReader) {
diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java
index 1113b9ff57a72..553b9847001c8 100644
--- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java
+++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java
@@ -41,13 +41,14 @@
 import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
 import org.elasticsearch.index.mapper.BlockDocValuesReader;
 import org.elasticsearch.index.mapper.BlockLoader;
+import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.SearchAfterTermsEnum;
-import org.elasticsearch.index.mapper.SortedSetDocValuesSyntheticFieldLoader;
+import org.elasticsearch.index.mapper.SortedSetDocValuesSyntheticFieldLoaderLayer;
 import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.TermBasedFieldType;
@@ -457,7 +458,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
                 "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        return new SortedSetDocValuesSyntheticFieldLoader(fullPath(), leafName(), null, false) {
+        return new CompositeSyntheticFieldLoader(leafName(), fullPath(), new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) {
             @Override
             protected BytesRef convert(BytesRef value) {
                 return VersionEncoder.decodeVersion(value);
@@ -468,6 +469,6 @@ protected BytesRef preserve(BytesRef value) {
                 // Convert copies the underlying bytes
                 return value;
             }
-        };
+        });
     }
 }
diff --git a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportGetTrainedModelPackageConfigAction.java b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportGetTrainedModelPackageConfigAction.java
index 6cdeb93d1e07d..ba50f2f6a6b74 100644
--- a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportGetTrainedModelPackageConfigAction.java
+++ b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportGetTrainedModelPackageConfigAction.java
@@ -19,9 +19,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java
index cdc3205f4197c..70dcee165d3f6 100644
--- a/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java
+++ b/x-pack/plugin/ml-package-loader/src/main/java/org/elasticsearch/xpack/ml/packageloader/action/TransportLoadTrainedModelPackage.java
@@ -22,8 +22,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskAwareRequest;
@@ -180,27 +180,37 @@ static void importModel(
     }
 
     private ModelDownloadTask createDownloadTask(Request request) {
-        return (ModelDownloadTask) taskManager.register(MODEL_IMPORT_TASK_TYPE, MODEL_IMPORT_TASK_ACTION, new TaskAwareRequest() {
-            @Override
-            public void setParentTask(TaskId taskId) {
-                request.setParentTask(taskId);
-            }
+        // Loading the model is done by a separate task, so needs a new trace context
+        try (var ignored = threadPool.getThreadContext().newTraceContext()) {
+            return (ModelDownloadTask) taskManager.register(MODEL_IMPORT_TASK_TYPE, MODEL_IMPORT_TASK_ACTION, new TaskAwareRequest() {
+                @Override
+                public void setParentTask(TaskId taskId) {
+                    request.setParentTask(taskId);
+                }
 
-            @Override
-            public void setRequestId(long requestId) {
-                request.setRequestId(requestId);
-            }
+                @Override
+                public void setRequestId(long requestId) {
+                    request.setRequestId(requestId);
+                }
 
-            @Override
-            public TaskId getParentTask() {
-                return request.getParentTask();
-            }
+                @Override
+                public TaskId getParentTask() {
+                    return request.getParentTask();
+                }
 
-            @Override
-            public ModelDownloadTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
-                return new ModelDownloadTask(id, type, action, downloadModelTaskDescription(request.getModelId()), parentTaskId, headers);
-            }
-        }, false);
+                @Override
+                public ModelDownloadTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+                    return new ModelDownloadTask(
+                        id,
+                        type,
+                        action,
+                        downloadModelTaskDescription(request.getModelId()),
+                        parentTaskId,
+                        headers
+                    );
+                }
+            }, false);
+        }
     }
 
     private static void recordError(
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java
index 46a7ca17714ae..623b676e0e0ee 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java
@@ -74,7 +74,6 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.oneOf;
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105239")
 public class DatafeedJobsIT extends MlNativeAutodetectIntegTestCase {
 
     @After
@@ -780,6 +779,7 @@ private void startRealtime(String jobId, Integer maxEmptySearches) throws Except
         }, 30, TimeUnit.SECONDS);
     }
 
+    @LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105239")
     public void testStartDatafeed_GivenTimeout_Returns408() throws Exception {
         client().admin().indices().prepareCreate("data-1").setMapping("time", "type=date").get();
         long numDocs = 100;
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java
index fec85730aaf2b..0effe5349d43a 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.xpack.core.ml.action.GetRecordsAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateFilterAction;
+import org.elasticsearch.xpack.core.ml.annotations.Annotation;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
 import org.elasticsearch.xpack.core.ml.job.config.DetectionRule;
@@ -20,7 +21,10 @@
 import org.elasticsearch.xpack.core.ml.job.config.JobUpdate;
 import org.elasticsearch.xpack.core.ml.job.config.MlFilter;
 import org.elasticsearch.xpack.core.ml.job.config.Operator;
+import org.elasticsearch.xpack.core.ml.job.config.RuleAction;
 import org.elasticsearch.xpack.core.ml.job.config.RuleCondition;
+import org.elasticsearch.xpack.core.ml.job.config.RuleParams;
+import org.elasticsearch.xpack.core.ml.job.config.RuleParamsForForceTimeShift;
 import org.elasticsearch.xpack.core.ml.job.config.RuleScope;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
@@ -39,7 +43,9 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.oneOf;
 
 /**
@@ -299,6 +305,67 @@ public void testScopeAndCondition() throws IOException {
         assertThat(records.get(0).getOverFieldValue(), equalTo("222.222.222.222"));
     }
 
+    public void testForceTimeShiftAction() throws Exception {
+        // The test ensures that the force time shift action works as expected.
+
+        long timeShiftAmount = 3600L;
+        long timestampStartMillis = 1491004800000L;
+        long bucketSpanMillis = 3600000L;
+        long timeShiftTimestamp = (timestampStartMillis + bucketSpanMillis) / 1000;
+
+        int totalBuckets = 2 * 24;
+
+        DetectionRule rule = new DetectionRule.Builder(
+            Arrays.asList(new RuleCondition(RuleCondition.AppliesTo.TIME, Operator.GTE, timeShiftTimestamp))
+        ).setActions(RuleAction.FORCE_TIME_SHIFT).setParams(new RuleParams(new RuleParamsForForceTimeShift(timeShiftAmount))).build();
+
+        Detector.Builder detector = new Detector.Builder("mean", "value");
+        detector.setRules(Arrays.asList(rule));
+        AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Arrays.asList(detector.build()));
+        analysisConfig.setBucketSpan(TimeValue.timeValueMillis(bucketSpanMillis));
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        Job.Builder job = new Job.Builder("detection-rules-it-test-force-time-shift");
+        job.setAnalysisConfig(analysisConfig);
+        job.setDataDescription(dataDescription);
+
+        putJob(job);
+        openJob(job.getId());
+
+        // post some data
+        int normalValue = 400;
+        List<String> data = new ArrayList<>();
+        long timestamp = timestampStartMillis;
+        for (int bucket = 0; bucket < totalBuckets; bucket++) {
+            Map<String, Object> record = new HashMap<>();
+            record.put("time", timestamp);
+            record.put("value", normalValue);
+            data.add(createJsonRecord(record));
+            timestamp += bucketSpanMillis;
+        }
+
+        postData(job.getId(), joinBetween(0, data.size(), data));
+        closeJob(job.getId());
+
+        List<Annotation> annotations = getAnnotations();
+        assertThat(annotations.size(), greaterThanOrEqualTo(1));
+        assertThat(annotations.size(), lessThanOrEqualTo(3));
+
+        // Check that annotation contain the expected time shift
+        boolean countingModelAnnotationFound = false;
+        boolean individualModelAnnotationFound = false;
+        for (Annotation annotation : annotations) {
+            if (annotation.getAnnotation().contains("Counting model shifted time by")) {
+                countingModelAnnotationFound = true;
+                assertThat(annotation.getAnnotation(), containsString(timeShiftAmount + " seconds"));
+            } else if (annotation.getAnnotation().contains("Model shifted time by")) {
+                individualModelAnnotationFound = true;
+                assertThat(annotation.getAnnotation(), containsString(timeShiftAmount + " seconds"));
+            }
+        }
+        assertThat("Counting model annotation with time shift not found", countingModelAnnotationFound, equalTo(true));
+        assertThat("Individual model annotation with time shift not found", individualModelAnnotationFound, equalTo(true));
+    }
+
     private String createIpRecord(long timestamp, String ip) throws IOException {
         Map<String, Object> record = new HashMap<>();
         record.put("time", timestamp);
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
index ce265d0e895aa..2e096f3262cb6 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
@@ -287,6 +287,27 @@ protected void assertThatNumberOfAnnotationsIsEqualTo(int expectedNumberOfAnnota
         });
     }
 
+    protected List<Annotation> getAnnotations() throws Exception {
+        List<Annotation> annotations = new ArrayList<>();
+        // Refresh the annotations index so that recently indexed annotation docs are visible.
+        indicesAdmin().prepareRefresh(AnnotationIndex.LATEST_INDEX_NAME)
+            .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
+            .get();
+
+        SearchRequest searchRequest = new SearchRequest(AnnotationIndex.READ_ALIAS_NAME).indicesOptions(
+            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN
+        );
+        assertCheckedResponse(client().search(searchRequest), searchResponse -> {
+
+            for (SearchHit hit : searchResponse.getHits().getHits()) {
+                try (XContentParser parser = createParser(jsonXContent, hit.getSourceRef())) {
+                    annotations.add(Annotation.fromXContent(parser, null));
+                }
+            }
+        });
+        return annotations;
+    }
+
     protected ForecastRequestStats getForecastStats(String jobId, String forecastId) throws Exception {
         SetOnce<ForecastRequestStats> forecastRequestStats = new SetOnce<>();
         assertCheckedResponse(
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
index afd150be5fd4c..ca5ecd80a83bb 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
@@ -118,8 +118,6 @@
 import java.util.Set;
 import java.util.function.Function;
 
-import static org.elasticsearch.test.XContentTestUtils.convertToMap;
-import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.elasticsearch.xpack.monitoring.MonitoringService.ELASTICSEARCH_COLLECTION_ENABLED;
@@ -408,45 +406,7 @@ protected void ensureClusterStateConsistency() throws IOException {
             entries.add(
                 new NamedWriteableRegistry.Entry(AutoscalingDeciderResult.Reason.class, MlScalingReason.NAME, MlScalingReason::new)
             );
-            final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
-            ClusterState masterClusterState = clusterAdmin().prepareState().all().get().getState();
-            byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState);
-            // remove local node reference
-            masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null, namedWriteableRegistry);
-            Map<String, Object> masterStateMap = convertToMap(masterClusterState);
-            int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length;
-            String masterId = masterClusterState.nodes().getMasterNodeId();
-            for (Client client : cluster().getClients()) {
-                ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState();
-                byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState);
-                // remove local node reference
-                localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null, namedWriteableRegistry);
-                final Map<String, Object> localStateMap = convertToMap(localClusterState);
-                final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length;
-                // Check that the non-master node has the same version of the cluster state as the master and
-                // that the master node matches the master (otherwise there is no requirement for the cluster state to match)
-                if (masterClusterState.version() == localClusterState.version()
-                    && masterId.equals(localClusterState.nodes().getMasterNodeId())) {
-                    try {
-                        assertEquals("clusterstate UUID does not match", masterClusterState.stateUUID(), localClusterState.stateUUID());
-                        // We cannot compare serialization bytes since serialization order of maps is not guaranteed
-                        // but we can compare serialization sizes - they should be the same
-                        assertEquals("clusterstate size does not match", masterClusterStateSize, localClusterStateSize);
-                        // Compare JSON serialization
-                        assertNull(
-                            "clusterstate JSON serialization does not match",
-                            differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap)
-                        );
-                    } catch (AssertionError error) {
-                        logger.error(
-                            "Cluster state from master:\n{}\nLocal cluster state:\n{}",
-                            masterClusterState.toString(),
-                            localClusterState.toString()
-                        );
-                        throw error;
-                    }
-                }
-            }
+            doEnsureClusterStateConsistency(new NamedWriteableRegistry(entries));
         }
     }
 
@@ -461,7 +421,10 @@ protected static void createDataStreamAndTemplate(String dataStreamName, String
                     .build()
             )
         ).actionGet();
-        client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet();
+        client().execute(
+            CreateDataStreamAction.INSTANCE,
+            new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName)
+        ).actionGet();
     }
 
     public static class MockPainlessScriptEngine extends MockScriptEngine {
diff --git a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearningToRankRescorerIT.java b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearningToRankRescorerIT.java
index b2a0b60aed7ba..4a703117c6551 100644
--- a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearningToRankRescorerIT.java
+++ b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearningToRankRescorerIT.java
@@ -16,7 +16,9 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -28,189 +30,11 @@ public class LearningToRankRescorerIT extends InferenceTestCase {
 
     @Before
     public void setupModelAndData() throws IOException {
-        putRegressionModel(MODEL_ID, """
-            {
-              "description": "super complex model for tests",
-              "inference_config": {
-                "learning_to_rank": {
-                  "feature_extractors": [
-                    {
-                      "query_extractor": {
-                        "feature_name": "cost",
-                        "query": {"script_score": {"query": {"match_all":{}}, "script": {"source": "return doc['cost'].value;"}}}
-                      }
-                    },
-                    {
-                      "query_extractor": {
-                        "feature_name": "type_tv",
-                        "query": {"constant_score": {"filter": {"term": { "product": "TV" }}, "boost": 1.0}}
-                      }
-                    },
-                    {
-                      "query_extractor": {
-                        "feature_name": "type_vcr",
-                        "query": {"constant_score": {"filter": {"term": { "product": "VCR" }}, "boost": 1.0}}
-                      }
-                    },
-                    {
-                      "query_extractor": {
-                        "feature_name": "type_laptop",
-                        "query": {"constant_score": {"filter": {"term": { "product": "Laptop" }}, "boost": 1.0}}
-                      }
-                    },
-                    {
-                        "query_extractor": {
-                            "feature_name": "two",
-                            "query": { "script_score": { "query": { "match_all": {} }, "script": { "source": "return 2.0;" } } }
-                        }
-                    },
-                    {
-                        "query_extractor": {
-                            "feature_name": "product_bm25",
-                            "query": { "term": { "product": "{{keyword}}" } }
-                        }
-                    }
-                  ]
-                }
-              },
-              "definition": {
-                "trained_model": {
-                  "ensemble": {
-                    "feature_names": ["cost", "type_tv", "type_vcr", "type_laptop", "two", "product_bm25"],
-                    "target_type": "regression",
-                    "trained_models": [
-                    {
-                      "tree": {
-                        "feature_names": [
-                          "cost"
-                        ],
-                        "tree_structure": [
-                        {
-                          "node_index": 0,
-                          "split_feature": 0,
-                          "split_gain": 12,
-                          "threshold": 400,
-                          "decision_type": "lte",
-                          "default_left": true,
-                          "left_child": 1,
-                          "right_child": 2
-                        },
-                        {
-                          "node_index": 1,
-                          "leaf_value": 5.0
-                        },
-                        {
-                          "node_index": 2,
-                          "leaf_value": 2.0
-                        }
-                        ],
-                        "target_type": "regression"
-                      }
-                    },
-                    {
-                      "tree": {
-                        "feature_names": [
-                          "type_tv"
-                        ],
-                        "tree_structure": [
-                        {
-                          "node_index": 0,
-                          "split_feature": 0,
-                          "split_gain": 12,
-                          "threshold": 1,
-                          "decision_type": "lt",
-                          "default_left": true,
-                          "left_child": 1,
-                          "right_child": 2
-                        },
-                        {
-                          "node_index": 1,
-                          "leaf_value": 1.0
-                        },
-                        {
-                          "node_index": 2,
-                          "leaf_value": 12.0
-                        }
-                        ],
-                        "target_type": "regression"
-                      }
-                    },
-                     {
-                      "tree": {
-                        "feature_names": [
-                          "two"
-                        ],
-                        "tree_structure": [
-                        {
-                          "node_index": 0,
-                          "split_feature": 0,
-                          "split_gain": 12,
-                          "threshold": 1,
-                          "decision_type": "lt",
-                          "default_left": true,
-                          "left_child": 1,
-                          "right_child": 2
-                        },
-                        {
-                          "node_index": 1,
-                          "leaf_value": 1.0
-                        },
-                        {
-                          "node_index": 2,
-                          "leaf_value": 2.0
-                        }
-                        ],
-                        "target_type": "regression"
-                      }
-                    },
-                     {
-                      "tree": {
-                        "feature_names": [
-                          "product_bm25"
-                        ],
-                        "tree_structure": [
-                        {
-                          "node_index": 0,
-                          "split_feature": 0,
-                          "split_gain": 12,
-                          "threshold": 1,
-                          "decision_type": "lt",
-                          "default_left": true,
-                          "left_child": 1,
-                          "right_child": 2
-                        },
-                        {
-                          "node_index": 1,
-                          "leaf_value": 1.0
-                        },
-                        {
-                          "node_index": 2,
-                          "leaf_value": 4.0
-                        }
-                        ],
-                        "target_type": "regression"
-                      }
-                    }
-                    ]
-                  }
-                }
-              }
-            }
-            """);
-        createIndex(INDEX_NAME, Settings.EMPTY, """
-            "properties":{
-              "product":{"type": "keyword"},
-              "cost":{"type": "integer"}
-            }""");
-        indexData("{ \"product\": \"TV\", \"cost\": 300}");
-        indexData("{ \"product\": \"TV\", \"cost\": 400}");
-        indexData("{ \"product\": \"TV\", \"cost\": 600}");
-        indexData("{ \"product\": \"VCR\", \"cost\": 15}");
-        indexData("{ \"product\": \"VCR\", \"cost\": 350}");
-        indexData("{ \"product\": \"VCR\", \"cost\": 580}");
-        indexData("{ \"product\": \"Laptop\", \"cost\": 100}");
-        indexData("{ \"product\": \"Laptop\", \"cost\": 300}");
-        indexData("{ \"product\": \"Laptop\", \"cost\": 500}");
+        putRegressionModel(MODEL_ID, testRegressionModel);
+        createIndex(INDEX_NAME, Settings.EMPTY, testIndexDefinition);
+        for (String testDataItem : testIndexData) {
+            indexData(testDataItem);
+        }
         adminClient().performRequest(new Request("POST", INDEX_NAME + "/_refresh"));
     }
 
@@ -249,6 +73,19 @@ public void testLearningToRankRescore() throws Exception {
         assertHitScores(client().performRequest(request), List.of(9.0, 9.0, 6.0));
     }
 
+    public void testLearningToRankRescoreWithExplain() throws Exception {
+        Request request = new Request("GET", "store/_search?size=3&explain=true&error_trace");
+        request.setJsonEntity("""
+            {
+              "rescore": {
+                "window_size": 10,
+                "learning_to_rank": { "model_id": "ltr-model" }
+              }
+            }""");
+        var response = client().performRequest(request);
+        assertExplainExtractedFeatures(response, List.of("type_tv", "cost", "two"));
+    }
+
     public void testLearningToRankRescoreSmallWindow() throws Exception {
         Request request = new Request("GET", "store/_search?size=5");
         request.setJsonEntity("""
@@ -336,4 +173,234 @@ private void indexData(String data) throws IOException {
     private static void assertHitScores(Response response, List<Double> expectedScores) throws IOException {
         assertThat((List<Double>) XContentMapValues.extractValue("hits.hits._score", responseAsMap(response)), equalTo(expectedScores));
     }
+
+    @SuppressWarnings("unchecked")
+    private static void assertExplainExtractedFeatures(Response response, List<String> expectedFeatures) throws IOException {
+        var explainValues = (ArrayList<Map<String, Object>>) XContentMapValues.extractValue(
+            "hits.hits._explanation",
+            responseAsMap(response)
+        );
+
+        assertThat(explainValues.size(), equalTo(3));
+        for (Map<String, Object> hit : explainValues) {
+            assertThat(hit.get("description"), equalTo("rescored using LTR model ltr-model"));
+
+            var queryDetails = (ArrayList<Map<String, Object>>) hit.get("details");
+            assertThat(queryDetails.size(), equalTo(2));
+
+            assertThat(queryDetails.get(0).get("description"), equalTo("first pass query score"));
+            assertThat(queryDetails.get(1).get("description"), equalTo("extracted features"));
+
+            var featureDetails = new ArrayList<>((ArrayList<Map<String, Object>>) queryDetails.get(1).get("details"));
+            assertThat(featureDetails.size(), equalTo(3));
+
+            var missingKeys = new ArrayList<String>();
+            for (String expectedFeature : expectedFeatures) {
+                var expectedDescription = Strings.format("feature value for [%s]", expectedFeature);
+
+                var wasFound = false;
+                for (Map<String, Object> detailItem : featureDetails) {
+                    if (detailItem.get("description").equals(expectedDescription)) {
+                        featureDetails.remove(detailItem);
+                        wasFound = true;
+                        break;
+                    }
+                }
+
+                if (wasFound == false) {
+                    missingKeys.add(expectedFeature);
+                }
+            }
+
+            assertThat(Strings.format("Could not find features: [%s]", String.join(", ", missingKeys)), featureDetails.size(), equalTo(0));
+        }
+    }
+
+    private static String testIndexDefinition = """
+        "properties":{
+          "product":{"type": "keyword"},
+          "cost":{"type": "integer"}
+        }""";
+
+    private static List<String> testIndexData = List.of(
+        "{ \"product\": \"TV\", \"cost\": 300}",
+        "{ \"product\": \"TV\", \"cost\": 400}",
+        "{ \"product\": \"TV\", \"cost\": 600}",
+        "{ \"product\": \"VCR\", \"cost\": 15}",
+        "{ \"product\": \"VCR\", \"cost\": 350}",
+        "{ \"product\": \"VCR\", \"cost\": 580}",
+        "{ \"product\": \"Laptop\", \"cost\": 100}",
+        "{ \"product\": \"Laptop\", \"cost\": 300}",
+        "{ \"product\": \"Laptop\", \"cost\": 500}"
+    );
+
+    private static String testRegressionModel = """
+        {
+          "description": "super complex model for tests",
+          "inference_config": {
+            "learning_to_rank": {
+              "feature_extractors": [
+                {
+                  "query_extractor": {
+                    "feature_name": "cost",
+                    "query": {"script_score": {"query": {"match_all":{}}, "script": {"source": "return doc['cost'].value;"}}}
+                  }
+                },
+                {
+                  "query_extractor": {
+                    "feature_name": "type_tv",
+                    "query": {"constant_score": {"filter": {"term": { "product": "TV" }}, "boost": 1.0}}
+                  }
+                },
+                {
+                  "query_extractor": {
+                    "feature_name": "type_vcr",
+                    "query": {"constant_score": {"filter": {"term": { "product": "VCR" }}, "boost": 1.0}}
+                  }
+                },
+                {
+                  "query_extractor": {
+                    "feature_name": "type_laptop",
+                    "query": {"constant_score": {"filter": {"term": { "product": "Laptop" }}, "boost": 1.0}}
+                  }
+                },
+                {
+                    "query_extractor": {
+                        "feature_name": "two",
+                        "query": { "script_score": { "query": { "match_all": {} }, "script": { "source": "return 2.0;" } } }
+                    }
+                },
+                {
+                    "query_extractor": {
+                        "feature_name": "product_bm25",
+                        "query": { "term": { "product": "{{keyword}}" } }
+                    }
+                }
+              ]
+            }
+          },
+          "definition": {
+            "trained_model": {
+              "ensemble": {
+                "feature_names": ["cost", "type_tv", "type_vcr", "type_laptop", "two", "product_bm25"],
+                "target_type": "regression",
+                "trained_models": [
+                {
+                  "tree": {
+                    "feature_names": [
+                      "cost"
+                    ],
+                    "tree_structure": [
+                    {
+                      "node_index": 0,
+                      "split_feature": 0,
+                      "split_gain": 12,
+                      "threshold": 400,
+                      "decision_type": "lte",
+                      "default_left": true,
+                      "left_child": 1,
+                      "right_child": 2
+                    },
+                    {
+                      "node_index": 1,
+                      "leaf_value": 5.0
+                    },
+                    {
+                      "node_index": 2,
+                      "leaf_value": 2.0
+                    }
+                    ],
+                    "target_type": "regression"
+                  }
+                },
+                {
+                  "tree": {
+                    "feature_names": [
+                      "type_tv"
+                    ],
+                    "tree_structure": [
+                    {
+                      "node_index": 0,
+                      "split_feature": 0,
+                      "split_gain": 12,
+                      "threshold": 1,
+                      "decision_type": "lt",
+                      "default_left": true,
+                      "left_child": 1,
+                      "right_child": 2
+                    },
+                    {
+                      "node_index": 1,
+                      "leaf_value": 1.0
+                    },
+                    {
+                      "node_index": 2,
+                      "leaf_value": 12.0
+                    }
+                    ],
+                    "target_type": "regression"
+                  }
+                },
+                 {
+                  "tree": {
+                    "feature_names": [
+                      "two"
+                    ],
+                    "tree_structure": [
+                    {
+                      "node_index": 0,
+                      "split_feature": 0,
+                      "split_gain": 12,
+                      "threshold": 1,
+                      "decision_type": "lt",
+                      "default_left": true,
+                      "left_child": 1,
+                      "right_child": 2
+                    },
+                    {
+                      "node_index": 1,
+                      "leaf_value": 1.0
+                    },
+                    {
+                      "node_index": 2,
+                      "leaf_value": 2.0
+                    }
+                    ],
+                    "target_type": "regression"
+                  }
+                },
+                 {
+                  "tree": {
+                    "feature_names": [
+                      "product_bm25"
+                    ],
+                    "tree_structure": [
+                    {
+                      "node_index": 0,
+                      "split_feature": 0,
+                      "split_gain": 12,
+                      "threshold": 1,
+                      "decision_type": "lt",
+                      "default_left": true,
+                      "left_child": 1,
+                      "right_child": 2
+                    },
+                    {
+                      "node_index": 1,
+                      "leaf_value": 1.0
+                    },
+                    {
+                      "node_index": 2,
+                      "leaf_value": 4.0
+                    }
+                    ],
+                    "target_type": "regression"
+                  }
+                }
+                ]
+              }
+            }
+          }
+        }
+        """;
 }
diff --git a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java
index cbdad9e86f4f8..321f34f81f16a 100644
--- a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java
+++ b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java
@@ -9,7 +9,6 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -186,9 +185,7 @@ static class TestConfiguration {
     }
 
     public void testIsolated() throws Exception {
-        Settings.Builder settings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0);
+        Settings.Builder settings = indexSettings(1, 0);
 
         createIndex("painless", settings.build());
         Request createDoc = new Request("PUT", "/painless/_doc/1");
@@ -282,9 +279,7 @@ public void testHRDSplit() throws Exception {
         client().performRequest(new Request("POST", BASE_PATH + "anomaly_detectors/hrd-split-job/_open"));
 
         // Create index to hold data
-        Settings.Builder settings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0);
+        Settings.Builder settings = indexSettings(1, 0);
 
         createIndex("painless", settings.build(), """
             "properties": { "domain": { "type": "keyword" },"time": { "type": "date" } }""");
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java
index 66f4797ef707c..fa94bf96c1167 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/DefaultMachineLearningExtension.java
@@ -51,11 +51,6 @@ public boolean isNlpEnabled() {
         return true;
     }
 
-    @Override
-    public boolean isLearningToRankEnabled() {
-        return true;
-    }
-
     @Override
     public String[] getAnalyticsDestIndexAllowedSettings() {
         return ANALYTICS_DEST_INDEX_ALLOWED_SETTINGS;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index 22a9c2dbcc281..5876836185ba3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -902,7 +902,7 @@ private static void reportClashingNodeAttribute(String attrName) {
 
     @Override
     public List<RescorerSpec<?>> getRescorers() {
-        if (enabled && machineLearningExtension.get().isLearningToRankEnabled()) {
+        if (enabled) {
             return List.of(
                 new RescorerSpec<>(
                     LearningToRankRescorerBuilder.NAME,
@@ -1283,6 +1283,7 @@ public Collection<?> createComponents(PluginServices services) {
             clusterService,
             client,
             inferenceAuditor,
+            telemetryProvider.getMeterRegistry(),
             mlAssignmentNotifier,
             machineLearningExtension.get().isAnomalyDetectionEnabled(),
             machineLearningExtension.get().isDataFrameAnalyticsEnabled(),
@@ -1863,10 +1864,8 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
             )
         );
         namedXContent.addAll(new CorrelationNamedContentProvider().getNamedXContentParsers());
-        // LTR Combine with Inference named content provider when feature flag is removed
-        if (machineLearningExtension.get().isLearningToRankEnabled()) {
-            namedXContent.addAll(new MlLTRNamedXContentProvider().getNamedXContentParsers());
-        }
+        namedXContent.addAll(new MlLTRNamedXContentProvider().getNamedXContentParsers());
+
         return namedXContent;
     }
 
@@ -1957,10 +1956,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         namedWriteables.addAll(MlAutoscalingNamedWritableProvider.getNamedWriteables());
         namedWriteables.addAll(new CorrelationNamedContentProvider().getNamedWriteables());
         namedWriteables.addAll(new ChangePointNamedContentProvider().getNamedWriteables());
-        // LTR Combine with Inference named content provider when feature flag is removed
-        if (machineLearningExtension.get().isLearningToRankEnabled()) {
-            namedWriteables.addAll(new MlLTRNamedXContentProvider().getNamedWriteables());
-        }
+        namedWriteables.addAll(new MlLTRNamedXContentProvider().getNamedWriteables());
+
         return namedWriteables;
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java
index 0f8024dd7207a..f46652978753c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningExtension.java
@@ -25,10 +25,6 @@ default void configure(Settings settings) {}
 
     boolean isNlpEnabled();
 
-    default boolean isLearningToRankEnabled() {
-        return false;
-    }
-
     default boolean disableInferenceProcessCache() {
         return false;
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportAction.java
index a9e026b9b4529..e1cadcb25d620 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportAction.java
@@ -7,8 +7,8 @@
 package org.elasticsearch.xpack.ml;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
index e5540ef7e8c5f..583965e76e542 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java
index a1664b7023fc0..2b3ed3f7a656c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex;
 import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsFeatureFlag;
@@ -68,6 +69,7 @@ public final class MlInitializationService implements ClusterStateListener {
         ClusterService clusterService,
         Client client,
         InferenceAuditor inferenceAuditor,
+        MeterRegistry meterRegistry,
         MlAssignmentNotifier mlAssignmentNotifier,
         boolean isAnomalyDetectionEnabled,
         boolean isDataFrameAnalyticsEnabled,
@@ -87,7 +89,7 @@ public final class MlInitializationService implements ClusterStateListener {
                 isDataFrameAnalyticsEnabled,
                 isNlpEnabled
             ),
-            new AdaptiveAllocationsScalerService(threadPool, clusterService, client, inferenceAuditor, isNlpEnabled),
+            new AdaptiveAllocationsScalerService(threadPool, clusterService, client, inferenceAuditor, meterRegistry, isNlpEnabled),
             clusterService
         );
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportAuditMlNotificationAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportAuditMlNotificationAction.java
index 70f3d54b704e0..cae2c9d5da2aa 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportAuditMlNotificationAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportAuditMlNotificationAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.common.notifications.AbstractAuditor;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCancelJobModelSnapshotUpgradeAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCancelJobModelSnapshotUpgradeAction.java
index 41b146f1d9adb..c4820112211b0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCancelJobModelSnapshotUpgradeAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCancelJobModelSnapshotUpgradeAction.java
@@ -15,9 +15,9 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java
index c3a09902de100..cca421801a394 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
index af18ba6170cda..306098f38bc08 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
@@ -21,13 +21,13 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.tasks.CancellableTask;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCoordinatedInferenceAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCoordinatedInferenceAction.java
index 3f95c4edc6d33..fd13e3de4e6cd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCoordinatedInferenceAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCoordinatedInferenceAction.java
@@ -16,12 +16,12 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCreateTrainedModelAssignmentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCreateTrainedModelAssignmentAction.java
index 30371fcbe115a..5932f404fc1a4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCreateTrainedModelAssignmentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCreateTrainedModelAssignmentAction.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java
index 729bed1709162..02f9f316509a6 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java
@@ -12,13 +12,13 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java
index e4c73106852ef..21aeb00ad72fb 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarEventAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
index 24c1623f7cc02..a4ff765c7debf 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
@@ -19,8 +19,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
index cddddc8d3c245..4fe24bbf468e2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
index ad85f22873cce..919f0a526b8ae 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java
index 4437a36318452..a127656103c45 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteFilterAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastAction.java
index 495d75b2de2cd..f25f36581d6fe 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastAction.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -34,6 +33,7 @@
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.index.reindex.ScrollableHitSource;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
index 9fcfc2472bf39..31aaf157d66ad 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
@@ -23,9 +23,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java
index fa1283a5097e6..987bbfb6cf16e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteModelSnapshotAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java
index 1be5a765d1735..fe7eec7623ac1 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java
@@ -25,12 +25,12 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskInfo;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java
index 2a4b3ea70a57f..643ceaae442d6 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java
@@ -22,11 +22,11 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAssignmentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAssignmentAction.java
index 01a58d64b0466..13c7e4f726678 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAssignmentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAssignmentAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
index 6e666b44a63ff..c201e77ac9617 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEstimateModelMemoryAction.java
@@ -9,9 +9,9 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.EstimateModelMemoryAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
index 92c9909441b14..5a71b0fb2e59f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
@@ -14,10 +14,10 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Predicates;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExplainDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExplainDataFrameAnalyticsAction.java
index b1f5eda679006..6c165d2ca445f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExplainDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExplainDataFrameAnalyticsAction.java
@@ -19,11 +19,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java
index f11c6098b7051..545dcfbefecec 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.InferModelAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java
index 5aed29fd6d152..cd519ba233c01 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java
@@ -21,8 +21,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushJobAction.java
index 17f1459984736..cee58f891e245 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushJobAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.FlushJobAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushTrainedModelCacheAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushTrainedModelCacheAction.java
index ab7c9f399fdc1..3d22ad6631a48 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushTrainedModelCacheAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushTrainedModelCacheAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java
index fa761c294a67b..208b775d79504 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java
@@ -12,11 +12,11 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.common.notifications.AbstractAuditMessage;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java
index 58de04146aa52..30ff100915275 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetBucketsAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.GetBucketsAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java
index 89527d2cd12d8..20c54cc1a0b9b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarEventsAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.action.util.QueryPage;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarsAction.java
index 4be53047a5df9..a13273bf87e97 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCalendarsAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.action.util.PageParams;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java
index bdcb4d3d0291f..8831bf43f9f1b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetCategoriesAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java
index eecc5999f842b..0971666682f00 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java
@@ -11,10 +11,10 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
index 07c0e3d7ea618..267f67d3155e9 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
@@ -22,10 +22,10 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.SortBuilders;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java
index a36fc2cdc1ee2..b62f03cfa20ce 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java
index db6c962abbf55..aeeed7d9aabba 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java
index 6a06c84de03c7..61e9a3f24894e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java
index 590aeded2b674..980cdc09252cb 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetFiltersAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetFiltersAction.java
index 0ca5c706e5b8c..512436752cad3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetFiltersAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetFiltersAction.java
@@ -11,9 +11,9 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java
index 5735d715c85cf..d0ec4cbd06658 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetInfluencersAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.GetInfluencersAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java
index 5ceb015198df5..997986d8ce76d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobModelSnapshotsUpgradeStatsAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java
index 0761de41bbd21..b87df826c551d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
index c5061b77e2c6a..12e5a1b32e7d8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
@@ -16,10 +16,10 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java
index ab5949412927c..8e47b5b40a24b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java
@@ -15,9 +15,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java
index 67838fcfa26df..d6b5eb5bc78a4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetModelSnapshotsAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java
index b37f82e45ec49..a47b67e490851 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetOverallBucketsAction.java
@@ -14,9 +14,9 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.InternalAggregations;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java
index 09f5c73d937a6..d246e12cd2b99 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetRecordsAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.GetRecordsAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsAction.java
index b250df8d5215f..1a1fe576757dc 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java
index cb37be3b1fcb7..0d58d6e45bb0c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java
@@ -9,7 +9,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestParameters;
@@ -25,7 +24,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.document.DocumentField;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
@@ -33,6 +31,7 @@
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.ingest.IngestStats;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.tasks.Task;
@@ -96,11 +95,29 @@ public TransportGetTrainedModelsStatsAction(
         TrainedModelProvider trainedModelProvider,
         Client client
     ) {
-        super(GetTrainedModelsStatsAction.NAME, actionFilters, transportService.getTaskManager());
+        this(
+            transportService,
+            actionFilters,
+            clusterService,
+            trainedModelProvider,
+            client,
+            threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)
+        );
+    }
+
+    private TransportGetTrainedModelsStatsAction(
+        TransportService transportService,
+        ActionFilters actionFilters,
+        ClusterService clusterService,
+        TrainedModelProvider trainedModelProvider,
+        Client client,
+        Executor executor
+    ) {
+        super(GetTrainedModelsStatsAction.NAME, actionFilters, transportService.getTaskManager(), executor);
         this.client = client;
         this.clusterService = clusterService;
         this.trainedModelProvider = trainedModelProvider;
-        this.executor = threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME);
+        this.executor = executor;
     }
 
     @Override
@@ -108,15 +125,6 @@ protected void doExecute(
         Task task,
         GetTrainedModelsStatsAction.Request request,
         ActionListener<GetTrainedModelsStatsAction.Response> listener
-    ) {
-        // workaround for https://github.com/elastic/elasticsearch/issues/97916 - TODO remove this when we can
-        executor.execute(ActionRunnable.wrap(listener, l -> doExecuteForked(task, request, l)));
-    }
-
-    protected void doExecuteForked(
-        Task task,
-        GetTrainedModelsStatsAction.Request request,
-        ActionListener<GetTrainedModelsStatsAction.Response> listener
     ) {
         final TaskId parentTaskId = new TaskId(clusterService.localNode().getId(), task.getId());
         final ModelAliasMetadata modelAliasMetadata = ModelAliasMetadata.fromState(clusterService.state());
@@ -151,7 +159,7 @@ protected void doExecuteForked(
             .andThenAccept(tuple -> responseBuilder.setExpandedModelIdsWithAliases(tuple.v2()).setTotalModelCount(tuple.v1()))
 
             .<NodesStatsResponse>andThen(
-                (l, ignored) -> executeAsyncWithOrigin(
+                l -> executeAsyncWithOrigin(
                     client,
                     ML_ORIGIN,
                     TransportNodesStatsAction.TYPE,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java
index c492ed1d804d8..c4915ef45c16d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java
@@ -14,10 +14,10 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.inference.InferenceResults;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
index 004d87d643962..0c4064348b3f6 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
@@ -13,12 +13,12 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.inference.InferenceResults;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
index 57ec0194bb918..ba857b9d3946f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
index dd575ce6c9a22..2ef7aeec7748b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java
index beab594e199ff..bc017915e00aa 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java
@@ -14,11 +14,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.Processors;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlMemoryAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlMemoryAction.java
index bf5b6dfbc53e7..cefc1f9ad02fa 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlMemoryAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlMemoryAction.java
@@ -23,10 +23,10 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.monitor.os.OsStats;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
index 52e9f93d7d31f..b220052baff0d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
@@ -22,9 +22,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java
index edea864eea779..0678ddc85e5c9 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.PersistJobAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java
index e94dae01f236b..9d43cb5366381 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.ToXContent;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java
index 7df6b35d272af..c55639c41b522 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java
@@ -9,9 +9,9 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.PostDataAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDataFrameAnalyticsAction.java
index 8fe7c3686dcb9..003bef914f72c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDataFrameAnalyticsAction.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java
index 3df8d36882ecc..5bf9014d233c5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java
@@ -16,13 +16,13 @@
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java
index 119b1d2e9ab1b..7e6a08bb77525 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutCalendarAction.java
@@ -15,9 +15,9 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.ToXContent;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java
index ece64865efe43..b5388cd1c9468 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java
@@ -18,11 +18,11 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
index 8cdb8050bd257..070afe941b166 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
@@ -14,9 +14,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java
index 9237c23e537c1..30567891fbcf2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutFilterAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.ToXContent;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java
index 55f89a993ce61..2327f81417e86 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java
@@ -17,10 +17,10 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java
index c4cefc1750c35..6aaa1e50f2e8a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAction.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.ml.action;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.TransportVersion;
@@ -25,7 +26,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -34,9 +34,11 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
@@ -143,61 +145,7 @@ protected void masterOperation(
         // NOTE: hasModelDefinition is false if we don't parse it. But, if the fully parsed model was already provided, continue
         boolean hasModelDefinition = config.getModelDefinition() != null;
         if (hasModelDefinition) {
-            try {
-                config.getModelDefinition().getTrainedModel().validate();
-            } catch (ElasticsearchException ex) {
-                finalResponseListener.onFailure(
-                    ExceptionsHelper.badRequestException("Definition for [{}] has validation failures.", ex, config.getModelId())
-                );
-                return;
-            }
-
-            TrainedModelType trainedModelType = TrainedModelType.typeFromTrainedModel(config.getModelDefinition().getTrainedModel());
-            if (trainedModelType == null) {
-                finalResponseListener.onFailure(
-                    ExceptionsHelper.badRequestException(
-                        "Unknown trained model definition class [{}]",
-                        config.getModelDefinition().getTrainedModel().getName()
-                    )
-                );
-                return;
-            }
-
-            if (config.getModelType() == null) {
-                // Set the model type from the definition
-                config = new TrainedModelConfig.Builder(config).setModelType(trainedModelType).build();
-            } else if (trainedModelType != config.getModelType()) {
-                finalResponseListener.onFailure(
-                    ExceptionsHelper.badRequestException(
-                        "{} [{}] does not match the model definition type [{}]",
-                        TrainedModelConfig.MODEL_TYPE.getPreferredName(),
-                        config.getModelType(),
-                        trainedModelType
-                    )
-                );
-                return;
-            }
-
-            if (config.getInferenceConfig().isTargetTypeSupported(config.getModelDefinition().getTrainedModel().targetType()) == false) {
-                finalResponseListener.onFailure(
-                    ExceptionsHelper.badRequestException(
-                        "Model [{}] inference config type [{}] does not support definition target type [{}]",
-                        config.getModelId(),
-                        config.getInferenceConfig().getName(),
-                        config.getModelDefinition().getTrainedModel().targetType()
-                    )
-                );
-                return;
-            }
-
-            TransportVersion minCompatibilityVersion = config.getModelDefinition().getTrainedModel().getMinimalCompatibilityVersion();
-            if (state.getMinTransportVersion().before(minCompatibilityVersion)) {
-                finalResponseListener.onFailure(
-                    ExceptionsHelper.badRequestException(
-                        "Cannot create model [{}] while cluster upgrade is in progress.",
-                        config.getModelId()
-                    )
-                );
+            if (validateModelDefinition(config, state, licenseState, finalResponseListener) == false) {
                 return;
             }
         }
@@ -507,6 +455,85 @@ private void checkTagsAgainstModelIds(List<String> tags, ActionListener<Void> li
         );
     }
 
+    public static boolean validateModelDefinition(
+        TrainedModelConfig config,
+        ClusterState state,
+        XPackLicenseState licenseState,
+        ActionListener<Response> finalResponseListener
+    ) {
+        try {
+            config.getModelDefinition().getTrainedModel().validate();
+        } catch (ElasticsearchException ex) {
+            finalResponseListener.onFailure(
+                ExceptionsHelper.badRequestException("Definition for [{}] has validation failures.", ex, config.getModelId())
+            );
+            return false;
+        }
+
+        TrainedModelType trainedModelType = TrainedModelType.typeFromTrainedModel(config.getModelDefinition().getTrainedModel());
+        if (trainedModelType == null) {
+            finalResponseListener.onFailure(
+                ExceptionsHelper.badRequestException(
+                    "Unknown trained model definition class [{}]",
+                    config.getModelDefinition().getTrainedModel().getName()
+                )
+            );
+            return false;
+        }
+
+        var configModelType = config.getModelType();
+        if (configModelType == null) {
+            // Set the model type from the definition
+            config = new TrainedModelConfig.Builder(config).setModelType(trainedModelType).build();
+        } else if (trainedModelType != configModelType) {
+            finalResponseListener.onFailure(
+                ExceptionsHelper.badRequestException(
+                    "{} [{}] does not match the model definition type [{}]",
+                    TrainedModelConfig.MODEL_TYPE.getPreferredName(),
+                    configModelType,
+                    trainedModelType
+                )
+            );
+            return false;
+        }
+
+        var inferenceConfig = config.getInferenceConfig();
+        if (inferenceConfig.isTargetTypeSupported(config.getModelDefinition().getTrainedModel().targetType()) == false) {
+            finalResponseListener.onFailure(
+                ExceptionsHelper.badRequestException(
+                    "Model [{}] inference config type [{}] does not support definition target type [{}]",
+                    config.getModelId(),
+                    config.getInferenceConfig().getName(),
+                    config.getModelDefinition().getTrainedModel().targetType()
+                )
+            );
+            return false;
+        }
+
+        var minLicenseSupported = inferenceConfig.getMinLicenseSupportedForAction(RestRequest.Method.PUT);
+        if (licenseState.isAllowedByLicense(minLicenseSupported) == false) {
+            finalResponseListener.onFailure(
+                new ElasticsearchSecurityException(
+                    "Model of type [{}] requires [{}] license level",
+                    RestStatus.FORBIDDEN,
+                    config.getInferenceConfig().getName(),
+                    minLicenseSupported
+                )
+            );
+            return false;
+        }
+
+        TransportVersion minCompatibilityVersion = config.getModelDefinition().getTrainedModel().getMinimalCompatibilityVersion();
+        if (state.getMinTransportVersion().before(minCompatibilityVersion)) {
+            finalResponseListener.onFailure(
+                ExceptionsHelper.badRequestException("Cannot create model [{}] while cluster upgrade is in progress.", config.getModelId())
+            );
+            return false;
+        }
+
+        return true;
+    }
+
     @Override
     protected ClusterBlockException checkBlock(Request request, ClusterState state) {
         return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAliasAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAliasAction.java
index 79560b8b8e94e..d38c923515203 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAliasAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAliasAction.java
@@ -21,11 +21,11 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelDefinitionPartAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelDefinitionPartAction.java
index 54c3f73f6a3f2..34bc555837840 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelDefinitionPartAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelDefinitionPartAction.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelVocabularyAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelVocabularyAction.java
index 57b5e269aa4c7..c74a9814af569 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelVocabularyAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelVocabularyAction.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportResetJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportResetJobAction.java
index 343bab2f55bba..13de0ad51ff1c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportResetJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportResetJobAction.java
@@ -24,9 +24,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
index 0f9a416121182..0cc82198f09d4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
@@ -20,8 +20,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetResetModeAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetResetModeAction.java
index 7c52e086ec43c..117e534490f9a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetResetModeAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetResetModeAction.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.action.AbstractTransportSetResetModeAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java
index 076573bb61b90..744d5dbd6974f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java
@@ -26,10 +26,10 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksClusterService;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
index 05f3d6311404a..9db8a72f0bb14 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
@@ -25,13 +25,13 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
index ab0ada00b8aaf..26d26d87e4cc7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
@@ -24,12 +24,12 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.license.XPackLicenseState;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java
index ae0da7dc9cc69..e130b13f4ec30 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java
@@ -27,13 +27,13 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.document.DocumentField;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
index 94cce17b4709f..7d39cd7f76e17 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
@@ -20,11 +20,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.tasks.CancellableTask;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
index b3178e7311447..f998701dbd4e0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
@@ -21,13 +21,13 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksClusterService;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
index dc95d548c5f1b..cbfa770f97e0f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
@@ -21,11 +21,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java
index 0dda155043556..5b76b46f66c6c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java
@@ -13,10 +13,10 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java
index 2ab40ca5b9ca4..3552385be50a7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateCalendarJobAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.PutCalendarAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java
index 66ce0df432c6c..4ca8d0f1a3749 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java
@@ -15,9 +15,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
index a71926a868c85..b97202826145a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
@@ -14,9 +14,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java
index 22d22cb9d0f73..1adcc89134a1f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateFilterAction.java
@@ -18,11 +18,11 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.ToXContent;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java
index 3781bdca41237..d00e7a2865cbe 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java
index a2bb420c1e705..6f30f85454eb4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateModelSnapshotAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.ToXContent;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateProcessAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateProcessAction.java
index 80846a74dfc61..856fe82c401b5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateProcessAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateProcessAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.UpdateProcessAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelAssignmentStateAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelAssignmentStateAction.java
index 900e0e5aa2a43..890811cd9953b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelAssignmentStateAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelAssignmentStateAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelDeploymentAction.java
index fa38b30ae8b84..4cbe09271540c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateTrainedModelDeploymentAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java
index dea6c53d39ab4..43b2f22ae79f0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java
@@ -20,9 +20,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportValidateDetectorAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportValidateDetectorAction.java
index 2307e93085f4d..87d57b13ffe50 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportValidateDetectorAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportValidateDetectorAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportValidateJobConfigAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportValidateJobConfigAction.java
index 0ad08c538a241..21f0295b0b1c3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportValidateJobConfigAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportValidateJobConfigAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java
index ad5a07387bc74..9a9fbfa0340a9 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java
@@ -3,6 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
+ *
+ * this file was contributed to by a Generative AI
  */
 
 package org.elasticsearch.xpack.ml.autoscaling;
@@ -20,7 +22,6 @@
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
 import org.elasticsearch.xpack.core.ml.autoscaling.MlAutoscalingStats;
-import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState;
 import org.elasticsearch.xpack.core.ml.inference.assignment.Priority;
 import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment;
 import org.elasticsearch.xpack.core.ml.utils.MemoryTrackedTaskState;
@@ -52,6 +53,11 @@
 public final class MlAutoscalingResourceTracker {
     private static final Logger logger = LogManager.getLogger(MlAutoscalingResourceTracker.class);
 
+    /**
+     * @param memory     (bytes)
+     * @param processors (count)
+     * @param jobs       (count)
+     */
     record MlJobRequirements(long memory, int processors, int jobs) {
         static MlJobRequirements of(long memory, int processors, int jobs) {
             return new MlJobRequirements(memory, processors, jobs);
@@ -110,6 +116,7 @@ public static void getMlAutoscalingStats(
             processorsAvailableFirstNode,
             maxOpenJobsPerNode,
             mlDummyAutoscalingEntity,
+            clusterSettings.get(MachineLearning.ALLOCATED_PROCESSORS_SCALE),
             listener
         );
     }
@@ -118,13 +125,15 @@ static void getMemoryAndProcessors(
         MlAutoscalingContext autoscalingContext,
         MlMemoryTracker mlMemoryTracker,
         Map<String, Long> nodeSizeByMlNode,
-        long perNodeAvailableModelMemoryInBytes,
+        long perNodeAvailableModelMemoryBytes,
         int perNodeAvailableProcessors,
         int maxOpenJobsPerNode,
         MlDummyAutoscalingEntity dummyAutoscalingEntity,
+        int allocatedProcessorsScale,
         ActionListener<MlAutoscalingStats> listener
     ) {
-        Map<String, List<MlJobRequirements>> perNodeModelMemoryInBytes = new HashMap<>();
+
+        Map<String, List<MlJobRequirements>> jobRequirementsByNode = new HashMap<>();
 
         int numberMlNodes = nodeSizeByMlNode.size();
 
@@ -134,12 +143,12 @@ static void getMemoryAndProcessors(
             ? 0L
             : nodeSizeByMlNode.values().iterator().next();
 
-        long modelMemoryBytesSum = 0;
-        long extraSingleNodeModelMemoryInBytes = 0;
+        long existingModelMemoryBytes = 0;
+        long extraPerNodeModelMemoryBytes = 0;
         long extraModelMemoryInBytes = 0;
-        int extraSingleNodeProcessors = 0;
+        int extraPerNodeProcessors = 0;
         int extraProcessors = 0;
-        int processorsSum = 0;
+        int sumOfCurrentlyExistingAndUsedProcessors = 0;
 
         logger.debug(
             "getting ml resources, found [{}] ad jobs, [{}] dfa jobs and [{}] inference deployments",
@@ -148,7 +157,7 @@ static void getMemoryAndProcessors(
             autoscalingContext.modelAssignments.size()
         );
 
-        // Start with `minNodes = 0`. If any ML job is started this will be increased to 1 in the loops below,
+        // Start with `wantedMinNodes = 0`. If any ML job is started this will be increased to 1 in the loops below,
         // and further adjustments are made for trained models depending on allocations.
         int minNodes = 0;
 
@@ -174,14 +183,14 @@ static void getMemoryAndProcessors(
 
                 // implementation decision: don't count processors for AD, if this gets a revisit, ensure to change it for the
                 // old autoscaling, too
-                extraSingleNodeModelMemoryInBytes = Math.max(extraSingleNodeModelMemoryInBytes, jobMemory);
+                extraPerNodeModelMemoryBytes = Math.max(extraPerNodeModelMemoryBytes, jobMemory);
                 extraModelMemoryInBytes += jobMemory;
             } else {
                 logger.debug("job [{}] assigned to [{}], memory required [{}]", jobId, task.getAssignment(), jobMemory);
 
-                modelMemoryBytesSum += jobMemory;
+                existingModelMemoryBytes += jobMemory;
 
-                perNodeModelMemoryInBytes.computeIfAbsent(task.getExecutorNode(), k -> new ArrayList<>())
+                jobRequirementsByNode.computeIfAbsent(task.getExecutorNode(), k -> new ArrayList<>())
                     .add(MlJobRequirements.of(jobMemory, 0));
             }
         }
@@ -208,91 +217,114 @@ static void getMemoryAndProcessors(
 
                 // implementation decision: don't count processors for DFA, if this gets a revisit, ensure to change it for the
                 // old autoscaling, too
-                extraSingleNodeModelMemoryInBytes = Math.max(extraSingleNodeModelMemoryInBytes, jobMemory);
+                extraPerNodeModelMemoryBytes = Math.max(extraPerNodeModelMemoryBytes, jobMemory);
                 extraModelMemoryInBytes += jobMemory;
             } else {
                 logger.debug("dfa job [{}] assigned to [{}], memory required [{}]", jobId, task.getAssignment(), jobMemory);
 
-                modelMemoryBytesSum += jobMemory;
-                perNodeModelMemoryInBytes.computeIfAbsent(task.getExecutorNode(), k -> new ArrayList<>())
+                existingModelMemoryBytes += jobMemory;
+                jobRequirementsByNode.computeIfAbsent(task.getExecutorNode(), k -> new ArrayList<>())
                     .add(MlJobRequirements.of(jobMemory, 0));
             }
         }
 
         // trained models
+        int numberOfAvailableProcessors = (int) Math.floor(
+            MlProcessors.getTotalMlNodeProcessors(autoscalingContext.mlNodes, allocatedProcessorsScale).count()
+        ) - totalAssignedProcessors(autoscalingContext);
         for (var modelAssignment : autoscalingContext.modelAssignments.entrySet()) {
             TrainedModelAssignment assignment = modelAssignment.getValue();
-            final int numberOfAllocations = assignment.getTaskParams().getNumberOfAllocations();
+            final int numberOfRequestedAllocations = assignment.getTaskParams().getNumberOfAllocations();
             final int numberOfThreadsPerAllocation = assignment.getTaskParams().getThreadsPerAllocation();
             final long estimatedMemoryUsage = assignment.getTaskParams().estimateMemoryUsageBytes();
+            final int numberOfTargetAllocationsOnExistingNodes = assignment.totalTargetAllocations();
+            final int numMissingAllocations = numberOfRequestedAllocations - numberOfTargetAllocationsOnExistingNodes;
+            final int numMissingProcessors = numMissingAllocations * numberOfThreadsPerAllocation;
+            int numExistingProcessorsToBeUsed = Math.min(numMissingProcessors, numberOfAvailableProcessors);
+
+            if (assignment.getNodeRoutingTable().isEmpty() == false
+                && assignment.getNodeRoutingTable().values().stream().allMatch(r -> r.getState().consumesMemory() == false)) {
+                // Ignore states that don't consume memory, for example all allocations are failed or stopped
+                // if the node routing table is empty, then it will match the above condition, but it needs to be handled in the next branch
+                continue;
+            } else {
 
-            if (AssignmentState.STARTING.equals(assignment.getAssignmentState()) && assignment.getNodeRoutingTable().isEmpty()) {
-
-                logger.debug(
-                    () -> format(
-                        "trained model [%s] lacks assignment , memory required [%d]",
-                        modelAssignment.getKey(),
-                        estimatedMemoryUsage
-                    )
-                );
-
-                extraSingleNodeModelMemoryInBytes = Math.max(extraSingleNodeModelMemoryInBytes, estimatedMemoryUsage);
-                extraModelMemoryInBytes += estimatedMemoryUsage;
+                if (assignment.getNodeRoutingTable().isEmpty() == false) {
+                    // if the routing table is non-empty, this is an existing model
+                    existingModelMemoryBytes += estimatedMemoryUsage;
+                } else {
+                    // only increase memory requirements for new models
+                    extraPerNodeModelMemoryBytes += Math.max(extraPerNodeModelMemoryBytes, estimatedMemoryUsage);
+                    extraModelMemoryInBytes += estimatedMemoryUsage;
+                }
 
-                // if not low priority, check processor requirements
+                // if not low priority, check processor requirements.
                 if (Priority.LOW.equals(modelAssignment.getValue().getTaskParams().getPriority()) == false) {
-                    // as assignments can be placed on different nodes, we only need numberOfThreadsPerAllocation here
-                    extraSingleNodeProcessors = Math.max(extraSingleNodeProcessors, numberOfThreadsPerAllocation);
-                    extraProcessors += numberOfAllocations * numberOfThreadsPerAllocation;
+                    if (numMissingProcessors > numberOfAvailableProcessors) {
+                        // as assignments can be placed on different nodes, we only need numberOfThreadsPerAllocation here
+                        extraProcessors += numMissingProcessors - numExistingProcessorsToBeUsed;
+                        extraPerNodeProcessors = Math.max(extraPerNodeProcessors, 1); // if extra processors >0, we need at least 1
+                                                                                      // extraPerNodeProcessors
+                    }
+                    if (perNodeAvailableProcessors < numberOfThreadsPerAllocation) {
+                        extraPerNodeProcessors = Math.max(extraPerNodeProcessors, numberOfThreadsPerAllocation);
+                    }
+                    numberOfAvailableProcessors -= numExistingProcessorsToBeUsed;
                 }
-            } else if (assignment.getNodeRoutingTable().values().stream().allMatch(r -> r.getState().consumesMemory() == false)) {
-                // Ignore states that don't consume memory, for example all allocations are failed
-                continue;
-            } else {
-                logger.debug(
-                    () -> format(
-                        "trained model [%s] assigned to [%s], memory required [%d]",
-                        modelAssignment.getKey(),
-                        Strings.arrayToCommaDelimitedString(modelAssignment.getValue().getStartedNodes()),
-                        estimatedMemoryUsage
-                    )
-                );
 
-                modelMemoryBytesSum += estimatedMemoryUsage;
-                processorsSum += numberOfAllocations * numberOfThreadsPerAllocation;
+                if (extraProcessors > 0 || extraPerNodeProcessors > 0 || extraModelMemoryInBytes > 0 || extraPerNodeModelMemoryBytes > 0) {
+                    logger.info(
+                        () -> format(
+                            "trained model [%s] assigned to [%s], waiting for [%d] allocations to start due to missing hardware",
+                            modelAssignment.getKey(),
+                            Strings.arrayToCommaDelimitedString(modelAssignment.getValue().getStartedNodes()),
+                            numMissingAllocations
+                        )
+                    );
+                }
 
                 for (String node : modelAssignment.getValue().getNodeRoutingTable().keySet()) {
-                    perNodeModelMemoryInBytes.computeIfAbsent(node, k -> new ArrayList<>())
+                    sumOfCurrentlyExistingAndUsedProcessors += modelAssignment.getValue()
+                        .getNodeRoutingTable()
+                        .get(node)
+                        .getTargetAllocations() * numberOfThreadsPerAllocation;
+
+                    jobRequirementsByNode.computeIfAbsent(node, k -> new ArrayList<>())
                         .add(
                             MlJobRequirements.of(
                                 estimatedMemoryUsage,
                                 Priority.LOW.equals(modelAssignment.getValue().getTaskParams().getPriority())
                                     ? 0
-                                    : numberOfThreadsPerAllocation
+                                    : modelAssignment.getValue().getNodeRoutingTable().get(node).getTargetAllocations()
+                                        * numberOfThreadsPerAllocation
                             )
                         );
                 }
-            }
 
-            // min(3, max(number of allocations over all deployed models)
-            minNodes = Math.min(3, Math.max(minNodes, numberOfAllocations));
+                // min(3, max(number of allocations over all deployed models)
+                // the minimum number of nodes is equal to the number of allocations, up to 3
+                // if the number of allocations is greater than 3, then wantedMinNodes is still 3
+                // in theory this should help availability for 2-3 allocations
+                // the planner should split over all available nodes
+                minNodes = Math.min(3, Math.max(minNodes, numberOfRequestedAllocations));
+            }
         }
 
         // dummy autoscaling entity
         if (dummyEntityFitsOnLeastLoadedNode(
-            perNodeModelMemoryInBytes,
-            perNodeAvailableModelMemoryInBytes,
+            jobRequirementsByNode,
+            perNodeAvailableModelMemoryBytes,
             perNodeAvailableProcessors,
             dummyAutoscalingEntity
         ) == false) {
-            logger.info(
+            logger.debug(
                 "Scaling up due to dummy entity: dummyEntityMemory: [{}], dummyEntityProcessors: [{}]",
                 dummyAutoscalingEntity.memory,
                 dummyAutoscalingEntity.processors
             );
 
-            modelMemoryBytesSum += dummyAutoscalingEntity.memory;
-            processorsSum += dummyAutoscalingEntity.processors;
+            existingModelMemoryBytes += dummyAutoscalingEntity.memory;
+            sumOfCurrentlyExistingAndUsedProcessors += dummyAutoscalingEntity.processors;
         }
 
         // check for downscaling
@@ -305,15 +337,15 @@ static void getMemoryAndProcessors(
         // - the total memory usage is less than memory usage after taking away 1 node
         // - the current number of nodes is greater than the minimum number of nodes
         if (perNodeMemoryInBytes > 0
-            && perNodeAvailableModelMemoryInBytes > 0
+            && perNodeAvailableModelMemoryBytes > 0
             && extraModelMemoryInBytes == 0
             && extraProcessors == 0
-            && modelMemoryBytesSum <= perNodeMemoryInBytes * (numberMlNodes - 1)
+            && existingModelMemoryBytes <= perNodeMemoryInBytes * (numberMlNodes - 1)
             && minNodes < numberMlNodes
-            && (perNodeModelMemoryInBytes.size() < numberMlNodes // a node has no assigned jobs
+            && (jobRequirementsByNode.size() < numberMlNodes // a node has no assigned jobs
                 || checkIfOneNodeCouldBeRemoved(
-                    perNodeModelMemoryInBytes,
-                    perNodeAvailableModelMemoryInBytes,
+                    jobRequirementsByNode,
+                    perNodeAvailableModelMemoryBytes,
                     perNodeAvailableProcessors,
                     maxOpenJobsPerNode,
                     dummyAutoscalingEntity
@@ -321,15 +353,18 @@ static void getMemoryAndProcessors(
             removeNodeMemoryInBytes = perNodeMemoryInBytes;
         }
 
+        // if we need extra processors, we need to tell the elasticsearch-autoscaler that we need at least 1 processor per node
+        assert extraProcessors == 0 || extraPerNodeProcessors > 0;
+
         listener.onResponse(
             new MlAutoscalingStats(
                 numberMlNodes,
                 perNodeMemoryInBytes,
-                modelMemoryBytesSum,
-                processorsSum,
+                existingModelMemoryBytes,
+                sumOfCurrentlyExistingAndUsedProcessors,
                 minNodes,
-                extraSingleNodeModelMemoryInBytes,
-                extraSingleNodeProcessors,
+                extraPerNodeModelMemoryBytes,
+                extraPerNodeProcessors,
                 extraModelMemoryInBytes,
                 extraProcessors,
                 removeNodeMemoryInBytes,
@@ -338,23 +373,27 @@ static void getMemoryAndProcessors(
         );
     }
 
+    private static int totalAssignedProcessors(MlAutoscalingContext autoscalingContext) {
+        return autoscalingContext.modelAssignments.values().stream().mapToInt(TrainedModelAssignment::totalTargetProcessors).sum();
+    }
+
     /**
      * Check if the dummy autoscaling entity task can be added by placing
      * the task on the least loaded node.
-     *
+     * <p>
      * If there exists a node that can accommodate the dummy entity then return true (nothing to do),
      * else return false and increment the memory and processor counts accordingly.
-     *
+     * <p>
      * We perform the calculation by identifying the least loaded node in terms of memory
      * and determining if the addition of the dummy entity's memory and processor requirements could
      * be accommodated on it.
-     *
+     * <p>
      * If the calculation returns false then treat the case as for a single trained model job
-     * that is already assigned, i.e. increment modelMemoryBytesSum and processorsSum appropriately.
+     * that is already assigned, i.e. increment modelMemoryBytesSum and currentTotalProcessorsInUse appropriately.
      *
      * @param perNodeJobRequirements per Node lists of requirements
-     * @param perNodeMemoryInBytes total model memory available on every node
-     * @param perNodeProcessors total processors on every node
+     * @param perNodeMemoryInBytes   total model memory available on every node
+     * @param perNodeProcessors      total processors on every node
      * @param dummyAutoscalingEntity "dummy" entity requirements used to potentially trigger a scaling event
      * @return true if the dummy entity can be accommodated, false if not
      */
@@ -369,7 +408,7 @@ static boolean dummyEntityFitsOnLeastLoadedNode(
             return true;
         }
 
-        if (perNodeJobRequirements.size() < 1) {
+        if (perNodeJobRequirements.isEmpty()) {
             return false;
         }
 
@@ -389,7 +428,6 @@ static boolean dummyEntityFitsOnLeastLoadedNode(
             )
             .min(Comparator.comparingLong(value -> value.memory));
 
-        assert (leastLoadedNodeRequirements.isPresent());
         assert leastLoadedNodeRequirements.get().memory >= 0L;
         assert leastLoadedNodeRequirements.get().processors >= 0;
 
@@ -433,8 +471,8 @@ private static MlAutoscalingStats noScaleStats(int numberMlNodes) {
      * Check if one node can be removed by placing the jobs of the least loaded node to others.
      *
      * @param perNodeJobRequirements per Node lists of requirements
-     * @param perNodeMemoryInBytes total model memory available on every node
-     * @param maxOpenJobsPerNode the maximum number of jobs per node
+     * @param perNodeMemoryInBytes   total model memory available on every node
+     * @param maxOpenJobsPerNode     the maximum number of jobs per node
      * @return true if a node can be removed, false if not
      */
     static boolean checkIfOneNodeCouldBeRemoved(
@@ -502,9 +540,9 @@ static boolean checkIfOneNodeCouldBeRemoved(
      * <p>Because the metric has no influence on how the jobs are placed in the end, it calculates the possibility of moving in the least
      * efficient way. That way we ensure that autoscaling is not looping between scaling down, up, down, up ... </p>
      *
-     * @param candidateJobRequirements list of job requirements given running on the candidate node
+     * @param candidateJobRequirements    list of job requirements given running on the candidate node
      * @param perNodeMlJobRequirementsSum other nodes requirements
-     * @param perNodeMemoryInBytes available memory per node
+     * @param perNodeMemoryInBytes        available memory per node
      * @return remaining memory, that could not be placed on other nodes, 0L if all jobs got placed
      */
     static long checkIfJobsCanBeMovedInLeastEfficientWay(
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java
index 15f647bc76697..044556d1b30ac 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java
@@ -35,10 +35,15 @@ public class AdaptiveAllocationsScaler {
     private final KalmanFilter1d inferenceTimeEstimator;
 
     private int numberOfAllocations;
+    private int neededNumberOfAllocations;
     private Integer minNumberOfAllocations;
     private Integer maxNumberOfAllocations;
     private boolean dynamicsChanged;
 
+    private Double lastMeasuredRequestRate;
+    private Double lastMeasuredInferenceTime;
+    private Long lastMeasuredQueueSize;
+
     AdaptiveAllocationsScaler(String deploymentId, int numberOfAllocations) {
         this.deploymentId = deploymentId;
         // A smoothing factor of 100 roughly means the last 100 measurements have an effect
@@ -51,9 +56,14 @@ public class AdaptiveAllocationsScaler {
         requestRateEstimator = new KalmanFilter1d(deploymentId + ":rate", 100, true);
         inferenceTimeEstimator = new KalmanFilter1d(deploymentId + ":time", 100, false);
         this.numberOfAllocations = numberOfAllocations;
-        this.minNumberOfAllocations = null;
-        this.maxNumberOfAllocations = null;
-        this.dynamicsChanged = false;
+        neededNumberOfAllocations = numberOfAllocations;
+        minNumberOfAllocations = null;
+        maxNumberOfAllocations = null;
+        dynamicsChanged = false;
+
+        lastMeasuredRequestRate = null;
+        lastMeasuredInferenceTime = null;
+        lastMeasuredQueueSize = null;
     }
 
     void setMinMaxNumberOfAllocations(Integer minNumberOfAllocations, Integer maxNumberOfAllocations) {
@@ -62,6 +72,8 @@ void setMinMaxNumberOfAllocations(Integer minNumberOfAllocations, Integer maxNum
     }
 
     void process(AdaptiveAllocationsScalerService.Stats stats, double timeIntervalSeconds, int numberOfAllocations) {
+        lastMeasuredQueueSize = stats.pendingCount();
+
         // The request rate (per second) is the request count divided by the time.
         // Assuming a Poisson process for the requests, the variance in the request
         // count equals the mean request count, and the variance in the request rate
@@ -74,6 +86,7 @@ void process(AdaptiveAllocationsScalerService.Stats stats, double timeIntervalSe
         double requestRateEstimate = requestRateEstimator.hasValue() ? requestRateEstimator.estimate() : requestRate;
         double requestRateVariance = Math.max(1.0, requestRateEstimate * timeIntervalSeconds) / Math.pow(timeIntervalSeconds, 2);
         requestRateEstimator.add(requestRate, requestRateVariance, false);
+        lastMeasuredRequestRate = requestRate;
 
         if (stats.requestCount() > 0 && Double.isNaN(stats.inferenceTime()) == false) {
             // The inference time distribution is unknown. For simplicity, we assume
@@ -86,6 +99,9 @@ void process(AdaptiveAllocationsScalerService.Stats stats, double timeIntervalSe
             double inferenceTimeEstimate = inferenceTimeEstimator.hasValue() ? inferenceTimeEstimator.estimate() : inferenceTime;
             double inferenceTimeVariance = Math.pow(inferenceTimeEstimate, 2) / stats.requestCount();
             inferenceTimeEstimator.add(inferenceTime, inferenceTimeVariance, dynamicsChanged);
+            lastMeasuredInferenceTime = inferenceTime;
+        } else {
+            lastMeasuredInferenceTime = null;
         }
 
         this.numberOfAllocations = numberOfAllocations;
@@ -104,6 +120,14 @@ void process(AdaptiveAllocationsScalerService.Stats stats, double timeIntervalSe
         return requestRateUpper * inferenceTimeUpper;
     }
 
+    Double getRequestRateEstimate() {
+        return requestRateEstimator.hasValue() ? requestRateEstimator.estimate() : null;
+    }
+
+    Double getInferenceTimeEstimate() {
+        return inferenceTimeEstimator.hasValue() ? inferenceTimeEstimator.estimate() : null;
+    }
+
     Integer scale() {
         if (requestRateEstimator.hasValue() == false) {
             return null;
@@ -121,6 +145,8 @@ Integer scale() {
             numberOfAllocations--;
         }
 
+        this.neededNumberOfAllocations = numberOfAllocations;
+
         if (maxNumberOfAllocations == null) {
             numberOfAllocations = Math.min(numberOfAllocations, MAX_NUMBER_OF_ALLOCATIONS_SAFEGUARD);
         }
@@ -161,4 +187,28 @@ Integer scale() {
             return null;
         }
     }
+
+    public String getDeploymentId() {
+        return deploymentId;
+    }
+
+    public long getNumberOfAllocations() {
+        return numberOfAllocations;
+    }
+
+    public long getNeededNumberOfAllocations() {
+        return neededNumberOfAllocations;
+    }
+
+    public Double getLastMeasuredRequestRate() {
+        return lastMeasuredRequestRate;
+    }
+
+    public Double getLastMeasuredInferenceTime() {
+        return lastMeasuredInferenceTime;
+    }
+
+    public Long getLastMeasuredQueueSize() {
+        return lastMeasuredQueueSize;
+    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java
index 30e3871ad5ad0..6c59add730052 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java
@@ -19,6 +19,9 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.telemetry.metric.DoubleWithAttributes;
+import org.elasticsearch.telemetry.metric.LongWithAttributes;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ClientHelper;
@@ -30,11 +33,15 @@
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
 
 /**
  * Periodically schedules adaptive allocations scaling. This process consists
@@ -75,6 +82,108 @@ Stats sub(Stats value) {
         }
     }
 
+    private class Metrics {
+
+        private final List<AutoCloseable> metrics = new ArrayList<>();
+
+        Metrics() {}
+
+        void init() {
+            if (metrics.isEmpty() == false) {
+                return;
+            }
+            metrics.add(
+                meterRegistry.registerLongsGauge(
+                    "es.ml.trained_models.adaptive_allocations.actual_number_of_allocations.current",
+                    "the actual number of allocations",
+                    "",
+                    () -> observeLong(AdaptiveAllocationsScaler::getNumberOfAllocations)
+                )
+            );
+            metrics.add(
+                meterRegistry.registerLongsGauge(
+                    "es.ml.trained_models.adaptive_allocations.needed_number_of_allocations.current",
+                    "the number of allocations needed according to the adaptive allocations scaler",
+                    "",
+                    () -> observeLong(AdaptiveAllocationsScaler::getNeededNumberOfAllocations)
+                )
+            );
+            metrics.add(
+                meterRegistry.registerDoublesGauge(
+                    "es.ml.trained_models.adaptive_allocations.measured_request_rate.current",
+                    "the request rate reported by the stats API",
+                    "1/s",
+                    () -> observeDouble(AdaptiveAllocationsScaler::getLastMeasuredRequestRate)
+                )
+            );
+            metrics.add(
+                meterRegistry.registerDoublesGauge(
+                    "es.ml.trained_models.adaptive_allocations.estimated_request_rate.current",
+                    "the request rate estimated by the adaptive allocations scaler",
+                    "1/s",
+                    () -> observeDouble(AdaptiveAllocationsScaler::getRequestRateEstimate)
+                )
+            );
+            metrics.add(
+                meterRegistry.registerDoublesGauge(
+                    "es.ml.trained_models.adaptive_allocations.measured_inference_time.current",
+                    "the inference time reported by the stats API",
+                    "s",
+                    () -> observeDouble(AdaptiveAllocationsScaler::getLastMeasuredInferenceTime)
+                )
+            );
+            metrics.add(
+                meterRegistry.registerDoublesGauge(
+                    "es.ml.trained_models.adaptive_allocations.estimated_inference_time.current",
+                    "the inference time estimated by the adaptive allocations scaler",
+                    "s",
+                    () -> observeDouble(AdaptiveAllocationsScaler::getInferenceTimeEstimate)
+                )
+            );
+            metrics.add(
+                meterRegistry.registerLongsGauge(
+                    "es.ml.trained_models.adaptive_allocations.queue_size.current",
+                    "the queue size reported by the stats API",
+                    "s",
+                    () -> observeLong(AdaptiveAllocationsScaler::getLastMeasuredQueueSize)
+                )
+            );
+        }
+
+        Collection<LongWithAttributes> observeLong(Function<AdaptiveAllocationsScaler, Long> getValue) {
+            List<LongWithAttributes> observations = new ArrayList<>();
+            for (AdaptiveAllocationsScaler scaler : scalers.values()) {
+                Long value = getValue.apply(scaler);
+                if (value != null) {
+                    observations.add(new LongWithAttributes(value, Map.of("deployment_id", scaler.getDeploymentId())));
+                }
+            }
+            return observations;
+        }
+
+        Collection<DoubleWithAttributes> observeDouble(Function<AdaptiveAllocationsScaler, Double> getValue) {
+            List<DoubleWithAttributes> observations = new ArrayList<>();
+            for (AdaptiveAllocationsScaler scaler : scalers.values()) {
+                Double value = getValue.apply(scaler);
+                if (value != null) {
+                    observations.add(new DoubleWithAttributes(value, Map.of("deployment_id", scaler.getDeploymentId())));
+                }
+            }
+            return observations;
+        }
+
+        void close() {
+            for (AutoCloseable metric : metrics) {
+                try {
+                    metric.close();
+                } catch (Exception e) {
+                    // do nothing
+                }
+            }
+            metrics.clear();
+        }
+    }
+
     /**
      * The time interval between the adaptive allocations triggers.
      */
@@ -92,6 +201,8 @@ Stats sub(Stats value) {
     private final ClusterService clusterService;
     private final Client client;
     private final InferenceAuditor inferenceAuditor;
+    private final MeterRegistry meterRegistry;
+    private final Metrics metrics;
     private final boolean isNlpEnabled;
     private final Map<String, Map<String, Stats>> lastInferenceStatsByDeploymentAndNode;
     private Long lastInferenceStatsTimestampMillis;
@@ -106,9 +217,10 @@ public AdaptiveAllocationsScalerService(
         ClusterService clusterService,
         Client client,
         InferenceAuditor inferenceAuditor,
+        MeterRegistry meterRegistry,
         boolean isNlpEnabled
     ) {
-        this(threadPool, clusterService, client, inferenceAuditor, isNlpEnabled, DEFAULT_TIME_INTERVAL_SECONDS);
+        this(threadPool, clusterService, client, inferenceAuditor, meterRegistry, isNlpEnabled, DEFAULT_TIME_INTERVAL_SECONDS);
     }
 
     // visible for testing
@@ -117,6 +229,7 @@ public AdaptiveAllocationsScalerService(
         ClusterService clusterService,
         Client client,
         InferenceAuditor inferenceAuditor,
+        MeterRegistry meterRegistry,
         boolean isNlpEnabled,
         int timeIntervalSeconds
     ) {
@@ -124,6 +237,7 @@ public AdaptiveAllocationsScalerService(
         this.clusterService = clusterService;
         this.client = client;
         this.inferenceAuditor = inferenceAuditor;
+        this.meterRegistry = meterRegistry;
         this.isNlpEnabled = isNlpEnabled;
         this.timeIntervalSeconds = timeIntervalSeconds;
 
@@ -131,11 +245,13 @@ public AdaptiveAllocationsScalerService(
         lastInferenceStatsTimestampMillis = null;
         lastScaleUpTimesMillis = new HashMap<>();
         scalers = new HashMap<>();
+        metrics = new Metrics();
         busy = new AtomicBoolean(false);
     }
 
     public synchronized void start() {
         updateAutoscalers(clusterService.state());
+        metrics.init();
         clusterService.addListener(this);
         if (scalers.isEmpty() == false) {
             startScheduling();
@@ -144,6 +260,7 @@ public synchronized void start() {
 
     public synchronized void stop() {
         stopScheduling();
+        metrics.close();
     }
 
     @Override
@@ -164,7 +281,8 @@ private synchronized void updateAutoscalers(ClusterState state) {
         TrainedModelAssignmentMetadata assignments = TrainedModelAssignmentMetadata.fromState(state);
         for (TrainedModelAssignment assignment : assignments.allAssignments().values()) {
             deploymentIds.add(assignment.getDeploymentId());
-            if (assignment.getAdaptiveAllocationsSettings() != null && assignment.getAdaptiveAllocationsSettings().getEnabled()) {
+            if (assignment.getAdaptiveAllocationsSettings() != null
+                && assignment.getAdaptiveAllocationsSettings().getEnabled() == Boolean.TRUE) {
                 AdaptiveAllocationsScaler adaptiveAllocationsScaler = scalers.computeIfAbsent(
                     assignment.getDeploymentId(),
                     key -> new AdaptiveAllocationsScaler(assignment.getDeploymentId(), assignment.totalTargetAllocations())
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java
index e86a9cfe94045..96490716c5c6c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java
@@ -833,7 +833,7 @@ private void updateDeployment(
             adaptiveAllocationsSettingsUpdates
         );
         if (adaptiveAllocationsSettings != null) {
-            if (isInternal == false && adaptiveAllocationsSettings.getEnabled() && numberOfAllocations != null) {
+            if (isInternal == false && adaptiveAllocationsSettings.getEnabled() == Boolean.TRUE && numberOfAllocations != null) {
                 ValidationException validationException = new ValidationException();
                 validationException.addValidationError("[" + NUMBER_OF_ALLOCATIONS + "] cannot be set if adaptive allocations is enabled");
                 listener.onFailure(validationException);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/AbstractPyTorchAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/AbstractPyTorchAction.java
index 64d3808e9ed8f..8aa3b310da21a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/AbstractPyTorchAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/AbstractPyTorchAction.java
@@ -65,12 +65,18 @@ public final void init() {
     }
 
     void onTimeout() {
+        onTimeout(new ElasticsearchStatusException("timeout [{}] waiting for inference result", RestStatus.REQUEST_TIMEOUT, timeout));
+    }
+
+    void onCancel() {
+        onTimeout(new ElasticsearchStatusException("inference task cancelled", RestStatus.BAD_REQUEST));
+    }
+
+    void onTimeout(Exception e) {
         if (notified.compareAndSet(false, true)) {
             processContext.getTimeoutCount().incrementAndGet();
             processContext.getResultProcessor().ignoreResponseWithoutNotifying(String.valueOf(requestId));
-            listener.onFailure(
-                new ElasticsearchStatusException("timeout [{}] waiting for inference result", RestStatus.REQUEST_TIMEOUT, timeout)
-            );
+            listener.onFailure(e);
             return;
         }
         getLogger().debug("[{}] request [{}] received timeout after [{}] but listener already alerted", deploymentId, requestId, timeout);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java
index 0442a37831d70..a0fc00af3f859 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/InferencePyTorchAction.java
@@ -84,12 +84,11 @@ protected void doRun() throws Exception {
             logger.debug(() -> format("[%s] skipping inference on request [%s] as it has timed out", getDeploymentId(), getRequestId()));
             return;
         }
+        final String requestIdStr = String.valueOf(getRequestId());
         if (isCancelled()) {
-            onFailure("inference task cancelled");
+            onCancel();
             return;
         }
-
-        final String requestIdStr = String.valueOf(getRequestId());
         try {
             String inputText = input.extractInput(getProcessContext().getModelInput().get());
             if (prefixType != TrainedModelPrefixStrings.PrefixType.NONE) {
@@ -141,7 +140,7 @@ protected void doRun() throws Exception {
 
             // Tokenization is non-trivial, so check for cancellation one last time before sending request to the native process
             if (isCancelled()) {
-                onFailure("inference task cancelled");
+                onCancel();
                 return;
             }
             getProcessContext().getResultProcessor()
@@ -196,9 +195,11 @@ private void processResult(
             return;
         }
         if (isCancelled()) {
-            onFailure("inference task cancelled");
+            onCancel();
             return;
         }
+
+        getProcessContext().getResultProcessor().updateStats(pyTorchResult);
         InferenceResults results = inferenceResultsProcessor.processResult(
             tokenization,
             pyTorchResult.inferenceResult(),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java
index 8a310ba2719f2..70d0b980bb3bf 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java
@@ -28,6 +28,7 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import static java.util.stream.Collectors.toUnmodifiableSet;
@@ -129,8 +130,58 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r
     @Override
     public Explanation explain(int topLevelDocId, IndexSearcher searcher, RescoreContext rescoreContext, Explanation sourceExplanation)
         throws IOException {
-        // TODO: Call infer again but with individual feature importance values and explaining the model (which features are used, etc.)
-        return null;
+        if (sourceExplanation == null) {
+            return Explanation.noMatch("no match found");
+        }
+
+        LearningToRankRescorerContext ltrContext = (LearningToRankRescorerContext) rescoreContext;
+        LocalModel localModelDefinition = ltrContext.regressionModelDefinition;
+
+        if (localModelDefinition == null) {
+            throw new IllegalStateException("local model reference is null, missing rewriteAndFetch before rescore phase?");
+        }
+
+        List<LeafReaderContext> leaves = ltrContext.executionContext.searcher().getIndexReader().leaves();
+
+        int endDoc = 0;
+        int readerUpto = -1;
+        LeafReaderContext currentSegment = null;
+
+        while (topLevelDocId >= endDoc) {
+            readerUpto++;
+            currentSegment = leaves.get(readerUpto);
+            endDoc = currentSegment.docBase + currentSegment.reader().maxDoc();
+        }
+
+        assert currentSegment != null : "Unexpected null segment";
+
+        int targetDoc = topLevelDocId - currentSegment.docBase;
+
+        List<FeatureExtractor> featureExtractors = ltrContext.buildFeatureExtractors(searcher);
+        int featureSize = featureExtractors.stream().mapToInt(fe -> fe.featureNames().size()).sum();
+
+        Map<String, Object> features = Maps.newMapWithExpectedSize(featureSize);
+
+        for (FeatureExtractor featureExtractor : featureExtractors) {
+            featureExtractor.setNextReader(currentSegment);
+            featureExtractor.addFeatures(features, targetDoc);
+        }
+
+        // Predicting the value
+        var ltrScore = ((Number) localModelDefinition.inferLtr(features, ltrContext.learningToRankConfig).predictedValue()).floatValue();
+
+        List<Explanation> featureExplanations = new ArrayList<>();
+        for (String featureName : features.keySet()) {
+            Number featureValue = Objects.requireNonNullElse((Number) features.get(featureName), 0);
+            featureExplanations.add(Explanation.match(featureValue, "feature value for [" + featureName + "]"));
+        }
+
+        return Explanation.match(
+            ltrScore,
+            "rescored using LTR model " + ltrContext.regressionModelDefinition.getModelId(),
+            Explanation.match(sourceExplanation.getValue(), "first pass query score", sourceExplanation),
+            Explanation.match(0f, "extracted features", featureExplanations)
+        );
     }
 
     /** Returns a new {@link TopDocs} with the topN from the incoming one, or the same TopDocs if the number of hits is already &lt;=
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java
index d2179a69ebc24..46edcf1f63c01 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java
@@ -9,6 +9,7 @@
 
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -303,8 +304,7 @@ public String getWriteableName() {
 
     @Override
     public TransportVersion getMinimalSupportedVersion() {
-        // TODO: update transport version when released!
-        return TransportVersion.current();
+        return TransportVersions.LTR_SERVERLESS_RELEASE;
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessor.java
index 3dbf941c8120d..05b97422bfb97 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/NerProcessor.java
@@ -172,6 +172,7 @@ static String buildAnnotatedText(String seq, List<NerResults.EntityGroup> entiti
     }
 
     record NerResultProcessor(IobTag[] iobMap, String resultsField, boolean ignoreCase) implements NlpTask.ResultProcessor {
+
         NerResultProcessor(IobTag[] iobMap, String resultsField, boolean ignoreCase) {
             this.iobMap = iobMap;
             this.resultsField = Optional.ofNullable(resultsField).orElse(DEFAULT_RESULTS_FIELD);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchResultProcessor.java
index 1bb2f1006822e..7dd0dae1e3ad7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchResultProcessor.java
@@ -153,18 +153,12 @@ private void notifyAndClearPendingResults(ErrorResult errorResult) {
     void processInferenceResult(PyTorchResult result) {
         PyTorchInferenceResult inferenceResult = result.inferenceResult();
         assert inferenceResult != null;
-        Long timeMs = result.timeMs();
-        if (timeMs == null) {
-            assert false : "time_ms should be set for an inference result";
-            timeMs = 0L;
-        }
 
         logger.debug(() -> format("[%s] Parsed inference result with id [%s]", modelId, result.requestId()));
         PendingResult pendingResult = pendingResults.remove(result.requestId());
         if (pendingResult == null) {
             logger.debug(() -> format("[%s] no pending result for inference [%s]", modelId, result.requestId()));
         } else {
-            updateStats(timeMs, Boolean.TRUE.equals(result.isCacheHit()));
             pendingResult.listener.onResponse(result);
         }
     }
@@ -273,7 +267,13 @@ private static LongSummaryStatistics cloneSummaryStats(LongSummaryStatistics sta
         return new LongSummaryStatistics(stats.getCount(), stats.getMin(), stats.getMax(), stats.getSum());
     }
 
-    private synchronized void updateStats(long timeMs, boolean isCacheHit) {
+    public synchronized void updateStats(PyTorchResult result) {
+        Long timeMs = result.timeMs();
+        if (timeMs == null) {
+            assert false : "time_ms should be set for an inference result";
+            timeMs = 0L;
+        }
+        boolean isCacheHit = Boolean.TRUE.equals(result.isCacheHit());
         timingStats.accept(timeMs);
 
         lastResultTimeMs = currentTimeMsSupplier.getAsLong();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java
index a93e50f5110e5..8f48950e135e4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java
@@ -91,7 +91,7 @@ private void writeJsonXContent(CategorizationAnalyzer categorizationAnalyzer, In
 
     private void writeSmileXContent(CategorizationAnalyzer categorizationAnalyzer, InputStream inputStream) throws IOException {
         while (true) {
-            byte[] nextObject = findNextObject(XContentType.SMILE.xContent().streamSeparator(), inputStream);
+            byte[] nextObject = findNextObject(XContentType.SMILE.xContent().bulkSeparator(), inputStream);
             if (nextObject.length == 0) {
                 break;
             }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/output/NormalizerResultHandler.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/output/NormalizerResultHandler.java
index 0b4f6765431da..4c800849fda65 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/output/NormalizerResultHandler.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/output/NormalizerResultHandler.java
@@ -57,7 +57,7 @@ public void process() throws IOException {
     }
 
     private BytesReference parseResults(XContent xContent, BytesReference bytesRef) throws IOException {
-        byte marker = xContent.streamSeparator();
+        byte marker = xContent.bulkSeparator();
         int from = 0;
         while (true) {
             int nextMarker = findNextMarker(marker, bytesRef, from);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandler.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandler.java
index 0416f4740f6ee..a32ef1e5ad1a4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandler.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandler.java
@@ -236,7 +236,7 @@ public String getErrors() {
     }
 
     private BytesReference parseMessages(XContent xContent, BytesReference bytesRef) {
-        byte marker = xContent.streamSeparator();
+        byte marker = xContent.bulkSeparator();
         int from = 0;
         while (true) {
             int nextMarker = findNextMarker(marker, bytesRef, from);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java
index fa9a15b246f84..3d7b2d392836a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java
@@ -53,7 +53,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
     }
 
     @Override
-    public boolean supportsContentStream() {
+    public boolean supportsBulkContent() {
         return true;
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/MlProcessors.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/MlProcessors.java
index 4d317ee7925fc..ee982dc395d32 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/MlProcessors.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/MlProcessors.java
@@ -60,7 +60,7 @@ public static Processors getMaxMlNodeProcessors(DiscoveryNodes nodes, Integer al
         return answer;
     }
 
-    public static Processors getTotalMlNodeProcessors(DiscoveryNodes nodes, Integer allocatedProcessorScale) {
+    public static Processors getTotalMlNodeProcessors(Iterable<DiscoveryNode> nodes, Integer allocatedProcessorScale) {
         int total = 0;
         for (DiscoveryNode node : nodes) {
             if (node.getRoles().contains(DiscoveryNodeRole.ML_ROLE)) {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java
index 2d7832d747de4..bab012afc3101 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java
@@ -12,11 +12,12 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.ActionPlugin;
@@ -121,7 +122,12 @@ public static class MockedRollupIndexCapsTransport extends TransportAction<
 
             @Inject
             public MockedRollupIndexCapsTransport(TransportService transportService) {
-                super(GetRollupIndexCapsAction.NAME, new ActionFilters(new HashSet<>()), transportService.getTaskManager());
+                super(
+                    GetRollupIndexCapsAction.NAME,
+                    new ActionFilters(new HashSet<>()),
+                    transportService.getTaskManager(),
+                    EsExecutors.DIRECT_EXECUTOR_SERVICE
+                );
             }
 
             @Override
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java
index 175a035a70f7e..3ff3a4a404f97 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningAdOnly.java
@@ -14,6 +14,6 @@
 
 public class LocalStateMachineLearningAdOnly extends LocalStateMachineLearning {
     public LocalStateMachineLearningAdOnly(final Settings settings, final Path configPath) {
-        super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, true, false, false, false)));
+        super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, true, false, false)));
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java
index f054e52dc29ec..1a72f27865d8a 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningDfaOnly.java
@@ -14,6 +14,6 @@
 
 public class LocalStateMachineLearningDfaOnly extends LocalStateMachineLearning {
     public LocalStateMachineLearningDfaOnly(final Settings settings, final Path configPath) {
-        super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, true, false, false)));
+        super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, true, false)));
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java
index a3d684011e932..0f11e8033b83d 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearningNlpOnly.java
@@ -14,6 +14,6 @@
 
 public class LocalStateMachineLearningNlpOnly extends LocalStateMachineLearning {
     public LocalStateMachineLearningNlpOnly(final Settings settings, final Path configPath) {
-        super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, false, true, false)));
+        super(settings, configPath, new MlTestExtensionLoader(new MlTestExtension(true, true, false, false, true)));
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java
index afa372fb94527..e5575abfeb020 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java
@@ -160,14 +160,7 @@ private MachineLearningUsageTransportAction newUsageAction(
             licenseState,
             jobManagerHolder,
             new MachineLearningExtensionHolder(
-                new MachineLearningTests.MlTestExtension(
-                    true,
-                    true,
-                    isAnomalyDetectionEnabled,
-                    isDataFrameAnalyticsEnabled,
-                    isNlpEnabled,
-                    true
-                )
+                new MachineLearningTests.MlTestExtension(true, true, isAnomalyDetectionEnabled, isDataFrameAnalyticsEnabled, isNlpEnabled)
             )
         );
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java
index c35b9da7b2bd2..8a05537917abe 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java
@@ -220,7 +220,7 @@ public void testNoAttributes_givenClash() throws IOException {
 
     public void testAnomalyDetectionOnly() throws IOException {
         Settings settings = Settings.builder().put("path.home", createTempDir()).build();
-        MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, true, false, false, false));
+        MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, true, false, false));
         try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) {
             List<RestHandler> restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null, null, null);
             assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class)));
@@ -240,7 +240,7 @@ public void testAnomalyDetectionOnly() throws IOException {
 
     public void testDataFrameAnalyticsOnly() throws IOException {
         Settings settings = Settings.builder().put("path.home", createTempDir()).build();
-        MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, true, false, false));
+        MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, true, false));
         try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) {
             List<RestHandler> restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null, null, null);
             assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class)));
@@ -260,7 +260,7 @@ public void testDataFrameAnalyticsOnly() throws IOException {
 
     public void testNlpOnly() throws IOException {
         Settings settings = Settings.builder().put("path.home", createTempDir()).build();
-        MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, false, true, false));
+        MlTestExtensionLoader loader = new MlTestExtensionLoader(new MlTestExtension(false, false, false, false, true));
         try (MachineLearning machineLearning = createTrialLicensedMachineLearning(settings, loader)) {
             List<RestHandler> restHandlers = machineLearning.getRestHandlers(settings, null, null, null, null, null, null, null, null);
             assertThat(restHandlers, hasItem(instanceOf(RestMlInfoAction.class)));
@@ -287,22 +287,19 @@ public static class MlTestExtension implements MachineLearningExtension {
         private final boolean isAnomalyDetectionEnabled;
         private final boolean isDataFrameAnalyticsEnabled;
         private final boolean isNlpEnabled;
-        private final boolean isLearningToRankEnabled;
 
         MlTestExtension(
             boolean useIlm,
             boolean includeNodeInfo,
             boolean isAnomalyDetectionEnabled,
             boolean isDataFrameAnalyticsEnabled,
-            boolean isNlpEnabled,
-            boolean isLearningToRankEnabled
+            boolean isNlpEnabled
         ) {
             this.useIlm = useIlm;
             this.includeNodeInfo = includeNodeInfo;
             this.isAnomalyDetectionEnabled = isAnomalyDetectionEnabled;
             this.isDataFrameAnalyticsEnabled = isDataFrameAnalyticsEnabled;
             this.isNlpEnabled = isNlpEnabled;
-            this.isLearningToRankEnabled = isLearningToRankEnabled;
         }
 
         @Override
@@ -330,11 +327,6 @@ public boolean isNlpEnabled() {
             return isNlpEnabled;
         }
 
-        @Override
-        public boolean isLearningToRankEnabled() {
-            return isLearningToRankEnabled;
-        }
-
         @Override
         public String[] getAnalyticsDestIndexAllowedSettings() {
             return ANALYTICS_DEST_INDEX_ALLOWED_SETTINGS;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java
index 2f251e3b0aee6..a5b9597886e15 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService;
@@ -40,6 +41,7 @@ public class MlInitializationServiceTests extends ESTestCase {
     private ClusterService clusterService;
     private Client client;
     private InferenceAuditor inferenceAuditor;
+    private MeterRegistry meterRegistry;
     private MlAssignmentNotifier mlAssignmentNotifier;
 
     @Before
@@ -49,6 +51,7 @@ public void setUpMocks() {
         clusterService = mock(ClusterService.class);
         client = mock(Client.class);
         inferenceAuditor = mock(InferenceAuditor.class);
+        meterRegistry = mock(MeterRegistry.class);
         mlAssignmentNotifier = mock(MlAssignmentNotifier.class);
 
         when(clusterService.getClusterName()).thenReturn(CLUSTER_NAME);
@@ -75,6 +78,7 @@ public void testInitialize() {
             clusterService,
             client,
             inferenceAuditor,
+            meterRegistry,
             mlAssignmentNotifier,
             true,
             true,
@@ -91,6 +95,7 @@ public void testInitialize_noMasterNode() {
             clusterService,
             client,
             inferenceAuditor,
+            meterRegistry,
             mlAssignmentNotifier,
             true,
             true,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
index f10df86cc23ae..9232d32e40a97 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.action.bulk.FailureStoreMetrics;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -136,7 +137,8 @@ public void setUpVariables() {
             Collections.singletonList(SKINNY_INGEST_PLUGIN),
             client,
             null,
-            DocumentParsingProvider.EMPTY_INSTANCE
+            DocumentParsingProvider.EMPTY_INSTANCE,
+            FailureStoreMetrics.NOOP
         );
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelActionTests.java
index 4a1a654a9a29f..4546058ddda13 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelActionTests.java
@@ -8,16 +8,21 @@
 package org.elasticsearch.xpack.ml.action;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.license.internal.XPackLicenseStatus;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -35,11 +40,13 @@
 import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfigTests;
+import org.elasticsearch.xpack.core.ml.inference.TrainedModelDefinition;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelInputTests;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelType;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfigTests;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.FillMaskConfigTests;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.LearningToRankConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ModelPackageConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ModelPackageConfigTests;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.NerConfigTests;
@@ -50,6 +57,7 @@
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigTests;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigTests;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfigTests;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.langident.LangIdentNeuralNetwork;
 import org.junit.After;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
@@ -60,10 +68,12 @@
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.elasticsearch.xpack.ml.utils.TaskRetrieverTests.getTaskInfoListOfOne;
 import static org.elasticsearch.xpack.ml.utils.TaskRetrieverTests.mockClientWithTasksResponse;
 import static org.elasticsearch.xpack.ml.utils.TaskRetrieverTests.mockListTasksClient;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.same;
@@ -73,6 +83,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class TransportPutTrainedModelActionTests extends ESTestCase {
     private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS);
@@ -273,6 +284,56 @@ public void testVerifyMlNodesAndModelArchitectures_GivenArchitecturesMatch_ThenT
         ensureNoWarnings();
     }
 
+    public void testValidateModelDefinition_FailsWhenLicenseIsNotSupported() throws IOException {
+        ModelPackageConfig packageConfig = ModelPackageConfigTests.randomModulePackageConfig();
+
+        TrainedModelConfig.Builder trainedModelConfigBuilder = new TrainedModelConfig.Builder().setModelId(
+            "." + packageConfig.getPackagedModelId()
+        ).setInput(TrainedModelInputTests.createRandomInput());
+
+        TransportPutTrainedModelAction.setTrainedModelConfigFieldsFromPackagedModel(
+            trainedModelConfigBuilder,
+            packageConfig,
+            xContentRegistry()
+        );
+
+        var mockTrainedModelDefinition = mock(TrainedModelDefinition.class);
+        when(mockTrainedModelDefinition.getTrainedModel()).thenReturn(mock(LangIdentNeuralNetwork.class));
+        var trainedModelConfig = trainedModelConfigBuilder.setLicenseLevel("basic").build();
+
+        var mockModelInferenceConfig = spy(new LearningToRankConfig(1, List.of(), Map.of()));
+        when(mockModelInferenceConfig.isTargetTypeSupported(any())).thenReturn(true);
+
+        var mockTrainedModelConfig = spy(trainedModelConfig);
+        when(mockTrainedModelConfig.getModelType()).thenReturn(TrainedModelType.LANG_IDENT);
+        when(mockTrainedModelConfig.getModelDefinition()).thenReturn(mockTrainedModelDefinition);
+        when(mockTrainedModelConfig.getInferenceConfig()).thenReturn(mockModelInferenceConfig);
+
+        ActionListener<PutTrainedModelAction.Response> responseListener = ActionListener.wrap(
+            response -> fail("Expected exception, but got response: " + response),
+            exception -> {
+                assertThat(exception, instanceOf(ElasticsearchSecurityException.class));
+                assertThat(exception.getMessage(), is("Model of type [learning_to_rank] requires [ENTERPRISE] license level"));
+            }
+        );
+
+        var mockClusterState = mock(ClusterState.class);
+
+        AtomicInteger currentTime = new AtomicInteger(100);
+        var mockXPackLicenseStatus = new XPackLicenseStatus(License.OperationMode.BASIC, true, "");
+        var mockLicenseState = new XPackLicenseState(currentTime::get, mockXPackLicenseStatus);
+
+        assertThat(
+            TransportPutTrainedModelAction.validateModelDefinition(
+                mockTrainedModelConfig,
+                mockClusterState,
+                mockLicenseState,
+                responseListener
+            ),
+            is(false)
+        );
+    }
+
     private static void prepareGetTrainedModelResponse(Client client, List<TrainedModelConfig> trainedModels) {
         doAnswer(invocationOnMock -> {
             @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerParameterizedTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerParameterizedTests.java
new file mode 100644
index 0000000000000..229926e0c9afb
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerParameterizedTests.java
@@ -0,0 +1,1526 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ml.autoscaling;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
+import org.elasticsearch.xpack.core.ml.autoscaling.MlAutoscalingStats;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState;
+import org.elasticsearch.xpack.core.ml.inference.assignment.Priority;
+import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo;
+import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState;
+import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment;
+import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentMetadata;
+import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static java.util.stream.Collectors.toList;
+import static org.elasticsearch.xpack.core.ml.MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT;
+import static org.elasticsearch.xpack.ml.MachineLearning.MACHINE_MEMORY_NODE_ATTR;
+import static org.elasticsearch.xpack.ml.MachineLearning.MAX_JVM_SIZE_NODE_ATTR;
+import static org.elasticsearch.xpack.ml.MachineLearning.MAX_MACHINE_MEMORY_PERCENT;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+
+public class MlAutoscalingResourceTrackerParameterizedTests extends ESTestCase {
+    private static final long MODEL_BYTES = ByteSizeValue.ofGb(2).getBytes();
+    private static final String NODE_NAME_PREFIX = "ML-Node-";
+    private final TestCase testCase;
+
+    @ParametersFactory(shuffle = true)
+    public static Iterable<Object[]> parameterizedTestCases() {
+        AtomicInteger testCounter = new AtomicInteger(0);
+        List<TestCase> testCases = new ArrayList<>(30);
+        testCases.addAll(testSuite(testCounter, 1, MODEL_BYTES, 0));
+        testCases.addAll(testSuite(testCounter, 2, MODEL_BYTES, 0));
+        testCases.addAll(testSuite(testCounter, 4, MODEL_BYTES, 0));
+        testCases.addAll(testSuite(testCounter, 1, 2 * MODEL_BYTES, 0));
+        testCases.addAll(testSuite(testCounter, 1, 3 * MODEL_BYTES, 0));
+        testCases.addAll(testSuite(testCounter, 1, MODEL_BYTES, ByteSizeValue.ofGb(1).getBytes()));
+
+        return testCases.stream().map(MlAutoscalingResourceTrackerParameterizedTests.TestCase::toArray).collect(toList());
+    }
+
+    private static List<TestCase> testSuite(AtomicInteger testCounter, int threadsPerAllocation, long modelBytes, long cacheSize) {
+        return List.of(
+            WhenStartTrainedModelDeployment_ThenScaleUp_GivenNoExistingDeployments(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            ),
+            WhenStartTrainedModelDeployment_ThenScaleUpDueToProcessors_GivenExistingDeployments(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            ),
+            WhenStartTrainedModelDeployment_ThenScaleUpDueToThreads_GivenExistingDeployments(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            ),
+            WhenStartTrainedModelDeployment_ThenScaleUpDueToMemory_GivenExistingDeployments(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            ),
+            WhenStartTrainedModelDeployment_ThenNoScale_GivenExistingDeployments(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            ),
+
+            WhenUpdateTrainedModelDeployment_ThenScaleUp_GivenDeploymentGetsLarger(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            ),
+            WhenUpdateTrainedModelDeployment_ThenNoScale_GivenDeploymentGetsLargerAndNodesAreSufficient(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            ),
+            WhenUpdateTrainedModelDeployment_ThenNoScale_GivenDeploymentGetsSmallerButAllNodesAreStillRequired(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            ),
+            // WhenUpdateTrainedModelDeployment_ThenScaleDown_GivenDeploymentGetsSmaller() TODO we don't currently
+            // support shrinking nodes
+
+            // Some of the below test cases test for states in between the Stop request being made and being completed
+            WhenStoppingTrainedModelDeployment_ThenNoScale_GivenAllNodesAreStillRequired(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            ),
+            WhenStopTrainedModelDeployment_ThenNoScale_GivenAllNodesAreStillRequired(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            ),
+            WhenStoppedTrainedModelDeployment_ThenNoScale_GivenAllNodesAreStillRequired(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            ),
+            WhenStopTrainedModelDeployment_ThenScaledown_GivenDeploymentRequiredAWholeNode(
+                testCounter.getAndAdd(1),
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            )
+        );
+    }
+
+    private record TestCase(
+        String testDescription,
+        ClusterState clusterState,
+        ClusterSettings clusterSettings,
+        MlMemoryTracker mlMemoryTracker,
+        Settings settings,
+        ActionListener<MlAutoscalingStats> verificationListener
+    ) {
+        Object[] toArray() {
+            return new Object[] { this };
+        }
+    }
+
+    public MlAutoscalingResourceTrackerParameterizedTests(MlAutoscalingResourceTrackerParameterizedTests.TestCase testCase) {
+        this.testCase = testCase;
+    }
+
+    static ActionListener<MlAutoscalingStats> createVerificationListener(String message, MlAutoscalingStats expectedStats) {
+        return new ActionListener<>() {
+            @Override
+            public void onResponse(MlAutoscalingStats actualMlAutoscalingStats) {
+                assertEquals(message, expectedStats, actualMlAutoscalingStats);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                fail("Unexpected failure" + e);
+            }
+        };
+    }
+
+    static ClusterState createClusterStateWithoutNodes(TrainedModelAssignmentMetadata trainedModelAssignmentMetadata) {
+        return createClusterState(trainedModelAssignmentMetadata, null);
+    }
+
+    static ClusterState createClusterState(TrainedModelAssignmentMetadata trainedModelAssignmentMetadata, DiscoveryNodes nodes) {
+        ClusterState.Builder csBuilder = new ClusterState.Builder(ClusterState.EMPTY_STATE);
+
+        Metadata.Builder metadataBuilder = Metadata.builder();
+        // TODO PersistentTasksCustomMetadata is required for jobs other than TrainedModels
+        // .customs(Map.of(PersistentTasksCustomMetadata.TYPE, PersistentTasksCustomMetadata.builder().build()))
+        if (trainedModelAssignmentMetadata != null) {
+            metadataBuilder.putCustom(TrainedModelAssignmentMetadata.NAME, trainedModelAssignmentMetadata);
+        }
+
+        Metadata metadata = metadataBuilder.build();
+        csBuilder.metadata(metadata);
+
+        if (nodes != null) {
+            csBuilder.nodes(nodes);
+        }
+        return csBuilder.build();
+    }
+
+    /**
+     * Create nodes of the same size which together have enough resources to satisfy the requirements. The smallest nodes which satisfy the
+     * requirements are used.
+     * <p>
+     * Using the smallest nodes is a business requirement to minimize costs and simplify the logic.
+     *
+     * @param minProcessorsPerNode
+     * @param minMemoryPerNode
+     * @param totalProcessors can be zero if total processors is not a requirement, but totalMemory must be non-zero
+     * @param totalMemory can be zero if total memory is not a requirement, but totalProcessors must be non-zero
+     * @return an iterable of the smallest nodes which satisfy the requirements
+     */
+    static DiscoveryNodes createMlNodesOfUniformSize(
+        int minProcessorsPerNode,
+        ByteSizeValue minMemoryPerNode,
+        int totalProcessors,
+        ByteSizeValue totalMemory
+    ) {
+
+        List<ByteSizeValue> nodeMemorySizes = List.of(
+            ByteSizeValue.ofGb(4),
+            ByteSizeValue.ofGb(8),
+            ByteSizeValue.ofGb(16),
+            ByteSizeValue.ofGb(32),
+            ByteSizeValue.ofGb(64),
+            ByteSizeValue.ofGb(128),
+            ByteSizeValue.ofGb(256),
+            ByteSizeValue.ofGb(512)
+
+        );
+        List<Integer> nodeProcessorSizes = List.of(2, 4, 8, 16, 32, 64, 128, 256);
+        assertEquals(
+            "Test misconfigured: nodeMemorySizes and nodeProcessorSizes must have the same size",
+            nodeMemorySizes.size(),
+            nodeProcessorSizes.size()
+        );
+
+        int smallestSufficientNodeIndex = nodeMemorySizes.size();
+
+        for (int i = 0; i < nodeMemorySizes.size(); i++) {
+            if (nodeMemorySizes.get(i).getBytes() >= minMemoryPerNode.getBytes() && nodeProcessorSizes.get(i) >= minProcessorsPerNode) {
+                smallestSufficientNodeIndex = i;
+                break;
+            }
+        }
+
+        double numProcessorsPerNode = nodeProcessorSizes.get(smallestSufficientNodeIndex);
+        ByteSizeValue memoryPerNode = nodeMemorySizes.get(smallestSufficientNodeIndex);
+
+        int assignedProcessors = 0;
+        ByteSizeValue assignedMemory = ByteSizeValue.ZERO;
+        DiscoveryNodes.Builder dnBuilder = DiscoveryNodes.builder();
+        int nodeCount = 0;
+
+        while (assignedProcessors < totalProcessors || assignedMemory.getBytes() < totalMemory.getBytes()) {
+            dnBuilder.add(buildDiscoveryNode(numProcessorsPerNode, String.valueOf(memoryPerNode.getBytes()), nodeCount));
+            assignedProcessors += (int) numProcessorsPerNode;
+            assignedMemory = ByteSizeValue.add(memoryPerNode, assignedMemory);
+            nodeCount += 1;
+        }
+
+        return dnBuilder.build();
+    }
+
+    private static DiscoveryNode buildDiscoveryNode(double numProcessorsPerNode, String memoryPerNode, int nodeNumber) {
+        Map<String, String> attributes = Map.of(
+            MACHINE_MEMORY_NODE_ATTR,
+            memoryPerNode,
+            MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR,
+            String.valueOf(numProcessorsPerNode),
+            MAX_JVM_SIZE_NODE_ATTR,
+            Long.toString(Runtime.getRuntime().maxMemory())
+        );
+
+        return DiscoveryNodeUtils.builder(NODE_NAME_PREFIX + nodeNumber)
+            .attributes(attributes)
+            .roles(Set.of(DiscoveryNodeRole.ML_ROLE))
+            .build();
+    }
+
+    private static ClusterSettings createClusterSettings() {
+        return new ClusterSettings(Settings.EMPTY, Set.of(MachineLearning.ALLOCATED_PROCESSORS_SCALE));
+    }
+
+    private static MlMemoryTracker createMlMemoryTracker() {
+        return mock(MlMemoryTracker.class);
+    }
+
+    private static Settings createSettings() {
+        return Settings.builder()
+            .put(MAX_MACHINE_MEMORY_PERCENT.getKey(), MAX_MACHINE_MEMORY_PERCENT.get(Settings.EMPTY))
+            .put(USE_AUTO_MACHINE_MEMORY_PERCENT.getKey(), USE_AUTO_MACHINE_MEMORY_PERCENT.get(Settings.EMPTY))
+            .build();
+    }
+
+    private static StartTrainedModelDeploymentAction.TaskParams createTaskParams(
+        int numAllocations,
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+
+        String modelId = "modelId" + seed;
+        String deploymentId = "deploymentId" + seed;
+        int queueCapacity = 1024;
+        Priority priority = Priority.NORMAL;
+        long perDeploymentMemoryBytes = modelBytes;
+        long perAllocationMemoryBytes = 0;
+
+        return new StartTrainedModelDeploymentAction.TaskParams(
+            modelId,
+            deploymentId,
+            modelBytes,
+            numAllocations,
+            threadsPerAllocation,
+            queueCapacity,
+            ByteSizeValue.ofBytes(cacheSize),
+            priority,
+            perDeploymentMemoryBytes,
+            perAllocationMemoryBytes
+        );
+    }
+
+    private static Map<String, TrainedModelAssignment> createModelAssignments(
+        int numAssignments,
+        int[] numAllocationsPerAssignment,
+        Map<String, RoutingInfo> routingInfo,
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+
+        for (int i = 0; i < numAssignments; i++) {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsPerAssignment[i],
+                i,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTING);
+            for (var entry : routingInfo.entrySet()) {
+                tmaBuilder.addRoutingEntry(entry.getKey(), entry.getValue());
+            }
+            assignments.put("TrainedModelAssignment-" + seed + "-" + i, tmaBuilder.build());
+        }
+
+        return assignments;
+    }
+
+    private static int calculateMaxThreadsPerAllocation(Map<String, TrainedModelAssignment> assignments) {
+        return assignments.values().stream().mapToInt(a -> a.getTaskParams().getThreadsPerAllocation()).max().orElseGet(() -> 0);
+    }
+
+    private static int calculateNodeSize(ClusterState clusterState) {
+        return (int) Double.parseDouble(
+            clusterState.nodes()
+                .getAllNodes()
+                .stream()
+                .findFirst()
+                .get()
+                .getAttributes()
+                .get(MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR)
+        );
+    }
+
+    private static long calculateExistingPerNodeMemoryBytes(ClusterState clusterState) {
+        return Long.parseLong(clusterState.nodes().getAllNodes().stream().findFirst().get().getAttributes().get(MACHINE_MEMORY_NODE_ATTR));
+    }
+
+    private static DiscoveryNodes createDiscoveryNode(int memoryGb, int processors) {
+        ByteSizeValue minMemoryPerNode = ByteSizeValue.ofGb(memoryGb);
+        int totalProcessors = processors;
+        ByteSizeValue totalMemory = ByteSizeValue.ofBytes(minMemoryPerNode.getBytes());
+        DiscoveryNodes nodes = createMlNodesOfUniformSize(processors, minMemoryPerNode, totalProcessors, totalMemory);
+        return nodes;
+    }
+
+    private static DiscoveryNodes createSmallDiscoveryNodes(int memoryGb, int totalProcessors) {
+        ByteSizeValue minMemoryPerNode = ByteSizeValue.ofGb(memoryGb);
+        ByteSizeValue totalMemory = ByteSizeValue.ofBytes(minMemoryPerNode.getBytes());
+        DiscoveryNodes nodes = createMlNodesOfUniformSize(2, minMemoryPerNode, totalProcessors, totalMemory);
+        return nodes;
+    }
+
+    private static long calculateExtraPerNodeModelMemoryBytes(Collection<TrainedModelAssignment> assignments) {
+        return assignments.stream().findFirst().get().getTaskParams().estimateMemoryUsageBytes();
+    }
+
+    private static int calculateTotalExistingAndUsedProcessors(Map<String, TrainedModelAssignment> assignments) {
+        return assignments.values().stream().mapToInt(TrainedModelAssignment::totalTargetProcessors).sum();
+    }
+
+    private static long calculateExistingTotalModelMemoryBytes(Map<String, TrainedModelAssignment> assignments) {
+        return assignments.values().stream().filter(tma -> {
+            if (tma.getAssignmentState() == AssignmentState.STARTED) {
+                return true;
+            } else if (tma.getAssignmentState() == AssignmentState.STARTING && tma.getNodeRoutingTable().isEmpty() == false) {
+                return true;
+            } else if (tma.getAssignmentState() == AssignmentState.STOPPING && tma.getNodeRoutingTable().isEmpty() == false) {
+                return true;
+            } else {
+                return false;
+            }
+        }).mapToLong(tma -> tma.getTaskParams().estimateMemoryUsageBytes()).sum();
+    }
+
+    /**
+     * This test is run for each of the supplied {@link TestCase} configurations.
+     * @throws IOException _
+     */
+    public void test() throws IOException {
+        SetOnce<Boolean> executeCalled = new SetOnce<>();
+
+        var executionVerificationListener = new ActionListener<MlAutoscalingStats>() {
+            @Override
+            public void onResponse(MlAutoscalingStats mlAutoscalingStats) {
+                executeCalled.set(true);
+                testCase.verificationListener.onResponse(mlAutoscalingStats);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                fail("Unexpected failure" + e);
+            }
+        };
+
+        MlAutoscalingResourceTracker.getMlAutoscalingStats(
+            testCase.clusterState,
+            testCase.clusterSettings,
+            testCase.mlMemoryTracker,
+            testCase.settings,
+            executionVerificationListener
+        );
+
+        assertThat(testCase.testDescription, executeCalled.get(), equalTo(true));
+        // other assertions are run in testCase.verificationListener
+    }
+
+    private static TestCase WhenStartTrainedModelDeployment_ThenScaleUp_GivenNoExistingDeployments(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription = "test scaling from zero";
+
+        // generic parameters
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        // TrainedModelAssignments
+        int numAllocationsRequested = 1;
+        Map<String, TrainedModelAssignment> assignments = createModelAssignments(
+            1,
+            new int[] { numAllocationsRequested },
+            Map.of(),
+            seed,
+            threadsPerAllocation,
+            modelBytes,
+            cacheSize
+        );
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state starts with zero nodes
+        ClusterState clusterState = createClusterStateWithoutNodes(trainedModelAssignmentMetadata);
+
+        // expected stats:
+        int existingNodes = 0;
+        long existingPerNodeMemoryBytes = 0;
+        long existingTotalModelMemoryBytes = 0;
+        int totalExistingProcessors = 0;
+        int minNodes = 1;
+        long extraSingleNodeModelMemoryInBytes = assignments.values()
+            .stream()
+            .mapToLong(tma -> tma.getTaskParams().estimateMemoryUsageBytes())
+            .sum();
+        int extraSingleNodeProcessors = threadsPerAllocation;
+        long extraModelMemoryInBytes = extraSingleNodeModelMemoryInBytes;
+        int extraProcessors = threadsPerAllocation;
+        long removeNodeMemoryInBytes = 0;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraSingleNodeModelMemoryInBytes,
+            extraSingleNodeProcessors,
+            extraModelMemoryInBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static TestCase WhenStartTrainedModelDeployment_ThenScaleUpDueToProcessors_GivenExistingDeployments(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription =
+            "test scaling up with existing deployment when the new deployment requires more processors than are available";
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        // TrainedModelAssignments
+        int numAllocationsRequestedPreviously = 8;
+        DiscoveryNodes nodes = createDiscoveryNode(0, numAllocationsRequestedPreviously * threadsPerAllocation);
+        int numProcessorsOnNodes = nodes.stream()
+            .mapToInt(n -> (int) Double.parseDouble(n.getAttributes().get(MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR)))
+            .sum();
+        int numAssignments = 2;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1 - already deployed
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsRequestedPreviously,
+                1,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(numAllocationsRequestedPreviously, numAllocationsRequestedPreviously, RoutingState.STARTED, null)
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        // asssignment 2 - not deployed yet
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsRequestedPreviously,
+                2,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTING);
+            tmaBuilder.clearNodeRoutingTable();
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 1, tmaBuilder.build());
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = Math.min(3, Math.max(1, numAllocationsRequestedPreviously));
+        long extraPerNodeModelMemoryBytes = calculateExtraPerNodeModelMemoryBytes(assignments.values());
+        int extraPerNodeProcessors = calculateExtraProcessorsPerNode(assignments, clusterState, true);
+        long extraModelMemoryBytes = extraPerNodeModelMemoryBytes;
+        int extraProcessors = (numAllocationsRequestedPreviously * threadsPerAllocation) * 2 - numProcessorsOnNodes;
+        long removeNodeMemoryInBytes = 0;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static TestCase WhenStartTrainedModelDeployment_ThenScaleUpDueToThreads_GivenExistingDeployments(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription =
+            "test scaling up with existing deployment when the new deployment requires more processers per node than are available";
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        // TrainedModelAssignments
+        int numAllocationsRequestedPreviously = 8;
+        DiscoveryNodes nodes = createMlNodesOfUniformSize(
+            threadsPerAllocation,
+            ByteSizeValue.ofBytes(modelBytes + cacheSize),
+            numAllocationsRequestedPreviously * threadsPerAllocation,
+            ByteSizeValue.ZERO
+        );
+        int numProcessorsOnNodes = nodes.stream()
+            .mapToInt(n -> (int) Double.parseDouble(n.getAttributes().get(MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR)))
+            .sum();
+        int numAssignments = 2;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1 - already deployed
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsRequestedPreviously,
+                1,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(numAllocationsRequestedPreviously, numAllocationsRequestedPreviously, RoutingState.STARTED, null)
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        // asssignment 2 - not deployed yet
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsRequestedPreviously,
+                2,
+                threadsPerAllocation * 2,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTING);
+            tmaBuilder.clearNodeRoutingTable();
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 1, tmaBuilder.build());
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = Math.min(3, Math.max(1, numAllocationsRequestedPreviously));
+        long extraPerNodeModelMemoryBytes = calculateExtraPerNodeModelMemoryBytes(assignments.values());
+        int extraPerNodeProcessors = calculateExtraProcessorsPerNode(assignments, clusterState, true);
+        long extraModelMemoryBytes = extraPerNodeModelMemoryBytes;
+        int extraProcessors = (numAllocationsRequestedPreviously * threadsPerAllocation) * 2;
+        long removeNodeMemoryInBytes = 0;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static int calculateExtraProcessorsPerNode(
+        Map<String, TrainedModelAssignment> assignments,
+        ClusterState clusterState,
+        boolean isScaleUp
+    ) {
+        int nodeSize = calculateNodeSize(clusterState);
+        int maxTPA = calculateMaxThreadsPerAllocation(assignments);
+        if (nodeSize >= maxTPA) {
+            return isScaleUp ? 1 : 0;
+        } else {
+            return maxTPA;
+        }
+    }
+
+    private static TestCase WhenStartTrainedModelDeployment_ThenScaleUpDueToMemory_GivenExistingDeployments(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription = "test scaling up with existing deployment when the new deployment requires more memory than is available";
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        // TrainedModelAssignments
+        int numAllocationsRequestedPreviously = 4;
+        DiscoveryNodes nodes = createMlNodesOfUniformSize(
+            threadsPerAllocation,
+            ByteSizeValue.ofBytes(modelBytes + cacheSize),
+            numAllocationsRequestedPreviously * threadsPerAllocation,
+            ByteSizeValue.ZERO
+        );
+        int numAssignments = 2;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1 - already deployed
+        {
+            int numAllocationsInAssignment1 = numAllocationsRequestedPreviously - 1;
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsInAssignment1,
+                1,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(numAllocationsInAssignment1, numAllocationsInAssignment1, RoutingState.STARTED, null)
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        // assignment 2 - not deployed yet
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                1,
+                2,
+                threadsPerAllocation,
+                modelBytes * 4,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTING);
+            tmaBuilder.clearNodeRoutingTable();
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 1, tmaBuilder.build());
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = 3; // TODO understand why this value
+        long extraPerNodeModelMemoryBytes = calculateExtraPerNodeModelMemoryBytes(
+            Collections.singleton(assignments.get("TrainedModelAssignment-" + seed + "-" + 1))
+        );
+        int extraPerNodeProcessors = 0;
+        long extraModelMemoryBytes = calculateExtraPerNodeModelMemoryBytes(
+            Collections.singleton(assignments.get("TrainedModelAssignment-" + seed + "-" + 1))
+        );
+        int extraProcessors = 0;
+        long removeNodeMemoryInBytes = 0;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static TestCase WhenStartTrainedModelDeployment_ThenNoScale_GivenExistingDeployments(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription = "test scaling when existing nodes have room for the new deployment";
+
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        // TrainedModelAssignments
+        int numAllocationsRequestedPreviously = seed;
+        DiscoveryNodes nodes = createDiscoveryNode(
+            (int) ByteSizeValue.ofBytes((modelBytes + cacheSize) * 2).getGb(),
+            numAllocationsRequestedPreviously * 2 * threadsPerAllocation
+        );
+        int numAssignments = 2;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1 - already deployed
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsRequestedPreviously,
+                1,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(numAllocationsRequestedPreviously, numAllocationsRequestedPreviously, RoutingState.STARTED, null)
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        // asssignment 2 - not deployed yet
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsRequestedPreviously,
+                2,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTING);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(0, numAllocationsRequestedPreviously, RoutingState.STARTING, null)
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 1, tmaBuilder.build());
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = 3; // TODO understand why this value
+        long extraPerNodeModelMemoryBytes = 0;
+        int extraPerNodeProcessors = 0;
+        long extraModelMemoryBytes = extraPerNodeModelMemoryBytes;
+        int extraProcessors = 0;
+        long removeNodeMemoryInBytes = 0;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static TestCase WhenUpdateTrainedModelDeployment_ThenScaleUp_GivenDeploymentGetsLarger(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription = "test scaling up when updating existing deployment to be larger";
+
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        // TrainedModelAssignments
+        int numAllocationsRequestedPreviously = 4;
+        int updatedNumAllocations = 8;
+        DiscoveryNodes nodes = createDiscoveryNode(
+            (int) ByteSizeValue.ofBytes(modelBytes + cacheSize).getGb(),
+            numAllocationsRequestedPreviously * threadsPerAllocation
+        );
+        int numAssignments = 1;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1 - already deployed - just updated to be larger
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                updatedNumAllocations,
+                seed,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(numAllocationsRequestedPreviously, numAllocationsRequestedPreviously, RoutingState.STARTED, null)
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = 3;
+        long extraPerNodeModelMemoryBytes = 0;
+        int extraPerNodeProcessors = 1;
+        long extraModelMemoryBytes = 0;
+        int extraProcessors = updatedNumAllocations * threadsPerAllocation - numAllocationsRequestedPreviously * threadsPerAllocation;
+        long removeNodeMemoryInBytes = 0;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static TestCase WhenUpdateTrainedModelDeployment_ThenNoScale_GivenDeploymentGetsLargerAndNodesAreSufficient(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription = "test scaling when updating existing deployment to be larger but still fits in existing nodes";
+        if (modelBytes > ByteSizeValue.ofGb(32).getBytes()) {
+            modelBytes = ByteSizeValue.ofGb(32).getBytes();
+            // this test case requires that the model fit on the existing nodes
+        }
+
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        // TrainedModelAssignments
+        int numAllocationsRequestedPreviously = 4;
+        int updatedNumAllocations = 8;
+        DiscoveryNodes nodes = createMlNodesOfUniformSize(
+            threadsPerAllocation * numAllocationsRequestedPreviously,
+            ByteSizeValue.ofBytes(modelBytes + cacheSize),
+            updatedNumAllocations * threadsPerAllocation,
+            ByteSizeValue.ZERO
+        );
+        int numAssignments = 1;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1 - already deployed - just updated to be larger
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                updatedNumAllocations,
+                seed,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(numAllocationsRequestedPreviously, numAllocationsRequestedPreviously, RoutingState.STARTED, null)
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = 3; // TODO understand why this value
+        long extraPerNodeModelMemoryBytes = 0;
+        int extraPerNodeProcessors = 0;
+        long extraModelMemoryBytes = 0;
+        int extraProcessors = 0;
+        long removeNodeMemoryInBytes = 0;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static TestCase WhenUpdateTrainedModelDeployment_ThenNoScale_GivenDeploymentGetsSmallerButAllNodesAreStillRequired(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription = "test scaling when updating existing deployment to be smaller but all nodes are still required";
+
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        // TrainedModelAssignments
+        int numAllocationsRequestedPreviously = 4;
+        int updatedNumAllocations = 3;
+        DiscoveryNodes nodes = createDiscoveryNode(
+            (int) ByteSizeValue.ofBytes(modelBytes + cacheSize).getGb(),
+            numAllocationsRequestedPreviously * threadsPerAllocation
+        );
+        int numAssignments = 1;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1 - already deployed - just updated to be smaller
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                updatedNumAllocations,
+                seed,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(numAllocationsRequestedPreviously, numAllocationsRequestedPreviously, RoutingState.STARTED, null)
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = 3;
+        long extraPerNodeModelMemoryBytes = 0;
+        int extraPerNodeProcessors = 0;
+        long extraModelMemoryBytes = 0;
+        int extraProcessors = 0;
+        long removeNodeMemoryInBytes = 0;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static TestCase WhenUpdateTrainedModelDeployment_ThenScaleDown_GivenDeploymentGetsSmaller(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription = "test scaling down when updating existing deployment to be smaller";
+        // TODO change this test to use threadsPerAllocation to accurately require a larger node size which then needs to be scaled down
+        // when
+        // threadsPerAllocation gets updated to a smaller number
+
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        // TrainedModelAssignments
+        int numAllocationsRequestedPreviously = 8;
+        int updatedNumAllocations = 2;
+        DiscoveryNodes nodes = createDiscoveryNode(16, numAllocationsRequestedPreviously);
+        int numAssignments = 1;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                updatedNumAllocations,
+                seed,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(updatedNumAllocations, updatedNumAllocations, RoutingState.STARTED, null)
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = 3;
+        long extraPerNodeModelMemoryBytes = 0;
+        int extraPerNodeProcessors = calculateMaxThreadsPerAllocation(assignments) == calculateNodeSize(clusterState)
+            ? 0
+            : calculateMaxThreadsPerAllocation(assignments);
+        long extraModelMemoryBytes = 0;
+        int extraProcessors = 0;
+        long removeNodeMemoryInBytes = 1;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static TestCase WhenStoppingTrainedModelDeployment_ThenNoScale_GivenAllNodesAreStillRequired(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription = "test scaling when the existing deployments require the same nodes when as small deployment is stopping";
+
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        // TrainedModelAssignments;
+        int numAllocationsOnAssignment1 = 3;
+        int numAllocationsOnAssignment2 = 1;
+        DiscoveryNodes nodes = createDiscoveryNode(
+            8,
+            numAllocationsOnAssignment1 * threadsPerAllocation + numAllocationsOnAssignment2 * threadsPerAllocation
+        );
+        int numAssignments = 2;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1 - has 3 allocations
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsOnAssignment1,
+                1,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(numAllocationsOnAssignment1, numAllocationsOnAssignment1, RoutingState.STARTED, null)
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        // assignment 2 - is stopping, has 1 allocation
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsOnAssignment2,
+                2,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STOPPING);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(numAllocationsOnAssignment2, numAllocationsOnAssignment2, RoutingState.STOPPING, "stopping deployment")
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 2, tmaBuilder.build());
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = 3; // TODO understand why this value
+        long extraPerNodeModelMemoryBytes = 0;
+        int extraPerNodeProcessors = 0;
+        long extraModelMemoryBytes = 0;
+        int extraProcessors = 0;
+        long removeNodeMemoryInBytes = 0;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static TestCase WhenStoppedTrainedModelDeployment_ThenNoScale_GivenAllNodesAreStillRequired(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription = "test scaling when the existing deployments require the same nodes when as small deployment is stopped";
+
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        // TrainedModelAssignments;
+        int numAllocationsOnAssignment1 = 3;
+        int numAllocationsOnAssignment2 = 1;
+        DiscoveryNodes nodes = createDiscoveryNode(
+            8,
+            numAllocationsOnAssignment1 * threadsPerAllocation + numAllocationsOnAssignment2 * threadsPerAllocation
+        );
+        int numAssignments = 2;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1 - has 3 allocations
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsOnAssignment1,
+                1,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(
+                NODE_NAME_PREFIX + 0,
+                new RoutingInfo(numAllocationsOnAssignment1, numAllocationsOnAssignment1, RoutingState.STARTED, null)
+            );
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        // assignment 2 - is stopping, has 1 allocation
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocationsOnAssignment2,
+                2,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STOPPING);
+            tmaBuilder.clearNodeRoutingTable();
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = 3; // TODO understand why this value
+        long extraPerNodeModelMemoryBytes = 0;
+        int extraPerNodeProcessors = 0;
+        long extraModelMemoryBytes = 0;
+        int extraProcessors = 0;
+        long removeNodeMemoryInBytes = 0;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static TestCase WhenStopTrainedModelDeployment_ThenNoScale_GivenAllNodesAreStillRequired(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription = "test scaling when the existing deployments require the same nodes after a small deployment was removed";
+
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+        int numAllocations = 3;
+
+        // TrainedModelAssignments;
+        DiscoveryNodes nodes = createDiscoveryNode(
+            (int) ByteSizeValue.ofBytes(modelBytes + cacheSize).getGb(),
+            numAllocations * threadsPerAllocation
+        );
+        int numAssignments = 1;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1 - has 3 allocations
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocations,
+                seed,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(NODE_NAME_PREFIX + 0, new RoutingInfo(numAllocations, numAllocations, RoutingState.STARTED, null));
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = 3;
+        long extraPerNodeModelMemoryBytes = 0;
+        int extraPerNodeProcessors = 0;
+        long extraModelMemoryBytes = 0;
+        int extraProcessors = 0;
+        long removeNodeMemoryInBytes = 0;
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+    private static TestCase WhenStopTrainedModelDeployment_ThenScaledown_GivenDeploymentRequiredAWholeNode(
+        int seed,
+        int threadsPerAllocation,
+        long modelBytes,
+        long cacheSize
+    ) {
+        String testDescription = "test scaling down when the removed deployment required a whole node";
+        // Generic settings
+        ClusterSettings clusterSettings = createClusterSettings();
+        MlMemoryTracker mlMemoryTracker = createMlMemoryTracker();
+        Settings settings = createSettings();
+
+        int numAllocations = 12;
+        // TrainedModelAssignments;
+        DiscoveryNodes nodes = createMlNodesOfUniformSize(
+            threadsPerAllocation,
+            ByteSizeValue.ofBytes(modelBytes + cacheSize),
+            numAllocations * threadsPerAllocation * 2,
+            ByteSizeValue.ZERO // any amount of total memory is ok
+        );
+        // more nodes than are needed, requiring a scaledown
+        int numAssignments = 2;
+        Map<String, TrainedModelAssignment> assignments = new HashMap<>(numAssignments);
+        // assignment 1 - has 12 allocations
+        {
+            StartTrainedModelDeploymentAction.TaskParams taskParams = createTaskParams(
+                numAllocations,
+                seed,
+                threadsPerAllocation,
+                modelBytes,
+                cacheSize
+            );
+            TrainedModelAssignment.Builder tmaBuilder = TrainedModelAssignment.Builder.empty(taskParams, null);
+            tmaBuilder.setAssignmentState(AssignmentState.STARTED);
+            tmaBuilder.addRoutingEntry(NODE_NAME_PREFIX + 0, new RoutingInfo(numAllocations, numAllocations, RoutingState.STARTED, null));
+            assignments.put("TrainedModelAssignment-" + seed + "-" + 0, tmaBuilder.build());
+        }
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = new TrainedModelAssignmentMetadata(assignments);
+
+        // Cluster state
+        ClusterState clusterState = createClusterState(trainedModelAssignmentMetadata, nodes);
+
+        // expected stats:
+        int existingNodes = clusterState.nodes().getSize();
+        long existingPerNodeMemoryBytes = calculateExistingPerNodeMemoryBytes(clusterState);
+        long existingTotalModelMemoryBytes = calculateExistingTotalModelMemoryBytes(assignments);
+        int totalExistingProcessors = calculateTotalExistingAndUsedProcessors(assignments);
+        int minNodes = 3; // TODO understand why this value
+        long extraPerNodeModelMemoryBytes = 0;
+        int extraPerNodeProcessors = 0;
+        long extraModelMemoryBytes = 0;
+        int extraProcessors = 0;
+        long removeNodeMemoryInBytes = calculateExistingPerNodeMemoryBytes(clusterState); // we need to remove a whole node
+        long perNodeMemoryOverheadInBytes = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+
+        MlAutoscalingStats expectedStats = new MlAutoscalingStats(
+            existingNodes,
+            existingPerNodeMemoryBytes,
+            existingTotalModelMemoryBytes,
+            totalExistingProcessors,
+            minNodes,
+            extraPerNodeModelMemoryBytes,
+            extraPerNodeProcessors,
+            extraModelMemoryBytes,
+            extraProcessors,
+            removeNodeMemoryInBytes,
+            perNodeMemoryOverheadInBytes
+        );
+
+        ActionListener<MlAutoscalingStats> verificationListener = createVerificationListener(testDescription, expectedStats);
+        return new TestCase(testDescription, clusterState, clusterSettings, mlMemoryTracker, settings, verificationListener);
+    }
+
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java
index 41a86e436f468..3674dda3934bd 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
@@ -21,16 +22,19 @@
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
 import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.autoscaling.MlAutoscalingStats;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState;
 import org.elasticsearch.xpack.core.ml.inference.assignment.Priority;
 import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingInfo;
 import org.elasticsearch.xpack.core.ml.inference.assignment.RoutingState;
 import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment;
+import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentTests;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
 import org.elasticsearch.xpack.ml.utils.NativeMemoryCalculator;
 
+import java.io.IOException;
 import java.net.InetAddress;
 import java.time.Instant;
 import java.util.Collections;
@@ -41,6 +45,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingResourceTracker.MlDummyAutoscalingEntity;
 import static org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingResourceTracker.MlJobRequirements;
@@ -64,14 +69,15 @@ public void testGetMemoryAndProcessors() throws InterruptedException {
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(0L, 0),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(2, stats.nodes());
-                assertEquals(0, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(2, stats.currentTotalNodes());
+                assertEquals(0, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
 
@@ -85,14 +91,15 @@ public void testGetMemoryAndProcessors() throws InterruptedException {
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(0L, 0),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(0, stats.perNodeMemoryInBytes());
-                assertEquals(2, stats.nodes());
-                assertEquals(0, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(0, stats.currentPerNodeMemoryBytes());
+                assertEquals(2, stats.currentTotalNodes());
+                assertEquals(0, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
 
@@ -107,16 +114,188 @@ public void testGetMemoryAndProcessors() throws InterruptedException {
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(memory / 2, 1),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(0, stats.perNodeMemoryInBytes());
-                assertEquals(2, stats.nodes());
-                assertEquals(0, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(0, stats.extraModelMemoryInBytes());
-                assertEquals(0, stats.extraSingleNodeModelMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(0, stats.currentPerNodeMemoryBytes());
+                assertEquals(2, stats.currentTotalNodes());
+                assertEquals(0, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(0, stats.wantedExtraModelMemoryBytes());
+                assertEquals(0, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
+            }
+        );
+    }
+
+    public void testScaleUpByProcessorsWhenAlreadyStarted() throws InterruptedException, IOException {
+        MlMemoryTracker mockTracker = mock(MlMemoryTracker.class);
+
+        long memory = randomLongBetween(100, 1_000_000);
+        var taskParams1 = new StartTrainedModelDeploymentAction.TaskParams(
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            memory,
+            2,
+            1,
+            randomIntBetween(1, 10000),
+            randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(0, memory)),
+            Priority.NORMAL,
+            memory,
+            memory
+        );
+
+        var taskParams2 = new StartTrainedModelDeploymentAction.TaskParams(
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            memory,
+            randomIntBetween(3, 80),
+            1,
+            randomIntBetween(1, 10000),
+            randomBoolean() ? null : ByteSizeValue.ofBytes(randomLongBetween(0, memory)),
+            Priority.NORMAL,
+            memory,
+            memory
+        );
+
+        var randomAssignment1 = TrainedModelAssignmentTests.randomInstanceBuilder(taskParams1, AssignmentState.STARTED)
+            .clearNodeRoutingTable()
+            .addRoutingEntry("ml-1", new RoutingInfo(2, 2, RoutingState.STARTED, ""))
+            .build();
+
+        var randomAssignment2 = TrainedModelAssignmentTests.randomInstanceBuilder(taskParams2, AssignmentState.STARTED)
+            .clearNodeRoutingTable()
+            .addRoutingEntry("ml-2", new RoutingInfo(2, 2, RoutingState.STARTED, ""))
+            .build();
+
+        List<DiscoveryNode> nodes = new java.util.ArrayList<>(
+            Stream.of(randomAssignment1.getNodeRoutingTable().values())
+                .map(r -> mock(DiscoveryNode.class))
+                .peek(n -> when(n.getRoles()).thenReturn(Set.of(DiscoveryNodeRole.ML_ROLE)))
+                .peek(n -> when(n.getAttributes()).thenReturn(Map.of(MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR, "2.0")))
+                .toList()
+        );
+        nodes.addAll(
+            Stream.of(randomAssignment2.getNodeRoutingTable().values())
+                .map(r -> mock(DiscoveryNode.class))
+                .peek(n -> when(n.getRoles()).thenReturn(Set.of(DiscoveryNodeRole.ML_ROLE)))
+                .peek(n -> when(n.getAttributes()).thenReturn(Map.of(MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR, "2.0")))
+                .toList()
+        );
+        MlAutoscalingContext scaleUpContext = new MlAutoscalingContext(
+            List.of(),
+            List.of(),
+            List.of(),
+            Map.of("deployment-1", randomAssignment1, "deployment-2", randomAssignment2),
+            nodes,
+            null
+        );
+
+        int expectedProcessorsPerNode = scaleUpContext.modelAssignments.values()
+            .stream()
+            .map(TrainedModelAssignment::getTaskParams)
+            .mapToInt(StartTrainedModelDeploymentAction.TaskParams::getThreadsPerAllocation)
+            .max()
+            .orElse(0);
+        int expectedTotalProccessors = scaleUpContext.modelAssignments.values()
+            .stream()
+            .map(TrainedModelAssignment::getTaskParams)
+            .mapToInt(tp -> tp.getNumberOfAllocations() * tp.getThreadsPerAllocation())
+            .sum();
+        int existantProccessors = scaleUpContext.modelAssignments.values()
+            .stream()
+            .map(TrainedModelAssignment::getNodeRoutingTable)
+            .mapToInt(m -> m.values().stream().mapToInt(RoutingInfo::getTargetAllocations).sum()) // threads == 1
+            .sum();
+        int extraProcessors = expectedTotalProccessors - existantProccessors;
+
+        this.<MlAutoscalingStats>assertAsync(
+            listener -> MlAutoscalingResourceTracker.getMemoryAndProcessors(
+                scaleUpContext,
+                mockTracker,
+                Map.of("ml-1", memory, "ml-2", memory),
+                memory,
+                2,
+                MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
+                MlDummyAutoscalingEntity.of(0, 0),
+                1,
+                listener
+            ),
+            stats -> {
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(2, stats.currentTotalNodes());
+                assertEquals(extraProcessors, stats.wantedExtraProcessors());
+                assertEquals(expectedProcessorsPerNode, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(0, stats.wantedExtraModelMemoryBytes());
+                assertEquals(0, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
+            }
+        );
+    }
+
+    public void testScaleUpByProcessorsWhenStarting() throws InterruptedException {
+        MlMemoryTracker mockTracker = mock(MlMemoryTracker.class);
+        long memory = randomLongBetween(100, 1_000_000);
+        long model_size = randomLongBetween(10, 10_000_000);
+        var taskParams = new StartTrainedModelDeploymentAction.TaskParams(
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            model_size,
+            randomIntBetween(5, 10),
+            1,
+            randomIntBetween(1, 10000),
+            randomBoolean() ? null : ByteSizeValue.ofBytes(randomNonNegativeLong()),
+            Priority.NORMAL,
+            model_size,
+            model_size
+        );
+
+        var randomAssignment = TrainedModelAssignmentTests.randomInstanceBuilder(taskParams, AssignmentState.STARTING)
+            .setAssignmentState(AssignmentState.STARTING)
+            .setNumberOfAllocations(0)
+            .clearNodeRoutingTable()
+            .build();
+
+        List<DiscoveryNode> nodes = Stream.of("ml-1", "ml-2")
+            .map(n -> mock(DiscoveryNode.class))
+            .peek(n -> when(n.getRoles()).thenReturn(Set.of(DiscoveryNodeRole.ML_ROLE)))
+            .peek(n -> when(n.getAttributes()).thenReturn(Map.of(MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR, "2.0")))
+            .toList();
+
+        MlAutoscalingContext scaleUpContext = new MlAutoscalingContext(List.of(), List.of(), List.of(), Map.of(), nodes, null);
+
+        int expectedTotalProccessors = scaleUpContext.modelAssignments.values()
+            .stream()
+            .map(TrainedModelAssignment::getTaskParams)
+            .mapToInt(tp -> tp.getNumberOfAllocations() * tp.getThreadsPerAllocation())
+            .sum();
+        int existantProccessors = scaleUpContext.modelAssignments.values()
+            .stream()
+            .map(TrainedModelAssignment::getNodeRoutingTable)
+            .mapToInt(m -> m.values().stream().mapToInt(RoutingInfo::getTargetAllocations).sum())
+            .sum();
+        int extraProcessors = expectedTotalProccessors - existantProccessors;
+
+        this.<MlAutoscalingStats>assertAsync(
+            listener -> MlAutoscalingResourceTracker.getMemoryAndProcessors(
+                scaleUpContext,
+                mockTracker,
+                Map.of("ml-1", memory, "ml-2", memory),
+                memory,
+                2,
+                MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
+                MlDummyAutoscalingEntity.of(0, 0),
+                1,
+                listener
+            ),
+            stats -> {
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(2, stats.currentTotalNodes());
+                assertEquals(Math.max(extraProcessors, 0), stats.wantedExtraProcessors());
+                assertEquals(extraProcessors > 0 ? 1 : 0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(0, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
     }
@@ -172,19 +351,20 @@ public void testGetMemoryAndProcessorsScaleUpGivenAwaitingLazyAssignment() throw
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(0L, 0),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(2, stats.nodes());
-                assertEquals(1, stats.minNodes());
-                assertEquals(0, stats.extraProcessors());
-                assertEquals(0, stats.modelMemoryInBytesSum());
-                assertEquals(0, stats.processorsSum());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(memory / 4, stats.extraSingleNodeModelMemoryInBytes());
-                assertEquals(memory / 4, stats.extraModelMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(2, stats.currentTotalNodes());
+                assertEquals(1, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraProcessors());
+                assertEquals(0, stats.currentTotalModelMemoryBytes());
+                assertEquals(0, stats.currentTotalProcessorsInUse());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(memory / 4, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(memory / 4, stats.wantedExtraModelMemoryBytes());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
 
@@ -198,19 +378,20 @@ public void testGetMemoryAndProcessorsScaleUpGivenAwaitingLazyAssignment() throw
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(memory / 4, 0),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(2, stats.nodes());
-                assertEquals(1, stats.minNodes());
-                assertEquals(0, stats.extraProcessors());
-                assertEquals(memory / 4, stats.modelMemoryInBytesSum());
-                assertEquals(0, stats.processorsSum());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(memory / 4, stats.extraSingleNodeModelMemoryInBytes());
-                assertEquals(memory / 4, stats.extraModelMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(2, stats.currentTotalNodes());
+                assertEquals(1, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraProcessors());
+                assertEquals(memory / 4, stats.currentTotalModelMemoryBytes());
+                assertEquals(0, stats.currentTotalProcessorsInUse());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(memory / 4, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(memory / 4, stats.wantedExtraModelMemoryBytes());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
 
@@ -224,19 +405,20 @@ public void testGetMemoryAndProcessorsScaleUpGivenAwaitingLazyAssignment() throw
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(memory / 4, 1),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(2, stats.nodes());
-                assertEquals(1, stats.minNodes());
-                assertEquals(0, stats.extraProcessors());
-                assertEquals(memory / 4, stats.modelMemoryInBytesSum());
-                assertEquals(1, stats.processorsSum());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(memory / 4, stats.extraSingleNodeModelMemoryInBytes());
-                assertEquals(memory / 4, stats.extraModelMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(2, stats.currentTotalNodes());
+                assertEquals(1, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraProcessors());
+                assertEquals(memory / 4, stats.currentTotalModelMemoryBytes());
+                assertEquals(1, stats.currentTotalProcessorsInUse());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(memory / 4, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(memory / 4, stats.wantedExtraModelMemoryBytes());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
     }
@@ -295,17 +477,18 @@ public void testGetMemoryAndProcessorsScaleUpGivenAwaitingLazyAssignmentButFaile
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(0L, 0),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(memory, stats.removeNodeMemoryInBytes());
-                assertEquals(2, stats.nodes());
-                assertEquals(0, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(0, stats.extraSingleNodeModelMemoryInBytes());
-                assertEquals(0, stats.extraModelMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(memory, stats.unwantedNodeMemoryBytesToRemove());
+                assertEquals(2, stats.currentTotalNodes());
+                assertEquals(0, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(0, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(0, stats.wantedExtraModelMemoryBytes());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
     }
@@ -1056,15 +1239,16 @@ public void testGetMemoryAndProcessorsScaleDownToZero() throws InterruptedExcept
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(0L, 0),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(1, stats.nodes());
-                assertEquals(0, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(memory, stats.removeNodeMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(1, stats.currentTotalNodes());
+                assertEquals(0, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(memory, stats.unwantedNodeMemoryBytesToRemove());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
 
@@ -1078,16 +1262,17 @@ public void testGetMemoryAndProcessorsScaleDownToZero() throws InterruptedExcept
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(0L, 1),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(1, stats.nodes());
-                assertEquals(0, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(0, stats.extraProcessors());
-                assertEquals(memory, stats.removeNodeMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(1, stats.currentTotalNodes());
+                assertEquals(0, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(0, stats.wantedExtraProcessors());
+                assertEquals(memory, stats.unwantedNodeMemoryBytesToRemove());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
 
@@ -1101,15 +1286,16 @@ public void testGetMemoryAndProcessorsScaleDownToZero() throws InterruptedExcept
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(0L, 0),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(3, stats.nodes());
-                assertEquals(0, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(memory, stats.removeNodeMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(3, stats.currentTotalNodes());
+                assertEquals(0, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(memory, stats.unwantedNodeMemoryBytesToRemove());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
     }
@@ -1122,7 +1308,9 @@ public void testGetMemoryAndProcessorsScaleDown() throws InterruptedException {
             MachineLearning.MAX_JVM_SIZE_NODE_ATTR,
             "400000000",
             MachineLearning.ML_CONFIG_VERSION_NODE_ATTR,
-            "7.2.0"
+            "7.2.0",
+            MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR,
+            "2.0"
         );
 
         MlAutoscalingContext mlAutoscalingContext = new MlAutoscalingContext(
@@ -1193,15 +1381,16 @@ public void testGetMemoryAndProcessorsScaleDown() throws InterruptedException {
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(0L, 0),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(3, stats.nodes());
-                assertEquals(1, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(memory, stats.removeNodeMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(3, stats.currentTotalNodes());
+                assertEquals(1, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(memory, stats.unwantedNodeMemoryBytesToRemove());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
     }
@@ -1295,15 +1484,16 @@ public void testGetMemoryAndProcessorsScaleDownPreventedByMinNodes() throws Inte
                 4,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(0L, 0),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(3, stats.nodes());
-                assertEquals(3, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(0, stats.removeNodeMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(3, stats.currentTotalNodes());
+                assertEquals(3, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(0, stats.unwantedNodeMemoryBytesToRemove());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
     }
@@ -1317,7 +1507,9 @@ public void testGetMemoryAndProcessorsScaleDownPreventedByDummyEntityMemory() th
             MachineLearning.MAX_JVM_SIZE_NODE_ATTR,
             "400000000",
             MachineLearning.ML_CONFIG_VERSION_NODE_ATTR,
-            "7.2.0"
+            "7.2.0",
+            MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR,
+            "2.0"
         );
 
         MlAutoscalingContext mlAutoscalingContext = new MlAutoscalingContext(
@@ -1388,22 +1580,24 @@ public void testGetMemoryAndProcessorsScaleDownPreventedByDummyEntityMemory() th
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(perNodeAvailableModelMemoryInBytes, 1),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(perNodeAvailableModelMemoryInBytes + 503318080, stats.modelMemoryInBytesSum()); // total model memory is that
-                                                                                                             // configured in the dummy
-                                                                                                             // entity plus that used by the
-                                                                                                             // trained models.
-                assertEquals(5, stats.processorsSum()); // account for the extra processor from the dummy entity
-                assertEquals(3, stats.nodes());
-                assertEquals(1, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(0, stats.extraProcessors());
-                assertEquals(0, stats.extraModelMemoryInBytes());
-                assertEquals(0, stats.extraSingleNodeModelMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(perNodeAvailableModelMemoryInBytes + 503318080, stats.currentTotalModelMemoryBytes()); // total model memory
+                                                                                                                    // is that
+                // configured in the dummy
+                // entity plus that used by the
+                // trained models.
+                assertEquals(5, stats.currentTotalProcessorsInUse()); // account for the extra processor from the dummy entity
+                assertEquals(3, stats.currentTotalNodes());
+                assertEquals(1, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(0, stats.wantedExtraProcessors());
+                assertEquals(0, stats.wantedExtraModelMemoryBytes());
+                assertEquals(0, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
     }
@@ -1417,7 +1611,9 @@ public void testGetMemoryAndProcessorsScaleDownNotPreventedByDummyEntityProcesso
             MachineLearning.MAX_JVM_SIZE_NODE_ATTR,
             "400000000",
             MachineLearning.ML_CONFIG_VERSION_NODE_ATTR,
-            "7.2.0"
+            "7.2.0",
+            MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR,
+            "2.0"
         );
 
         MlAutoscalingContext mlAutoscalingContext = new MlAutoscalingContext(
@@ -1488,19 +1684,20 @@ public void testGetMemoryAndProcessorsScaleDownNotPreventedByDummyEntityProcesso
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(0L, 9),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(503318080, stats.modelMemoryInBytesSum());
-                assertEquals(13, stats.processorsSum()); // account for the extra processors from the dummy entity
-                assertEquals(3, stats.nodes());
-                assertEquals(1, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(0, stats.extraProcessors());
-                assertEquals(0, stats.extraModelMemoryInBytes());
-                assertEquals(0, stats.extraSingleNodeModelMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(503318080, stats.currentTotalModelMemoryBytes());
+                assertEquals(13, stats.currentTotalProcessorsInUse()); // account for the extra processors from the dummy entity
+                assertEquals(3, stats.currentTotalNodes());
+                assertEquals(1, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(0, stats.wantedExtraProcessors());
+                assertEquals(0, stats.wantedExtraModelMemoryBytes());
+                assertEquals(0, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
     }
@@ -1512,7 +1709,9 @@ public void testGetMemoryAndProcessorsScaleDownNotPreventedByDummyEntityAsMemory
             MachineLearning.MAX_JVM_SIZE_NODE_ATTR,
             "400000000",
             MachineLearning.ML_CONFIG_VERSION_NODE_ATTR,
-            "7.2.0"
+            "7.2.0",
+            MachineLearning.ALLOCATED_PROCESSORS_NODE_ATTR,
+            "2.0"
         );
 
         MlAutoscalingContext mlAutoscalingContext = new MlAutoscalingContext(
@@ -1583,19 +1782,20 @@ public void testGetMemoryAndProcessorsScaleDownNotPreventedByDummyEntityAsMemory
                 10,
                 MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE,
                 MlDummyAutoscalingEntity.of(1024, 0),
+                1,
                 listener
             ),
             stats -> {
-                assertEquals(memory, stats.perNodeMemoryInBytes());
-                assertEquals(503318080, stats.modelMemoryInBytesSum());
-                assertEquals(4, stats.processorsSum());
-                assertEquals(3, stats.nodes());
-                assertEquals(1, stats.minNodes());
-                assertEquals(0, stats.extraSingleNodeProcessors());
-                assertEquals(0, stats.extraProcessors());
-                assertEquals(0, stats.extraModelMemoryInBytes());
-                assertEquals(0, stats.extraSingleNodeModelMemoryInBytes());
-                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes());
+                assertEquals(memory, stats.currentPerNodeMemoryBytes());
+                assertEquals(503318080, stats.currentTotalModelMemoryBytes());
+                assertEquals(4, stats.currentTotalProcessorsInUse());
+                assertEquals(3, stats.currentTotalNodes());
+                assertEquals(1, stats.wantedMinNodes());
+                assertEquals(0, stats.wantedExtraPerNodeNodeProcessors());
+                assertEquals(0, stats.wantedExtraProcessors());
+                assertEquals(0, stats.wantedExtraModelMemoryBytes());
+                assertEquals(0, stats.wantedExtraPerNodeMemoryBytes());
+                assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.currentPerNodeMemoryOverheadBytes());
             }
         );
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java
index 3ad44f256dc66..4aaddc91231f3 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerServiceTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ScalingExecutorBuilder;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -55,6 +56,7 @@ public class AdaptiveAllocationsScalerServiceTests extends ESTestCase {
     private ClusterService clusterService;
     private Client client;
     private InferenceAuditor inferenceAuditor;
+    private MeterRegistry meterRegistry;
 
     @Override
     @Before
@@ -66,6 +68,7 @@ public void setUp() throws Exception {
         clusterService = mock(ClusterService.class);
         client = mock(Client.class);
         inferenceAuditor = mock(InferenceAuditor.class);
+        meterRegistry = mock(MeterRegistry.class);
     }
 
     @Override
@@ -156,6 +159,7 @@ public void test() throws IOException {
             clusterService,
             client,
             inferenceAuditor,
+            meterRegistry,
             true,
             1
         );
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchResultProcessorTests.java
index 7eb9d7e940dda..69a9d4e9430c3 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchResultProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/pytorch/process/PyTorchResultProcessorTests.java
@@ -210,6 +210,8 @@ public void testsStats() {
         var c = wrapInferenceResult("c", true, 200L); // cache hit
 
         processor.processInferenceResult(a);
+        processor.updateStats(a);
+
         var stats = processor.getResultStats();
         assertThat(stats.errorCount(), equalTo(0));
         assertThat(stats.cacheHitCount(), equalTo(0L));
@@ -220,6 +222,8 @@ public void testsStats() {
         assertThat(stats.timingStatsExcludingCacheHits().getSum(), equalTo(1000L));
 
         processor.processInferenceResult(b);
+        processor.updateStats(b);
+
         stats = processor.getResultStats();
         assertThat(stats.errorCount(), equalTo(0));
         assertThat(stats.cacheHitCount(), equalTo(0L));
@@ -230,6 +234,8 @@ public void testsStats() {
         assertThat(stats.timingStatsExcludingCacheHits().getSum(), equalTo(1900L));
 
         processor.processInferenceResult(c);
+        processor.updateStats(c);
+
         stats = processor.getResultStats();
         assertThat(stats.errorCount(), equalTo(0));
         assertThat(stats.cacheHitCount(), equalTo(1L));
@@ -284,6 +290,9 @@ public void testsTimeDependentStats() {
         processor.processInferenceResult(wrapInferenceResult("foo0", false, 200L));
         processor.processInferenceResult(wrapInferenceResult("foo1", false, 200L));
         processor.processInferenceResult(wrapInferenceResult("foo2", false, 200L));
+        processor.updateStats(wrapInferenceResult("foo0", false, 200L));
+        processor.updateStats(wrapInferenceResult("foo1", false, 200L));
+        processor.updateStats(wrapInferenceResult("foo2", false, 200L));
 
         // first call has no results as is in the same period
         var stats = processor.getResultStats();
@@ -299,6 +308,7 @@ public void testsTimeDependentStats() {
 
         // 2nd period
         processor.processInferenceResult(wrapInferenceResult("foo3", false, 100L));
+        processor.updateStats(wrapInferenceResult("foo3", false, 100L));
         stats = processor.getResultStats();
         assertNotNull(stats.recentStats());
         assertThat(stats.recentStats().requestsProcessed(), equalTo(1L));
@@ -311,6 +321,7 @@ public void testsTimeDependentStats() {
 
         // 4th period
         processor.processInferenceResult(wrapInferenceResult("foo4", false, 300L));
+        processor.updateStats(wrapInferenceResult("foo4", false, 300L));
         stats = processor.getResultStats();
         assertNotNull(stats.recentStats());
         assertThat(stats.recentStats().requestsProcessed(), equalTo(1L));
@@ -320,6 +331,8 @@ public void testsTimeDependentStats() {
         // 7th period
         processor.processInferenceResult(wrapInferenceResult("foo5", false, 410L));
         processor.processInferenceResult(wrapInferenceResult("foo6", false, 390L));
+        processor.updateStats(wrapInferenceResult("foo5", false, 410L));
+        processor.updateStats(wrapInferenceResult("foo6", false, 390L));
         stats = processor.getResultStats();
         assertThat(stats.recentStats().requestsProcessed(), equalTo(0L));
         assertThat(stats.recentStats().avgInferenceTime(), nullValue());
@@ -333,6 +346,9 @@ public void testsTimeDependentStats() {
         processor.processInferenceResult(wrapInferenceResult("foo7", false, 510L));
         processor.processInferenceResult(wrapInferenceResult("foo8", false, 500L));
         processor.processInferenceResult(wrapInferenceResult("foo9", false, 490L));
+        processor.updateStats(wrapInferenceResult("foo7", false, 510L));
+        processor.updateStats(wrapInferenceResult("foo8", false, 500L));
+        processor.updateStats(wrapInferenceResult("foo9", false, 490L));
         stats = processor.getResultStats();
         assertNotNull(stats.recentStats());
         assertThat(stats.recentStats().requestsProcessed(), equalTo(3L));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriterTests.java
index 82a4ddea6460c..8a6881ad57b88 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriterTests.java
@@ -373,7 +373,7 @@ public void testWrite_Smile() throws Exception {
         xsonGen.writeStringField("value", "1.0");
         xsonGen.writeEndObject();
         xsonGen.close();
-        xsonOs.writeByte(XContentType.SMILE.xContent().streamSeparator());
+        xsonOs.writeByte(XContentType.SMILE.xContent().bulkSeparator());
 
         xsonGen = XContentFactory.xContent(XContentType.SMILE).createGenerator(xsonOs);
         xsonGen.writeStartObject();
@@ -382,7 +382,7 @@ public void testWrite_Smile() throws Exception {
         xsonGen.writeStringField("value", "2.0");
         xsonGen.writeEndObject();
         xsonGen.flush();
-        xsonOs.writeByte(XContentType.SMILE.xContent().streamSeparator());
+        xsonOs.writeByte(XContentType.SMILE.xContent().bulkSeparator());
 
         InputStream inputStream = new ByteArrayInputStream(BytesReference.toBytes(xsonOs.bytes()));
         JsonDataToProcessWriter writer = createWriter();
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportAction.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportAction.java
index 00e7ab603f834..5663de0cc4c45 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportAction.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.monitoring;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageTransportAction.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageTransportAction.java
index 8553e63e380fe..e3fe12ca2a9e7 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageTransportAction.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageTransportAction.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkAction.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkAction.java
index 6482c4d77c019..78ad42dbcbb65 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkAction.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsAction.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsAction.java
index d3794201caec8..bf2825155530c 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsAction.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsAction.java
@@ -22,9 +22,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java
index 0b223249dcaaf..7bcba0fdbafea 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java
@@ -176,7 +176,7 @@ private void writeDocument(MonitoringDoc doc, OutputStream out) throws IOExcepti
         }
 
         // Adds action metadata line bulk separator
-        out.write(xContent.streamSeparator());
+        out.write(xContent.bulkSeparator());
 
         // Adds the source of the monitoring document
         try (XContentBuilder builder = new XContentBuilder(xContent, out)) {
@@ -184,7 +184,7 @@ private void writeDocument(MonitoringDoc doc, OutputStream out) throws IOExcepti
         }
 
         // Adds final bulk separator
-        out.write(xContent.streamSeparator());
+        out.write(xContent.bulkSeparator());
 
         logger.trace("http exporter [{}] - added index request [index={}, id={}, monitoring data type={}]", name, index, id, doc.getType());
     }
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java
index 343353a9b3a39..300ac5a650a8d 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java
@@ -102,7 +102,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
     }
 
     @Override
-    public boolean supportsContentStream() {
+    public boolean supportsBulkContent() {
         return true;
     }
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/LocalStateMonitoring.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/LocalStateMonitoring.java
index e9e8a195656fd..1b08e88f9ce03 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/LocalStateMonitoring.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/LocalStateMonitoring.java
@@ -16,10 +16,10 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java
index 37fe08b302e7d..e7357672bbeda 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java
@@ -105,16 +105,16 @@ public void testAddRequestContent() throws IOException {
                 builder.endObject();
 
                 builder.flush();
-                content.write(xContentType.xContent().streamSeparator());
+                content.write(xContentType.xContent().bulkSeparator());
 
                 sources[i] = RandomObjects.randomSource(random(), xContentType);
                 BytesRef bytes = sources[i].toBytesRef();
                 content.write(bytes.bytes, bytes.offset, bytes.length);
 
-                content.write(xContentType.xContent().streamSeparator());
+                content.write(xContentType.xContent().bulkSeparator());
             }
 
-            content.write(xContentType.xContent().streamSeparator());
+            content.write(xContentType.xContent().bulkSeparator());
         }
 
         final MonitoredSystem system = randomFrom(MonitoredSystem.values());
@@ -146,7 +146,7 @@ public void testAddRequestContentWithEmptySource() throws IOException {
         final int totalDocs = nbDocs + nbEmptyDocs;
 
         final XContentType xContentType = XContentType.JSON;
-        final byte separator = xContentType.xContent().streamSeparator();
+        final byte separator = xContentType.xContent().bulkSeparator();
 
         final BytesStreamOutput content = new BytesStreamOutput();
         try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, content)) {
@@ -192,7 +192,7 @@ public void testAddRequestContentWithUnrecognizedIndexName() throws IOException
         final String indexName = randomAlphaOfLength(10);
 
         final XContentType xContentType = XContentType.JSON;
-        final byte separator = xContentType.xContent().streamSeparator();
+        final byte separator = xContentType.xContent().bulkSeparator();
 
         final BytesStreamOutput content = new BytesStreamOutput();
         try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, content)) {
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
index c89638045a5a8..4a695f7c51e4c 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodeResponse;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
 import org.elasticsearch.action.admin.cluster.stats.MappingStats;
+import org.elasticsearch.action.admin.cluster.stats.RepositoryUsageStats;
 import org.elasticsearch.action.admin.cluster.stats.SearchUsageStats;
 import org.elasticsearch.action.admin.cluster.stats.VersionStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
@@ -420,6 +421,7 @@ public void testToXContent() throws IOException {
         when(mockNodeResponse.nodeStats()).thenReturn(mockNodeStats);
         when(mockNodeResponse.shardsStats()).thenReturn(new ShardStats[] { mockShardStats });
         when(mockNodeResponse.searchUsageStats()).thenReturn(new SearchUsageStats());
+        when(mockNodeResponse.repositoryUsageStats()).thenReturn(RepositoryUsageStats.EMPTY);
 
         final Metadata metadata = testClusterState.metadata();
         final ClusterStatsResponse clusterStatsResponse = new ClusterStatsResponse(
@@ -533,7 +535,9 @@ public void testToXContent() throws IOException {
                   "fielddata": {
                     "memory_size_in_bytes": 1,
                     "evictions": 0,
-                    "global_ordinals":{"build_time_in_millis":1}
+                    "global_ordinals": {
+                      "build_time_in_millis": 1
+                    }
                   },
                   "query_cache": {
                     "memory_size_in_bytes": 0,
@@ -563,9 +567,9 @@ public void testToXContent() throws IOException {
                     "file_sizes": {}
                   },
                   "mappings": {
-                    "total_field_count" : 0,
-                    "total_deduplicated_field_count" : 0,
-                    "total_deduplicated_mapping_size_in_bytes" : 0,
+                    "total_field_count": 0,
+                    "total_deduplicated_field_count": 0,
+                    "total_deduplicated_mapping_size_in_bytes": 0,
                     "field_types": [],
                     "runtime_field_types": []
                   },
@@ -581,11 +585,11 @@ public void testToXContent() throws IOException {
                     "synonyms": {}
                   },
                   "versions": [],
-                  "search" : {
-                    "total" : 0,
-                    "queries" : {},
-                    "rescorers" : {},
-                    "sections" : {}
+                  "search": {
+                    "total": 0,
+                    "queries": {},
+                    "rescorers": {},
+                    "sections": {}
                   },
                   "dense_vector": {
                     "value_count": 0
@@ -749,7 +753,8 @@ public void testToXContent() throws IOException {
                     "cleanups": 0
                   },
                   "repositories": {}
-                }
+                },
+                "repositories": {}
               },
               "cluster_state": {
                 "nodes_hash": 1314980060,
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java
index 3d1034cd815e7..bf3ea4492a52c 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java
@@ -45,9 +45,9 @@ public void testGetName() {
         assertThat(action.getName(), is("monitoring_bulk"));
     }
 
-    public void testSupportsContentStream() {
+    public void testSupportsBulkContent() {
         // if you change this, it's a very breaking change for Monitoring
-        assertThat(action.supportsContentStream(), is(true));
+        assertThat(action.supportsBulkContent(), is(true));
     }
 
     public void testMissingSystemId() {
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringMigrateAlertsActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringMigrateAlertsActionTests.java
index a92cf9c775aba..776e7937fe69d 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringMigrateAlertsActionTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringMigrateAlertsActionTests.java
@@ -35,7 +35,7 @@ public void testGetName() {
     }
 
     public void testSupportsContentStream() {
-        assertThat(action.supportsContentStream(), is(false));
+        assertThat(action.supportsBulkContent(), is(false));
     }
 
     public void testRestActionCompletion() throws Exception {
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveInfoTransportAction.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveInfoTransportAction.java
index 702559a4810d8..f56d63a72a110 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveInfoTransportAction.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveInfoTransportAction.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.lucene.bwc;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTransportAction.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTransportAction.java
index e4e4c67adbc04..ae3de7c6c588d 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTransportAction.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/ArchiveUsageTransportAction.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/otel-data/README.md b/x-pack/plugin/otel-data/README.md
new file mode 100644
index 0000000000000..7cab6bfa453d8
--- /dev/null
+++ b/x-pack/plugin/otel-data/README.md
@@ -0,0 +1,33 @@
+## OpenTelemetry Ingest plugin
+
+The OpenTelemetry Ingest plugin installs index templates and component templates for OpenTelemetry data.
+
+All resources are defined as YAML under [src/main/resources](src/main/resources).
+
+The OpenTelemetry index templates rely on mappings from `x-pack-core`.
+See [x-pack/plugin/core/src/main/resources](../core/src/main/resources).
+
+## Adding/Removing/Updating a resource
+
+All resources are defined as YAML under [src/main/resources](src/main/resources).
+
+For a resource to be known to the plugin it must be added to
+[src/main/resources/resources.yaml](src/main/resources/resources.yaml) in the
+appropriate section.
+
+Any update to resources included by this package also requires a bump to the
+`version` property included in the resources file.
+
+## Testing
+
+## Integration testing
+
+The index templates and ingest pipeline functionality is tested using YAML REST tests.
+These can be run with:
+
+```
+./gradlew :x-pack:plugin:otel-data:yamlRestTest
+```
+
+Refer to the [rest-api-spec documentation](../../../rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc)
+for information about writing YAML REST tests.
diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle
new file mode 100644
index 0000000000000..f56efe21acccc
--- /dev/null
+++ b/x-pack/plugin/otel-data/build.gradle
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
+
+esplugin {
+  name 'x-pack-otel-data'
+  description 'The OTEL plugin defines OTEL data streams and ingest pipelines.'
+  classname 'org.elasticsearch.xpack.oteldata.OTelPlugin'
+  extendedPlugins = ['x-pack-core']
+}
+
+dependencies {
+  compileOnly project(path: xpackModule('core'))
+  testImplementation project(path: ':x-pack:plugin:stack')
+  testImplementation(testArtifact(project(xpackModule('core'))))
+  testImplementation project(':modules:data-streams')
+  clusterModules project(':modules:data-streams')
+  clusterModules project(':modules:ingest-common')
+  clusterModules project(':modules:ingest-geoip')
+  clusterModules project(':modules:ingest-user-agent')
+  clusterModules project(':modules:lang-mustache')
+  clusterModules project(':modules:mapper-extras')
+  clusterModules project(xpackModule('analytics'))
+  clusterModules project(xpackModule('ilm'))
+  clusterModules project(xpackModule('mapper-aggregate-metric'))
+  clusterModules project(xpackModule('mapper-constant-keyword'))
+  clusterModules project(xpackModule('mapper-counted-keyword'))
+  clusterModules project(xpackModule('stack'))
+  clusterModules project(xpackModule('wildcard'))
+  clusterModules project(xpackModule('mapper-version'))
+}
diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java
new file mode 100644
index 0000000000000..435530542c857
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelIndexTemplateRegistry.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.oteldata;
+
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.template.YamlTemplateRegistry;
+
+import static org.elasticsearch.xpack.oteldata.OTelPlugin.OTEL_DATA_REGISTRY_ENABLED;
+
+public class OTelIndexTemplateRegistry extends YamlTemplateRegistry {
+
+    public static final String OTEL_TEMPLATE_VERSION_VARIABLE = "xpack.oteldata.template.version";
+
+    public OTelIndexTemplateRegistry(
+        Settings nodeSettings,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        Client client,
+        NamedXContentRegistry xContentRegistry,
+        FeatureService featureService
+    ) {
+        super(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService);
+    }
+
+    @Override
+    public void initialize() {
+        super.initialize();
+        if (isEnabled()) {
+            clusterService.getClusterSettings().addSettingsUpdateConsumer(OTEL_DATA_REGISTRY_ENABLED, this::setEnabled);
+        }
+    }
+
+    @Override
+    protected String getOrigin() {
+        return ClientHelper.OTEL_ORIGIN;
+    }
+
+    @Override
+    public String getName() {
+        return "OpenTelemetry";
+    }
+
+    @Override
+    protected String getVersionProperty() {
+        return OTEL_TEMPLATE_VERSION_VARIABLE;
+    }
+}
diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java
new file mode 100644
index 0000000000000..cece2b5373631
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.oteldata;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+public class OTelPlugin extends Plugin implements ActionPlugin {
+    private static final Logger logger = LogManager.getLogger(OTelPlugin.class);
+
+    final SetOnce<OTelIndexTemplateRegistry> registry = new SetOnce<>();
+
+    private final boolean enabled;
+
+    // OTEL_DATA_REGISTRY_ENABLED controls enabling the index template registry.
+    //
+    // This setting will be ignored if the plugin is disabled.
+    static final Setting<Boolean> OTEL_DATA_REGISTRY_ENABLED = Setting.boolSetting(
+        "xpack.otel_data.registry.enabled",
+        // OTel-data is under development, and we start with opt-in first.
+        // Furthermore, this could help with staged rollout in serverless
+        false,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    public OTelPlugin(Settings settings) {
+        this.enabled = XPackSettings.OTEL_DATA_ENABLED.get(settings);
+    }
+
+    @Override
+    public Collection<?> createComponents(PluginServices services) {
+        logger.info("OTel ingest plugin is {}", enabled ? "enabled" : "disabled");
+        Settings settings = services.environment().settings();
+        ClusterService clusterService = services.clusterService();
+        registry.set(
+            new OTelIndexTemplateRegistry(
+                settings,
+                clusterService,
+                services.threadPool(),
+                services.client(),
+                services.xContentRegistry(),
+                services.featureService()
+            )
+        );
+        if (enabled) {
+            OTelIndexTemplateRegistry registryInstance = registry.get();
+            registryInstance.setEnabled(OTEL_DATA_REGISTRY_ENABLED.get(settings));
+            registryInstance.initialize();
+        }
+        return Collections.emptyList();
+    }
+
+    @Override
+    public void close() {
+        registry.get().close();
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return List.of(OTEL_DATA_REGISTRY_ENABLED);
+    }
+}
diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/package-info.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/package-info.java
new file mode 100644
index 0000000000000..98c6c9a3999c4
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/package-info.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * This package contains index templates for OpenTelemetry data. It covers traces (spans), metrics, and logs.
+ * The plugin is expected to be used in combination with the Elasticsearch exporter defined as the exporter
+ * within an OpenTelemetry collector with the mapping mode `otel`.
+ * For more information about the Elasticsearch exporter
+ * @see <a href="https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/elasticsearchexporter/README.md">
+ * https://github.com/open-telemetry/opentelemetry-collector-contrib</a>.
+ *
+ */
+package org.elasticsearch.xpack.oteldata;
diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml
new file mode 100644
index 0000000000000..a0971f45ccf4f
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml
@@ -0,0 +1,42 @@
+---
+version: ${xpack.oteldata.template.version}
+_meta:
+  description: Default mappings for OpenTelemetry logs index template installed by x-pack
+  managed: true
+template:
+  settings:
+    index:
+      mode: logsdb
+      sort:
+        field: [ "resource.attributes.host.name" ]
+  mappings:
+    properties:
+      data_stream.type:
+        type: constant_keyword
+        value: logs
+      observed_timestamp:
+        type: date_nanos
+      severity_number:
+        type: byte
+      severity_text:
+        type: keyword
+      log.level:
+        type: alias
+        path: severity_text
+      body_text:
+        type: match_only_text
+      message:
+        type: alias
+        path: body_text
+      body_structured:
+        type: flattened
+      trace_id:
+        type: keyword
+      trace.id:
+        type: alias
+        path: trace_id
+      span_id:
+        type: keyword
+      span.id:
+        type: alias
+        path: span_id
diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml
new file mode 100644
index 0000000000000..b7a17dba973f8
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/metrics-otel@mappings.yaml
@@ -0,0 +1,17 @@
+version: ${xpack.oteldata.template.version}
+_meta:
+  description: Default mappings for the OpenTelemetry metrics index template installed by x-pack
+  managed: true
+template:
+  mappings:
+    properties:
+      start_timestamp:
+        type: date_nanos
+      metrics:
+        type: passthrough
+        dynamic: true
+        priority: 1
+      unit:
+        type: keyword
+        time_series_dimension: true
+        ignore_above: 1024
diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml
new file mode 100644
index 0000000000000..fad85661203d6
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/otel@mappings.yaml
@@ -0,0 +1,64 @@
+---
+version: ${xpack.oteldata.template.version}
+_meta:
+  description: Default mappings for all OpenTelemetry data streams
+  managed: true
+template:
+  mappings:
+    date_detection: false
+    dynamic: false
+    properties:
+      "@timestamp":
+#Ultimeately we aim to use date_nanos. Waiting for https://github.com/elastic/elasticsearch/issues/109352
+        type: date
+      data_stream.type:
+        type: constant_keyword
+      data_stream.dataset:
+        type: constant_keyword
+      data_stream.namespace:
+        type: constant_keyword
+      attributes:
+        type: passthrough
+        dynamic: true
+        priority: 10
+        time_series_dimension: true
+      dropped_attributes_count:
+        type: long
+      scope:
+        properties:
+          name:
+            type: keyword
+            ignore_above: 1024
+          version:
+            type: version
+          schema_url:
+            type: keyword
+            ignore_above: 1024
+          dropped_attributes_count:
+            type: long
+          attributes:
+            type: passthrough
+            dynamic: true
+            priority: 20
+            time_series_dimension: true
+      resource:
+        properties:
+          schema_url:
+            type: keyword
+            ignore_above: 1024
+          dropped_attributes_count:
+            type: long
+          attributes:
+            type: passthrough
+            dynamic: true
+            priority: 30
+            time_series_dimension: true
+    dynamic_templates:
+      - complex_attributes:
+          path_match:
+            - resource.attributes.*
+            - scope.attributes.*
+            - attributes.*
+          match_mapping_type: object
+          mapping:
+            type: flattened
diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml
new file mode 100644
index 0000000000000..711f72ae95220
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/semconv-resource-to-ecs@mappings.yaml
@@ -0,0 +1,128 @@
+---
+version: ${xpack.oteldata.template.version}
+_meta:
+  description: Aliases from OpenTelemetry SemConv fields to ECS (and some non-ECS) fields
+  managed: true
+template:
+  mappings:
+    properties:
+      resource:
+        properties:
+          attributes:
+            type: passthrough
+            dynamic: true
+            priority: 30
+            time_series_dimension: true
+            properties:
+              host.name:
+                type: keyword
+                ignore_above: 1024
+              telemetry.sdk.language:
+                type: keyword
+                ignore_above: 1024
+              service.instance.id:
+                type: keyword
+                ignore_above: 1024
+              deployment.environment:
+                type: keyword
+                ignore_above: 1024
+              cloud.platform:
+                type: keyword
+                ignore_above: 1024
+              container.image.tags:
+                type: keyword
+                ignore_above: 1024
+              host.arch:
+                type: keyword
+                ignore_above: 1024
+              process.executable.path:
+                type: keyword
+                ignore_above: 1024
+              process.runtime.name:
+                type: keyword
+                ignore_above: 1024
+              process.runtime.version:
+                type: keyword
+                ignore_above: 1024
+              os.name:
+                type: keyword
+                ignore_above: 1024
+              os.type:
+                type: keyword
+                ignore_above: 1024
+              os.description:
+                type: keyword
+                ignore_above: 1024
+              os.version:
+                type: keyword
+                ignore_above: 1024
+              k8s.deployment.name:
+                type: keyword
+                ignore_above: 1024
+              k8s.namespace.name:
+                type: keyword
+                ignore_above: 1024
+              k8s.node.name:
+                type: keyword
+                ignore_above: 1024
+              k8s.pod.name:
+                type: keyword
+                ignore_above: 1024
+              k8s.pod.uid:
+                type: keyword
+                ignore_above: 1024
+      service.node.name:
+        type: alias
+        path: resource.attributes.service.instance.id
+      service.environment:
+        type: alias
+        path: resource.attributes.deployment.environment
+      cloud.service.name:
+        type: alias
+        path: resource.attributes.cloud.platform
+      container.image.tag:
+        type: alias
+        path: resource.attributes.container.image.tags
+      host.architecture:
+        type: alias
+        path: resource.attributes.host.arch
+      process.executable:
+        type: alias
+        path: resource.attributes.process.executable.path
+      service.runtime.name:
+        type: alias
+        path: resource.attributes.process.runtime.name
+      service.runtime.version:
+        type: alias
+        path: resource.attributes.process.runtime.version
+      host.os.name:
+        type: alias
+        path: resource.attributes.os.name
+      host.os.platform:
+        type: alias
+        path: resource.attributes.os.type
+      host.os.full:
+        type: alias
+        path: resource.attributes.os.description
+      host.os.version:
+        type: alias
+        path: resource.attributes.os.version
+      kubernetes.deployment.name:
+        type: alias
+        path: resource.attributes.k8s.deployment.name
+      kubernetes.namespace:
+        type: alias
+        path: resource.attributes.k8s.namespace.name
+      kubernetes.node.name:
+        type: alias
+        path: resource.attributes.k8s.node.name
+      kubernetes.pod.name:
+        type: alias
+        path: resource.attributes.k8s.pod.name
+      kubernetes.pod.uid:
+        type: alias
+        path: resource.attributes.k8s.pod.uid
+# Below are non-ECS fields that may be used by Kibana.
+      service.language.name:
+        type: alias
+        path: resource.attributes.telemetry.sdk.language
diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml
new file mode 100644
index 0000000000000..a4c62efeed7a4
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml
@@ -0,0 +1,68 @@
+---
+version: ${xpack.oteldata.template.version}
+_meta:
+  description: Default mappings for OpenTelemetry traces
+  managed: true
+template:
+  settings:
+    index:
+      mode: logsdb
+      sort:
+        field: [ "resource.attributes.host.name" ]
+  mappings:
+    _source:
+      mode: synthetic
+    properties:
+      trace_id:
+        type: keyword
+      trace.id:
+        type: alias
+        path: trace_id
+      span_id:
+        type: keyword
+      span.id:
+        type: alias
+        path: span_id
+      trace_state:
+        type: keyword
+      parent_span_id:
+        type: keyword
+      parent.id:
+        type: alias
+        path: parent_span_id
+      name:
+        type: keyword
+      span.name:
+        type: alias
+        path: name
+      kind:
+        type: keyword
+      duration:
+        type: long
+        meta:
+          unit: nanos
+      dropped_events_count:
+        type: long
+      links:
+        store_array_source: true
+        properties:
+          trace_id:
+            type: keyword
+          span_id:
+            type: keyword
+          trace_state:
+            type: keyword
+          attributes:
+            type: object
+            subobjects: false
+            dynamic: true
+          dropped_attributes_count:
+            type: long
+      dropped_links_count:
+        type: long
+      status:
+        properties:
+          message:
+            type: keyword
+          code:
+            type: keyword
diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml
new file mode 100644
index 0000000000000..6772ec5bc65d4
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/logs-otel@template.yaml
@@ -0,0 +1,27 @@
+---
+version: ${xpack.oteldata.template.version}
+index_patterns: ["logs-*.otel-*"]
+priority: 120
+data_stream: {}
+allow_auto_create: true
+_meta:
+  description: default OpenTelemetry logs template installed by x-pack
+  managed: true
+composed_of:
+  - logs@mappings
+  - logs@settings
+  - otel@mappings
+  - logs-otel@mappings
+  - semconv-resource-to-ecs@mappings
+  - logs@custom
+  - logs-otel@custom
+  - ecs@mappings
+ignore_missing_component_templates:
+  - logs@custom
+  - logs-otel@custom
+template:
+  mappings:
+    properties:
+      data_stream.type:
+        type: constant_keyword
+        value: logs
diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml
new file mode 100644
index 0000000000000..89ff28249aabb
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml
@@ -0,0 +1,36 @@
+---
+version: ${xpack.oteldata.template.version}
+index_patterns: ["metrics-*.otel-*"]
+priority: 120
+data_stream: {}
+allow_auto_create: true
+_meta:
+  description: default OpenTelemetry metrics template installed by x-pack
+  managed: true
+composed_of:
+  - metrics@mappings
+  - metrics@tsdb-settings
+  - otel@mappings
+  - metrics-otel@mappings
+  - semconv-resource-to-ecs@mappings
+  - metrics@custom
+  - metrics-otel@custom
+ignore_missing_component_templates:
+  - metrics@custom
+  - metrics-otel@custom
+template:
+  settings:
+    index:
+      mode: time_series
+  mappings:
+    properties:
+      data_stream.type:
+        type: constant_keyword
+        value: metrics
+    dynamic_templates:
+      - all_strings_to_keywords:
+          mapping:
+            ignore_above: 1024
+            type: keyword
+          match_mapping_type: string
+
diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml
new file mode 100644
index 0000000000000..370b9351c16f5
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/traces-otel@template.yaml
@@ -0,0 +1,27 @@
+---
+version: ${xpack.oteldata.template.version}
+index_patterns: ["traces-*.otel-*"]
+priority: 120
+data_stream: {}
+allow_auto_create: true
+_meta:
+  description: default OpenTelemetry traces template installed by x-pack
+  managed: true
+composed_of:
+  - traces@mappings
+  - traces@settings
+  - otel@mappings
+  - traces-otel@mappings
+  - semconv-resource-to-ecs@mappings
+  - traces@custom
+  - traces-otel@custom
+  - ecs@mappings
+ignore_missing_component_templates:
+  - traces@custom
+  - traces-otel@custom
+template:
+  mappings:
+    properties:
+      data_stream.type:
+        type: constant_keyword
+        value: traces
diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml
new file mode 100644
index 0000000000000..8e0a7606cbd05
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml
@@ -0,0 +1,15 @@
+# "version" holds the version of the templates and ingest pipelines installed
+# by xpack-plugin otel-data. This must be increased whenever an existing template is
+# changed, in order for it to be updated on Elasticsearch upgrade.
+version: 1
+
+component-templates:
+  - otel@mappings
+  - logs-otel@mappings
+  - semconv-resource-to-ecs@mappings
+  - metrics-otel@mappings
+  - traces-otel@mappings
+index-templates:
+  - logs-otel@template
+  - metrics-otel@template
+  - traces-otel@template
diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java b/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..4a5f7d03b12a2
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/yamlRestTest/java/org/elasticsearch/xpack/oteldata/OTelYamlTestSuiteIT.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.oteldata;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
+
+public class OTelYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("constant-keyword")
+        .module("counted-keyword")
+        .module("data-streams")
+        .module("ingest-common")
+        .module("ingest-geoip")
+        .module("ingest-user-agent")
+        .module("lang-mustache")
+        .module("mapper-extras")
+        .module("wildcard")
+        .module("x-pack-analytics")
+        .module("x-pack-otel-data")
+        .module("x-pack-aggregate-metric")
+        .module("x-pack-ilm")
+        .module("x-pack-stack")
+        .module("mapper-version")
+        .setting("ingest.geoip.downloader.enabled", "false")
+        .setting("xpack.otel_data.registry.enabled", "true")
+        .build();
+
+    public OTelYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}
diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/10_otel.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/10_otel.yml
new file mode 100644
index 0000000000000..72b7a127dcd02
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/10_otel.yml
@@ -0,0 +1,38 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_events: languid
+
+---
+"Test traces-otel* template installation":
+  - skip:
+      reason: contains is a newly added assertion
+      features: contains
+  - do:
+      indices.get_index_template:
+        name: traces-otel*
+  - length: {index_templates: 1}
+  - contains: {index_templates: {name: traces-otel@template}}
+
+---
+"Test metrics-otel* template installation":
+  - skip:
+      reason: contains is a newly added assertion
+      features: contains
+  - do:
+      indices.get_index_template:
+        name: metrics-otel*
+  - length: {index_templates: 1}
+  - contains: {index_templates: {name: metrics-otel@template}}
+
+---
+"Test logs-otel* template installation":
+  - skip:
+      reason: contains is a newly added assertion
+      features: contains
+  - do:
+      indices.get_index_template:
+        name: logs-otel*
+  - length: {index_templates: 1}
+  - contains: {index_templates: {name: logs-otel@template}}
diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml
new file mode 100644
index 0000000000000..d87c2a80deab8
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml
@@ -0,0 +1,22 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_events: languid
+---
+"Default data_stream.type must be logs":
+  - do:
+      bulk:
+        index: logs-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "attributes": { "foo": "bar"}, "body_text":"Error: Unable to connect to the database.","severity_text":"ERROR","severity_number":3,"trace_id":"abc123xyz456def789ghi012jkl345"}'
+  - is_false: errors
+  - do:
+      search:
+        index: logs-generic.otel-default
+        body:
+          fields: ["data_stream.type"]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.data_stream\.type: ["logs"] }
diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml
new file mode 100644
index 0000000000000..a6591d6c32210
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml
@@ -0,0 +1,149 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_events: languid
+  - do:
+      cluster.put_component_template:
+        name: metrics-otel@custom
+        body:
+          template:
+            settings:
+              index:
+                routing_path: [unit, attributes.*, resource.attributes.*]
+                mode: time_series
+                time_series:
+                  start_time: 2024-07-01T13:03:08.138Z
+---
+"Test push service overview metric":
+  - do:
+      indices.get_index_template:
+        name: metrics-otel@template
+  - length: {index_templates: 1}
+  - do:
+      bulk:
+        index: metrics-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0},"attributes":{"processor.event":"metric"}}'
+  - is_false: errors
+  - do:
+      search:
+        index: metrics-generic.otel-default
+        body:
+          fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name" ]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.service\.name: [ "OtelSample" ] }
+  - match: { hits.hits.0.fields.telemetry\.sdk\.language: [ "dotnet" ] }
+  - match: { hits.hits.0.fields.telemetry\.sdk\.name: [ "opentelemetry" ] }
+---
+"Query resource attributes as top level":
+  - do:
+      bulk:
+        index: metrics-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"processor":{"event":"metric"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}'
+  - is_false: errors
+  - do:
+      search:
+        index: metrics-generic.otel-default
+        body:
+          fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name"]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.service\.name: [ "OtelSample" ] }
+  - match: { hits.hits.0.fields.telemetry\.sdk\.language: [ "dotnet" ] }
+  - match: { hits.hits.0.fields.telemetry\.sdk\.name: [ "opentelemetry" ] }
+---
+"Query attributes as top level":
+  - do:
+      bulk:
+        index: metrics-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{"processor.event":"metric", "foo": "bar"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}'
+  - is_false: errors
+  - do:
+      search:
+        index: metrics-generic.otel-default
+        body:
+          fields: ["foo"]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.foo\: ["bar"] }
+---
+"Boolean as dimension":
+  - do:
+      bulk:
+        index: metrics-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{"processor.event":"metric","transaction.root":false},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}}}'
+  - is_false: errors
+  - do:
+      search:
+        index: metrics-generic.otel-default
+        body:
+          fields: ["transaction.root"]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.transaction\.root: [false] }
+---
+"Default data_stream.type must be metrics":
+  - do:
+      bulk:
+        index: metrics-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"},"attributes":{"processor.event":"metric","transaction.root":false},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}}}'
+  - is_false: errors
+  - do:
+      search:
+        index: metrics-generic.otel-default
+        body:
+          fields: ["data_stream.type"]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.data_stream\.type: ["metrics"] }
+
+---
+"Custom dynamic template":
+  - do:
+      cluster.put_component_template:
+        name: metrics-otel@custom
+        body:
+          template:
+            settings:
+              index:
+                routing_path: [unit, attributes.*, resource.attributes.*]
+                mode: time_series
+                time_series:
+                  start_time: 2024-07-01T13:03:08.138Z
+            mappings:
+              dynamic_templates:
+                - ip_fields:
+                    mapping:
+                      type: ip
+                    match_mapping_type: string
+                    path_match: "*.ip"
+  - do:
+      bulk:
+        index: metrics-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"},"attributes":{"host.ip":"127.0.0.1","foo":"bar"}}'
+  - is_false: errors
+  - do:
+      indices.get_data_stream:
+        name: metrics-generic.otel-default
+  - set: { data_streams.0.indices.0.index_name: idx0name }
+
+  - do:
+      indices.get_mapping:
+        index: $idx0name
+        expand_wildcards: hidden
+  - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'ip' }
+  - match: { .$idx0name.mappings.properties.attributes.properties.foo.type: "keyword" }
diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml
new file mode 100644
index 0000000000000..abdb8d49d774c
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml
@@ -0,0 +1,94 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_events: languid
+---
+"Test pushing simple trace":
+  - do:
+      bulk:
+        index: traces-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}'
+  - is_false: errors
+  - do:
+      search:
+        index: traces-generic.otel-default
+  - length: { hits.hits: 1 }
+
+---
+"Query resource attributes as top level":
+  - do:
+      bulk:
+        index: traces-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}'
+  - is_false: errors
+  - do:
+      search:
+        index: traces-generic.otel-default
+        body:
+          fields: ["service.name", "telemetry.sdk.language", "telemetry.sdk.name" ]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.service\.name: [ "OtelSample" ] }
+  - match: { hits.hits.0.fields.telemetry\.sdk\.language: [ "dotnet" ] }
+  - match: { hits.hits.0.fields.telemetry\.sdk\.name: [ "opentelemetry" ] }
+---
+"Query attributes as top level":
+  - do:
+      bulk:
+        index: traces-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}'
+  - is_false: errors
+  - do:
+      search:
+        index: traces-generic.otel-default
+        body:
+          fields: ["db.type", "db.name", "db.operation", "db.statement"]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.db\.type: [ "mssql" ] }
+  - match: { hits.hits.0.fields.db\.operation: [ "SELECT" ] }
+  - match: { hits.hits.0.fields.db\.statement: [ "SELECT * FROM wuser_table" ] }
+---
+"Span links test":
+  - do:
+      bulk:
+        index: traces-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"attributes":{"db.type":"mssql","db.name":"foo","db.operation":"SELECT","db.statement":"SELECT * FROM wuser_table"},"links":[{"trace_id":"4aaa9f33312b3dbb8b2c2c62bb7abe1a1","span_id":"086e83747d0e381e","attributes":{"foo":"bar"}},{"trace_id":"4aaa9f33312b3dbb8b2c2c62bb7abe1a1","span_id":"b3b7d1f1f1b4e1e1"}],"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}'
+  - is_false: errors
+  - do:
+      search:
+        index: traces-generic.otel-default
+  - length: { hits.hits.0._source.links: 2 }
+  - match: { hits.hits.0._source.links.0.trace_id: "4aaa9f33312b3dbb8b2c2c62bb7abe1a1"  }
+  - match: { hits.hits.0._source.links.0.span_id: "086e83747d0e381e"  }
+  - match: { hits.hits.0._source.links.0.attributes.foo: "bar"  }
+  - match: { hits.hits.0._source.links.1.trace_id: "4aaa9f33312b3dbb8b2c2c62bb7abe1a1"  }
+  - match: { hits.hits.0._source.links.1.span_id: "b3b7d1f1f1b4e1e1"  }
+---
+"Default data_stream.type must be traces":
+  - do:
+      bulk:
+        index: traces-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}'
+  - is_false: errors
+  - do:
+      search:
+        index: traces-generic.otel-default
+        body:
+          fields: ["data_stream.type"]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.data_stream\.type: ["traces"] }
diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml
new file mode 100644
index 0000000000000..d80c52c756b54
--- /dev/null
+++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_non_ecs_alias_tests.yml
@@ -0,0 +1,37 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_events: languid
+  - do:
+      cluster.put_component_template:
+        name: metrics-otel@custom
+        body:
+          template:
+            settings:
+              index:
+                routing_path: [unit, attributes.*, resource.attributes.*]
+                mode: time_series
+                time_series:
+                  start_time: 2024-07-01T13:03:08.138Z
+---
+"Test alias from service.language.name non-ecs field to telemetry.sdk.language":
+  - do:
+      indices.get_index_template:
+        name: metrics-otel@template
+  - length: {index_templates: 1}
+  - do:
+      bulk:
+        index: metrics-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"}, "attributes": {"processor.event":"metric"}, "resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"},"dropped_attributes_count":0}}'
+  - is_false: errors
+  - do:
+      search:
+        index: metrics-generic.otel-default
+        body:
+          fields: ["service.language.name"]
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0.fields.service\.language\.name: [ "dotnet" ] }
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/ProfilingInfoTransportAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/ProfilingInfoTransportAction.java
index 1a6809774f7f6..67369b8428200 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/ProfilingInfoTransportAction.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/ProfilingInfoTransportAction.java
@@ -8,8 +8,8 @@
 package org.elasticsearch.xpack.profiling.action;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/ProfilingUsageTransportAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/ProfilingUsageTransportAction.java
index 738a7a4e52ddb..05799d14f4614 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/ProfilingUsageTransportAction.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/ProfilingUsageTransportAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java
index 4f3778081563b..66c9eaea3ec41 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetFlamegraphAction.java
@@ -15,7 +15,8 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
@@ -33,7 +34,7 @@ public class TransportGetFlamegraphAction extends TransportAction<GetStackTraces
 
     @Inject
     public TransportGetFlamegraphAction(NodeClient nodeClient, TransportService transportService, ActionFilters actionFilters) {
-        super(GetFlamegraphAction.NAME, actionFilters, transportService.getTaskManager());
+        super(GetFlamegraphAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.nodeClient = nodeClient;
         this.transportService = transportService;
     }
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java
index 6efab6e99da6f..48673d2002170 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java
@@ -22,13 +22,14 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.countedterms.CountedTermsAggregationBuilder;
@@ -138,7 +139,7 @@ public TransportGetStackTracesAction(
         ProfilingLicenseChecker licenseChecker,
         IndexNameExpressionResolver resolver
     ) {
-        super(GetStackTracesAction.NAME, actionFilters, transportService.getTaskManager());
+        super(GetStackTracesAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.nodeClient = nodeClient;
         this.licenseChecker = licenseChecker;
         this.clusterService = clusterService;
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStatusAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStatusAction.java
index 2105535e71432..9dd46e778fb9a 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStatusAction.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStatusAction.java
@@ -24,10 +24,10 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsAction.java
index f8c8d24a0864b..c9c83560ed03c 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsAction.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetTopNFunctionsAction.java
@@ -15,7 +15,8 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
@@ -33,7 +34,7 @@ public class TransportGetTopNFunctionsAction extends TransportAction<GetStackTra
 
     @Inject
     public TransportGetTopNFunctionsAction(NodeClient nodeClient, TransportService transportService, ActionFilters actionFilters) {
-        super(GetTopNFunctionsAction.NAME, actionFilters, transportService.getTaskManager());
+        super(GetTopNFunctionsAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.nodeClient = nodeClient;
         this.transportService = transportService;
     }
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingDataStreamManager.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingDataStreamManager.java
index 331d93b066da5..a048d23f188ee 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingDataStreamManager.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingDataStreamManager.java
@@ -141,8 +141,11 @@ public void onFailure(Exception e) {
     private void createDataStream(ProfilingDataStream dataStream, final ActionListener<? super ActionResponse> listener) {
         final Executor executor = threadPool.generic();
         executor.execute(() -> {
-            CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(dataStream.getName());
-            request.masterNodeTimeout(TimeValue.timeValueMinutes(1));
+            CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(
+                TimeValue.ONE_MINUTE /* TODO should we wait longer? */,
+                TimeValue.THIRTY_SECONDS /* TODO should we wait longer? */,
+                dataStream.getName()
+            );
             executeAsyncWithOrigin(
                 client.threadPool().getThreadContext(),
                 ClientHelper.PROFILING_ORIGIN,
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java
index 3b361748abf67..7d8a474453c4c 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java
@@ -53,10 +53,11 @@ public class ProfilingIndexTemplateRegistry extends IndexTemplateRegistry {
     // version 10: changed mapping profiling-events @timestamp to 'date_nanos' from 'date'
     // version 11: Added 'profiling.agent.protocol' keyword mapping to profiling-hosts
     // version 12: Added 'profiling.agent.env_https_proxy' keyword mapping to profiling-hosts
-    public static final int INDEX_TEMPLATE_VERSION = 12;
+    // version 13: Added 'container.id' keyword mapping to profiling-events
+    public static final int INDEX_TEMPLATE_VERSION = 13;
 
     // history for individual indices / index templates. Only bump these for breaking changes that require to create a new index
-    public static final int PROFILING_EVENTS_VERSION = 4;
+    public static final int PROFILING_EVENTS_VERSION = 5;
     public static final int PROFILING_EXECUTABLES_VERSION = 1;
     public static final int PROFILING_METRICS_VERSION = 2;
     public static final int PROFILING_HOSTS_VERSION = 2;
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
index e891e575e7de3..10aff2f4d68cd 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
@@ -46,9 +46,6 @@ public class RRFRankBuilder extends RankBuilder {
     static final ConstructingObjectParser<RRFRankBuilder, Void> PARSER = new ConstructingObjectParser<>(RRFRankPlugin.NAME, args -> {
         int windowSize = args[0] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[0];
         int rankConstant = args[1] == null ? DEFAULT_RANK_CONSTANT : (int) args[1];
-        if (rankConstant < 1) {
-            throw new IllegalArgumentException("[rank_constant] must be greater than [0] for [rrf]");
-        }
         return new RRFRankBuilder(windowSize, rankConstant);
     });
 
@@ -73,6 +70,11 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio
 
     public RRFRankBuilder(int rankWindowSize, int rankConstant) {
         super(rankWindowSize);
+
+        if (rankConstant < 1) {
+            throw new IllegalArgumentException("[rank_constant] must be greater or equal to [1] for [rrf]");
+        }
+
         this.rankConstant = rankConstant;
     }
 
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java
index 8f078c0c4d116..4dbc9a6a54dcf 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java
@@ -7,9 +7,11 @@
 
 package org.elasticsearch.xpack.rank.rrf;
 
+import org.apache.lucene.search.Explanation;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -54,6 +56,32 @@ public RRFRankDoc(StreamInput in) throws IOException {
         scores = in.readFloatArray();
     }
 
+    @Override
+    public Explanation explain() {
+        // ideally we'd need access to the rank constant to provide score info for this one
+        int queries = positions.length;
+        Explanation[] details = new Explanation[queries];
+        for (int i = 0; i < queries; i++) {
+            final String queryIndex = "at index [" + i + "]";
+            if (positions[i] == RRFRankDoc.NO_RANK) {
+                final String description = "rrf score: [0], result not found in query " + queryIndex;
+                details[i] = Explanation.noMatch(description);
+            } else {
+                final int rank = positions[i] + 1;
+                details[i] = Explanation.match(rank, "rank [" + (rank) + "] in query " + queryIndex);
+            }
+        }
+        return Explanation.match(
+            score,
+            "rrf score: ["
+                + score
+                + "] computed for initial ranks "
+                + Arrays.toString(Arrays.stream(positions).map(x -> x + 1).toArray())
+                + "] as sum of [1 / (rank + rankConstant)] for each query",
+            details
+        );
+    }
+
     @Override
     public void doWriteTo(StreamOutput out) throws IOException {
         out.writeVInt(rank);
@@ -96,4 +124,10 @@ public String toString() {
     public String getWriteableName() {
         return NAME;
     }
+
+    @Override
+    protected void doToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("positions", positions);
+        builder.field("scores", scores);
+    }
 }
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
index e5a7983107278..0d6208e474eea 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.retriever.RetrieverBuilder;
@@ -74,6 +75,11 @@ public static RRFRetrieverBuilder fromXContent(XContentParser parser, RetrieverP
     int rankWindowSize = RRFRankBuilder.DEFAULT_RANK_WINDOW_SIZE;
     int rankConstant = RRFRankBuilder.DEFAULT_RANK_CONSTANT;
 
+    @Override
+    public QueryBuilder topDocsQuery() {
+        throw new IllegalStateException("{" + getName() + "} cannot be nested");
+    }
+
     @Override
     public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) {
         if (compoundUsed) {
diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java
index 8c0eafe3ab022..0b8ee30fe0680 100644
--- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java
+++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java
@@ -71,4 +71,9 @@ protected RRFRankDoc mutateInstance(RRFRankDoc instance) throws IOException {
         }
         return mutated;
     }
+
+    public void testExplain() {
+        RRFRankDoc instance = createTestRRFRankDoc();
+        assertEquals(instance.explain().toString(), instance.explain().toString());
+    }
 }
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml
index a4972d0557dab..4f76f52409810 100644
--- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml
@@ -33,7 +33,7 @@ setup:
         body:
           text: "term term"
           keyword: "other"
-          vector: [0.0]
+          vector: [ 0.0 ]
 
   - do:
       index:
@@ -42,7 +42,7 @@ setup:
         body:
           text: "other"
           keyword: "other"
-          vector: [1.0]
+          vector: [ 1.0 ]
 
   - do:
       index:
@@ -51,10 +51,10 @@ setup:
         body:
           text: "term"
           keyword: "keyword"
-          vector: [2.0]
+          vector: [ 2.0 ]
 
   - do:
-      indices.refresh: {}
+      indices.refresh: { }
 
 ---
 "Simple rank with bm25 search and kNN search":
@@ -67,7 +67,7 @@ setup:
           fields: [ "text", "keyword" ]
           knn:
             field: vector
-            query_vector: [0.0]
+            query_vector: [ 0.0 ]
             k: 3
             num_candidates: 3
           query:
@@ -125,7 +125,7 @@ setup:
               rank_constant: 1
           size: 10
 
-  - match: { hits.total.value : 2 }
+  - match: { hits.total.value: 2 }
 
   - match: { hits.hits.0._id: "3" }
   - match: { hits.hits.0._rank: 1 }
@@ -173,7 +173,7 @@ setup:
               rank_constant: 1
           size: 10
 
-  - match: { hits.total.value : 3 }
+  - match: { hits.total.value: 3 }
 
   - match: { hits.hits.0._id: "3" }
   - match: { hits.hits.0._rank: 1 }
@@ -227,3 +227,43 @@ setup:
               rank_window_size: 2
               rank_constant: 1
           size: 10
+
+---
+"RRF rank should fail if rank_constant < 1":
+  - requires:
+      cluster_features: "gte_v8.16.0"
+      reason: 'validation fixed in 8.16.0'
+
+  - do:
+      catch: "/\\[rank_constant\\] must be greater or equal to \\[1\\] for \\[rrf\\]/"
+      search:
+        index: test
+        body:
+          track_total_hits: true
+          fields: [ "text", "keyword" ]
+          knn:
+            field: vector
+            query_vector: [ 0.0 ]
+            k: 3
+            num_candidates: 3
+          sub_searches: [
+            {
+              "query": {
+                "term": {
+                  "text": "term"
+                }
+              }
+            },
+            {
+              "query": {
+                "match": {
+                  "keyword": "keyword"
+                }
+              }
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 10
+              rank_constant: 0.3
+          size: 10
diff --git a/x-pack/plugin/repositories-metering-api/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java b/x-pack/plugin/repositories-metering-api/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java
index 882e4c342592c..7029a38edcb5a 100644
--- a/x-pack/plugin/repositories-metering-api/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java
+++ b/x-pack/plugin/repositories-metering-api/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/repositories/metering/azure/AzureRepositoriesMeteringIT.java
@@ -10,6 +10,8 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestTrustStore;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.xpack.repositories.metering.AbstractRepositoriesMeteringAPIRestTestCase;
 import org.junit.ClassRule;
@@ -21,12 +23,26 @@
 
 public class AzureRepositoriesMeteringIT extends AbstractRepositoriesMeteringAPIRestTestCase {
     private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.azure.fixture", "true"));
+    private static final boolean USE_HTTPS_FIXTURE = USE_FIXTURE && ESTestCase.inFipsJvm() == false;
+    // TODO when https://github.com/elastic/elasticsearch/issues/111532 addressed, use a HTTPS fixture in FIPS mode too
+
     private static final String AZURE_TEST_ACCOUNT = System.getProperty("test.azure.account");
     private static final String AZURE_TEST_CONTAINER = System.getProperty("test.azure.container");
     private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key");
     private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token");
 
-    private static AzureHttpFixture fixture = new AzureHttpFixture(USE_FIXTURE, AZURE_TEST_ACCOUNT, AZURE_TEST_CONTAINER);
+    private static AzureHttpFixture fixture = new AzureHttpFixture(
+        USE_HTTPS_FIXTURE ? AzureHttpFixture.Protocol.HTTPS : USE_FIXTURE ? AzureHttpFixture.Protocol.HTTP : AzureHttpFixture.Protocol.NONE,
+        AZURE_TEST_ACCOUNT,
+        AZURE_TEST_CONTAINER,
+        System.getProperty("test.azure.tenant_id"),
+        System.getProperty("test.azure.client_id"),
+        AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_TEST_ACCOUNT)
+    );
+
+    private static TestTrustStore trustStore = new TestTrustStore(
+        () -> AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem")
+    );
 
     private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-azure")
@@ -44,13 +60,15 @@ public class AzureRepositoriesMeteringIT extends AbstractRepositoriesMeteringAPI
         )
         .setting(
             "azure.client.repositories_metering.endpoint_suffix",
-            () -> "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + fixture.getAddress(),
+            () -> "ignored;DefaultEndpointsProtocol=https;BlobEndpoint=" + fixture.getAddress(),
             s -> USE_FIXTURE
         )
+        .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_HTTPS_FIXTURE)
+        .systemProperty("javax.net.ssl.trustStoreType", () -> "jks", s -> USE_HTTPS_FIXTURE)
         .build();
 
-    @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
+    @ClassRule(order = 1)
+    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster);
 
     @Override
     protected String repositoryType() {
diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportClearRepositoriesStatsArchiveAction.java
index d68903ae857ee..a7ffc096f6ffb 100644
--- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportClearRepositoriesStatsArchiveAction.java
+++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportClearRepositoriesStatsArchiveAction.java
@@ -12,9 +12,9 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.RepositoryStatsSnapshot;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportRepositoriesStatsAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportRepositoriesStatsAction.java
index 87a990be5f8af..cb7d274814483 100644
--- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportRepositoriesStatsAction.java
+++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportRepositoriesStatsAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupInfoTransportAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupInfoTransportAction.java
index 0bbd27c7281de..3a26889de5846 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupInfoTransportAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupInfoTransportAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.rollup.action;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupUsageTransportAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupUsageTransportAction.java
index 4d3a9ef933255..63264a2a9dd02 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupUsageTransportAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupUsageTransportAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.Predicates;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java
index 6923ab2a1f618..0d5a9c86cc3b8 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java
index 7a390a789d7d4..b4e468ac0bffe 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java
index 8f41db3810f0c..c2a81c6bb16ef 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java
index 646efed572df5..12cea1c305020 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupJobAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
index b6f91c71445a4..6618f3199debf 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.time.DateUtils;
@@ -40,6 +39,7 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.rest.RestStatus;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
index 4108b0f6d3c83..34d788d5f094d 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -39,6 +38,7 @@
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.query.TermsQueryBuilder;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationReduceContext;
@@ -95,7 +95,7 @@ public TransportRollupSearchAction(
         ClusterService clusterService,
         IndexNameExpressionResolver resolver
     ) {
-        super(RollupSearchAction.NAME, actionFilters, transportService.getTaskManager());
+        super(RollupSearchAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.client = client;
         this.registry = registry;
         this.bigArrays = bigArrays;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStartRollupAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStartRollupAction.java
index acbe8a0ed7ce1..24a04cac40092 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStartRollupAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStartRollupAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.rollup.action.StartRollupJobAction;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStopRollupAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStopRollupAction.java
index 6b518a68e4936..e4fe926f18feb 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStopRollupAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportStopRollupAction.java
@@ -14,9 +14,9 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
index f4c420db47ac3..5704d7837268b 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
@@ -463,8 +463,8 @@ public synchronized void onCancelled() {
     public synchronized void triggered(SchedulerEngine.Event event) {
         // Verify this is actually the event that we care about, then trigger the indexer.
         // Note that the status of the indexer is checked in the indexer itself
-        if (event.getJobName().equals(SCHEDULE_NAME + "_" + job.getConfig().getId())) {
-            logger.debug("Rollup indexer [" + event.getJobName() + "] schedule has triggered, state: [" + indexer.getState() + "]");
+        if (event.jobName().equals(SCHEDULE_NAME + "_" + job.getConfig().getId())) {
+            logger.debug("Rollup indexer [" + event.jobName() + "] schedule has triggered, state: [" + indexer.getState() + "]");
             indexer.maybeTriggerAsyncJob(System.currentTimeMillis());
         }
     }
diff --git a/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java b/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
index 9a28e11ee1cbb..2a17a4a1152cf 100644
--- a/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
+++ b/x-pack/plugin/search-business-rules/src/internalClusterTest/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder.Item;
 
 import java.util.Collection;
 import java.util.Collections;
@@ -90,11 +89,11 @@ public void testPinnedPromotions() throws Exception {
             int numPromotions = randomIntBetween(0, totalDocs);
 
             LinkedHashSet<String> idPins = new LinkedHashSet<>();
-            LinkedHashSet<Item> docPins = new LinkedHashSet<>();
+            LinkedHashSet<SpecifiedDocument> docPins = new LinkedHashSet<>();
             for (int j = 0; j < numPromotions; j++) {
                 String id = Integer.toString(randomIntBetween(0, totalDocs));
                 idPins.add(id);
-                docPins.add(new Item("test", id));
+                docPins.add(new SpecifiedDocument("test", id));
             }
             QueryBuilder organicQuery = null;
             if (i % 5 == 0) {
@@ -105,7 +104,12 @@ public void testPinnedPromotions() throws Exception {
             }
 
             assertPinnedPromotions(new PinnedQueryBuilder(organicQuery, idPins.toArray(new String[0])), idPins, i, numRelevantDocs);
-            assertPinnedPromotions(new PinnedQueryBuilder(organicQuery, docPins.toArray(new Item[0])), idPins, i, numRelevantDocs);
+            assertPinnedPromotions(
+                new PinnedQueryBuilder(organicQuery, docPins.toArray(new SpecifiedDocument[0])),
+                idPins,
+                i,
+                numRelevantDocs
+            );
         }
 
     }
@@ -184,7 +188,7 @@ public void testExhaustiveScoring() throws Exception {
 
         QueryBuilder organicQuery = QueryBuilders.queryStringQuery("foo");
         assertExhaustiveScoring(new PinnedQueryBuilder(organicQuery, "2"));
-        assertExhaustiveScoring(new PinnedQueryBuilder(organicQuery, new Item("test", "2")));
+        assertExhaustiveScoring(new PinnedQueryBuilder(organicQuery, new SpecifiedDocument("test", "2")));
     }
 
     private void assertExhaustiveScoring(PinnedQueryBuilder pqb) {
@@ -218,7 +222,7 @@ public void testExplain() throws Exception {
 
         QueryBuilder organicQuery = QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR);
         assertExplain(new PinnedQueryBuilder(organicQuery, "2"));
-        assertExplain(new PinnedQueryBuilder(organicQuery, new Item("test", "2")));
+        assertExplain(new PinnedQueryBuilder(organicQuery, new SpecifiedDocument("test", "2")));
     }
 
     private void assertExplain(PinnedQueryBuilder pqb) {
@@ -259,7 +263,7 @@ public void testHighlight() throws Exception {
 
         QueryBuilder organicQuery = QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR);
         assertHighlight(new PinnedQueryBuilder(organicQuery, "2"));
-        assertHighlight(new PinnedQueryBuilder(organicQuery, new Item("test", "2")));
+        assertHighlight(new PinnedQueryBuilder(organicQuery, new SpecifiedDocument("test", "2")));
     }
 
     private void assertHighlight(PinnedQueryBuilder pqb) {
@@ -320,9 +324,9 @@ public void testMultiIndexDocs() throws Exception {
 
         PinnedQueryBuilder pqb = new PinnedQueryBuilder(
             QueryBuilders.queryStringQuery("foo"),
-            new Item("test2", "a"),
-            new Item("test1", "a"),
-            new Item("test1", "b")
+            new SpecifiedDocument("test2", "a"),
+            new SpecifiedDocument("test1", "a"),
+            new SpecifiedDocument("test1", "b")
         );
 
         assertResponse(prepareSearch().setQuery(pqb).setTrackTotalHits(true).setSearchType(DFS_QUERY_THEN_FETCH), searchResponse -> {
@@ -360,9 +364,9 @@ public void testMultiIndexWithAliases() throws Exception {
 
         PinnedQueryBuilder pqb = new PinnedQueryBuilder(
             QueryBuilders.queryStringQuery("document"),
-            new Item("test", "b"),
-            new Item("test-alias", "a"),
-            new Item("test", "a")
+            new SpecifiedDocument("test", "b"),
+            new SpecifiedDocument("test-alias", "a"),
+            new SpecifiedDocument("test", "a")
         );
 
         assertResponse(prepareSearch().setQuery(pqb).setTrackTotalHits(true).setSearchType(DFS_QUERY_THEN_FETCH), searchResponse -> {
@@ -416,11 +420,11 @@ public void testMultiIndexWithAliasesAndDuplicateIds() throws Exception {
 
         PinnedQueryBuilder pqb = new PinnedQueryBuilder(
             QueryBuilders.queryStringQuery("document"),
-            new Item("test1", "b"),
-            new Item(null, "a"),
-            new Item("test1", "c"),
-            new Item("test1", "a"),
-            new Item("test-alias", "a")
+            new SpecifiedDocument("test1", "b"),
+            new SpecifiedDocument(null, "a"),
+            new SpecifiedDocument("test1", "c"),
+            new SpecifiedDocument("test1", "a"),
+            new SpecifiedDocument("test-alias", "a")
         );
 
         assertResponse(prepareSearch().setQuery(pqb).setTrackTotalHits(true).setSearchType(DFS_QUERY_THEN_FETCH), searchResponse -> {
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java
index 45f1fc4939bc1..0ef7eebffadaf 100644
--- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java
+++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java
@@ -16,11 +16,8 @@
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
@@ -29,7 +26,6 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
@@ -57,132 +53,26 @@ public class PinnedQueryBuilder extends AbstractQueryBuilder<PinnedQueryBuilder>
     public static final ParseField DOCS_FIELD = new ParseField("docs");
     public static final ParseField ORGANIC_QUERY_FIELD = new ParseField("organic");
 
-    private static final TransportVersion OPTIONAL_INDEX_IN_DOCS_VERSION = TransportVersions.V_8_11_X;
-
     private final List<String> ids;
-    private final List<Item> docs;
+    private final List<SpecifiedDocument> docs;
     private QueryBuilder organicQuery;
 
     // Organic queries will have their scores capped to this number range,
     // We reserve the highest float exponent for scores of pinned queries
     private static final float MAX_ORGANIC_SCORE = Float.intBitsToFloat((0xfe << 23)) - 1;
 
-    /**
-     * A single item to be used for a {@link PinnedQueryBuilder}.
-     */
-    public static final class Item implements ToXContentObject, Writeable {
-        public static final String NAME = "item";
-
-        public static final ParseField INDEX_FIELD = new ParseField("_index");
-        public static final ParseField ID_FIELD = new ParseField("_id");
-
-        private final String index;
-        private final String id;
-
-        /**
-         * Constructor for a given item request
-         *
-         * @param index the index where the document is located
-         * @param id and its id
-         */
-        public Item(String index, String id) {
-            if (index != null && Regex.isSimpleMatchPattern(index)) {
-                throw new IllegalArgumentException("Item index cannot contain wildcard expressions");
-            }
-            if (id == null) {
-                throw new IllegalArgumentException("Item requires id to be non-null");
-            }
-            this.index = index;
-            this.id = id;
-        }
-
-        private Item(String id) {
-            this.index = null;
-            this.id = id;
-        }
-
-        /**
-         * Read from a stream.
-         */
-        public Item(StreamInput in) throws IOException {
-            if (in.getTransportVersion().onOrAfter(OPTIONAL_INDEX_IN_DOCS_VERSION)) {
-                index = in.readOptionalString();
-            } else {
-                index = in.readString();
-            }
-            id = in.readString();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            if (out.getTransportVersion().onOrAfter(OPTIONAL_INDEX_IN_DOCS_VERSION)) {
-                out.writeOptionalString(index);
-            } else {
-                if (index == null) {
-                    throw new IllegalArgumentException(
-                        "[_index] needs to be specified for docs elements when cluster nodes are not in the same version"
-                    );
-                }
-                out.writeString(index);
-            }
-            out.writeString(id);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            if (this.index != null) {
-                builder.field(INDEX_FIELD.getPreferredName(), this.index);
-            }
-            builder.field(ID_FIELD.getPreferredName(), this.id);
-            return builder.endObject();
-        }
-
-        private static final ConstructingObjectParser<Item, Void> PARSER = new ConstructingObjectParser<>(
-            NAME,
-            a -> new Item((String) a[0], (String) a[1])
-        );
-
-        static {
-            PARSER.declareString(optionalConstructorArg(), INDEX_FIELD);
-            PARSER.declareString(constructorArg(), ID_FIELD);
-        }
-
-        @Override
-        public String toString() {
-            return Strings.toString(this, true, true);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(index, id);
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) {
-                return true;
-            }
-            if ((o instanceof Item) == false) {
-                return false;
-            }
-            Item other = (Item) o;
-            return Objects.equals(index, other.index) && Objects.equals(id, other.id);
-        }
-    }
-
     public PinnedQueryBuilder(QueryBuilder organicQuery, String... ids) {
         this(organicQuery, Arrays.asList(ids), null);
     }
 
-    public PinnedQueryBuilder(QueryBuilder organicQuery, Item... docs) {
+    public PinnedQueryBuilder(QueryBuilder organicQuery, SpecifiedDocument... docs) {
         this(organicQuery, null, Arrays.asList(docs));
     }
 
     /**
      * Creates a new PinnedQueryBuilder
      */
-    private PinnedQueryBuilder(QueryBuilder organicQuery, List<String> ids, List<Item> docs) {
+    private PinnedQueryBuilder(QueryBuilder organicQuery, List<String> ids, List<SpecifiedDocument> docs) {
         if (organicQuery == null) {
             throw new IllegalArgumentException("[" + NAME + "] organicQuery cannot be null");
         }
@@ -215,8 +105,8 @@ private PinnedQueryBuilder(QueryBuilder organicQuery, List<String> ids, List<Ite
                     "[" + NAME + "] Max of " + MAX_NUM_PINNED_HITS + " docs exceeded: " + docs.size() + " provided."
                 );
             }
-            LinkedHashSet<Item> deduped = new LinkedHashSet<>();
-            for (Item doc : docs) {
+            LinkedHashSet<SpecifiedDocument> deduped = new LinkedHashSet<>();
+            for (SpecifiedDocument doc : docs) {
                 if (doc == null) {
                     throw new IllegalArgumentException("[" + NAME + "] doc cannot be null");
                 }
@@ -239,7 +129,7 @@ public PinnedQueryBuilder(StreamInput in) throws IOException {
             docs = null;
         } else {
             ids = in.readOptionalStringCollectionAsList();
-            docs = in.readBoolean() ? in.readCollectionAsList(Item::new) : null;
+            docs = in.readBoolean() ? in.readCollectionAsList(SpecifiedDocument::new) : null;
         }
         organicQuery = in.readNamedWriteable(QueryBuilder.class);
     }
@@ -280,7 +170,7 @@ public List<String> ids() {
     /**
      * @return the pinned docs for the query.
      */
-    public List<Item> docs() {
+    public List<SpecifiedDocument> docs() {
         if (this.docs == null) {
             return Collections.emptyList();
         }
@@ -303,8 +193,8 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         }
         if (docs != null) {
             builder.startArray(DOCS_FIELD.getPreferredName());
-            for (Item item : docs) {
-                builder.value(item);
+            for (SpecifiedDocument specifiedDocument : docs) {
+                builder.value(specifiedDocument);
             }
             builder.endArray();
         }
@@ -317,13 +207,13 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         @SuppressWarnings("unchecked")
         List<String> ids = (List<String>) a[1];
         @SuppressWarnings("unchecked")
-        List<Item> docs = (List<Item>) a[2];
+        List<SpecifiedDocument> docs = (List<SpecifiedDocument>) a[2];
         return new PinnedQueryBuilder(organicQuery, ids, docs);
     });
     static {
         PARSER.declareObject(constructorArg(), (p, c) -> parseInnerQueryBuilder(p), ORGANIC_QUERY_FIELD);
         PARSER.declareStringArray(optionalConstructorArg(), IDS_FIELD);
-        PARSER.declareObjectArray(optionalConstructorArg(), Item.PARSER, DOCS_FIELD);
+        PARSER.declareObjectArray(optionalConstructorArg(), SpecifiedDocument.PARSER, DOCS_FIELD);
         declareStandardFields(PARSER);
     }
 
@@ -357,24 +247,26 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException {
         if (idField == null) {
             return new MatchNoDocsQuery("No mappings");
         }
-        List<Item> items = (docs != null) ? docs : ids.stream().map(id -> new Item(id)).toList();
-        if (items.isEmpty()) {
+        List<SpecifiedDocument> specifiedDocuments = (docs != null)
+            ? docs
+            : ids.stream().map(id -> new SpecifiedDocument(null, id)).toList();
+        if (specifiedDocuments.isEmpty()) {
             return new CappedScoreQuery(organicQuery.toQuery(context), MAX_ORGANIC_SCORE);
         } else {
             List<Query> pinnedQueries = new ArrayList<>();
 
             // Ensure each pin order using a Boost query with the relevant boost factor
             int minPin = NumericUtils.floatToSortableInt(MAX_ORGANIC_SCORE) + 1;
-            int boostNum = minPin + items.size();
+            int boostNum = minPin + specifiedDocuments.size();
             float lastScore = Float.MAX_VALUE;
-            for (Item item : items) {
+            for (SpecifiedDocument specifiedDocument : specifiedDocuments) {
                 float pinScore = NumericUtils.sortableIntToFloat(boostNum);
                 assert pinScore < lastScore;
                 lastScore = pinScore;
                 boostNum--;
-                if (item.index == null || context.indexMatches(item.index)) {
+                if (specifiedDocument.index() == null || context.indexMatches(specifiedDocument.index())) {
                     // Ensure the pin order using a Boost query with the relevant boost factor
-                    Query idQuery = new BoostQuery(new ConstantScoreQuery(idField.termQuery(item.id, context)), pinScore);
+                    Query idQuery = new BoostQuery(new ConstantScoreQuery(idField.termQuery(specifiedDocument.id(), context)), pinScore);
                     pinnedQueries.add(idQuery);
                 }
             }
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java
new file mode 100644
index 0000000000000..8c3801630fb2b
--- /dev/null
+++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SpecifiedDocument.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.searchbusinessrules;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * A single specified document to be used for a {@link PinnedQueryBuilder} or query rules.
+ */
+public final class SpecifiedDocument implements ToXContentObject, Writeable {
+
+    public static final TransportVersion OPTIONAL_INDEX_IN_DOCS_VERSION = TransportVersions.V_8_11_X;
+
+    public static final String NAME = "specified_document";
+
+    public static final ParseField INDEX_FIELD = new ParseField("_index");
+    public static final ParseField ID_FIELD = new ParseField("_id");
+
+    private final String index;
+    private final String id;
+
+    /**
+     * Constructor for a given specified document request
+     *
+     * @param index the index where the document is located
+     * @param id and its id
+     */
+    public SpecifiedDocument(String index, String id) {
+        if (index != null && Regex.isSimpleMatchPattern(index)) {
+            throw new IllegalArgumentException("Specified document index cannot contain wildcard expressions");
+        }
+        if (id == null) {
+            throw new IllegalArgumentException("Specified document requires id to be non-null");
+        }
+        this.index = index;
+        this.id = id;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public SpecifiedDocument(StreamInput in) throws IOException {
+        if (in.getTransportVersion().onOrAfter(OPTIONAL_INDEX_IN_DOCS_VERSION)) {
+            index = in.readOptionalString();
+        } else {
+            index = in.readString();
+        }
+        id = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        if (out.getTransportVersion().onOrAfter(OPTIONAL_INDEX_IN_DOCS_VERSION)) {
+            out.writeOptionalString(index);
+        } else {
+            if (index == null) {
+                throw new IllegalArgumentException(
+                    "[_index] needs to be specified for docs elements when cluster nodes are not in the same version"
+                );
+            }
+            out.writeString(index);
+        }
+        out.writeString(id);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (this.index != null) {
+            builder.field(INDEX_FIELD.getPreferredName(), this.index);
+        }
+        builder.field(ID_FIELD.getPreferredName(), this.id);
+        return builder.endObject();
+    }
+
+    static final ConstructingObjectParser<SpecifiedDocument, Void> PARSER = new ConstructingObjectParser<>(
+        NAME,
+        a -> new SpecifiedDocument((String) a[0], (String) a[1])
+    );
+
+    static {
+        PARSER.declareString(optionalConstructorArg(), INDEX_FIELD);
+        PARSER.declareString(constructorArg(), ID_FIELD);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(index, id);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if ((o instanceof SpecifiedDocument) == false) {
+            return false;
+        }
+        SpecifiedDocument other = (SpecifiedDocument) o;
+        return Objects.equals(index, other.index) && Objects.equals(id, other.id);
+    }
+
+    public String id() {
+        return id;
+    }
+
+    public String index() {
+        return index;
+    }
+}
diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java
index 3f41d186d9f8a..56d652b05f100 100644
--- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java
+++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xcontent.json.JsonStringEncoder;
-import org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilder.Item;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -91,8 +90,13 @@ private static QueryBuilder createTestTermQueryBuilder() {
         return new TermQueryBuilder(fieldName, value);
     }
 
-    private static Item[] generateRandomItems() {
-        return randomArray(1, 100, Item[]::new, () -> new Item(randomBoolean() ? null : randomAlphaOfLength(64), randomAlphaOfLength(256)));
+    private static SpecifiedDocument[] generateRandomItems() {
+        return randomArray(
+            1,
+            100,
+            SpecifiedDocument[]::new,
+            () -> new SpecifiedDocument(randomBoolean() ? null : randomAlphaOfLength(64), randomAlphaOfLength(256))
+        );
     }
 
     @Override
@@ -121,24 +125,29 @@ public void testIllegalArguments() {
         expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), (String) null));
         expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(null, "1"));
         expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), "1", null, "2"));
+        expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), (SpecifiedDocument) null));
+        expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(null, new SpecifiedDocument("test", "1")));
         expectThrows(
             IllegalArgumentException.class,
-            () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), (PinnedQueryBuilder.Item) null)
+            () -> new PinnedQueryBuilder(
+                new MatchAllQueryBuilder(),
+                new SpecifiedDocument("test", "1"),
+                null,
+                new SpecifiedDocument("test", "2")
+            )
         );
-        expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(null, new Item("test", "1")));
         expectThrows(
             IllegalArgumentException.class,
-            () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), new Item("test", "1"), null, new Item("test", "2"))
+            () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), new SpecifiedDocument("test*", "1"))
         );
-        expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), new Item("test*", "1")));
         String[] bigIdList = new String[PinnedQueryBuilder.MAX_NUM_PINNED_HITS + 1];
-        Item[] bigItemList = new Item[PinnedQueryBuilder.MAX_NUM_PINNED_HITS + 1];
+        SpecifiedDocument[] bigSpecifiedDocumentList = new SpecifiedDocument[PinnedQueryBuilder.MAX_NUM_PINNED_HITS + 1];
         for (int i = 0; i < bigIdList.length; i++) {
             bigIdList[i] = String.valueOf(i);
-            bigItemList[i] = new Item("test", String.valueOf(i));
+            bigSpecifiedDocumentList[i] = new SpecifiedDocument("test", String.valueOf(i));
         }
         expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), bigIdList));
-        expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), bigItemList));
+        expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), bigSpecifiedDocumentList));
 
     }
 
@@ -214,7 +223,7 @@ public void testIdsRewrite() throws IOException {
     }
 
     public void testDocsRewrite() throws IOException {
-        PinnedQueryBuilder pinnedQueryBuilder = new PinnedQueryBuilder(new TermQueryBuilder("foo", 1), new Item("test", "1"));
+        PinnedQueryBuilder pinnedQueryBuilder = new PinnedQueryBuilder(new TermQueryBuilder("foo", 1), new SpecifiedDocument("test", "1"));
         QueryBuilder rewritten = pinnedQueryBuilder.rewrite(createSearchExecutionContext());
         assertThat(rewritten, instanceOf(PinnedQueryBuilder.class));
     }
@@ -239,12 +248,16 @@ public void testIdInsertionOrderRetained() {
     }
 
     public void testDocInsertionOrderRetained() {
-        Item[] items = randomArray(10, Item[]::new, () -> new Item(randomAlphaOfLength(64), randomAlphaOfLength(256)));
-        PinnedQueryBuilder pqb = new PinnedQueryBuilder(new MatchAllQueryBuilder(), items);
-        List<Item> addedDocs = pqb.docs();
+        SpecifiedDocument[] specifiedDocuments = randomArray(
+            10,
+            SpecifiedDocument[]::new,
+            () -> new SpecifiedDocument(randomAlphaOfLength(64), randomAlphaOfLength(256))
+        );
+        PinnedQueryBuilder pqb = new PinnedQueryBuilder(new MatchAllQueryBuilder(), specifiedDocuments);
+        List<SpecifiedDocument> addedDocs = pqb.docs();
         int pos = 0;
-        for (Item item : addedDocs) {
-            assertEquals(items[pos++], item);
+        for (SpecifiedDocument specifiedDocument : addedDocs) {
+            assertEquals(specifiedDocuments[pos++], specifiedDocument);
         }
     }
 }
diff --git a/x-pack/plugin/searchable-snapshots/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/AzureSearchableSnapshotsIT.java b/x-pack/plugin/searchable-snapshots/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/AzureSearchableSnapshotsIT.java
index 35f00f8f0e022..610b58453716c 100644
--- a/x-pack/plugin/searchable-snapshots/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/AzureSearchableSnapshotsIT.java
+++ b/x-pack/plugin/searchable-snapshots/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/searchablesnapshots/AzureSearchableSnapshotsIT.java
@@ -11,6 +11,8 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestTrustStore;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.ClassRule;
@@ -22,12 +24,26 @@
 
 public class AzureSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase {
     private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.azure.fixture", "true"));
+    private static final boolean USE_HTTPS_FIXTURE = USE_FIXTURE && ESTestCase.inFipsJvm() == false;
+    // TODO when https://github.com/elastic/elasticsearch/issues/111532 addressed, use a HTTPS fixture in FIPS mode too
+
     private static final String AZURE_TEST_ACCOUNT = System.getProperty("test.azure.account");
     private static final String AZURE_TEST_CONTAINER = System.getProperty("test.azure.container");
     private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key");
     private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token");
 
-    private static AzureHttpFixture fixture = new AzureHttpFixture(USE_FIXTURE, AZURE_TEST_ACCOUNT, AZURE_TEST_CONTAINER);
+    private static AzureHttpFixture fixture = new AzureHttpFixture(
+        USE_HTTPS_FIXTURE ? AzureHttpFixture.Protocol.HTTPS : USE_FIXTURE ? AzureHttpFixture.Protocol.HTTP : AzureHttpFixture.Protocol.NONE,
+        AZURE_TEST_ACCOUNT,
+        AZURE_TEST_CONTAINER,
+        System.getProperty("test.azure.tenant_id"),
+        System.getProperty("test.azure.client_id"),
+        AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_TEST_ACCOUNT)
+    );
+
+    private static TestTrustStore trustStore = new TestTrustStore(
+        () -> AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem")
+    );
 
     private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-azure")
@@ -53,10 +69,12 @@ public class AzureSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestT
         .setting("xpack.searchable.snapshot.shared_cache.size", "16MB")
         .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB")
         .setting("xpack.searchable_snapshots.cache_fetch_async_thread_pool.keep_alive", "0ms")
+        .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_HTTPS_FIXTURE)
+        .systemProperty("javax.net.ssl.trustStoreType", () -> "jks", s -> USE_HTTPS_FIXTURE)
         .build();
 
-    @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
+    @ClassRule(order = 1)
+    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster);
 
     @Override
     protected final Settings restClientSettings() {
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
index 5204bdfcc78e6..d5e87558d1ced 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.shard.IndexLongFieldRange;
+import org.elasticsearch.indices.DateFieldRangeInfo;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.Plugin;
@@ -100,11 +101,11 @@ public void testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQuerying
 
         final String indexOutsideSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         final int indexOutsideSearchRangeShardCount = randomIntBetween(1, 3);
-        createIndexWithTimestamp(indexOutsideSearchRange, indexOutsideSearchRangeShardCount, Settings.EMPTY);
+        createIndexWithTimestampAndEventIngested(indexOutsideSearchRange, indexOutsideSearchRangeShardCount, Settings.EMPTY);
 
         final String indexWithinSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         final int indexWithinSearchRangeShardCount = randomIntBetween(1, 3);
-        createIndexWithTimestamp(
+        createIndexWithTimestampAndEventIngested(
             indexWithinSearchRange,
             indexWithinSearchRangeShardCount,
             Settings.builder()
@@ -117,11 +118,10 @@ public void testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQuerying
         // Either add data outside of the range, or documents that don't have timestamp data
         final boolean indexDataWithTimestamp = randomBoolean();
         // Add enough documents to have non-metadata segment files in all shards,
-        // otherwise the mount operation might go through as the read won't be
-        // blocked
+        // otherwise the mount operation might go through as the read won't be blocked
         final int numberOfDocsInIndexOutsideSearchRange = between(350, 1000);
         if (indexDataWithTimestamp) {
-            indexDocumentsWithTimestampWithinDate(
+            indexDocumentsWithTimestampAndEventIngestedDates(
                 indexOutsideSearchRange,
                 numberOfDocsInIndexOutsideSearchRange,
                 TIMESTAMP_TEMPLATE_OUTSIDE_RANGE
@@ -132,7 +132,7 @@ public void testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQuerying
 
         // Index enough documents to ensure that all shards have at least some documents
         int numDocsWithinRange = between(100, 1000);
-        indexDocumentsWithTimestampWithinDate(indexWithinSearchRange, numDocsWithinRange, TIMESTAMP_TEMPLATE_WITHIN_RANGE);
+        indexDocumentsWithTimestampAndEventIngestedDates(indexWithinSearchRange, numDocsWithinRange, TIMESTAMP_TEMPLATE_WITHIN_RANGE);
 
         final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         createRepository(repositoryName, "mock");
@@ -166,9 +166,10 @@ public void testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQuerying
 
         final IndexMetadata indexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
         assertThat(indexMetadata.getTimestampRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
+        assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
 
-        DateFieldMapper.DateFieldType timestampFieldType = indicesService.getTimestampFieldType(indexMetadata.getIndex());
-        assertThat(timestampFieldType, nullValue());
+        DateFieldRangeInfo timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(indexMetadata.getIndex());
+        assertThat(timestampFieldTypeInfo, nullValue());
 
         final boolean includeIndexCoveringSearchRangeInSearchRequest = randomBoolean();
         List<String> indicesToSearch = new ArrayList<>();
@@ -176,7 +177,9 @@ public void testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQuerying
             indicesToSearch.add(indexWithinSearchRange);
         }
         indicesToSearch.add(searchableSnapshotIndexOutsideSearchRange);
-        RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(DataStream.TIMESTAMP_FIELD_NAME)
+
+        String timeField = randomFrom(IndexMetadata.EVENT_INGESTED_FIELD_NAME, DataStream.TIMESTAMP_FIELD_NAME);
+        RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(timeField)
             .from("2020-11-28T00:00:00.000000000Z", true)
             .to("2020-11-29T00:00:00.000000000Z");
 
@@ -250,20 +253,44 @@ public void testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQuerying
         ensureGreen(searchableSnapshotIndexOutsideSearchRange);
 
         final IndexMetadata updatedIndexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
+
+        // check that @timestamp and 'event.ingested' are now in cluster state
         final IndexLongFieldRange updatedTimestampMillisRange = updatedIndexMetadata.getTimestampRange();
-        final DateFieldMapper.DateFieldType dateFieldType = indicesService.getTimestampFieldType(updatedIndexMetadata.getIndex());
-        assertThat(dateFieldType, notNullValue());
-        final DateFieldMapper.Resolution resolution = dateFieldType.resolution();
         assertThat(updatedTimestampMillisRange.isComplete(), equalTo(true));
+        final IndexLongFieldRange updatedEventIngestedRange = updatedIndexMetadata.getEventIngestedRange();
+        assertThat(updatedEventIngestedRange.isComplete(), equalTo(true));
+
+        timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(updatedIndexMetadata.getIndex());
+        final DateFieldMapper.DateFieldType timestampDataFieldType = timestampFieldTypeInfo.timestampFieldType();
+        assertThat(timestampDataFieldType, notNullValue());
+        final DateFieldMapper.DateFieldType eventIngestedDataFieldType = timestampFieldTypeInfo.eventIngestedFieldType();
+        assertThat(eventIngestedDataFieldType, notNullValue());
+
+        final DateFieldMapper.Resolution timestampResolution = timestampDataFieldType.resolution();
+        final DateFieldMapper.Resolution eventIngestedResolution = eventIngestedDataFieldType.resolution();
         if (indexDataWithTimestamp) {
             assertThat(updatedTimestampMillisRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
             assertThat(
                 updatedTimestampMillisRange.getMin(),
-                greaterThanOrEqualTo(resolution.convert(Instant.parse("2020-11-26T00:00:00Z")))
+                greaterThanOrEqualTo(timestampResolution.convert(Instant.parse("2020-11-26T00:00:00Z")))
+            );
+            assertThat(
+                updatedTimestampMillisRange.getMax(),
+                lessThanOrEqualTo(timestampResolution.convert(Instant.parse("2020-11-27T00:00:00Z")))
+            );
+
+            assertThat(updatedEventIngestedRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
+            assertThat(
+                updatedEventIngestedRange.getMin(),
+                greaterThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-26T00:00:00Z")))
+            );
+            assertThat(
+                updatedEventIngestedRange.getMax(),
+                lessThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-27T00:00:00Z")))
             );
-            assertThat(updatedTimestampMillisRange.getMax(), lessThanOrEqualTo(resolution.convert(Instant.parse("2020-11-27T00:00:00Z"))));
         } else {
             assertThat(updatedTimestampMillisRange, sameInstance(IndexLongFieldRange.EMPTY));
+            assertThat(updatedEventIngestedRange, sameInstance(IndexLongFieldRange.EMPTY));
         }
 
         // Stop the node holding the searchable snapshots, and since we defined
@@ -383,6 +410,171 @@ public void testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQuerying
         }
     }
 
+    /**
+     * Test shard skipping when only 'event.ingested' is in the index and cluster state.
+     */
+    public void testEventIngestedRangeInSearchAgainstSearchableSnapshotShards() throws Exception {
+        internalCluster().startMasterOnlyNode();
+        internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
+        final String dataNodeHoldingRegularIndex = internalCluster().startDataOnlyNode();
+        final String dataNodeHoldingSearchableSnapshot = internalCluster().startDataOnlyNode();
+        final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, dataNodeHoldingSearchableSnapshot);
+
+        final String indexOutsideSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        final int indexOutsideSearchRangeShardCount = randomIntBetween(1, 3);
+
+        final String timestampField = IndexMetadata.EVENT_INGESTED_FIELD_NAME;
+
+        createIndexWithOnlyOneTimestampField(timestampField, indexOutsideSearchRange, indexOutsideSearchRangeShardCount, Settings.EMPTY);
+
+        final String indexWithinSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        final int indexWithinSearchRangeShardCount = randomIntBetween(1, 3);
+        createIndexWithOnlyOneTimestampField(
+            timestampField,
+            indexWithinSearchRange,
+            indexWithinSearchRangeShardCount,
+            Settings.builder()
+                .put(INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey(), dataNodeHoldingRegularIndex)
+                .build()
+        );
+
+        final int totalShards = indexOutsideSearchRangeShardCount + indexWithinSearchRangeShardCount;
+
+        // Add enough documents to have non-metadata segment files in all shards,
+        // otherwise the mount operation might go through as the read won't be blocked
+        final int numberOfDocsInIndexOutsideSearchRange = between(350, 1000);
+
+        indexDocumentsWithOnlyOneTimestampField(
+            timestampField,
+            indexOutsideSearchRange,
+            numberOfDocsInIndexOutsideSearchRange,
+            TIMESTAMP_TEMPLATE_OUTSIDE_RANGE
+        );
+
+        // Index enough documents to ensure that all shards have at least some documents
+        int numDocsWithinRange = between(100, 1000);
+        indexDocumentsWithOnlyOneTimestampField(
+            timestampField,
+            indexWithinSearchRange,
+            numDocsWithinRange,
+            TIMESTAMP_TEMPLATE_WITHIN_RANGE
+        );
+
+        final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        createRepository(repositoryName, "mock");
+
+        final SnapshotId snapshotId = createSnapshot(repositoryName, "snapshot-1", List.of(indexOutsideSearchRange)).snapshotId();
+        assertAcked(indicesAdmin().prepareDelete(indexOutsideSearchRange));
+
+        final String searchableSnapshotIndexOutsideSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+
+        // Block the repository for the node holding the searchable snapshot shards
+        // to delay its restore
+        blockDataNode(repositoryName, dataNodeHoldingSearchableSnapshot);
+
+        // Force the searchable snapshot to be allocated in a particular node
+        Settings restoredIndexSettings = Settings.builder()
+            .put(INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey(), dataNodeHoldingSearchableSnapshot)
+            .build();
+
+        final MountSearchableSnapshotRequest mountRequest = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
+            searchableSnapshotIndexOutsideSearchRange,
+            repositoryName,
+            snapshotId.getName(),
+            indexOutsideSearchRange,
+            restoredIndexSettings,
+            Strings.EMPTY_ARRAY,
+            false,
+            randomFrom(MountSearchableSnapshotRequest.Storage.values())
+        );
+        client().execute(MountSearchableSnapshotAction.INSTANCE, mountRequest).actionGet();
+
+        final IndexMetadata indexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
+        assertThat(indexMetadata.getTimestampRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
+        assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
+
+        // Allow the searchable snapshots to be finally mounted
+        unblockNode(repositoryName, dataNodeHoldingSearchableSnapshot);
+        waitUntilRecoveryIsDone(searchableSnapshotIndexOutsideSearchRange);
+        ensureGreen(searchableSnapshotIndexOutsideSearchRange);
+
+        IndexMetadata updatedIndexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
+        IndexLongFieldRange updatedTimestampMillisRange = updatedIndexMetadata.getTimestampRange();
+        IndexLongFieldRange updatedEventIngestedMillisRange = updatedIndexMetadata.getEventIngestedRange();
+
+        // @timestamp range should be null since it was not included in the index or indexed docs
+        assertThat(updatedTimestampMillisRange, equalTo(IndexLongFieldRange.UNKNOWN));
+        assertThat(updatedEventIngestedMillisRange, not(equalTo(IndexLongFieldRange.UNKNOWN)));
+
+        DateFieldRangeInfo timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(updatedIndexMetadata.getIndex());
+
+        DateFieldMapper.DateFieldType timestampDataFieldType = timestampFieldTypeInfo.timestampFieldType();
+        assertThat(timestampDataFieldType, nullValue());
+
+        DateFieldMapper.DateFieldType eventIngestedFieldType = timestampFieldTypeInfo.eventIngestedFieldType();
+        assertThat(eventIngestedFieldType, notNullValue());
+
+        DateFieldMapper.Resolution eventIngestedResolution = eventIngestedFieldType.resolution();
+        assertThat(updatedEventIngestedMillisRange.isComplete(), equalTo(true));
+        assertThat(
+            updatedEventIngestedMillisRange.getMin(),
+            greaterThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-26T00:00:00Z")))
+        );
+        assertThat(
+            updatedEventIngestedMillisRange.getMax(),
+            lessThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-27T00:00:00Z")))
+        );
+
+        // now do a search against event.ingested
+        List<String> indicesToSearch = new ArrayList<>();
+        indicesToSearch.add(indexWithinSearchRange);
+        indicesToSearch.add(searchableSnapshotIndexOutsideSearchRange);
+
+        {
+            RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(timestampField)
+                .from("2020-11-28T00:00:00.000000000Z", true)
+                .to("2020-11-29T00:00:00.000000000Z");
+
+            SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0]))
+                .source(new SearchSourceBuilder().query(rangeQuery));
+
+            assertResponse(client().search(request), searchResponse -> {
+                // All the regular index searches succeeded
+                assertThat(searchResponse.getSuccessfulShards(), equalTo(totalShards));
+                assertThat(searchResponse.getFailedShards(), equalTo(0));
+                // All the searchable snapshots shards were skipped
+                assertThat(searchResponse.getSkippedShards(), equalTo(indexOutsideSearchRangeShardCount));
+                assertThat(searchResponse.getTotalShards(), equalTo(totalShards));
+            });
+
+            SearchShardAPIResult searchShardResult = doSearchShardAPIQuery(indicesToSearch, rangeQuery, true, totalShards);
+            assertThat(searchShardResult.skipped().size(), equalTo(indexOutsideSearchRangeShardCount));
+            assertThat(searchShardResult.notSkipped().size(), equalTo(indexWithinSearchRangeShardCount));
+        }
+
+        // query a range that covers both indexes - all shards should be searched, none skipped
+        {
+            RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(timestampField)
+                .from("2019-11-28T00:00:00.000000000Z", true)
+                .to("2021-11-29T00:00:00.000000000Z");
+
+            SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0]))
+                .source(new SearchSourceBuilder().query(rangeQuery));
+
+            assertResponse(client().search(request), searchResponse -> {
+                assertThat(searchResponse.getSuccessfulShards(), equalTo(totalShards));
+                assertThat(searchResponse.getFailedShards(), equalTo(0));
+                assertThat(searchResponse.getSkippedShards(), equalTo(0));
+                assertThat(searchResponse.getTotalShards(), equalTo(totalShards));
+            });
+
+            SearchShardAPIResult searchShardResult = doSearchShardAPIQuery(indicesToSearch, rangeQuery, true, totalShards);
+            assertThat(searchShardResult.skipped().size(), equalTo(0));
+            assertThat(searchShardResult.notSkipped().size(), equalTo(totalShards));
+        }
+    }
+
     /**
      * Can match against searchable snapshots is tested via both the Search API and the SearchShards (transport-only) API.
      * The latter is a way to do only a can-match rather than all search phases.
@@ -396,7 +588,7 @@ public void testQueryPhaseIsExecutedInAnAvailableNodeWhenAllShardsCanBeSkipped()
 
         final String indexOutsideSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         final int indexOutsideSearchRangeShardCount = randomIntBetween(1, 3);
-        createIndexWithTimestamp(
+        createIndexWithTimestampAndEventIngested(
             indexOutsideSearchRange,
             indexOutsideSearchRangeShardCount,
             Settings.builder()
@@ -404,7 +596,7 @@ public void testQueryPhaseIsExecutedInAnAvailableNodeWhenAllShardsCanBeSkipped()
                 .build()
         );
 
-        indexDocumentsWithTimestampWithinDate(indexOutsideSearchRange, between(1, 1000), TIMESTAMP_TEMPLATE_OUTSIDE_RANGE);
+        indexDocumentsWithTimestampAndEventIngestedDates(indexOutsideSearchRange, between(1, 1000), TIMESTAMP_TEMPLATE_OUTSIDE_RANGE);
 
         final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         createRepository(repositoryName, "mock");
@@ -438,11 +630,14 @@ public void testQueryPhaseIsExecutedInAnAvailableNodeWhenAllShardsCanBeSkipped()
 
         final IndexMetadata indexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
         assertThat(indexMetadata.getTimestampRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
+        assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
 
-        DateFieldMapper.DateFieldType timestampFieldType = indicesService.getTimestampFieldType(indexMetadata.getIndex());
-        assertThat(timestampFieldType, nullValue());
+        DateFieldRangeInfo timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(indexMetadata.getIndex());
+        assertThat(timestampFieldTypeInfo, nullValue());
 
-        RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(DataStream.TIMESTAMP_FIELD_NAME)
+        final String timestampField = randomFrom(DataStream.TIMESTAMP_FIELD_NAME, IndexMetadata.EVENT_INGESTED_FIELD_NAME);
+
+        RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(timestampField)
             .from("2020-11-28T00:00:00.000000000Z", true)
             .to("2020-11-29T00:00:00.000000000Z");
 
@@ -500,14 +695,29 @@ public void testQueryPhaseIsExecutedInAnAvailableNodeWhenAllShardsCanBeSkipped()
         ensureGreen(searchableSnapshotIndexOutsideSearchRange);
 
         final IndexMetadata updatedIndexMetadata = getIndexMetadata(searchableSnapshotIndexOutsideSearchRange);
-        final IndexLongFieldRange updatedTimestampMillisRange = updatedIndexMetadata.getTimestampRange();
-        final DateFieldMapper.DateFieldType dateFieldType = indicesService.getTimestampFieldType(updatedIndexMetadata.getIndex());
-        assertThat(dateFieldType, notNullValue());
-        final DateFieldMapper.Resolution resolution = dateFieldType.resolution();
-        assertThat(updatedTimestampMillisRange.isComplete(), equalTo(true));
-        assertThat(updatedTimestampMillisRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
-        assertThat(updatedTimestampMillisRange.getMin(), greaterThanOrEqualTo(resolution.convert(Instant.parse("2020-11-26T00:00:00Z"))));
-        assertThat(updatedTimestampMillisRange.getMax(), lessThanOrEqualTo(resolution.convert(Instant.parse("2020-11-27T00:00:00Z"))));
+        timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(updatedIndexMetadata.getIndex());
+        assertThat(timestampFieldTypeInfo, notNullValue());
+
+        final IndexLongFieldRange updatedTimestampRange = updatedIndexMetadata.getTimestampRange();
+        DateFieldMapper.Resolution tsResolution = timestampFieldTypeInfo.timestampFieldType().resolution();
+        ;
+        assertThat(updatedTimestampRange.isComplete(), equalTo(true));
+        assertThat(updatedTimestampRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
+        assertThat(updatedTimestampRange.getMin(), greaterThanOrEqualTo(tsResolution.convert(Instant.parse("2020-11-26T00:00:00Z"))));
+        assertThat(updatedTimestampRange.getMax(), lessThanOrEqualTo(tsResolution.convert(Instant.parse("2020-11-27T00:00:00Z"))));
+
+        final IndexLongFieldRange updatedEventIngestedRange = updatedIndexMetadata.getEventIngestedRange();
+        DateFieldMapper.Resolution eventIngestedResolution = timestampFieldTypeInfo.eventIngestedFieldType().resolution();
+        assertThat(updatedEventIngestedRange.isComplete(), equalTo(true));
+        assertThat(updatedEventIngestedRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
+        assertThat(
+            updatedEventIngestedRange.getMin(),
+            greaterThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-26T00:00:00Z")))
+        );
+        assertThat(
+            updatedEventIngestedRange.getMax(),
+            lessThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-27T00:00:00Z")))
+        );
 
         // Stop the node holding the searchable snapshots, and since we defined
         // the index allocation criteria to require the searchable snapshot
@@ -579,7 +789,7 @@ public void testSearchableSnapshotShardsThatHaveMatchingDataAreNotSkippedOnTheCo
 
         final String indexWithinSearchRange = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         final int indexWithinSearchRangeShardCount = randomIntBetween(1, 3);
-        createIndexWithTimestamp(
+        createIndexWithTimestampAndEventIngested(
             indexWithinSearchRange,
             indexWithinSearchRangeShardCount,
             Settings.builder()
@@ -587,7 +797,7 @@ public void testSearchableSnapshotShardsThatHaveMatchingDataAreNotSkippedOnTheCo
                 .build()
         );
 
-        indexDocumentsWithTimestampWithinDate(indexWithinSearchRange, between(1, 1000), TIMESTAMP_TEMPLATE_WITHIN_RANGE);
+        indexDocumentsWithTimestampAndEventIngestedDates(indexWithinSearchRange, between(1, 1000), TIMESTAMP_TEMPLATE_WITHIN_RANGE);
 
         final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
         createRepository(repositoryName, "mock");
@@ -621,11 +831,13 @@ public void testSearchableSnapshotShardsThatHaveMatchingDataAreNotSkippedOnTheCo
 
         final IndexMetadata indexMetadata = getIndexMetadata(searchableSnapshotIndexWithinSearchRange);
         assertThat(indexMetadata.getTimestampRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
+        assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.NO_SHARDS));
 
-        DateFieldMapper.DateFieldType timestampFieldType = indicesService.getTimestampFieldType(indexMetadata.getIndex());
-        assertThat(timestampFieldType, nullValue());
+        DateFieldRangeInfo timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(indexMetadata.getIndex());
+        assertThat(timestampFieldTypeInfo, nullValue());
 
-        RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(DataStream.TIMESTAMP_FIELD_NAME)
+        String timeField = randomFrom(IndexMetadata.EVENT_INGESTED_FIELD_NAME, DataStream.TIMESTAMP_FIELD_NAME);
+        RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(timeField)
             .from("2020-11-28T00:00:00.000000000Z", true)
             .to("2020-11-29T00:00:00.000000000Z");
 
@@ -680,13 +892,32 @@ public void testSearchableSnapshotShardsThatHaveMatchingDataAreNotSkippedOnTheCo
 
         final IndexMetadata updatedIndexMetadata = getIndexMetadata(searchableSnapshotIndexWithinSearchRange);
         final IndexLongFieldRange updatedTimestampMillisRange = updatedIndexMetadata.getTimestampRange();
-        final DateFieldMapper.DateFieldType dateFieldType = indicesService.getTimestampFieldType(updatedIndexMetadata.getIndex());
-        assertThat(dateFieldType, notNullValue());
-        final DateFieldMapper.Resolution resolution = dateFieldType.resolution();
+        timestampFieldTypeInfo = indicesService.getTimestampFieldTypeInfo(updatedIndexMetadata.getIndex());
+        assertThat(timestampFieldTypeInfo, notNullValue());
+        final DateFieldMapper.Resolution timestampResolution = timestampFieldTypeInfo.timestampFieldType().resolution();
         assertThat(updatedTimestampMillisRange.isComplete(), equalTo(true));
         assertThat(updatedTimestampMillisRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
-        assertThat(updatedTimestampMillisRange.getMin(), greaterThanOrEqualTo(resolution.convert(Instant.parse("2020-11-28T00:00:00Z"))));
-        assertThat(updatedTimestampMillisRange.getMax(), lessThanOrEqualTo(resolution.convert(Instant.parse("2020-11-29T00:00:00Z"))));
+        assertThat(
+            updatedTimestampMillisRange.getMin(),
+            greaterThanOrEqualTo(timestampResolution.convert(Instant.parse("2020-11-28T00:00:00Z")))
+        );
+        assertThat(
+            updatedTimestampMillisRange.getMax(),
+            lessThanOrEqualTo(timestampResolution.convert(Instant.parse("2020-11-29T00:00:00Z")))
+        );
+
+        final IndexLongFieldRange updatedEventIngestedMillisRange = updatedIndexMetadata.getEventIngestedRange();
+        final DateFieldMapper.Resolution eventIngestedResolution = timestampFieldTypeInfo.eventIngestedFieldType().resolution();
+        assertThat(updatedEventIngestedMillisRange.isComplete(), equalTo(true));
+        assertThat(updatedEventIngestedMillisRange, not(sameInstance(IndexLongFieldRange.EMPTY)));
+        assertThat(
+            updatedEventIngestedMillisRange.getMin(),
+            greaterThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-28T00:00:00Z")))
+        );
+        assertThat(
+            updatedEventIngestedMillisRange.getMax(),
+            lessThanOrEqualTo(eventIngestedResolution.convert(Instant.parse("2020-11-29T00:00:00Z")))
+        );
 
         // Stop the node holding the searchable snapshots, and since we defined
         // the index allocation criteria to require the searchable snapshot
@@ -724,17 +955,24 @@ public void testSearchableSnapshotShardsThatHaveMatchingDataAreNotSkippedOnTheCo
         }
     }
 
-    private void createIndexWithTimestamp(String indexName, int numShards, Settings extraSettings) throws IOException {
+    private void createIndexWithTimestampAndEventIngested(String indexName, int numShards, Settings extraSettings) throws IOException {
         assertAcked(
             indicesAdmin().prepareCreate(indexName)
                 .setMapping(
                     XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("properties")
+
                         .startObject(DataStream.TIMESTAMP_FIELD_NAME)
                         .field("type", randomFrom("date", "date_nanos"))
                         .field("format", "strict_date_optional_time_nanos")
                         .endObject()
+
+                        .startObject(IndexMetadata.EVENT_INGESTED_FIELD_NAME)
+                        .field("type", randomFrom("date", "date_nanos"))
+                        .field("format", "strict_date_optional_time_nanos")
+                        .endObject()
+
                         .endObject()
                         .endObject()
                 )
@@ -743,12 +981,70 @@ private void createIndexWithTimestamp(String indexName, int numShards, Settings
         ensureGreen(indexName);
     }
 
-    private void indexDocumentsWithTimestampWithinDate(String indexName, int docCount, String timestampTemplate) throws Exception {
+    private void createIndexWithOnlyOneTimestampField(String timestampField, String index, int numShards, Settings extraSettings)
+        throws IOException {
+        assertAcked(
+            indicesAdmin().prepareCreate(index)
+                .setMapping(
+                    XContentFactory.jsonBuilder()
+                        .startObject()
+                        .startObject("properties")
+
+                        .startObject(timestampField)
+                        .field("type", randomFrom("date", "date_nanos"))
+                        .field("format", "strict_date_optional_time_nanos")
+                        .endObject()
+
+                        .endObject()
+                        .endObject()
+                )
+                .setSettings(indexSettingsNoReplicas(numShards).put(INDEX_SOFT_DELETES_SETTING.getKey(), true).put(extraSettings))
+        );
+        ensureGreen(index);
+    }
+
+    private void indexDocumentsWithOnlyOneTimestampField(String timestampField, String index, int docCount, String timestampTemplate)
+        throws Exception {
+        final List<IndexRequestBuilder> indexRequestBuilders = new ArrayList<>();
+        for (int i = 0; i < docCount; i++) {
+            indexRequestBuilders.add(
+                prepareIndex(index).setSource(
+                    timestampField,
+                    String.format(
+                        Locale.ROOT,
+                        timestampTemplate,
+                        between(0, 23),
+                        between(0, 59),
+                        between(0, 59),
+                        randomLongBetween(0, 999999999L)
+                    )
+                )
+            );
+        }
+        indexRandom(true, false, indexRequestBuilders);
+
+        assertThat(indicesAdmin().prepareForceMerge(index).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(), equalTo(0));
+        refresh(index);
+        forceMerge();
+    }
+
+    private void indexDocumentsWithTimestampAndEventIngestedDates(String indexName, int docCount, String timestampTemplate)
+        throws Exception {
+
         final List<IndexRequestBuilder> indexRequestBuilders = new ArrayList<>();
         for (int i = 0; i < docCount; i++) {
             indexRequestBuilders.add(
                 prepareIndex(indexName).setSource(
                     DataStream.TIMESTAMP_FIELD_NAME,
+                    String.format(
+                        Locale.ROOT,
+                        timestampTemplate,
+                        between(0, 23),
+                        between(0, 59),
+                        between(0, 59),
+                        randomLongBetween(0, 999999999L)
+                    ),
+                    IndexMetadata.EVENT_INGESTED_FIELD_NAME,
                     String.format(
                         Locale.ROOT,
                         timestampTemplate,
@@ -789,4 +1085,39 @@ private void waitUntilRecoveryIsDone(String index) throws Exception {
     private void waitUntilAllShardsAreUnassigned(Index index) throws Exception {
         awaitClusterState(state -> state.getRoutingTable().index(index).allPrimaryShardsUnassigned());
     }
+
+    record SearchShardAPIResult(List<SearchShardsGroup> skipped, List<SearchShardsGroup> notSkipped) {}
+
+    private static SearchShardAPIResult doSearchShardAPIQuery(
+        List<String> indicesToSearch,
+        RangeQueryBuilder rangeQuery,
+        boolean allowPartialSearchResults,
+        int expectedTotalShards
+    ) {
+        SearchShardsRequest searchShardsRequest = new SearchShardsRequest(
+            indicesToSearch.toArray(new String[0]),
+            SearchRequest.DEFAULT_INDICES_OPTIONS,
+            rangeQuery,
+            null,
+            null,
+            allowPartialSearchResults,
+            null
+        );
+
+        SearchShardsResponse searchShardsResponse = client().execute(TransportSearchShardsAction.TYPE, searchShardsRequest).actionGet();
+        assertThat(searchShardsResponse.getGroups().size(), equalTo(expectedTotalShards));
+        List<List<SearchShardsGroup>> partitionedBySkipped = searchShardsResponse.getGroups()
+            .stream()
+            .collect(
+                Collectors.teeing(
+                    Collectors.filtering(g -> g.skipped(), Collectors.toList()),
+                    Collectors.filtering(g -> g.skipped() == false, Collectors.toList()),
+                    List::of
+                )
+            );
+
+        List<SearchShardsGroup> skipped = partitionedBySkipped.get(0);
+        List<SearchShardsGroup> notSkipped = partitionedBySkipped.get(1);
+        return new SearchShardAPIResult(skipped, notSkipped);
+    }
 }
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
index 56aec13cbab29..c99f2be0a6cad 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
@@ -371,12 +371,8 @@ public void testCanMountSnapshotTakenWhileConcurrentlyIndexing() throws Exceptio
             for (int i = between(10, 10_000); i >= 0; i--) {
                 indexRequestBuilders.add(prepareIndex(indexName).setSource("foo", randomBoolean() ? "bar" : "baz"));
             }
-            try {
-                safeAwait(cyclicBarrier);
-                indexRandom(true, true, indexRequestBuilders);
-            } catch (InterruptedException e) {
-                throw new AssertionError(e);
-            }
+            safeAwait(cyclicBarrier);
+            indexRandom(true, true, indexRequestBuilders);
             refresh(indexName);
             assertThat(
                 indicesAdmin().prepareForceMerge(indexName).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(),
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java
index 0811ee86b3c32..0a0cce5dd87ed 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java
@@ -20,7 +20,6 @@
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING;
-import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING;
 import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest.Storage;
@@ -37,11 +36,7 @@ public void setUp() throws Exception {
         assertAcked(
             prepareCreate(
                 "index",
-                Settings.builder()
-                    .put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                    .put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 2)
-                    .put(INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey(), 4)
-                    .put(INDEX_SOFT_DELETES_SETTING.getKey(), true)
+                indexSettings(2, 0).put(INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey(), 4).put(INDEX_SOFT_DELETES_SETTING.getKey(), true)
             )
         );
         indexRandomDocs("index", scaledRandomIntBetween(0, 1_000));
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java
index 18ebe65d87986..4eea006b4c2f2 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java
@@ -331,7 +331,7 @@ public Collection<?> createComponents(PluginServices services) {
                 nodeEnvironment,
                 settings,
                 threadPool,
-                SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME,
+                threadPool.executor(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME),
                 new BlobCacheMetrics(services.telemetryProvider().getMeterRegistry())
             );
             this.frozenCacheService.set(sharedBlobCacheService);
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsInfoTransportAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsInfoTransportAction.java
index 35247b7606449..6be8e3828802d 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsInfoTransportAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsInfoTransportAction.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.searchablesnapshots.action;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsUsageTransportAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsUsageTransportAction.java
index 0c09a0e31091d..53a3a26d2542f 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsUsageTransportAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsUsageTransportAction.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java
index 077ee165d58ef..e781514526bde 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java
index bdcce1e518700..4dfb982f01d0d 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -31,6 +30,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.ShardLimitValidator;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java
index e4632b697f160..7ee60df53a578 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java
@@ -11,10 +11,10 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoAction.java
index cc99e9514df67..9d12b6d941a26 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoAction.java
@@ -15,9 +15,9 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponseHandler;
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoNodeAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoNodeAction.java
index 756f9c191a561..b24af3c876b7a 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoNodeAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoNodeAction.java
@@ -13,10 +13,10 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java
index 5e9ef177bdc6f..446f0f433fe33 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java
@@ -17,10 +17,10 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java
index 78d520e984bcf..c28948b4101e2 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java
@@ -19,10 +19,10 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java
index 56efc72f2f6f7..d7cf22a05981f 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java
@@ -11,6 +11,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.blobcache.BlobCacheUtils;
 import org.elasticsearch.blobcache.common.ByteBufferReference;
 import org.elasticsearch.blobcache.common.ByteRange;
@@ -146,32 +147,38 @@ private void readWithoutBlobCacheSlow(ByteBuffer b, long position, int length) t
                 final int read = SharedBytes.readCacheFile(channel, pos, relativePos, len, byteBufferReference);
                 stats.addCachedBytesRead(read);
                 return read;
-            }, (channel, channelPos, streamFactory, relativePos, len, progressUpdater) -> {
-                assert streamFactory == null : streamFactory;
-                final long startTimeNanos = stats.currentTimeNanos();
-                try (InputStream input = openInputStreamFromBlobStore(rangeToWrite.start() + relativePos, len)) {
-                    assert ThreadPool.assertCurrentThreadPool(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME);
-                    logger.trace(
-                        "{}: writing channel {} pos {} length {} (details: {})",
-                        fileInfo.physicalName(),
-                        channelPos,
-                        relativePos,
-                        len,
-                        cacheFile
-                    );
-                    SharedBytes.copyToCacheFileAligned(
-                        channel,
-                        input,
-                        channelPos,
-                        relativePos,
-                        len,
-                        progressUpdater,
-                        writeBuffer.get().clear()
-                    );
-                    final long endTimeNanos = stats.currentTimeNanos();
-                    stats.addCachedBytesWritten(len, endTimeNanos - startTimeNanos);
-                }
-            });
+            },
+                (channel, channelPos, streamFactory, relativePos, len, progressUpdater, completionListener) -> ActionListener.completeWith(
+                    completionListener,
+                    () -> {
+                        assert streamFactory == null : streamFactory;
+                        final long startTimeNanos = stats.currentTimeNanos();
+                        try (InputStream input = openInputStreamFromBlobStore(rangeToWrite.start() + relativePos, len)) {
+                            assert ThreadPool.assertCurrentThreadPool(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME);
+                            logger.trace(
+                                "{}: writing channel {} pos {} length {} (details: {})",
+                                fileInfo.physicalName(),
+                                channelPos,
+                                relativePos,
+                                len,
+                                cacheFile
+                            );
+                            SharedBytes.copyToCacheFileAligned(
+                                channel,
+                                input,
+                                channelPos,
+                                relativePos,
+                                len,
+                                progressUpdater,
+                                writeBuffer.get().clear()
+                            );
+                            final long endTimeNanos = stats.currentTimeNanos();
+                            stats.addCachedBytesWritten(len, endTimeNanos - startTimeNanos);
+                            return null;
+                        }
+                    }
+                )
+            );
             assert bytesRead == length : bytesRead + " vs " + length;
             byteBufferReference.finish(bytesRead);
         } finally {
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java
index 5f083d568fed8..41121453e41a4 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java
@@ -144,7 +144,7 @@ protected SharedBlobCacheService<CacheKey> defaultFrozenCacheService() {
             nodeEnvironment,
             Settings.EMPTY,
             threadPool,
-            SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME,
+            threadPool.executor(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME),
             BlobCacheMetrics.NOOP
         );
     }
@@ -167,7 +167,7 @@ protected SharedBlobCacheService<CacheKey> randomFrozenCacheService() {
             singlePathNodeEnvironment,
             cacheSettings.build(),
             threadPool,
-            SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME,
+            threadPool.executor(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME),
             BlobCacheMetrics.NOOP
         );
     }
@@ -192,7 +192,7 @@ protected SharedBlobCacheService<CacheKey> createFrozenCacheService(final ByteSi
                 .put(SharedBlobCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), cacheRangeSize)
                 .build(),
             threadPool,
-            SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME,
+            threadPool.executor(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME),
             BlobCacheMetrics.NOOP
         );
     }
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java
index 81e9c06a149b9..53ea908ad8801 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java
@@ -111,7 +111,7 @@ public void testRandomReads() throws IOException {
                 nodeEnvironment,
                 settings,
                 threadPool,
-                SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME,
+                threadPool.executor(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME),
                 BlobCacheMetrics.NOOP
             );
             CacheService cacheService = randomCacheService();
diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle
index 07308d5d29a9a..d3697eade8b24 100644
--- a/x-pack/plugin/security/build.gradle
+++ b/x-pack/plugin/security/build.gradle
@@ -79,12 +79,19 @@ dependencies {
   runtimeOnly "joda-time:joda-time:2.10.10"
 
   // Dependencies for oidc
-  api "com.nimbusds:oauth2-oidc-sdk:9.37"
-  api "com.nimbusds:nimbus-jose-jwt:9.23"
+  api "com.nimbusds:oauth2-oidc-sdk:11.10.1"
+  api project(path: xpackModule('security:lib:nimbus-jose-jwt-modified'), configuration: 'shadow')
+  if (isEclipse) {
+    /*
+     * Eclipse can't pick up the shadow dependency so we point it at the unmodified version of the library
+     * so it can compile things.
+     */
+    api "com.nimbusds:nimbus-jose-jwt:9.37.3"
+  }
   api "com.nimbusds:lang-tag:1.4.4"
   api "com.sun.mail:jakarta.mail:1.6.3"
   api "net.jcip:jcip-annotations:1.0"
-  api "net.minidev:json-smart:2.4.10"
+  api "net.minidev:json-smart:2.5.1"
   api "net.minidev:accessors-smart:2.4.2"
   api "org.ow2.asm:asm:8.0.1"
 
@@ -103,7 +110,6 @@ dependencies {
   testImplementation('org.apache.kerby:kerb-crypto:1.1.1')
   testImplementation('org.apache.kerby:kerb-util:1.1.1')
   testImplementation('org.apache.kerby:token-provider:1.1.1')
-  testImplementation('com.nimbusds:nimbus-jose-jwt:9.23')
   testImplementation('net.jcip:jcip-annotations:1.0')
   testImplementation('org.apache.kerby:kerb-admin:1.1.1')
   testImplementation('org.apache.kerby:kerb-server:1.1.1')
@@ -225,6 +231,9 @@ tasks.named("thirdPartyAudit").configure {
     'javax.servlet.http.HttpSession',
     'javax.servlet.http.HttpUpgradeHandler',
     'javax.servlet.http.Part',
+    'jakarta.servlet.ServletRequest',
+    'jakarta.servlet.http.HttpServletRequest',
+    'jakarta.servlet.http.HttpServletResponse',
     // [missing classes] Shibboleth + OpenSAML have velocity support that we don't use
     'org.apache.velocity.VelocityContext',
     'org.apache.velocity.app.VelocityEngine',
@@ -274,112 +283,103 @@ tasks.named("thirdPartyAudit").configure {
     // [missing classes] Http Client cache has optional ehcache support
     'net.sf.ehcache.Ehcache',
     'net.sf.ehcache.Element',
-        // Bouncycastle is an optional dependency for apache directory, cryptacular and opensaml packages. We
-        // acknowledge them here instead of adding bouncy castle as a compileOnly dependency
-        'org.bouncycastle.asn1.ASN1Encodable',
-        'org.bouncycastle.asn1.ASN1InputStream',
-        'org.bouncycastle.asn1.ASN1Integer',
-        'org.bouncycastle.asn1.ASN1ObjectIdentifier',
-        'org.bouncycastle.asn1.ASN1OctetString',
-        'org.bouncycastle.asn1.ASN1Primitive',
-        'org.bouncycastle.asn1.ASN1Sequence',
-        'org.bouncycastle.asn1.ASN1TaggedObject',
-        // 'org.bouncycastle.asn1.DEROctetString',
-        'org.bouncycastle.asn1.pkcs.EncryptedPrivateKeyInfo',
-        'org.bouncycastle.asn1.pkcs.EncryptionScheme',
-        'org.bouncycastle.asn1.pkcs.KeyDerivationFunc',
-        'org.bouncycastle.asn1.pkcs.PBEParameter',
-        'org.bouncycastle.asn1.pkcs.PBES2Parameters',
-        'org.bouncycastle.asn1.pkcs.PBKDF2Params',
-        'org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers',
-        'org.bouncycastle.asn1.pkcs.PrivateKeyInfo',
-        'org.bouncycastle.asn1.x500.AttributeTypeAndValue',
-        'org.bouncycastle.asn1.x500.RDN',
-        'org.bouncycastle.asn1.x500.X500Name',
-        'org.bouncycastle.asn1.x509.AccessDescription',
-        'org.bouncycastle.asn1.x509.AlgorithmIdentifier',
-        'org.bouncycastle.asn1.x509.AuthorityKeyIdentifier',
-        'org.bouncycastle.asn1.x509.BasicConstraints',
-        'org.bouncycastle.asn1.x509.DistributionPoint',
-        'org.bouncycastle.asn1.x509.Extension',
-        'org.bouncycastle.asn1.x509.GeneralName',
-        'org.bouncycastle.asn1.x509.GeneralNames',
-        'org.bouncycastle.asn1.x509.GeneralNamesBuilder',
-        'org.bouncycastle.asn1.x509.KeyPurposeId',
-        'org.bouncycastle.asn1.x509.KeyUsage',
-        'org.bouncycastle.asn1.x509.PolicyInformation',
-        'org.bouncycastle.asn1.x509.SubjectKeyIdentifier',
-        'org.bouncycastle.asn1.x509.SubjectPublicKeyInfo',
-        // 'org.bouncycastle.asn1.x9.DomainParameters',
-        // 'org.bouncycastle.asn1.x9.ECNamedCurveTable',
-        'org.bouncycastle.asn1.x9.X9ECParameters',
-        'org.bouncycastle.cert.X509v3CertificateBuilder',
-        'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
-        'org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils',
-        'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
-        'org.bouncycastle.crypto.BlockCipher',
-        'org.bouncycastle.crypto.BufferedBlockCipher',
-        'org.bouncycastle.crypto.CipherParameters',
-        'org.bouncycastle.crypto.Digest',
-        'org.bouncycastle.crypto.PBEParametersGenerator',
-        'org.bouncycastle.crypto.StreamCipher',
-        'org.bouncycastle.crypto.agreement.kdf.ConcatenationKDFGenerator',
-        // 'org.bouncycastle.crypto.ec.CustomNamedCurves',
-        'org.bouncycastle.crypto.engines.AESEngine',
-        'org.bouncycastle.crypto.generators.BCrypt',
-        'org.bouncycastle.crypto.generators.OpenSSLPBEParametersGenerator',
-        'org.bouncycastle.crypto.generators.PKCS5S1ParametersGenerator',
-        'org.bouncycastle.crypto.generators.PKCS5S2ParametersGenerator',
-        'org.bouncycastle.crypto.macs.HMac',
-        'org.bouncycastle.crypto.modes.AEADBlockCipher',
-        'org.bouncycastle.crypto.modes.GCMBlockCipher',
-        'org.bouncycastle.crypto.paddings.BlockCipherPadding',
-        'org.bouncycastle.crypto.paddings.PaddedBufferedBlockCipher',
-        'org.bouncycastle.crypto.params.AsymmetricKeyParameter',
-        'org.bouncycastle.crypto.params.DSAKeyParameters',
-        'org.bouncycastle.crypto.params.DSAParameters',
-        'org.bouncycastle.crypto.params.DSAPrivateKeyParameters',
-        'org.bouncycastle.crypto.params.DSAPublicKeyParameters',
-        'org.bouncycastle.crypto.params.ECDomainParameters',
-        'org.bouncycastle.crypto.params.ECKeyParameters',
-        'org.bouncycastle.crypto.params.ECPrivateKeyParameters',
-        'org.bouncycastle.crypto.params.ECPublicKeyParameters',
-        // 'org.bouncycastle.crypto.params.KDFParameters',
-        'org.bouncycastle.crypto.params.KeyParameter',
-        'org.bouncycastle.crypto.params.RSAKeyParameters',
-        'org.bouncycastle.crypto.params.RSAPrivateCrtKeyParameters',
-        'org.bouncycastle.crypto.prng.EntropySource',
-        'org.bouncycastle.crypto.prng.SP800SecureRandom',
-        'org.bouncycastle.crypto.prng.SP800SecureRandomBuilder',
-        'org.bouncycastle.crypto.prng.drbg.SP80090DRBG',
-        'org.bouncycastle.crypto.signers.DSASigner',
-        'org.bouncycastle.crypto.signers.ECDSASigner',
-        'org.bouncycastle.crypto.signers.RSADigestSigner',
-        'org.bouncycastle.crypto.util.PrivateKeyFactory',
-        'org.bouncycastle.crypto.util.PrivateKeyInfoFactory',
-        'org.bouncycastle.crypto.util.PublicKeyFactory',
-        'org.bouncycastle.crypto.util.SubjectPublicKeyInfoFactory',
-        'org.bouncycastle.jcajce.provider.asymmetric.dsa.KeyPairGeneratorSpi',
-        'org.bouncycastle.jcajce.provider.asymmetric.ec.KeyPairGeneratorSpi$EC',
-        'org.bouncycastle.jcajce.provider.asymmetric.rsa.KeyPairGeneratorSpi',
-        'org.bouncycastle.jcajce.provider.asymmetric.util.EC5Util',
-        'org.bouncycastle.jcajce.provider.asymmetric.util.ECUtil',
-        // 'org.bouncycastle.jce.ECNamedCurveTable',
-        // 'org.bouncycastle.jce.spec.ECNamedCurveParameterSpec',
-        'org.bouncycastle.math.ec.ECFieldElement',
-        'org.bouncycastle.math.ec.ECPoint',
-        'org.bouncycastle.openssl.jcajce.JcaPEMWriter',
-        'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
-        'org.bouncycastle.util.Arrays',
-        'org.bouncycastle.util.io.Streams',
-        'org.bouncycastle.cert.jcajce.JcaX509CertificateHolder',
-        'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider',
-        'org.bouncycastle.cert.X509CertificateHolder',
-        'org.bouncycastle.openssl.PEMKeyPair',
-        'org.bouncycastle.openssl.PEMParser',
-        'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
-        'org.bouncycastle.crypto.InvalidCipherTextException',
-        'org.bouncycastle.jce.provider.BouncyCastleProvider',
+    // Bouncycastle is an optional dependency for apache directory, cryptacular and opensaml packages. We
+    // acknowledge them here instead of adding bouncy castle as a compileOnly dependency
+    'org.bouncycastle.asn1.ASN1Encodable',
+    'org.bouncycastle.asn1.ASN1InputStream',
+    'org.bouncycastle.asn1.ASN1Integer',
+    'org.bouncycastle.asn1.ASN1ObjectIdentifier',
+    'org.bouncycastle.asn1.ASN1OctetString',
+    'org.bouncycastle.asn1.ASN1Primitive',
+    'org.bouncycastle.asn1.ASN1Sequence',
+    'org.bouncycastle.asn1.ASN1TaggedObject',
+    // 'org.bouncycastle.asn1.DEROctetString',
+    'org.bouncycastle.asn1.pkcs.EncryptedPrivateKeyInfo',
+    'org.bouncycastle.asn1.pkcs.EncryptionScheme',
+    'org.bouncycastle.asn1.pkcs.KeyDerivationFunc',
+    'org.bouncycastle.asn1.pkcs.PBEParameter',
+    'org.bouncycastle.asn1.pkcs.PBES2Parameters',
+    'org.bouncycastle.asn1.pkcs.PBKDF2Params',
+    'org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers',
+    'org.bouncycastle.asn1.pkcs.PrivateKeyInfo',
+    'org.bouncycastle.asn1.x500.AttributeTypeAndValue',
+    'org.bouncycastle.asn1.x500.RDN',
+    'org.bouncycastle.asn1.x500.X500Name',
+    'org.bouncycastle.asn1.x509.AccessDescription',
+    'org.bouncycastle.asn1.x509.AlgorithmIdentifier',
+    'org.bouncycastle.asn1.x509.AuthorityKeyIdentifier',
+    'org.bouncycastle.asn1.x509.BasicConstraints',
+    'org.bouncycastle.asn1.x509.DistributionPoint',
+    'org.bouncycastle.asn1.x509.Extension',
+    'org.bouncycastle.asn1.x509.GeneralName',
+    'org.bouncycastle.asn1.x509.GeneralNames',
+    'org.bouncycastle.asn1.x509.GeneralNamesBuilder',
+    'org.bouncycastle.asn1.x509.KeyPurposeId',
+    'org.bouncycastle.asn1.x509.KeyUsage',
+    'org.bouncycastle.asn1.x509.PolicyInformation',
+    'org.bouncycastle.asn1.x509.SubjectKeyIdentifier',
+    'org.bouncycastle.asn1.x509.SubjectPublicKeyInfo',
+    // 'org.bouncycastle.asn1.x9.DomainParameters',
+    // 'org.bouncycastle.asn1.x9.ECNamedCurveTable',
+    'org.bouncycastle.asn1.x9.X9ECParameters',
+    'org.bouncycastle.cert.X509v3CertificateBuilder',
+    'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
+    'org.bouncycastle.cert.jcajce.JcaX509ExtensionUtils',
+    'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
+    'org.bouncycastle.crypto.BlockCipher',
+    'org.bouncycastle.crypto.BufferedBlockCipher',
+    'org.bouncycastle.crypto.CipherParameters',
+    'org.bouncycastle.crypto.Digest',
+    'org.bouncycastle.crypto.PBEParametersGenerator',
+    'org.bouncycastle.crypto.StreamCipher',
+    'org.bouncycastle.crypto.agreement.kdf.ConcatenationKDFGenerator',
+    // 'org.bouncycastle.crypto.ec.CustomNamedCurves',
+    'org.bouncycastle.crypto.generators.BCrypt',
+    'org.bouncycastle.crypto.generators.OpenSSLPBEParametersGenerator',
+    'org.bouncycastle.crypto.generators.PKCS5S1ParametersGenerator',
+    'org.bouncycastle.crypto.generators.PKCS5S2ParametersGenerator',
+    'org.bouncycastle.crypto.macs.HMac',
+    'org.bouncycastle.crypto.modes.AEADBlockCipher',
+    'org.bouncycastle.crypto.paddings.BlockCipherPadding',
+    'org.bouncycastle.crypto.paddings.PaddedBufferedBlockCipher',
+    'org.bouncycastle.crypto.params.AsymmetricKeyParameter',
+    'org.bouncycastle.crypto.params.DSAKeyParameters',
+    'org.bouncycastle.crypto.params.DSAParameters',
+    'org.bouncycastle.crypto.params.DSAPrivateKeyParameters',
+    'org.bouncycastle.crypto.params.DSAPublicKeyParameters',
+    'org.bouncycastle.crypto.params.ECDomainParameters',
+    'org.bouncycastle.crypto.params.ECKeyParameters',
+    'org.bouncycastle.crypto.params.ECPrivateKeyParameters',
+    'org.bouncycastle.crypto.params.ECPublicKeyParameters',
+    // 'org.bouncycastle.crypto.params.KDFParameters',
+    'org.bouncycastle.crypto.params.KeyParameter',
+    'org.bouncycastle.crypto.params.RSAKeyParameters',
+    'org.bouncycastle.crypto.params.RSAPrivateCrtKeyParameters',
+    'org.bouncycastle.crypto.prng.EntropySource',
+    'org.bouncycastle.crypto.prng.SP800SecureRandom',
+    'org.bouncycastle.crypto.prng.SP800SecureRandomBuilder',
+    'org.bouncycastle.crypto.prng.drbg.SP80090DRBG',
+    'org.bouncycastle.crypto.signers.DSASigner',
+    'org.bouncycastle.crypto.signers.ECDSASigner',
+    'org.bouncycastle.crypto.signers.RSADigestSigner',
+    'org.bouncycastle.crypto.util.PrivateKeyFactory',
+    'org.bouncycastle.crypto.util.PrivateKeyInfoFactory',
+    'org.bouncycastle.crypto.util.PublicKeyFactory',
+    'org.bouncycastle.crypto.util.SubjectPublicKeyInfoFactory',
+    'org.bouncycastle.jcajce.provider.asymmetric.dsa.KeyPairGeneratorSpi',
+    'org.bouncycastle.jcajce.provider.asymmetric.ec.KeyPairGeneratorSpi$EC',
+    'org.bouncycastle.jcajce.provider.asymmetric.rsa.KeyPairGeneratorSpi',
+    'org.bouncycastle.jcajce.provider.asymmetric.util.EC5Util',
+    'org.bouncycastle.jcajce.provider.asymmetric.util.ECUtil',
+    // 'org.bouncycastle.jce.ECNamedCurveTable',
+    // 'org.bouncycastle.jce.spec.ECNamedCurveParameterSpec',
+    'org.bouncycastle.math.ec.ECFieldElement',
+    'org.bouncycastle.math.ec.ECPoint',
+    'org.bouncycastle.openssl.jcajce.JcaPEMWriter',
+    'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+    'org.bouncycastle.util.Arrays',
+    'org.bouncycastle.util.io.Streams',
+    'org.bouncycastle.cert.X509CertificateHolder',
   )
 
   ignoreViolations(
@@ -402,26 +402,21 @@ tasks.named("thirdPartyAudit").configure {
 
 tasks.named("thirdPartyAudit").configure {
   ignoreMissingClasses(
-          'javax.xml.bind.JAXBContext',
-          'javax.xml.bind.JAXBElement',
-          'javax.xml.bind.JAXBException',
-          'javax.xml.bind.Unmarshaller',
-          'javax.xml.bind.UnmarshallerHandler',
-          // Optional dependency of oauth2-oidc-sdk that we don't need since we do not support AES-SIV for JWE
-          'org.cryptomator.siv.SivMode',
-          // Optional dependency of nimbus-jose-jwt for handling Ed25519 signatures and ECDH with X25519 (RFC 8037)
-          'com.google.crypto.tink.subtle.Ed25519Sign',
-          'com.google.crypto.tink.subtle.Ed25519Sign$KeyPair',
-          'com.google.crypto.tink.subtle.Ed25519Verify',
-          'com.google.crypto.tink.subtle.X25519',
-          'com.google.crypto.tink.subtle.XChaCha20Poly1305',
-          'com.nimbusds.common.contenttype.ContentType',
-          'javax.activation.ActivationDataFlavor',
-          'javax.activation.DataContentHandler',
-          'javax.activation.DataHandler',
-          'javax.activation.DataSource',
-          'javax.activation.FileDataSource',
-          'javax.activation.FileTypeMap'
+    'javax.xml.bind.JAXBContext',
+    'javax.xml.bind.JAXBElement',
+    'javax.xml.bind.JAXBException',
+    'javax.xml.bind.Unmarshaller',
+    'javax.xml.bind.UnmarshallerHandler',
+    // Optional dependency of oauth2-oidc-sdk that we don't need since we do not support AES-SIV for JWE
+    'org.cryptomator.siv.SivMode',
+    'com.nimbusds.common.contenttype.ContentType',
+    'com.nimbusds.common.contenttype.ContentType$Parameter',
+    'javax.activation.ActivationDataFlavor',
+    'javax.activation.DataContentHandler',
+    'javax.activation.DataHandler',
+    'javax.activation.DataSource',
+    'javax.activation.FileDataSource',
+    'javax.activation.FileTypeMap'
   )
 }
 
diff --git a/x-pack/plugin/security/lib/build.gradle b/x-pack/plugin/security/lib/build.gradle
new file mode 100644
index 0000000000000..7bc94f348e781
--- /dev/null
+++ b/x-pack/plugin/security/lib/build.gradle
@@ -0,0 +1,13 @@
+// This build deserves an explanation. Nimbus-jose-jwt uses gson internally, which is unfriendly
+// to our usage of the security manager, to a degree that it makes the library extremely difficult
+// to work with safely. The purpose of this build is to create a version of nimbus-jose-jwt with
+// a couple classes replaced with wrappers which work with the security manager, the source files
+// in this directory.
+
+// Because we want to include the original class files so that we can reference them without
+// modification, there are a couple intermediate steps:
+// nimbus-jose-jwt-modified-part1: Create a version of the JAR in which the relevant class files are moved to a different package.
+//      This is not immediately usable as this process rewrites the rest of the JAR to "correctly" reference the new classes. So, we need to...
+// nimbus-jose-jwt-modified-part2: Create a JAR from the result of part 1 which contains *only* the relevant class files by removing everything else.
+// nimbus-jose-jwt-modified: Use the result of part 2 here, combined with the original library, so that we can use our
+//      replacement classes which wrap the original class files.
diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle
new file mode 100644
index 0000000000000..f751fcd0a655d
--- /dev/null
+++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/build.gradle
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+apply plugin: 'elasticsearch.build'
+apply plugin: 'com.github.johnrengelman.shadow'
+
+// See the build.gradle file in the parent directory for an explanation of this unusual build
+
+dependencies {
+  implementation "com.nimbusds:nimbus-jose-jwt:9.37.3"
+}
+
+tasks.named('shadowJar').configure {
+  // Attempting to exclude all of the classes we *don't* move here ought to be possible per the
+  // shadowJar docs, but actually attempting to do so results in an empty JAR. May be a bug in the shadowJar plugin.
+  relocate 'com.nimbusds.jose.util.JSONObjectUtils', 'org.elasticsearch.nimbus.jose.util.JSONObjectUtils'
+  relocate 'com.nimbusds.jose.util.JSONStringUtils', 'org.elasticsearch.nimbus.jose.util.JSONStringUtils'
+}
+
+['jarHell', 'thirdPartyAudit', 'forbiddenApisMain', 'splitPackagesAudit', 'licenseHeaders'].each {
+  tasks.named(it).configure {
+    enabled = false
+  }
+}
+
diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/licenses/nimbus-jose-jwt-LICENSE.txt b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/licenses/nimbus-jose-jwt-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/licenses/nimbus-jose-jwt-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/licenses/nimbus-jose-jwt-NOTICE.txt b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/licenses/nimbus-jose-jwt-NOTICE.txt
new file mode 100644
index 0000000000000..cb9ad94f662a6
--- /dev/null
+++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part1/licenses/nimbus-jose-jwt-NOTICE.txt
@@ -0,0 +1,14 @@
+Nimbus JOSE + JWT
+
+Copyright 2012 - 2018, Connect2id Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part2/build.gradle b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part2/build.gradle
new file mode 100644
index 0000000000000..c4c0f2ebd2fe1
--- /dev/null
+++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified-part2/build.gradle
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+apply plugin: 'elasticsearch.build'
+apply plugin: 'com.github.johnrengelman.shadow'
+
+// See the build.gradle file in the parent directory for an explanation of this unusual build
+
+dependencies {
+  implementation project(path: xpackModule('security:lib:nimbus-jose-jwt-modified-part1'), configuration: 'shadow')
+}
+
+tasks.named('shadowJar').configure {
+  // Drop everything in the original namespace, as the classes we want to modify have already been moved to another package by part 1
+  exclude 'com/nimbusds/'
+}
+
+['jarHell', 'thirdPartyAudit', 'forbiddenApisMain', 'splitPackagesAudit', 'licenseHeaders'].each {
+  tasks.named(it).configure {
+    enabled = false
+  }
+}
diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle
new file mode 100644
index 0000000000000..3438c067d8ab5
--- /dev/null
+++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/build.gradle
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+apply plugin: 'elasticsearch.build'
+apply plugin: 'com.github.johnrengelman.shadow'
+
+// See the build.gradle file in the parent directory for an explanation of this unusual build
+
+dependencies {
+  implementation "com.nimbusds:nimbus-jose-jwt:9.37.3"
+  implementation project(path: xpackModule('security:lib:nimbus-jose-jwt-modified-part2'), configuration: 'shadow')
+}
+
+tasks.named('shadowJar').configure {
+  manifest {
+    // The original library uses this and it gets stripped by shadowJar
+    attributes 'Automatic-Module-Name': 'com.nimbusds.jose.jwt'
+  }
+}
+
+['jarHell', 'thirdPartyAudit', 'forbiddenApisMain', 'splitPackagesAudit', 'licenseHeaders'].each {
+  tasks.named(it).configure {
+    enabled = false
+  }
+}
diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/licenses/nimbus-jose-jwt-LICENSE.txt b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/licenses/nimbus-jose-jwt-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/licenses/nimbus-jose-jwt-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/licenses/nimbus-jose-jwt-NOTICE.txt b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/licenses/nimbus-jose-jwt-NOTICE.txt
new file mode 100644
index 0000000000000..cb9ad94f662a6
--- /dev/null
+++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/licenses/nimbus-jose-jwt-NOTICE.txt
@@ -0,0 +1,14 @@
+Nimbus JOSE + JWT
+
+Copyright 2012 - 2018, Connect2id Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/src/main/java/com/nimbusds/jose/util/JSONObjectUtils.java b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/src/main/java/com/nimbusds/jose/util/JSONObjectUtils.java
new file mode 100644
index 0000000000000..1ea11f5c280ef
--- /dev/null
+++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/src/main/java/com/nimbusds/jose/util/JSONObjectUtils.java
@@ -0,0 +1,208 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package com.nimbusds.jose.util;
+
+import java.net.URI;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.text.ParseException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This class wraps {@link org.elasticsearch.nimbus.jose.util.JSONObjectUtils}, which is copied directly from the source
+ * library, and delegates to that class as quickly as possible. This layer is only here to provide a point at which we
+ * can insert {@link java.security.AccessController#doPrivileged(PrivilegedAction)} calls as necessary. We don't do
+ * anything here other than ensure gson has the proper security manager permissions.
+ */
+public class JSONObjectUtils {
+
+    public static Map<String, Object> parse(final String s) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<Map<String, Object>>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.parse(s)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static Map<String, Object> parse(final String s, final int sizeLimit) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<Map<String, Object>>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.parse(
+                    s,
+                    sizeLimit
+                )
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    @Deprecated
+    public static Map<String, Object> parseJSONObject(final String s) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<Map<String, Object>>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.parseJSONObject(s)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static boolean getBoolean(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<Boolean>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getBoolean(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static int getInt(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<Integer>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getInt(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static long getLong(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<Long>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getLong(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static float getFloat(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<Float>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getFloat(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static double getDouble(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<Double>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getDouble(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static String getString(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<String>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getString(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static URI getURI(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<URI>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getURI(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static List<Object> getJSONArray(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<List<Object>>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getJSONArray(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static String[] getStringArray(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<String[]>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getStringArray(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static Map<String, Object>[] getJSONObjectArray(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<Map<String, Object>[]>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils
+                    .getJSONObjectArray(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static List<String> getStringList(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<List<String>>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getStringList(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static Map<String, Object> getJSONObject(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<Map<String, Object>>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getJSONObject(
+                    o,
+                    key
+                )
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static Base64URL getBase64URL(final Map<String, Object> o, final String key) throws ParseException {
+        try {
+            return AccessController.doPrivileged(
+                (PrivilegedExceptionAction<Base64URL>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.getBase64URL(o, key)
+            );
+        } catch (PrivilegedActionException e) {
+            throw (ParseException) e.getException();
+        }
+    }
+
+    public static String toJSONString(final Map<String, ?> o) {
+        return AccessController.doPrivileged(
+            (PrivilegedAction<String>) () -> org.elasticsearch.nimbus.jose.util.JSONObjectUtils.toJSONString(o)
+        );
+    }
+
+    public static Map<String, Object> newJSONObject() {
+        return AccessController.doPrivileged(
+            (PrivilegedAction<Map<String, Object>>) org.elasticsearch.nimbus.jose.util.JSONObjectUtils::newJSONObject
+        );
+    }
+
+    private JSONObjectUtils() {}
+}
diff --git a/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/src/main/java/com/nimbusds/jose/util/JSONStringUtils.java b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/src/main/java/com/nimbusds/jose/util/JSONStringUtils.java
new file mode 100644
index 0000000000000..e9e34d21ce7d6
--- /dev/null
+++ b/x-pack/plugin/security/lib/nimbus-jose-jwt-modified/src/main/java/com/nimbusds/jose/util/JSONStringUtils.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package com.nimbusds.jose.util;
+
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+
+/**
+ * This class wraps {@link JSONStringUtils}, which is copied directly from the source library, and delegates to
+ * that class as quickly as possible. This layer is only here to provide a point at which we can insert
+ * {@link java.security.AccessController#doPrivileged(PrivilegedAction)} calls as necessary. We don't do anything here
+ * other than ensure gson has the proper security manager permissions.
+ */
+public class JSONStringUtils {
+
+    public static String toJSONString(final String string) {
+        return AccessController.doPrivileged((PrivilegedAction<String>) () -> JSONStringUtils.toJSONString(string));
+    }
+
+    private JSONStringUtils() {}
+}
diff --git a/x-pack/plugin/security/qa/consistency-checks/src/test/java/org/elasticsearch/xpack/security/CrossClusterShardTests.java b/x-pack/plugin/security/qa/consistency-checks/src/test/java/org/elasticsearch/xpack/security/CrossClusterShardTests.java
index d9e870b031877..ab5be0f48f5f3 100644
--- a/x-pack/plugin/security/qa/consistency-checks/src/test/java/org/elasticsearch/xpack/security/CrossClusterShardTests.java
+++ b/x-pack/plugin/security/qa/consistency-checks/src/test/java/org/elasticsearch/xpack/security/CrossClusterShardTests.java
@@ -11,12 +11,12 @@
 import org.elasticsearch.action.search.TransportSearchShardsAction;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
-import org.elasticsearch.common.inject.Binding;
-import org.elasticsearch.common.inject.TypeLiteral;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
 import org.elasticsearch.index.rankeval.RankEvalPlugin;
 import org.elasticsearch.ingest.IngestTestPlugin;
 import org.elasticsearch.ingest.common.IngestCommonPlugin;
+import org.elasticsearch.injection.guice.Binding;
+import org.elasticsearch.injection.guice.TypeLiteral;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.reindex.ReindexPlugin;
@@ -61,7 +61,10 @@ public class CrossClusterShardTests extends ESSingleNodeTestCase {
         DownsampleShardPersistentTaskExecutor.DelegatingAction.NAME,
 
         // These actions do not have any references to shard IDs in their requests.
-        TransportClusterSearchShardsAction.TYPE.name()
+        TransportClusterSearchShardsAction.TYPE.name(),
+
+        // forked search_shards for ES|QL
+        "indices:data/read/esql/search_shards"
     );
 
     Set<Class<?>> CHECKED_ABSTRACT_CLASSES = Set.of(
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityTestCase.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityTestCase.java
index c4a058013caf2..2c61707f57a5a 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityTestCase.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityTestCase.java
@@ -48,11 +48,12 @@ public abstract class AbstractRemoteClusterSecurityTestCase extends ESRestTestCa
     protected static final String USER = "test_user";
     protected static final SecureString PASS = new SecureString("x-pack-test-password".toCharArray());
     protected static final String REMOTE_SEARCH_USER = "remote_search_user";
+    protected static final String MANAGE_USER = "manage_user";
     protected static final String REMOTE_METRIC_USER = "remote_metric_user";
     protected static final String REMOTE_TRANSFORM_USER = "remote_transform_user";
     protected static final String REMOTE_SEARCH_ROLE = "remote_search";
     protected static final String REMOTE_CLUSTER_ALIAS = "my_remote_cluster";
-    private static final String KEYSTORE_PASSWORD = "keystore-password";
+    static final String KEYSTORE_PASSWORD = "keystore-password";
 
     protected static LocalClusterConfigProvider commonClusterConfig = cluster -> cluster.module("analysis-common")
         .keystorePassword(KEYSTORE_PASSWORD)
@@ -217,7 +218,7 @@ protected void removeRemoteClusterCredentials(String clusterAlias, MutableSettin
     }
 
     @SuppressWarnings("unchecked")
-    private void reloadSecureSettings() throws IOException {
+    protected void reloadSecureSettings() throws IOException {
         final Request request = new Request("POST", "/_nodes/reload_secure_settings");
         request.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}");
         final Response reloadResponse = adminClient().performRequest(request);
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java
index cbf735c66462c..17acd258ed34b 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java
@@ -113,6 +113,12 @@ public void testBwcWithLegacyCrossClusterSearch() throws Exception {
                       "privileges": ["read", "read_cross_cluster"],
                       "clusters": ["my_remote_cluster"]
                     }
+                  ],
+                  "remote_cluster": [
+                    {
+                      "privileges": ["monitor_enrich"],
+                      "clusters": ["*"]
+                    }
                   ]
                 }""");
             assertOK(adminClient().performRequest(putRoleRequest));
@@ -157,6 +163,12 @@ public void testBwcWithLegacyCrossClusterSearch() throws Exception {
                           "privileges": ["read", "read_cross_cluster"],
                           "clusters": ["my_remote_*", "non_existing_remote_cluster"]
                         }
+                      ],
+                      "remote_cluster": [
+                        {
+                          "privileges": ["monitor_enrich"],
+                          "clusters": ["*"]
+                        }
                       ]
                     }
                   }
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java
index b7198a6a88984..f5f9410a145cc 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.junit.RunnableTestRuleAdapter;
@@ -85,7 +86,7 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe
                         {
                           "search": [
                             {
-                                "names": ["index*", "not_found_index", "employees", "employees2"]
+                                "names": ["index*", "alias*", "not_found_index", "employees", "employees2"]
                             },
                             {
                                 "names": ["employees3"],
@@ -347,21 +348,6 @@ public void testCrossClusterQuery() throws Exception {
             | LIMIT 10"""));
         assertRemoteAndLocalResults(response);
 
-        // query remote cluster only - but also include employees2 which the user does not have access to
-        response = performRequestWithRemoteSearchUser(esqlRequest("""
-            FROM my_remote_cluster:employees,my_remote_cluster:employees2
-            | SORT emp_id ASC
-            | LIMIT 2
-            | KEEP emp_id, department"""));
-        assertRemoteOnlyResults(response); // same as above since the user only has access to employees
-
-        // query remote and local cluster - but also include employees2 which the user does not have access to
-        response = performRequestWithRemoteSearchUser(esqlRequest("""
-            FROM my_remote_cluster:employees,my_remote_cluster:employees2,employees,employees2
-            | SORT emp_id ASC
-            | LIMIT 10"""));
-        assertRemoteAndLocalResults(response); // same as above since the user only has access to employees
-
         // update role to include both employees and employees2 for the remote cluster
         final var putRoleRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE);
         putRoleRequest.setJsonEntity("""
@@ -618,6 +604,37 @@ public void testCrossClusterQueryWithOnlyRemotePrivs() throws Exception {
                     + "this action is granted by the index privileges [read,read_cross_cluster,all]"
             )
         );
+
+        // query remote cluster only - but also include employees2 which the user does not have access to
+        error = expectThrows(ResponseException.class, () -> { performRequestWithRemoteSearchUser(esqlRequest("""
+            FROM my_remote_cluster:employees,my_remote_cluster:employees2
+            | SORT emp_id ASC
+            | LIMIT 2
+            | KEEP emp_id, department""")); });
+
+        assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(403));
+        assertThat(
+            error.getMessage(),
+            containsString(
+                "action [indices:data/read/esql] is unauthorized for user [remote_search_user] with effective roles "
+                    + "[remote_search], this action is granted by the index privileges [read,read_cross_cluster,all]"
+            )
+        );
+
+        // query remote and local cluster - but also include employees2 which the user does not have access to
+        error = expectThrows(ResponseException.class, () -> { performRequestWithRemoteSearchUser(esqlRequest("""
+            FROM my_remote_cluster:employees,my_remote_cluster:employees2,employees,employees2
+            | SORT emp_id ASC
+            | LIMIT 10""")); });
+
+        assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(403));
+        assertThat(
+            error.getMessage(),
+            containsString(
+                "action [indices:data/read/esql] is unauthorized for user [remote_search_user] with effective roles "
+                    + "[remote_search], this action is granted by the index privileges [read,read_cross_cluster,all]"
+            )
+        );
     }
 
     @SuppressWarnings("unchecked")
@@ -740,6 +757,159 @@ public void testCrossClusterEnrichWithOnlyRemotePrivs() throws Exception {
         assertThat(flatList, containsInAnyOrder(1, 3, "usa", "germany"));
     }
 
+    private void createAliases() throws Exception {
+        Request createAlias = new Request("POST", "_aliases");
+        createAlias.setJsonEntity("""
+            {
+                "actions": [
+                    {
+                        "add": {
+                            "index": "employees",
+                            "alias": "alias-employees"
+                        }
+                    },
+                    {
+                        "add": {
+                            "index": "employees",
+                            "alias": "alias-engineering",
+                            "filter": { "match": { "department": "engineering" }}
+                        }
+                    },
+                    {
+                        "add": {
+                            "index": "employees",
+                            "alias": "alias-management",
+                            "filter": { "match": { "department": "management" }}
+                        }
+                    },
+                    {
+                        "add": {
+                           "index": "employees2",
+                           "alias": "alias-employees2"
+                        }
+                    }
+                ]
+            }
+            """);
+        assertOK(performRequestAgainstFulfillingCluster(createAlias));
+    }
+
+    private void removeAliases() throws Exception {
+        var removeAlias = new Request("POST", "/_aliases/");
+        removeAlias.setJsonEntity("""
+            {
+                "actions": [
+                    {
+                        "remove": {
+                            "index": "employees",
+                            "alias": "alias-employees"
+                        }
+                    },
+                    {
+                        "remove": {
+                            "index": "employees",
+                            "alias": "alias-engineering"
+                        }
+                    },
+                    {
+                        "remove": {
+                            "index": "employees",
+                            "alias": "alias-management"
+                        }
+                    },
+                    {
+                        "remove": {
+                           "index": "employees2",
+                           "alias": "alias-employees2"
+                        }
+                    }
+                ]
+            }
+            """);
+        assertOK(performRequestAgainstFulfillingCluster(removeAlias));
+    }
+
+    public void testAlias() throws Exception {
+        configureRemoteCluster();
+        populateData();
+        createAliases();
+        var putRoleRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE);
+        putRoleRequest.setJsonEntity("""
+            {
+              "indices": [{"names": [""], "privileges": ["read"]}],
+              "cluster": ["cross_cluster_search"],
+              "remote_indices": [
+                {
+                  "names": ["alias-engineering"],
+                  "privileges": ["read"],
+                  "clusters": ["my_remote_cluster"]
+                },
+                {
+                  "names": ["employees2"],
+                  "privileges": ["read"],
+                  "clusters": ["my_remote_cluster"]
+                },
+                {
+                  "names": ["employees3"],
+                  "privileges": ["view_index_metadata", "read_cross_cluster"],
+                  "clusters": ["my_remote_cluster"]
+                }
+              ]
+            }""");
+        assertOK(adminClient().performRequest(putRoleRequest));
+        // query `employees2`
+        for (String index : List.of("*:employees2", "*:employee*")) {
+            Request request = esqlRequest("FROM " + index + " | KEEP emp_id | SORT emp_id | LIMIT 100");
+            Response response = performRequestWithRemoteSearchUser(request);
+            assertOK(response);
+            Map<String, Object> responseAsMap = entityAsMap(response);
+            List<?> ids = (List<?>) responseAsMap.get("values");
+            assertThat(ids, equalTo(List.of(List.of("11"), List.of("13"))));
+        }
+
+        // query `employees2` and `alias-engineering`
+        for (var index : List.of("*:employees2,*:alias-engineering", "*:emp*,*:alias-engineering", "*:emp*,my*:alias*")) {
+            Request request = esqlRequest("FROM " + index + " | KEEP emp_id | SORT emp_id | LIMIT 100");
+            Response response = performRequestWithRemoteSearchUser(request);
+            assertOK(response);
+            Map<String, Object> responseAsMap = entityAsMap(response);
+            List<?> ids = (List<?>) responseAsMap.get("values");
+            assertThat(ids, equalTo(List.of(List.of("1"), List.of("11"), List.of("13"), List.of("7"))));
+        }
+        // none
+        for (var index : List.of("*:employees1", "*:employees3", "*:employees1,employees3", "*:alias-employees,*:alias-management")) {
+            Request request = esqlRequest("FROM " + index + " | KEEP emp_id | SORT emp_id | LIMIT 100");
+            ResponseException error = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(request));
+            assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400));
+            assertThat(error.getMessage(), containsString(" Unknown index [" + index + "]"));
+        }
+
+        for (var index : List.of(
+            Tuple.tuple("*:employee*,*:alias-employees,*:employees3", "alias-employees,employees3"),
+            Tuple.tuple("*:alias*,my*:employees1", "employees1"),
+            Tuple.tuple("*:alias*,my*:employees3", "employees3")
+        )) {
+            Request request = esqlRequest("FROM " + index.v1() + " | KEEP emp_id | SORT emp_id | LIMIT 100");
+            ResponseException error = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(request));
+            assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(403));
+            assertThat(
+                error.getMessage(),
+                containsString("unauthorized for user [remote_search_user] with assigned roles [remote_search]")
+            );
+            assertThat(error.getMessage(), containsString("user [test_user] on indices [" + index.v2() + "]"));
+        }
+
+        // query `alias-engineering`
+        Request request = esqlRequest("FROM *:alias* | KEEP emp_id | SORT emp_id | LIMIT 100");
+        Response response = performRequestWithRemoteSearchUser(request);
+        assertOK(response);
+        Map<String, Object> responseAsMap = entityAsMap(response);
+        List<?> ids = (List<?>) responseAsMap.get("values");
+        assertThat(ids, equalTo(List.of(List.of("1"), List.of("7"))));
+
+        removeAliases();
+    }
+
     protected Request esqlRequest(String command) throws IOException {
         XContentBuilder body = JsonXContent.contentBuilder();
         body.startObject();
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java
index a721605d228db..42982e6183613 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java
@@ -34,7 +34,12 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
 
 // account for slow stored secure settings updates (involves removing and re-creating the keystore)
 @TimeoutSuite(millis = 10 * TimeUnits.MINUTE)
@@ -78,6 +83,7 @@ public class RemoteClusterSecurityReloadCredentialsRestIT extends AbstractRemote
             })
             .rolesFile(Resource.fromClasspath("roles.yml"))
             .user(REMOTE_SEARCH_USER, PASS.toString(), "read_remote_shared_logs", false)
+            .user(MANAGE_USER, PASS.toString(), "manage_role", false)
             .build();
     }
 
@@ -237,4 +243,28 @@ private Response performRequestWithRemoteSearchUser(final Request request) throw
         return client().performRequest(request);
     }
 
+    @Override
+    @SuppressWarnings("unchecked")
+    protected void reloadSecureSettings() throws IOException {
+        final Request request = new Request("POST", "/_nodes/reload_secure_settings");
+        request.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}");
+        // execute as user with only `manage` cluster privilege
+        final Response reloadResponse = performRequestWithManageUser(request);
+        assertOK(reloadResponse);
+        final Map<String, Object> map = entityAsMap(reloadResponse);
+        assertThat(map.get("nodes"), instanceOf(Map.class));
+        final Map<String, Object> nodes = (Map<String, Object>) map.get("nodes");
+        assertThat(nodes, is(not(anEmptyMap())));
+        for (Map.Entry<String, Object> entry : nodes.entrySet()) {
+            assertThat(entry.getValue(), instanceOf(Map.class));
+            final Map<String, Object> node = (Map<String, Object>) entry.getValue();
+            assertThat(node.get("reload_exception"), nullValue());
+        }
+    }
+
+    private Response performRequestWithManageUser(final Request request) throws IOException {
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", headerFromRandomAuthMethod(MANAGE_USER, PASS)));
+        return client().performRequest(request);
+    }
+
 }
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml
index d0487f378d34e..b61daa068ed1a 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml
@@ -38,3 +38,6 @@ ccr_user_role:
     - names: [ 'leader-index', 'shared-*', 'metrics-*' ]
       privileges: [ 'cross_cluster_replication' ]
       clusters: [ "*" ]
+
+manage_role:
+  cluster: [ 'manage' ]
diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
index 9eee5b0bd7a6f..c5304d8313df2 100644
--- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
+++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
@@ -22,8 +22,6 @@ public class Constants {
         "cluster:admin/component_template/delete",
         "cluster:admin/component_template/get",
         "cluster:admin/component_template/put",
-        "cluster:admin/data_stream/global_retention/delete",
-        "cluster:admin/data_stream/global_retention/put",
         "cluster:admin/deprecation/cache/reset",
         "cluster:admin/fleet/secrets/delete",
         "cluster:admin/fleet/secrets/get",
@@ -54,6 +52,7 @@ public class Constants {
         "cluster:admin/migration/get_system_feature",
         "cluster:admin/migration/post_system_feature",
         "cluster:admin/nodes/reload_secure_settings",
+        "cluster:admin/nodes/reload_secure_settings/security/remote_cluster_credentials",
         "cluster:admin/persistent/completion",
         "cluster:admin/persistent/remove",
         "cluster:admin/persistent/start",
@@ -278,7 +277,6 @@ public class Constants {
         "cluster:admin/xpack/security/profile/suggest",
         "cluster:admin/xpack/security/profile/set_enabled",
         "cluster:admin/xpack/security/realm/cache/clear",
-        "cluster:admin/xpack/security/remote_cluster_credentials/reload",
         "cluster:admin/xpack/security/role/delete",
         "cluster:admin/xpack/security/role/get",
         "cluster:admin/xpack/security/role/query",
@@ -344,7 +342,6 @@ public class Constants {
         "cluster:monitor/ccr/follow_info",
         "cluster:monitor/ccr/follow_stats",
         "cluster:monitor/ccr/stats",
-        "cluster:monitor/data_stream/global_retention/get",
         "cluster:monitor/data_stream/lifecycle/stats",
         "cluster:monitor/eql/async/status",
         "cluster:monitor/fetch/health/info",
@@ -557,6 +554,7 @@ public class Constants {
         "indices:data/read/esql",
         "indices:data/read/esql/async/get",
         "indices:data/read/esql/resolve_fields",
+        "indices:data/read/esql/search_shards",
         "indices:data/read/explain",
         "indices:data/read/field_caps",
         "indices:data/read/get",
diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/LicenseDLSFLSRoleIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/LicenseDLSFLSRoleIT.java
index f81bab4866bdf..552e9f5cba578 100644
--- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/LicenseDLSFLSRoleIT.java
+++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/LicenseDLSFLSRoleIT.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.SecureString;
@@ -21,6 +20,8 @@
 import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.ClassRule;
 
 import java.io.IOException;
@@ -50,8 +51,6 @@ public final class LicenseDLSFLSRoleIT extends ESRestTestCase {
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .nodes(1)
         .distribution(DistributionType.DEFAULT)
-        // start as "trial"
-        .setting("xpack.license.self_generated.type", "trial")
         .setting("xpack.security.enabled", "true")
         .setting("xpack.security.http.ssl.enabled", "false")
         .setting("xpack.security.transport.ssl.enabled", "false")
@@ -61,6 +60,23 @@ public final class LicenseDLSFLSRoleIT extends ESRestTestCase {
         .user(READ_SECURITY_USER, READ_SECURITY_PASSWORD.toString(), "read_security_user_role", false)
         .build();
 
+    @Before
+    public void setupLicense() throws IOException {
+        // start with trial license
+        Request request = new Request("POST", "/_license/start_trial?acknowledge=true");
+        Response response = adminClient().performRequest(request);
+        assertOK(response);
+        assertTrue((boolean) responseAsMap(response).get("trial_was_started"));
+    }
+
+    @After
+    public void removeLicense() throws IOException {
+        // start with trial license
+        Request request = new Request("DELETE", "/_license");
+        Response response = adminClient().performRequest(request);
+        assertOK(response);
+    }
+
     @Override
     protected String getTestRestCluster() {
         return cluster.getHttpAddresses();
@@ -78,10 +94,7 @@ protected Settings restClientSettings() {
         return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
     }
 
-    @SuppressWarnings("unchecked")
     public void testQueryDLSFLSRolesShowAsDisabled() throws Exception {
-        // auto-generated "trial"
-        waitForLicense(adminClient(), "trial");
         // neither DLS nor FLS role
         {
             RoleDescriptor.IndicesPrivileges[] indicesPrivileges = new RoleDescriptor.IndicesPrivileges[] {
@@ -138,7 +151,6 @@ public void testQueryDLSFLSRolesShowAsDisabled() throws Exception {
         Map<String, Object> responseMap = responseAsMap(response);
         assertTrue(((Boolean) responseMap.get("basic_was_started")));
         assertTrue(((Boolean) responseMap.get("acknowledged")));
-        waitForLicense(adminClient(), "basic");
         // now the same roles show up as disabled ("enabled" is "false")
         assertQuery(client(), "", 4, roles -> {
             roles.sort(Comparator.comparing(o -> ((String) o.get("name"))));
@@ -175,22 +187,4 @@ private static void assertRoleEnabled(Map<String, Object> roleMap, boolean enabl
         assertThat(roleMap.get("transient_metadata"), instanceOf(Map.class));
         assertThat(((Map<String, Object>) roleMap.get("transient_metadata")).get("enabled"), equalTo(enabled));
     }
-
-    @SuppressWarnings("unchecked")
-    private static void waitForLicense(RestClient adminClient, String type) throws Exception {
-        final Request request = new Request("GET", "_license");
-        assertBusy(() -> {
-            Response response;
-            try {
-                response = adminClient.performRequest(request);
-            } catch (ResponseException e) {
-                throw new AssertionError("license not yet installed", e);
-            }
-            assertOK(response);
-            Map<String, Object> responseMap = responseAsMap(response);
-            assertTrue(responseMap.containsKey("license"));
-            assertThat(((Map<String, Object>) responseMap.get("license")).get("status"), equalTo("active"));
-            assertThat(((Map<String, Object>) responseMap.get("license")).get("type"), equalTo(type));
-        });
-    }
 }
diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/ManageRolesPrivilegeIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/ManageRolesPrivilegeIT.java
new file mode 100644
index 0000000000000..728f068adcae4
--- /dev/null
+++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/ManageRolesPrivilegeIT.java
@@ -0,0 +1,211 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.test.TestSecurityClient;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.core.StringContains.containsString;
+
+public class ManageRolesPrivilegeIT extends SecurityInBasicRestTestCase {
+
+    private TestSecurityClient adminSecurityClient;
+    private static final SecureString TEST_PASSWORD = new SecureString("100%-secure-password".toCharArray());
+
+    @Before
+    public void setupClient() {
+        adminSecurityClient = new TestSecurityClient(adminClient());
+    }
+
+    public void testManageRoles() throws Exception {
+        createManageRolesRole("manage-roles-role", new String[0], Set.of("*-allowed-suffix"), Set.of("read", "write"));
+        createUser("test-user", Set.of("manage-roles-role"));
+
+        String authHeader = basicAuthHeaderValue("test-user", TEST_PASSWORD);
+
+        createRole(
+            authHeader,
+            new RoleDescriptor(
+                "manage-roles-role",
+                new String[0],
+                new RoleDescriptor.IndicesPrivileges[] {
+                    RoleDescriptor.IndicesPrivileges.builder().indices("test-allowed-suffix").privileges(Set.of("read", "write")).build() },
+                new RoleDescriptor.ApplicationResourcePrivileges[0],
+                new ConfigurableClusterPrivilege[0],
+                new String[0],
+                Map.of(),
+                Map.of()
+            )
+        );
+
+        {
+            ResponseException responseException = assertThrows(
+                ResponseException.class,
+                () -> createRole(
+                    authHeader,
+                    new RoleDescriptor(
+                        "manage-roles-role",
+                        new String[0],
+                        new RoleDescriptor.IndicesPrivileges[] {
+                            RoleDescriptor.IndicesPrivileges.builder().indices("test-suffix-not-allowed").privileges("write").build() },
+                        new RoleDescriptor.ApplicationResourcePrivileges[0],
+                        new ConfigurableClusterPrivilege[0],
+                        new String[0],
+                        Map.of(),
+                        Map.of()
+                    )
+                )
+            );
+
+            assertThat(
+                responseException.getMessage(),
+                containsString("this action is granted by the cluster privileges [manage_security,all]")
+            );
+        }
+
+        {
+            ResponseException responseException = assertThrows(
+                ResponseException.class,
+                () -> createRole(
+                    authHeader,
+                    new RoleDescriptor(
+                        "manage-roles-role",
+                        new String[0],
+                        new RoleDescriptor.IndicesPrivileges[] {
+                            RoleDescriptor.IndicesPrivileges.builder().indices("test-allowed-suffix").privileges("manage").build() },
+                        new RoleDescriptor.ApplicationResourcePrivileges[0],
+                        new ConfigurableClusterPrivilege[0],
+                        new String[0],
+                        Map.of(),
+                        Map.of()
+                    )
+                )
+            );
+            assertThat(
+                responseException.getMessage(),
+                containsString("this action is granted by the cluster privileges [manage_security,all]")
+            );
+        }
+    }
+
+    public void testManageSecurityNullifiesManageRoles() throws Exception {
+        createManageRolesRole("manage-roles-no-manage-security", new String[0], Set.of("allowed"));
+        createManageRolesRole("manage-roles-manage-security", new String[] { "manage_security" }, Set.of("allowed"));
+
+        createUser("test-user-no-manage-security", Set.of("manage-roles-no-manage-security"));
+        createUser("test-user-manage-security", Set.of("manage-roles-manage-security"));
+
+        String authHeaderNoManageSecurity = basicAuthHeaderValue("test-user-no-manage-security", TEST_PASSWORD);
+        String authHeaderManageSecurity = basicAuthHeaderValue("test-user-manage-security", TEST_PASSWORD);
+
+        createRole(
+            authHeaderNoManageSecurity,
+            new RoleDescriptor(
+                "test-role-allowed-by-manage-roles",
+                new String[0],
+                new RoleDescriptor.IndicesPrivileges[] {
+                    RoleDescriptor.IndicesPrivileges.builder().indices("allowed").privileges("read").build() },
+                new RoleDescriptor.ApplicationResourcePrivileges[0],
+                new ConfigurableClusterPrivilege[0],
+                new String[0],
+                Map.of(),
+                Map.of()
+            )
+        );
+
+        ResponseException responseException = assertThrows(
+            ResponseException.class,
+            () -> createRole(
+                authHeaderNoManageSecurity,
+                new RoleDescriptor(
+                    "test-role-not-allowed-by-manage-roles",
+                    new String[0],
+                    new RoleDescriptor.IndicesPrivileges[] {
+                        RoleDescriptor.IndicesPrivileges.builder().indices("not-allowed").privileges("read").build() },
+                    new RoleDescriptor.ApplicationResourcePrivileges[0],
+                    new ConfigurableClusterPrivilege[0],
+                    new String[0],
+                    Map.of(),
+                    Map.of()
+                )
+            )
+        );
+
+        assertThat(
+            responseException.getMessage(),
+            // TODO Should the new global role/manage privilege be listed here? Probably not because it's not documented
+            containsString("this action is granted by the cluster privileges [manage_security,all]")
+        );
+
+        createRole(
+            authHeaderManageSecurity,
+            new RoleDescriptor(
+                "test-role-not-allowed-by-manage-roles",
+                new String[0],
+                new RoleDescriptor.IndicesPrivileges[] {
+                    RoleDescriptor.IndicesPrivileges.builder().indices("not-allowed").privileges("read").build() },
+                new RoleDescriptor.ApplicationResourcePrivileges[0],
+                new ConfigurableClusterPrivilege[0],
+                new String[0],
+                Map.of(),
+                Map.of()
+            )
+        );
+    }
+
+    private void createRole(String authHeader, RoleDescriptor descriptor) throws IOException {
+        TestSecurityClient userAuthSecurityClient = new TestSecurityClient(
+            adminClient(),
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authHeader).build()
+        );
+        userAuthSecurityClient.putRole(descriptor);
+    }
+
+    private void createUser(String username, Set<String> roles) throws IOException {
+        adminSecurityClient.putUser(new User(username, roles.toArray(String[]::new)), TEST_PASSWORD);
+    }
+
+    private void createManageRolesRole(String roleName, String[] clusterPrivileges, Set<String> indexPatterns) throws IOException {
+        createManageRolesRole(roleName, clusterPrivileges, indexPatterns, Set.of("read"));
+    }
+
+    private void createManageRolesRole(String roleName, String[] clusterPrivileges, Set<String> indexPatterns, Set<String> privileges)
+        throws IOException {
+        adminSecurityClient.putRole(
+            new RoleDescriptor(
+                roleName,
+                clusterPrivileges,
+                new RoleDescriptor.IndicesPrivileges[0],
+                new RoleDescriptor.ApplicationResourcePrivileges[0],
+                new ConfigurableClusterPrivilege[] {
+                    new ConfigurableClusterPrivileges.ManageRolesPrivilege(
+                        List.of(
+                            new ConfigurableClusterPrivileges.ManageRolesPrivilege.ManageRolesIndexPermissionGroup(
+                                indexPatterns.toArray(String[]::new),
+                                privileges.toArray(String[]::new)
+                            )
+                        )
+                    ) },
+                new String[0],
+                Map.of(),
+                Map.of()
+            )
+        );
+    }
+}
diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecuritySettingsIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecuritySettingsIT.java
index 5651538b2757c..f776c4834cc5f 100644
--- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecuritySettingsIT.java
+++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecuritySettingsIT.java
@@ -19,6 +19,7 @@
 import static org.elasticsearch.test.XContentTestUtils.createJsonMapView;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 public class SecuritySettingsIT extends SecurityInBasicRestTestCase {
 
@@ -70,6 +71,54 @@ public void testBasicWorkflow() throws IOException {
         assertOK(getResp);
         final XContentTestUtils.JsonMapView mapView = createJsonMapView(getResp.getEntity().getContent());
         assertThat(mapView.get("security.index.auto_expand_replicas"), equalTo("0-all"));
+        assertThat(mapView.get("security-profile.index.auto_expand_replicas"), equalTo("0-all"));
+    }
+
+    public void testTierPreference() throws IOException {
+        {
+            Request req = new Request("PUT", "/_security/settings");
+            req.setJsonEntity("""
+                {
+                    "security": {
+                        "index.routing.allocation.include._tier_preference": "data_hot"
+                    },
+                    "security-profile": {
+                        "index.routing.allocation.include._tier_preference": "data_hot"
+                    }
+                }
+                """);
+            Response resp = adminClient().performRequest(req);
+            assertOK(resp);
+            Request getRequest = new Request("GET", "/_security/settings");
+            Response getResp = adminClient().performRequest(getRequest);
+            assertOK(getResp);
+            final XContentTestUtils.JsonMapView mapView = createJsonMapView(getResp.getEntity().getContent());
+            assertThat(mapView.get("security.index.routing.allocation.include._tier_preference"), equalTo("data_hot"));
+            assertThat(mapView.get("security-profile.index.routing.allocation.include._tier_preference"), equalTo("data_hot"));
+        }
+
+        {
+            Request req = new Request("PUT", "/_security/settings");
+            req.setJsonEntity("""
+                {
+                    "security": {
+                        "index.routing.allocation.include._tier_preference": null
+                    },
+                    "security-profile": {
+                        "index.routing.allocation.include._tier_preference": null
+                    }
+                }
+                """);
+            Response resp = adminClient().performRequest(req);
+            assertOK(resp);
+            Request getRequest = new Request("GET", "/_security/settings");
+            Response getResp = adminClient().performRequest(getRequest);
+            assertOK(getResp);
+            final XContentTestUtils.JsonMapView mapView = createJsonMapView(getResp.getEntity().getContent());
+            assertThat(mapView.get("security.index.routing.allocation.include._tier_preference"), nullValue());
+            assertThat(mapView.get("security-profile.index.routing.allocation.include._tier_preference"), nullValue());
+        }
+
     }
 
     public void testNoUpdatesThrowsException() throws IOException {
@@ -85,7 +134,7 @@ public void testDisallowedSettingThrowsException() throws IOException {
         ResponseException ex = expectThrows(ResponseException.class, () -> adminClient().performRequest(req));
         assertThat(
             EntityUtils.toString(ex.getResponse().getEntity()),
-            containsString("illegal settings for index [security]: " + "[index.max_ngram_diff], these settings may not be configured.")
+            containsString("illegal setting for index [security]: " + "[index.max_ngram_diff], this setting may not be configured.")
         );
     }
 
diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
index 5ae84517202d4..667140b849951 100644
--- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
+++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java
@@ -31,6 +31,8 @@
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissionGroup;
 import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
 import org.elasticsearch.xpack.security.SecurityOnTrialLicenseRestTestCase;
 import org.junit.After;
 import org.junit.Before;
@@ -385,6 +387,50 @@ public void testGrantApiKeyWithOnlyManageOwnApiKeyPrivilegeFails() throws IOExce
         assertThat(e.getMessage(), containsString("action [" + GrantApiKeyAction.NAME + "] is unauthorized for user"));
     }
 
+    public void testApiKeyWithManageRoles() throws IOException {
+        RoleDescriptor role = roleWithManageRoles("manage-roles-role", new String[] { "manage_own_api_key" }, "allowed-prefix*");
+        getSecurityClient().putRole(role);
+        createUser("test-user", END_USER_PASSWORD, List.of("manage-roles-role"));
+
+        final Request createApiKeyrequest = new Request("POST", "_security/api_key");
+        createApiKeyrequest.setOptions(
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", basicAuthHeaderValue("test-user", END_USER_PASSWORD))
+        );
+        final Map<String, Object> requestBody = Map.of(
+            "name",
+            "test-api-key",
+            "role_descriptors",
+            Map.of(
+                "test-role",
+                XContentTestUtils.convertToMap(roleWithManageRoles("test-role", new String[0], "allowed-prefix*")),
+                "another-test-role",
+                // This is not allowed by the limited-by-role (creator of the api key), so should not grant access to not-allowed=prefix*
+                XContentTestUtils.convertToMap(roleWithManageRoles("another-test-role", new String[0], "not-allowed-prefix*"))
+            )
+        );
+
+        createApiKeyrequest.setJsonEntity(XContentTestUtils.convertToXContent(requestBody, XContentType.JSON).utf8ToString());
+        Map<String, Object> responseMap = responseAsMap(client().performRequest(createApiKeyrequest));
+        String encodedApiKey = responseMap.get("encoded").toString();
+
+        final Request createRoleRequest = new Request("POST", "_security/role/test-role");
+        createRoleRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", "ApiKey " + encodedApiKey));
+        // Allowed role by manage roles permission
+        {
+            createRoleRequest.setJsonEntity("""
+                {"indices": [{"names": ["allowed-prefix-test"],"privileges": ["read"]}]}""");
+            assertOK(client().performRequest(createRoleRequest));
+        }
+        // Not allowed role by manage roles permission
+        {
+            createRoleRequest.setJsonEntity("""
+                {"indices": [{"names": ["not-allowed-prefix-test"],"privileges": ["read"]}]}""");
+            final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(createRoleRequest));
+            assertEquals(403, e.getResponse().getStatusLine().getStatusCode());
+            assertThat(e.getMessage(), containsString("this action is granted by the cluster privileges [manage_security,all]"));
+        }
+    }
+
     public void testUpdateApiKey() throws IOException {
         final var apiKeyName = "my-api-key-name";
         final Map<String, Object> apiKeyMetadata = Map.of("not", "returned");
@@ -2393,6 +2439,27 @@ private void createRole(String name, Collection<String> localClusterPrivileges,
         getSecurityClient().putRole(role);
     }
 
+    private RoleDescriptor roleWithManageRoles(String name, String[] clusterPrivileges, String indexPattern) {
+        return new RoleDescriptor(
+            name,
+            clusterPrivileges,
+            null,
+            null,
+            new ConfigurableClusterPrivilege[] {
+                new ConfigurableClusterPrivileges.ManageRolesPrivilege(
+                    List.of(
+                        new ConfigurableClusterPrivileges.ManageRolesPrivilege.ManageRolesIndexPermissionGroup(
+                            new String[] { indexPattern },
+                            new String[] { "read" }
+                        )
+                    )
+                ) },
+            null,
+            null,
+            null
+        );
+    }
+
     protected void createRoleWithDescription(String name, Collection<String> clusterPrivileges, String description) throws IOException {
         final RoleDescriptor role = new RoleDescriptor(
             name,
diff --git a/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java b/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java
index e790866cf3d77..c1686a500fb2c 100644
--- a/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java
+++ b/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java
@@ -80,6 +80,33 @@ public class ServiceAccountIT extends ESRestTestCase {
         }
         """;
 
+    private static final String ELASTIC_AUTO_OPS_ROLE_DESCRIPTOR = """
+        {
+            "cluster": [
+                "monitor",
+                "read_ilm",
+                "read_slm"
+            ],
+            "indices": [
+                {
+                    "names": [
+                        "*"
+                    ],
+                    "privileges": [
+                        "monitor",
+                        "view_index_metadata"
+                    ],
+                    "allow_restricted_indices": true
+                }
+            ],
+            "applications": [],
+            "run_as": [],
+            "metadata": {},
+            "transient_metadata": {
+                "enabled": true
+            }
+        }""";
+
     private static final String ELASTIC_FLEET_SERVER_ROLE_DESCRIPTOR = """
         {
               "cluster": [
@@ -400,6 +427,10 @@ public void testGetServiceAccount() throws IOException {
         assertOK(getServiceAccountResponse3);
         assertServiceAccountRoleDescriptor(getServiceAccountResponse3, "elastic/fleet-server", ELASTIC_FLEET_SERVER_ROLE_DESCRIPTOR);
 
+        final Request getServiceAccountRequestAutoOps = new Request("GET", "_security/service/elastic/auto-ops");
+        final Response getServiceAccountResponseAutoOps = client().performRequest(getServiceAccountRequestAutoOps);
+        assertServiceAccountRoleDescriptor(getServiceAccountResponseAutoOps, "elastic/auto-ops", ELASTIC_AUTO_OPS_ROLE_DESCRIPTOR);
+
         final Request getServiceAccountRequestKibana = new Request("GET", "_security/service/elastic/kibana");
         final Response getServiceAccountResponseKibana = client().performRequest(getServiceAccountRequestKibana);
         assertOK(getServiceAccountResponseKibana);
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java
index 57a2272e38ce9..5f8744ace090d 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java
@@ -257,7 +257,10 @@ private Map<String, String> collectErrorsFromStoreAsMap() {
     }
 
     private List<Index> getDataStreamBackingIndices(String dataStreamName) {
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStreamName }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java
index 96b5a4445ed7e..2ab51bece41ea 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java
@@ -183,13 +183,20 @@ private Map<String, String> collectErrorsFromStoreAsMap() {
     private void prepareDataStreamAndIndex(String dataStreamName, DataStreamLifecycle lifecycle) throws IOException, InterruptedException,
         ExecutionException {
         putComposableIndexTemplate("id1", null, List.of(dataStreamName + "*"), null, null, lifecycle);
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            dataStreamName
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         indexDoc(dataStreamName);
     }
 
     private List<Index> getDataStreamBackingIndices(String dataStreamName) {
-        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
+        GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            new String[] { dataStreamName }
+        );
         GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
             .actionGet();
         assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamSecurityIT.java
index e37823f8d3c4c..2039b72c4f49f 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamSecurityIT.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamSecurityIT.java
@@ -68,7 +68,7 @@ public void testRemoveGhostReference() throws Exception {
         assertAcked(client.execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest).actionGet());
 
         String dataStreamName = "logs-es";
-        var request = new CreateDataStreamAction.Request(dataStreamName);
+        var request = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName);
         assertAcked(client.execute(CreateDataStreamAction.INSTANCE, request).actionGet());
         assertAcked(client.admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).actionGet());
 
@@ -125,7 +125,11 @@ public void onFailure(Exception e) {
         assertAcked(
             client.execute(
                 ModifyDataStreamsAction.INSTANCE,
-                new ModifyDataStreamsAction.Request(List.of(DataStreamAction.removeBackingIndex(dataStreamName, ghostReference.getName())))
+                new ModifyDataStreamsAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    TEST_REQUEST_TIMEOUT,
+                    List.of(DataStreamAction.removeBackingIndex(dataStreamName, ghostReference.getName()))
+                )
             )
         );
         ClusterState after = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state();
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java
index 3fbcd00690e82..b0572b265a45b 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java
@@ -8,6 +8,8 @@
 package org.elasticsearch.integration;
 
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
@@ -348,8 +350,17 @@ public void testRequestCacheWithTemplateRoleQuery() {
     private void prepareIndices() {
         final Client client = client();
 
-        assertAcked(client.admin().cluster().preparePutStoredScript().setId("my-script").setContent(new BytesArray("""
-            {"script":{"source":"{\\"match\\":{\\"username\\":\\"{{_user.username}}\\"}}","lang":"mustache"}}"""), XContentType.JSON));
+        assertAcked(
+            safeExecute(
+                TransportPutStoredScriptAction.TYPE,
+                new PutStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).id("my-script")
+                    .content(
+                        new BytesArray("""
+                            {"script":{"source":"{\\"match\\":{\\"username\\":\\"{{_user.username}}\\"}}","lang":"mustache"}}"""),
+                        XContentType.JSON
+                    )
+            )
+        );
 
         assertAcked(indicesAdmin().prepareCreate(DLS_INDEX).addAlias(new Alias("dls-alias")).get());
         client.prepareIndex(DLS_INDEX).setId("101").setSource("number", 101, "letter", "A").get();
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java
index 73897fc38633a..1bf7d89347755 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java
@@ -13,13 +13,16 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.junit.BeforeClass;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
@@ -29,9 +32,12 @@ public class DocumentLevelSecurityRandomTests extends SecurityIntegTestCase {
 
     protected static final SecureString USERS_PASSWD = SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING;
 
-    // can't add a second test method, because each test run creates a new instance of this class and that will will result
-    // in a new random value:
-    private final int numberOfRoles = scaledRandomIntBetween(3, 99);
+    private static volatile int numberOfRoles;
+
+    @BeforeClass
+    public static void setupRoleCount() throws Exception {
+        numberOfRoles = scaledRandomIntBetween(3, 99);
+    }
 
     @Override
     protected String configUsers() {
@@ -119,4 +125,74 @@ public void testDuelWithAliasFilters() throws Exception {
         }
     }
 
+    public void testWithRuntimeFields() throws Exception {
+        assertAcked(
+            indicesAdmin().prepareCreate("test")
+                .setMapping(
+                    XContentFactory.jsonBuilder()
+                        .startObject()
+                        .startObject("runtime")
+                        .startObject("field1")
+                        .field("type", "keyword")
+                        .endObject()
+                        .endObject()
+                        .startObject("properties")
+                        .startObject("field2")
+                        .field("type", "keyword")
+                        .endObject()
+                        .endObject()
+                        .endObject()
+                )
+        );
+        doTestWithRuntimeFieldsInTestIndex();
+    }
+
+    public void testWithRuntimeFieldsAndSyntheticSource() throws Exception {
+        assertAcked(
+            indicesAdmin().prepareCreate("test")
+                .setMapping(
+                    XContentFactory.jsonBuilder()
+                        .startObject()
+                        .startObject("_source")
+                        .field("mode", "synthetic")
+                        .endObject()
+                        .startObject("runtime")
+                        .startObject("field1")
+                        .field("type", "keyword")
+                        .endObject()
+                        .startObject("field2")
+                        .field("type", "keyword")
+                        .endObject()
+                        .endObject()
+                        .startObject("properties")
+                        .startObject("field1")
+                        .field("type", "text")
+                        .field("store", true)
+                        .endObject()
+                        .startObject("field2")
+                        .field("type", "text")
+                        .field("store", true)
+                        .endObject()
+                        .endObject()
+                        .endObject()
+                )
+        );
+        doTestWithRuntimeFieldsInTestIndex();
+    }
+
+    private void doTestWithRuntimeFieldsInTestIndex() {
+        List<IndexRequestBuilder> requests = new ArrayList<>(47);
+        for (int i = 1; i <= 42; i++) {
+            requests.add(prepareIndex("test").setSource("field1", "value1", "field2", "foo" + i));
+        }
+        for (int i = 42; i <= 57; i++) {
+            requests.add(prepareIndex("test").setSource("field1", "value2", "field2", "foo" + i));
+        }
+        indexRandom(true, requests);
+        assertHitCount(
+            client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                .prepareSearch("test"),
+            42L
+        );
+    }
 }
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java
index 0fdb494ce500a..ba3481832052b 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java
@@ -199,7 +199,11 @@ public void testSearchResolveWildcardsRegexs() throws Exception {
 
     public void testSearchResolveDataStreams() throws Exception {
         putComposableIndexTemplate("id1", List.of("test*"));
-        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("test");
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "test"
+        );
         client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
 
         IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java
index d04c2a4b0c578..3c639471f80b5 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java
@@ -54,7 +54,7 @@ public void setupClusterBeforeSnapshot() throws IOException {
         logger.info("-->  creating ordinary index");
         final int shards = between(1, 10);
         ordinaryIndex = randomAlphaOfLength(4).toLowerCase(Locale.ROOT);
-        assertAcked(prepareCreate(ordinaryIndex, 0, Settings.builder().put("number_of_shards", shards).put("number_of_replicas", 0)));
+        assertAcked(prepareCreate(ordinaryIndex, 0, indexSettings(shards, 0)));
         ensureGreen();
 
         logger.info("-->  creating snapshot_user user");
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java
index 963c42c55aa60..4b8fbfd41acdf 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java
@@ -133,7 +133,7 @@ public void testProfileIndexAutoCreation() {
         final Settings settings = getIndexResponse.getSettings().get(INTERNAL_SECURITY_PROFILE_INDEX_8);
         assertThat(settings.get("index.number_of_shards"), equalTo("1"));
         assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-1"));
-        assertThat(settings.get("index.routing.allocation.include._tier_preference"), equalTo("data_content"));
+        assertThat(settings.get("index.routing.allocation.include._tier_preference"), equalTo("data_hot,data_content"));
 
         final Map<String, Object> mappings = getIndexResponse.getMappings().get(INTERNAL_SECURITY_PROFILE_INDEX_8).getSourceAsMap();
 
@@ -451,7 +451,7 @@ public void testSuggestProfilesWithHint() throws IOException {
         final List<String> spaces = List.of("space1", "space2", "space3", "space4", "*");
         final List<Profile> profiles = spaces.stream().map(space -> {
             final PlainActionFuture<Profile> future1 = new PlainActionFuture<>();
-            final String lastName = randomAlphaOfLengthBetween(3, 8);
+            final String lastName = randomAlphaOfLengthBetween(3, 8) + space;
             final Authentication.RealmRef realmRef = randomBoolean()
                 ? AuthenticationTestHelper.randomRealmRef(false)
                 : new Authentication.RealmRef(
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java
index ea9e7059c7ea8..c4c583a2c76c9 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java
@@ -165,7 +165,7 @@ protected InboundPipeline getInboundPipeline(Channel channel, boolean isRemoteCl
         } else {
             return new InboundPipeline(
                 getStatsTracker(),
-                threadPool::relativeTimeInMillis,
+                threadPool.relativeTimeInMillisSupplier(),
                 new InboundDecoder(recycler, RemoteClusterPortSettings.MAX_REQUEST_HEADER_SIZE.get(settings), SERVER),
                 new InboundAggregator(getInflightBreaker(), getRequestHandlers()::getHandler, ignoreDeserializationErrors()),
                 this::inboundMessage
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 11c688e9ee5eb..7e44d6d8b1c99 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -584,6 +584,7 @@ public class Security extends Plugin
 
     private Settings settings;
     private final boolean enabled;
+    private final SetOnce<Boolean> dlsFlsEnabled = new SetOnce<>();
     private final SecuritySystemIndices systemIndices;
     private final ListenableFuture<Void> nodeStartedListenable;
 
@@ -1106,12 +1107,13 @@ Collection<Object> createComponents(
         );
         components.add(authcService.get());
         systemIndices.getMainIndexManager().addStateListener(authcService.get()::onSecurityIndexStateChange);
-
+        dlsFlsEnabled.set(XPackSettings.DLS_FLS_ENABLED.get(settings));
         Set<RequestInterceptor> requestInterceptors = Sets.newHashSet(
-            new ResizeRequestInterceptor(threadPool, getLicenseState(), auditTrailService),
-            new IndicesAliasesRequestInterceptor(threadPool.getThreadContext(), getLicenseState(), auditTrailService)
+            new ResizeRequestInterceptor(threadPool, getLicenseState(), auditTrailService, dlsFlsEnabled.get()),
+            new IndicesAliasesRequestInterceptor(threadPool.getThreadContext(), getLicenseState(), auditTrailService, dlsFlsEnabled.get())
         );
-        if (XPackSettings.DLS_FLS_ENABLED.get(settings)) {
+
+        if (dlsFlsEnabled.get()) {
             requestInterceptors.addAll(
                 Arrays.asList(
                     new SearchRequestInterceptor(threadPool, getLicenseState()),
@@ -2131,6 +2133,9 @@ public Function<String, FieldPredicate> getFieldFilter() {
                 XPackLicenseState licenseState = getLicenseState();
                 IndicesAccessControl indicesAccessControl = threadContext.get()
                     .getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
+                if (dlsFlsEnabled.get() == false) {
+                    return FieldPredicate.ACCEPT_ALL;
+                }
                 if (indicesAccessControl == null) {
                     return FieldPredicate.ACCEPT_ALL;
                 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java
index 1b11737eac7bf..50236cad4fc39 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java
@@ -7,8 +7,8 @@
 package org.elasticsearch.xpack.security;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
index 3fa5e0e5319c7..4166633906f8a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
@@ -12,10 +12,10 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportClearSecurityCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportClearSecurityCacheAction.java
index f8df2461ec8e5..56965274c6fab 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportClearSecurityCacheAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportClearSecurityCacheAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportDelegatePkiAuthenticationAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportDelegatePkiAuthenticationAction.java
index 3010ce0b4e646..1f0861e52bbf3 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportDelegatePkiAuthenticationAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportDelegatePkiAuthenticationAction.java
@@ -12,10 +12,10 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantAction.java
index fffcb476abaa4..ea3cdab66beff 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGrantAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
@@ -50,7 +51,7 @@ public TransportGrantAction(
         AuthorizationService authorizationService,
         ThreadContext threadContext
     ) {
-        super(actionName, actionFilters, transportService.getTaskManager());
+        super(actionName, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.authenticationService = authenticationService;
         this.authorizationService = authorizationService;
         this.threadContext = threadContext;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBaseUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBaseUpdateApiKeyAction.java
index 33b1e44004454..95aa383faa246 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBaseUpdateApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBaseUpdateApiKeyAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.SecurityContext;
@@ -32,7 +33,7 @@ protected TransportBaseUpdateApiKeyAction(
         final ActionFilters actionFilters,
         final SecurityContext context
     ) {
-        super(actionName, actionFilters, transportService.getTaskManager());
+        super(actionName, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.securityContext = context;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBulkUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBulkUpdateApiKeyAction.java
index 3b978c3e44b4c..0d698628f46e7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBulkUpdateApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportBulkUpdateApiKeyAction.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateApiKeyAction.java
index 268afc7f0b32f..52b5d6d64e72d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateApiKeyAction.java
@@ -10,7 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -42,7 +43,7 @@ public TransportCreateApiKeyAction(
         CompositeRolesStore rolesStore,
         NamedXContentRegistry xContentRegistry
     ) {
-        super(CreateApiKeyAction.NAME, actionFilters, transportService.getTaskManager());
+        super(CreateApiKeyAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.apiKeyService = apiKeyService;
         this.resolver = new ApiKeyUserRoleDescriptorResolver(rolesStore, xContentRegistry);
         this.securityContext = context;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyAction.java
index eeccd4b833a23..e49746fbe87a0 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyAction.java
@@ -10,7 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.SecurityContext;
@@ -37,7 +38,7 @@ public TransportCreateCrossClusterApiKeyAction(
         ApiKeyService apiKeyService,
         SecurityContext context
     ) {
-        super(CreateCrossClusterApiKeyAction.NAME, actionFilters, transportService.getTaskManager());
+        super(CreateCrossClusterApiKeyAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.apiKeyService = apiKeyService;
         this.securityContext = context;
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGetApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGetApiKeyAction.java
index 2ae790aaa94e7..a8c4def137427 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGetApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGetApiKeyAction.java
@@ -11,7 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.SecurityContext;
@@ -36,7 +37,7 @@ public TransportGetApiKeyAction(
         SecurityContext context,
         ProfileService profileService
     ) {
-        super(GetApiKeyAction.NAME, actionFilters, transportService.getTaskManager());
+        super(GetApiKeyAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.apiKeyService = apiKeyService;
         this.securityContext = context;
         this.profileService = profileService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyAction.java
index 54e073906b815..a25566c9e3205 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyAction.java
@@ -9,8 +9,8 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportInvalidateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportInvalidateApiKeyAction.java
index 8b938fed34d56..5571afdef59e8 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportInvalidateApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportInvalidateApiKeyAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.SecurityContext;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java
index 1454b9e480a39..b1de7d5b2e7e7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java
@@ -11,7 +11,8 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
@@ -58,7 +59,7 @@ public TransportQueryApiKeyAction(
         SecurityContext context,
         ProfileService profileService
     ) {
-        super(QueryApiKeyAction.NAME, actionFilters, transportService.getTaskManager());
+        super(QueryApiKeyAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.apiKeyService = apiKeyService;
         this.securityContext = context;
         this.profileService = profileService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java
index b6e0854d6c443..8600f4bf4a839 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateApiKeyAction.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyAction.java
index f4578bf7a737c..2c2bc1c952d37 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyAction.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.SecurityContext;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportKibanaEnrollmentAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportKibanaEnrollmentAction.java
index fd81606f42e05..e12db61b1cfd5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportKibanaEnrollmentAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportKibanaEnrollmentAction.java
@@ -15,11 +15,11 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.ssl.SslKeyConfig;
 import org.elasticsearch.common.ssl.StoreKeyConfig;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.enrollment.KibanaEnrollmentAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentAction.java
index 6483b754f8ae9..b0daf15cd4764 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentAction.java
@@ -16,12 +16,12 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.ssl.SslKeyConfig;
 import org.elasticsearch.common.ssl.StoreKeyConfig;
 import org.elasticsearch.common.ssl.StoredCertificate;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportInfo;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectAuthenticateAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectAuthenticateAction.java
index 4b80b1266672d..32208483a5587 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectAuthenticateAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectAuthenticateAction.java
@@ -14,11 +14,11 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutAction.java
index 9c5db3a25558e..eaf45642d5af5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.oidc.OpenIdConnectLogoutAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectPrepareAuthenticationAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectPrepareAuthenticationAction.java
index 183ad92069242..519ee229d6063 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectPrepareAuthenticationAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectPrepareAuthenticationAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.oidc.OpenIdConnectPrepareAuthenticationAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportClearPrivilegesCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportClearPrivilegesCacheAction.java
index 282733c64d37b..852144d1c2777 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportClearPrivilegesCacheAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportClearPrivilegesCacheAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportDeletePrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportDeletePrivilegesAction.java
index 69366911f3655..412bfc12f12d0 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportDeletePrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportDeletePrivilegesAction.java
@@ -9,9 +9,9 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java
index 62f0087c1a2d2..c9555ce7e6394 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java
@@ -9,7 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesAction;
@@ -28,7 +29,7 @@ public class TransportGetBuiltinPrivilegesAction extends TransportAction<GetBuil
 
     @Inject
     public TransportGetBuiltinPrivilegesAction(ActionFilters actionFilters, TransportService transportService) {
-        super(GetBuiltinPrivilegesAction.NAME, actionFilters, transportService.getTaskManager());
+        super(GetBuiltinPrivilegesAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java
index a73fb13b5e2d2..d54aa585861b2 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportPutPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportPutPrivilegesAction.java
index f13e599ef741c..e8f4ef2093f89 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportPutPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportPutPrivilegesAction.java
@@ -10,9 +10,9 @@
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportActivateProfileAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportActivateProfileAction.java
index 4d76205d29021..02b0786048963 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportActivateProfileAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportActivateProfileAction.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportGetProfilesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportGetProfilesAction.java
index 611a31997fbb1..78a5e4afc7d80 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportGetProfilesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportGetProfilesAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.profile.GetProfilesAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesAction.java
index 1506fba51089e..29364ffdc6010 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskCancelledException;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportSetProfileEnabledAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportSetProfileEnabledAction.java
index 10c3f87389d52..ad419cc585c73 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportSetProfileEnabledAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportSetProfileEnabledAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.profile.SetProfileEnabledAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportSuggestProfilesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportSuggestProfilesAction.java
index 347b64586fd89..9dd63db21532f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportSuggestProfilesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportSuggestProfilesAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportUpdateProfileDataAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportUpdateProfileDataAction.java
index 4be262ab100ee..14c9fbeeee9d5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportUpdateProfileDataAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/profile/TransportUpdateProfileDataAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.profile.UpdateProfileDataAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java
index b17cf624bf83d..4d574c6b6c0ac 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportBulkDeleteRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportBulkDeleteRolesAction.java
index 1bd9e6e108e45..01fe03e4bbe0c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportBulkDeleteRolesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportBulkDeleteRolesAction.java
@@ -9,7 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.ActionTypes;
@@ -23,7 +24,7 @@ public class TransportBulkDeleteRolesAction extends TransportAction<BulkDeleteRo
 
     @Inject
     public TransportBulkDeleteRolesAction(ActionFilters actionFilters, NativeRolesStore rolesStore, TransportService transportService) {
-        super(ActionTypes.BULK_DELETE_ROLES.name(), actionFilters, transportService.getTaskManager());
+        super(ActionTypes.BULK_DELETE_ROLES.name(), actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.rolesStore = rolesStore;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportBulkPutRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportBulkPutRolesAction.java
index 19972e90bdbbe..6d719d9420853 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportBulkPutRolesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportBulkPutRolesAction.java
@@ -9,7 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.ActionTypes;
@@ -22,7 +23,7 @@ public class TransportBulkPutRolesAction extends TransportAction<BulkPutRolesReq
 
     @Inject
     public TransportBulkPutRolesAction(ActionFilters actionFilters, NativeRolesStore rolesStore, TransportService transportService) {
-        super(ActionTypes.BULK_PUT_ROLES.name(), actionFilters, transportService.getTaskManager());
+        super(ActionTypes.BULK_PUT_ROLES.name(), actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.rolesStore = rolesStore;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java
index a879f35e096da..82e62187f7f47 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java
index 5548fc7497fd3..e8d248233415c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java
@@ -10,7 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.role.DeleteRoleAction;
@@ -31,7 +32,7 @@ public TransportDeleteRoleAction(
         TransportService transportService,
         ReservedRoleNameChecker reservedRoleNameChecker
     ) {
-        super(DeleteRoleAction.NAME, actionFilters, transportService.getTaskManager());
+        super(DeleteRoleAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.rolesStore = rolesStore;
         this.reservedRoleNameChecker = reservedRoleNameChecker;
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java
index 229ca9f74024d..e019f168cf8c0 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportGetRolesAction.java
@@ -9,7 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.role.GetRolesAction;
@@ -33,7 +34,7 @@ public class TransportGetRolesAction extends TransportAction<GetRolesRequest, Ge
 
     @Inject
     public TransportGetRolesAction(ActionFilters actionFilters, NativeRolesStore nativeRolesStore, TransportService transportService) {
-        super(GetRolesAction.NAME, actionFilters, transportService.getTaskManager());
+        super(GetRolesAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.nativeRolesStore = nativeRolesStore;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportPutRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportPutRoleAction.java
index c22e460728dca..0f36036a2bdb8 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportPutRoleAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportPutRoleAction.java
@@ -9,7 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.role.PutRoleAction;
@@ -23,7 +24,7 @@ public class TransportPutRoleAction extends TransportAction<PutRoleRequest, PutR
 
     @Inject
     public TransportPutRoleAction(ActionFilters actionFilters, NativeRolesStore rolesStore, TransportService transportService) {
-        super(PutRoleAction.NAME, actionFilters, transportService.getTaskManager());
+        super(PutRoleAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.rolesStore = rolesStore;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportQueryRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportQueryRoleAction.java
index d6bdfea29a676..837db14030824 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportQueryRoleAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportQueryRoleAction.java
@@ -11,7 +11,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
@@ -40,7 +41,7 @@ public class TransportQueryRoleAction extends TransportAction<QueryRoleRequest,
 
     @Inject
     public TransportQueryRoleAction(ActionFilters actionFilters, NativeRolesStore nativeRolesStore, TransportService transportService) {
-        super(ActionTypes.QUERY_ROLE_ACTION.name(), actionFilters, transportService.getTaskManager());
+        super(ActionTypes.QUERY_ROLE_ACTION.name(), actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.nativeRolesStore = nativeRolesStore;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java
index b4e8d5d6db83f..74129facae70a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java
index 17db5c555da91..ac0d3177cca09 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java
index 44c72bc13a54b..82a3b4f000064 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlAuthenticateAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlAuthenticateAction.java
index 590757f9d7d32..a91f8a5c82825 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlAuthenticateAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlAuthenticateAction.java
@@ -10,10 +10,10 @@
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlCompleteLogoutAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlCompleteLogoutAction.java
index 500ef7c51f4bc..41785fed124df 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlCompleteLogoutAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlCompleteLogoutAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.saml.SamlCompleteLogoutRequest;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java
index ade43984c6bab..f151cf2c5afe7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java
index f5a55da7b1916..7966a0d45ca1d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlPrepareAuthenticationAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlPrepareAuthenticationAction.java
index e18ca43c018f5..8ad0f505bba40 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlPrepareAuthenticationAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlPrepareAuthenticationAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlSpMetadataAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlSpMetadataAction.java
index 8a9df48767a4c..213e72b7d99c9 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlSpMetadataAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlSpMetadataAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.saml.SamlSpMetadataAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenAction.java
index a194452b8652a..fe556002076f8 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.SecurityContext;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenAction.java
index 772371b3ca6c3..82799203b5021 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.service.DeleteServiceAccountTokenAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountAction.java
index 372a550eedbc9..2e882546cb5a6 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountAction.java
@@ -10,9 +10,9 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Predicates;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsAction.java
index 17227873bbb14..a8e2405cd1c99 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountCredentialsAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountNodesCredentialsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountNodesCredentialsAction.java
index e9e90ec855d4d..228c606dd1e35 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountNodesCredentialsAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountNodesCredentialsAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportGetSecuritySettingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportGetSecuritySettingsAction.java
index 73abfffcd3a2f..96363d5a099da 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportGetSecuritySettingsAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportGetSecuritySettingsAction.java
@@ -17,10 +17,10 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -85,7 +85,7 @@ private static Settings getFilteredSettingsForIndex(String indexName, ClusterSta
             .map(IndexMetadata::getSettings)
             .map(settings -> {
                 Settings.Builder builder = Settings.builder();
-                for (String settingName : UpdateSecuritySettingsAction.ALLOWED_SETTING_KEYS) {
+                for (String settingName : UpdateSecuritySettingsAction.ALLOWED_SETTING_VALIDATORS.keySet()) {
                     if (settings.hasValue(settingName)) {
                         builder.put(settingName, settings.get(settingName));
                     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportReloadRemoteClusterCredentialsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportReloadRemoteClusterCredentialsAction.java
index 22dcf1b4f9daa..fefd47f9e352e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportReloadRemoteClusterCredentialsAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportReloadRemoteClusterCredentialsAction.java
@@ -17,9 +17,10 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportService;
@@ -53,7 +54,12 @@ public TransportReloadRemoteClusterCredentialsAction(
         ClusterService clusterService,
         ActionFilters actionFilters
     ) {
-        super(ActionTypes.RELOAD_REMOTE_CLUSTER_CREDENTIALS_ACTION.name(), actionFilters, transportService.getTaskManager());
+        super(
+            ActionTypes.RELOAD_REMOTE_CLUSTER_CREDENTIALS_ACTION.name(),
+            actionFilters,
+            transportService.getTaskManager(),
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
+        );
         this.remoteClusterService = transportService.getRemoteClusterService();
         this.clusterService = clusterService;
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportUpdateSecuritySettingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportUpdateSecuritySettingsAction.java
index 20bab85a50921..49f8846c36e1f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportUpdateSecuritySettingsAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/settings/TransportUpdateSecuritySettingsAction.java
@@ -21,11 +21,11 @@
 import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java
index 7718bd634f9d0..54eb43020e8ef 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java
@@ -10,11 +10,11 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenAction.java
index 0213faa2e2ee6..a9c9a85d0b738 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportRefreshTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportRefreshTokenAction.java
index e1665fb7776ab..83e48ec723ad1 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportRefreshTokenAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportRefreshTokenAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequest;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java
index fa063f79cf3e3..878bbf6a7134d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.SecurityContext;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java
index b696c93cf6899..96323836aa005 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java
@@ -11,9 +11,9 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackSettings;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserAction.java
index 3a3ed5faf5a0b..9a7c409ce4e36 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserAction.java
@@ -9,9 +9,9 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.user.DeleteUserAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUserPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUserPrivilegesAction.java
index 5b230f00b54e4..d1c27029da68e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUserPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUserPrivilegesAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.SecurityContext;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
index 5f47cb9223f70..63aefbef9965e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
@@ -11,10 +11,10 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java
index 0e642754f9287..0adb8ccde3150 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.SecurityContext;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportPutUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportPutUserAction.java
index ec81213c6e02e..5aa09e71b0082 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportPutUserAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportPutUserAction.java
@@ -11,9 +11,9 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.user.PutUserAction;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserAction.java
index 72f89209b0b79..8ce2e809973f6 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserAction.java
@@ -12,8 +12,9 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
@@ -49,7 +50,7 @@ public TransportQueryUserAction(
         ProfileService profileService,
         Realms realms
     ) {
-        super(ActionTypes.QUERY_USER_ACTION.name(), actionFilters, transportService.getTaskManager());
+        super(ActionTypes.QUERY_USER_ACTION.name(), actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.usersStore = usersStore;
         this.profileService = profileService;
         this.nativeRealmRef = realms.getNativeRealmRef();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledAction.java
index 70670840a912d..320b7a7880b2a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledAction.java
@@ -11,9 +11,9 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.SecurityContext;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index aaa1841bd2354..90566e25b4ea5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -100,6 +100,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.authz.store.RoleReference;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
@@ -137,6 +138,7 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE;
 import static org.elasticsearch.TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING;
@@ -363,29 +365,10 @@ public void createApiKey(
             listener.onFailure(new IllegalArgumentException("authentication must be provided"));
         } else {
             final TransportVersion transportVersion = getMinTransportVersion();
-            if (transportVersion.before(TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY)
-                && hasRemoteIndices(request.getRoleDescriptors())) {
-                // Creating API keys with roles which define remote indices privileges is not allowed in a mixed cluster.
-                listener.onFailure(
-                    new IllegalArgumentException(
-                        "all nodes must have version ["
-                            + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
-                            + "] or higher to support remote indices privileges for API keys"
-                    )
-                );
-                return;
-            }
-            if (transportVersion.before(ROLE_REMOTE_CLUSTER_PRIVS) && hasRemoteCluster(request.getRoleDescriptors())) {
-                // Creating API keys with roles which define remote cluster privileges is not allowed in a mixed cluster.
-                listener.onFailure(
-                    new IllegalArgumentException(
-                        "all nodes must have version ["
-                            + ROLE_REMOTE_CLUSTER_PRIVS
-                            + "] or higher to support remote cluster privileges for API keys"
-                    )
-                );
+            if (validateRoleDescriptorsForMixedCluster(listener, request.getRoleDescriptors(), transportVersion) == false) {
                 return;
             }
+
             if (transportVersion.before(TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY)
                 && request.getType() == ApiKey.Type.CROSS_CLUSTER) {
                 listener.onFailure(
@@ -407,15 +390,63 @@ && hasRemoteIndices(request.getRoleDescriptors())) {
                 return;
             }
 
-            final Set<RoleDescriptor> userRolesWithoutDescription = removeUserRoleDescriptorDescriptions(userRoleDescriptors);
-            final Set<RoleDescriptor> filteredUserRoleDescriptors = maybeRemoveRemotePrivileges(
-                userRolesWithoutDescription,
+            Set<RoleDescriptor> filteredRoleDescriptors = filterRoleDescriptorsForMixedCluster(
+                userRoleDescriptors,
                 transportVersion,
                 request.getId()
             );
 
-            createApiKeyAndIndexIt(authentication, request, filteredUserRoleDescriptors, listener);
+            createApiKeyAndIndexIt(authentication, request, filteredRoleDescriptors, listener);
+        }
+    }
+
+    private Set<RoleDescriptor> filterRoleDescriptorsForMixedCluster(
+        final Set<RoleDescriptor> userRoleDescriptors,
+        final TransportVersion transportVersion,
+        final String... apiKeyIds
+    ) {
+        final Set<RoleDescriptor> userRolesWithoutDescription = removeUserRoleDescriptorDescriptions(userRoleDescriptors);
+        return maybeRemoveRemotePrivileges(userRolesWithoutDescription, transportVersion, apiKeyIds);
+    }
+
+    private boolean validateRoleDescriptorsForMixedCluster(
+        final ActionListener<?> listener,
+        final List<RoleDescriptor> roleDescriptors,
+        final TransportVersion transportVersion
+    ) {
+        if (transportVersion.before(TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY) && hasRemoteIndices(roleDescriptors)) {
+            // API keys with roles which define remote indices privileges is not allowed in a mixed cluster.
+            listener.onFailure(
+                new IllegalArgumentException(
+                    "all nodes must have version ["
+                        + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
+                        + "] or higher to support remote indices privileges for API keys"
+                )
+            );
+            return false;
+        }
+        if (transportVersion.before(ROLE_REMOTE_CLUSTER_PRIVS) && hasRemoteCluster(roleDescriptors)) {
+            // API keys with roles which define remote cluster privileges is not allowed in a mixed cluster.
+            listener.onFailure(
+                new IllegalArgumentException(
+                    "all nodes must have version ["
+                        + ROLE_REMOTE_CLUSTER_PRIVS
+                        + "] or higher to support remote cluster privileges for API keys"
+                )
+            );
+            return false;
+        }
+        if (transportVersion.before(ADD_MANAGE_ROLES_PRIVILEGE) && hasGlobalManageRolesPrivilege(roleDescriptors)) {
+            listener.onFailure(
+                new IllegalArgumentException(
+                    "all nodes must have version ["
+                        + ADD_MANAGE_ROLES_PRIVILEGE
+                        + "] or higher to support the manage roles privilege for API keys"
+                )
+            );
+            return false;
         }
+        return true;
     }
 
     /**
@@ -458,6 +489,13 @@ private static boolean hasRemoteCluster(Collection<RoleDescriptor> roleDescripto
         return roleDescriptors != null && roleDescriptors.stream().anyMatch(RoleDescriptor::hasRemoteClusterPermissions);
     }
 
+    private static boolean hasGlobalManageRolesPrivilege(Collection<RoleDescriptor> roleDescriptors) {
+        return roleDescriptors != null
+            && roleDescriptors.stream()
+                .flatMap(roleDescriptor -> Arrays.stream(roleDescriptor.getConditionalClusterPrivileges()))
+                .anyMatch(privilege -> privilege instanceof ConfigurableClusterPrivileges.ManageRolesPrivilege);
+    }
+
     private static IllegalArgumentException validateWorkflowsRestrictionConstraints(
         TransportVersion transportVersion,
         List<RoleDescriptor> requestRoleDescriptors,
@@ -594,28 +632,11 @@ public void updateApiKeys(
         }
 
         final TransportVersion transportVersion = getMinTransportVersion();
-        if (transportVersion.before(TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY) && hasRemoteIndices(request.getRoleDescriptors())) {
-            // Updating API keys with roles which define remote indices privileges is not allowed in a mixed cluster.
-            listener.onFailure(
-                new IllegalArgumentException(
-                    "all nodes must have version ["
-                        + TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY.toReleaseVersion()
-                        + "] or higher to support remote indices privileges for API keys"
-                )
-            );
-            return;
-        }
-        if (transportVersion.before(ROLE_REMOTE_CLUSTER_PRIVS) && hasRemoteCluster(request.getRoleDescriptors())) {
-            // Updating API keys with roles which define remote cluster privileges is not allowed in a mixed cluster.
-            listener.onFailure(
-                new IllegalArgumentException(
-                    "all nodes must have version ["
-                        + ROLE_REMOTE_CLUSTER_PRIVS
-                        + "] or higher to support remote indices privileges for API keys"
-                )
-            );
+
+        if (validateRoleDescriptorsForMixedCluster(listener, request.getRoleDescriptors(), transportVersion) == false) {
             return;
         }
+
         final Exception workflowsValidationException = validateWorkflowsRestrictionConstraints(
             transportVersion,
             request.getRoleDescriptors(),
@@ -627,22 +648,22 @@ public void updateApiKeys(
         }
 
         final String[] apiKeyIds = request.getIds().toArray(String[]::new);
-        final Set<RoleDescriptor> userRolesWithoutDescription = removeUserRoleDescriptorDescriptions(userRoleDescriptors);
-        final Set<RoleDescriptor> filteredUserRoleDescriptors = maybeRemoveRemotePrivileges(
-            userRolesWithoutDescription,
-            transportVersion,
-            apiKeyIds
-        );
 
         if (logger.isDebugEnabled()) {
             logger.debug("Updating [{}] API keys", buildDelimitedStringWithLimit(10, apiKeyIds));
         }
 
+        Set<RoleDescriptor> filteredRoleDescriptors = filterRoleDescriptorsForMixedCluster(
+            userRoleDescriptors,
+            transportVersion,
+            apiKeyIds
+        );
+
         findVersionedApiKeyDocsForSubject(
             authentication,
             apiKeyIds,
             ActionListener.wrap(
-                versionedDocs -> updateApiKeys(authentication, request, filteredUserRoleDescriptors, versionedDocs, listener),
+                versionedDocs -> updateApiKeys(authentication, request, filteredRoleDescriptors, versionedDocs, listener),
                 ex -> listener.onFailure(traceLog("bulk update", ex))
             )
         );
@@ -1315,7 +1336,18 @@ void validateApiKeyCredentials(
                                 AuthenticationResult.unsuccessful("invalid credentials for API key [" + credentials.getId() + "]", null)
                             );
                         }
-                    }, listener::onFailure));
+                    }, exception -> {
+                        // Crypto threadpool queue is full, invalidate this cache entry and make sure nothing is going to wait on it
+                        logger.warn(
+                            Strings.format(
+                                "rejecting possibly valid API key authentication because the [%s] threadpool is full",
+                                SECURITY_CRYPTO_THREAD_POOL_NAME
+                            )
+                        );
+                        apiKeyAuthCache.invalidate(credentials.getId(), listenableCacheEntry);
+                        listenableCacheEntry.onFailure(exception);
+                        listener.onFailure(exception);
+                    }));
                 }
             } else {
                 verifyKeyAgainstHash(apiKeyDoc.hash, credentials, ActionListener.wrap(verified -> {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticator.java
index 51af3a7eda665..c1995d0a26c5c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/RealmsAuthenticator.java
@@ -284,7 +284,16 @@ private static void consumeNullUser(
     ) {
         messages.forEach((realm, tuple) -> {
             final String message = tuple.v1();
-            final String cause = tuple.v2() == null ? "" : " (Caused by " + tuple.v2() + ")";
+            Throwable ex = tuple.v2();
+            var cause = new StringBuilder();
+            while (ex != null) {
+                cause.append(cause.isEmpty() ? " (" : "; ");
+                cause.append("Caused by ").append(ex);
+                ex = ex.getCause();
+            }
+            if (cause.isEmpty() == false) {
+                cause.append(')');
+            }
             logger.warn("Authentication to realm {} failed - {}{}", realm.name(), message, cause);
         });
         if (context.getUnlicensedRealms().isEmpty() == false) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java
index 928ecd7fa265d..b345178e205c3 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java
@@ -11,7 +11,6 @@
 import com.nimbusds.jose.jwk.JWK;
 import com.nimbusds.jose.jwk.JWKSet;
 import com.nimbusds.jose.util.Base64URL;
-import com.nimbusds.jose.util.JSONObjectUtils;
 import com.nimbusds.jwt.JWT;
 import com.nimbusds.jwt.SignedJWT;
 
@@ -33,6 +32,7 @@
 import org.apache.http.nio.reactor.ConnectingIOReactor;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.action.ActionListener;
@@ -45,11 +45,14 @@
 import org.elasticsearch.common.ssl.SslConfiguration;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
@@ -64,6 +67,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.function.Supplier;
 
@@ -237,7 +241,13 @@ public static String serializeJwkSet(final JWKSet jwkSet, final boolean publicKe
         if (jwkSet == null) {
             return null;
         }
-        return JSONObjectUtils.toJSONString(jwkSet.toJSONObject(publicKeysOnly));
+        Map<String, Object> jwkJson = jwkSet.toJSONObject(publicKeysOnly);
+        try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+            builder.map(jwkJson);
+            return Strings.toString(builder);
+        } catch (IOException e) {
+            throw new ElasticsearchException(e);
+        }
     }
 
     public static String serializeJwkHmacOidc(final JWK key) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java
index 0f34850b861b7..c2e0caf7234cb 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java
@@ -718,7 +718,7 @@ private CloseableHttpAsyncClient createHttpClient() {
                 connectionManager.setMaxTotal(realmConfig.getSetting(HTTP_MAX_CONNECTIONS));
                 final RequestConfig requestConfig = RequestConfig.custom()
                     .setConnectTimeout(Math.toIntExact(realmConfig.getSetting(HTTP_CONNECT_TIMEOUT).getMillis()))
-                    .setConnectionRequestTimeout(Math.toIntExact(realmConfig.getSetting(HTTP_CONNECTION_READ_TIMEOUT).getSeconds()))
+                    .setConnectionRequestTimeout(Math.toIntExact(realmConfig.getSetting(HTTP_CONNECTION_READ_TIMEOUT).getMillis()))
                     .setSocketTimeout(Math.toIntExact(realmConfig.getSetting(HTTP_SOCKET_TIMEOUT).getMillis()))
                     .build();
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java
index abd586920f2d8..b62ce28422a9c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java
@@ -22,6 +22,25 @@ final class ElasticServiceAccounts {
 
     static final String NAMESPACE = "elastic";
 
+    private static final ServiceAccount AUTO_OPS_ACCOUNT = new ElasticServiceAccount(
+        "auto-ops",
+        new RoleDescriptor(
+            NAMESPACE + "/auto-ops",
+            new String[] { "monitor", "read_ilm", "read_slm" },
+            new RoleDescriptor.IndicesPrivileges[] {
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .allowRestrictedIndices(true)
+                    .indices("*")
+                    .privileges("monitor", "view_index_metadata")
+                    .build(), },
+            null,
+            null,
+            null,
+            null,
+            null
+        )
+    );
+
     private static final ServiceAccount ENTERPRISE_SEARCH_ACCOUNT = new ElasticServiceAccount(
         "enterprise-search-server",
         new RoleDescriptor(
@@ -173,6 +192,7 @@ final class ElasticServiceAccounts {
     );
 
     static final Map<String, ServiceAccount> ACCOUNTS = Stream.of(
+        AUTO_OPS_ACCOUNT,
         ENTERPRISE_SEARCH_ACCOUNT,
         FLEET_ACCOUNT,
         FLEET_REMOTE_ACCOUNT,
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
index 194440722545a..4173f3db45409 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
@@ -39,6 +39,7 @@
 import static org.elasticsearch.xpack.core.ClientHelper.LOGSTASH_MANAGEMENT_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.MONITORING_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.OTEL_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.PROFILING_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.ROLLUP_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.SEARCHABLE_SNAPSHOTS_ORIGIN;
@@ -151,6 +152,7 @@ public static void switchUserBasedOnActionOriginAndExecute(
             case INGEST_ORIGIN:
             case PROFILING_ORIGIN:
             case APM_ORIGIN:
+            case OTEL_ORIGIN:
             case STACK_ORIGIN:
             case SEARCHABLE_SNAPSHOTS_ORIGIN:
             case LOGSTASH_MANAGEMENT_ORIGIN:
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java
index 6d4f02ee7e893..01cc1e9dd4cdf 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java
@@ -38,15 +38,18 @@ public final class IndicesAliasesRequestInterceptor implements RequestIntercepto
     private final ThreadContext threadContext;
     private final XPackLicenseState licenseState;
     private final AuditTrailService auditTrailService;
+    private final boolean dlsFlsEnabled;
 
     public IndicesAliasesRequestInterceptor(
         ThreadContext threadContext,
         XPackLicenseState licenseState,
-        AuditTrailService auditTrailService
+        AuditTrailService auditTrailService,
+        boolean dlsFlsEnabled
     ) {
         this.threadContext = threadContext;
         this.licenseState = licenseState;
         this.auditTrailService = auditTrailService;
+        this.dlsFlsEnabled = dlsFlsEnabled;
     }
 
     @Override
@@ -61,7 +64,7 @@ public void intercept(
             final boolean isDlsLicensed = DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
             final boolean isFlsLicensed = FIELD_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
             IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
-            if (isDlsLicensed || isFlsLicensed) {
+            if (dlsFlsEnabled && (isDlsLicensed || isFlsLicensed)) {
                 for (IndicesAliasesRequest.AliasActions aliasAction : request.getAliasActions()) {
                     if (aliasAction.actionType() == IndicesAliasesRequest.AliasActions.Type.ADD) {
                         for (String index : aliasAction.indices()) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java
index 4be327b1f5049..c2bea70613b46 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java
@@ -34,11 +34,18 @@ public final class ResizeRequestInterceptor implements RequestInterceptor {
     private final ThreadContext threadContext;
     private final XPackLicenseState licenseState;
     private final AuditTrailService auditTrailService;
+    private final boolean dlsFlsEnabled;
 
-    public ResizeRequestInterceptor(ThreadPool threadPool, XPackLicenseState licenseState, AuditTrailService auditTrailService) {
+    public ResizeRequestInterceptor(
+        ThreadPool threadPool,
+        XPackLicenseState licenseState,
+        AuditTrailService auditTrailService,
+        boolean dlsFlsEnabled
+    ) {
         this.threadContext = threadPool.getThreadContext();
         this.licenseState = licenseState;
         this.auditTrailService = auditTrailService;
+        this.dlsFlsEnabled = dlsFlsEnabled;
     }
 
     @Override
@@ -52,7 +59,7 @@ public void intercept(
             final AuditTrail auditTrail = auditTrailService.get();
             final boolean isDlsLicensed = DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
             final boolean isFlsLicensed = FIELD_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
-            if (isDlsLicensed || isFlsLicensed) {
+            if (dlsFlsEnabled && (isDlsLicensed || isFlsLicensed)) {
                 IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
                 IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(
                     request.getSourceIndex()
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
index a2d2b21b489ea..9ddda193dba39 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
@@ -60,6 +60,7 @@
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
 import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
 import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator;
 import org.elasticsearch.xpack.core.security.support.NativeRealmValidationUtil;
@@ -476,7 +477,15 @@ private Exception validateRoleDescriptor(RoleDescriptor role) {
                                 + TransportVersions.SECURITY_ROLE_DESCRIPTION.toReleaseVersion()
                                 + "] or higher to support specifying role description"
                         );
-                    }
+                    } else if (Arrays.stream(role.getConditionalClusterPrivileges())
+                        .anyMatch(privilege -> privilege instanceof ConfigurableClusterPrivileges.ManageRolesPrivilege)
+                        && clusterService.state().getMinTransportVersion().before(TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE)) {
+                            return new IllegalStateException(
+                                "all nodes must have version ["
+                                    + TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE.toReleaseVersion()
+                                    + "] or higher to support the manage roles privilege"
+                            );
+                        }
         try {
             DLSRoleQueryValidator.validateQueryField(role.getIndicesPrivileges(), xContentRegistry);
         } catch (ElasticsearchException | IllegalArgumentException e) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java
index 217afdb3cfea2..29ffc76425060 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java
@@ -55,8 +55,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        CreateApiKeyRequestBuilder builder = builderFactory.create(client, request.hasParam(RestRequest.PATH_RESTRICTED))
-            .source(request.requiredContent(), request.getXContentType());
+        CreateApiKeyRequestBuilder builder = builderFactory.create(client).source(request.requiredContent(), request.getXContentType());
         String refresh = request.param("refresh");
         if (refresh != null) {
             builder.setRefreshPolicy(WriteRequest.RefreshPolicy.parse(request.param("refresh")));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java
index d804559bba0ec..5f0657079e694 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java
@@ -62,14 +62,13 @@ public String getName() {
 
     @Override
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
-        final boolean restrictResponse = request.hasParam(RestRequest.PATH_RESTRICTED);
         return channel -> client.execute(
             GetBuiltinPrivilegesAction.INSTANCE,
             new GetBuiltinPrivilegesRequest(),
             new RestBuilderListener<>(channel) {
                 @Override
                 public RestResponse buildResponse(GetBuiltinPrivilegesResponse response, XContentBuilder builder) throws Exception {
-                    final var translatedResponse = responseTranslator.translate(response, restrictResponse);
+                    final var translatedResponse = responseTranslator.translate(response);
                     builder.startObject();
                     builder.array("cluster", translatedResponse.getClusterPrivileges());
                     builder.array("index", translatedResponse.getIndexPrivileges());
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java
index 82596738e95a7..6a819c098e9f1 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java
@@ -55,8 +55,7 @@ public String getName() {
 
     @Override
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
-        final boolean restrictRequest = request.hasParam(RestRequest.PATH_RESTRICTED);
-        final PutRoleRequestBuilder requestBuilder = builderFactory.create(client, restrictRequest)
+        final PutRoleRequestBuilder requestBuilder = builderFactory.create(client)
             .source(request.param("name"), request.requiredContent(), request.getXContentType())
             .setRefreshPolicy(request.param("refresh"));
         return channel -> requestBuilder.execute(new RestBuilderListener<>(channel) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
index 4c5ce703f48ad..36ea14c6e101b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.VersionId;
 import org.elasticsearch.common.settings.Settings;
@@ -36,6 +37,7 @@
 import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_PROFILE_ORIGIN;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_VERSION_STRING;
+import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion.ADD_MANAGE_ROLES_PRIVILEGE;
 
 /**
  * Responsible for handling system indices for the Security plugin
@@ -153,6 +155,7 @@ private static Settings getMainIndexSettings() {
         return Settings.builder()
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+            .put(DataTier.TIER_PREFERENCE, "data_hot,data_content")
             .put(IndexMetadata.SETTING_PRIORITY, 1000)
             .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), INTERNAL_MAIN_INDEX_FORMAT)
             .put("analysis.filter.email.type", "pattern_capture")
@@ -409,6 +412,40 @@ private XContentBuilder getMainIndexMappings(SecurityMainIndexMappingVersion map
                                 builder.endObject();
                             }
                             builder.endObject();
+                            if (mappingVersion.onOrAfter(ADD_MANAGE_ROLES_PRIVILEGE)) {
+                                builder.startObject("role");
+                                {
+                                    builder.field("type", "object");
+                                    builder.startObject("properties");
+                                    {
+                                        builder.startObject("manage");
+                                        {
+                                            builder.field("type", "object");
+                                            builder.startObject("properties");
+                                            {
+                                                builder.startObject("indices");
+                                                {
+                                                    builder.startObject("properties");
+                                                    {
+                                                        builder.startObject("names");
+                                                        builder.field("type", "keyword");
+                                                        builder.endObject();
+                                                        builder.startObject("privileges");
+                                                        builder.field("type", "keyword");
+                                                        builder.endObject();
+                                                    }
+                                                    builder.endObject();
+                                                }
+                                                builder.endObject();
+                                            }
+                                            builder.endObject();
+                                        }
+                                        builder.endObject();
+                                    }
+                                    builder.endObject();
+                                }
+                                builder.endObject();
+                            }
                         }
                         builder.endObject();
                     }
@@ -667,6 +704,7 @@ private static Settings getTokenIndexSettings() {
         return Settings.builder()
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+            .put(DataTier.TIER_PREFERENCE, "data_hot,data_content")
             .put(IndexMetadata.SETTING_PRIORITY, 1000)
             .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), INTERNAL_TOKENS_INDEX_FORMAT)
             .build();
@@ -867,6 +905,7 @@ private static Settings getProfileIndexSettings(Settings settings) {
         final Settings.Builder settingsBuilder = Settings.builder()
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+            .put(DataTier.TIER_PREFERENCE, "data_hot,data_content")
             .put(IndexMetadata.SETTING_PRIORITY, 1000)
             .put(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), INTERNAL_PROFILE_INDEX_FORMAT)
             .put("analysis.filter.email.type", "pattern_capture")
@@ -1050,6 +1089,11 @@ public enum SecurityMainIndexMappingVersion implements VersionId<SecurityMainInd
          */
         ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS(2),
 
+        /**
+         * Mapping for global manage role privilege
+         */
+        ADD_MANAGE_ROLES_PRIVILEGE(3),
+
         ;
 
         private static final SecurityMainIndexMappingVersion LATEST = Arrays.stream(values())
diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
index 2c9d38e5ae55e..b3d5e80e09dcd 100644
--- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
@@ -51,3 +51,9 @@ grant codeBase "${codebase.netty-transport}" {
    // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
    permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
 };
+
+grant codeBase "${codebase.nimbus-jose-jwt-modified}" {
+    // for JSON serialization based on a shaded GSON dependency
+    permission java.lang.RuntimePermission "accessDeclaredMembers";
+    permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+};
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java
index 6f98a2b5bba09..7d82d286cda4c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SettingsFilterTests.java
@@ -7,13 +7,13 @@
 package org.elasticsearch.test;
 
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Guice;
-import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.injection.guice.Guice;
+import org.elasticsearch.injection.guice.Injector;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java
index 064c38557f6cf..98fc8a8a49c4a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.Plugin;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
index 400bc35b93fd5..a07a7a3a5dd27 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -152,7 +151,6 @@
 import static org.mockito.Mockito.when;
 
 public class SecurityTests extends ESTestCase {
-
     private Security security = null;
     private ThreadContext threadContext = null;
     private SecurityContext securityContext = null;
@@ -960,8 +958,11 @@ public <T> List<T> loadExtensions(Class<T> extensionPointType) {
     public void testReload() throws Exception {
         final Settings settings = Settings.builder().put("xpack.security.enabled", true).put("path.home", createTempDir()).build();
 
-        final PlainActionFuture<ActionResponse.Empty> value = new PlainActionFuture<>();
+        final ThreadPool threadPool = mock(ThreadPool.class);
+        threadContext = new ThreadContext(Settings.EMPTY);
+        when(threadPool.getThreadContext()).thenReturn(threadContext);
         final Client mockedClient = mock(Client.class);
+        when(mockedClient.threadPool()).thenReturn(threadPool);
 
         final JwtRealm mockedJwtRealm = mock(JwtRealm.class);
         final List<ReloadableSecurityComponent> reloadableComponents = List.of(mockedJwtRealm);
@@ -992,11 +993,17 @@ protected List<ReloadableSecurityComponent> getReloadableSecurityComponents() {
         verify(mockedJwtRealm).reload(same(inputSettings));
     }
 
-    public void testReloadWithFailures() throws Exception {
+    public void testReloadWithFailures() {
         final Settings settings = Settings.builder().put("xpack.security.enabled", true).put("path.home", createTempDir()).build();
 
         final boolean failRemoteClusterCredentialsReload = randomBoolean();
+
+        final ThreadPool threadPool = mock(ThreadPool.class);
+        threadContext = new ThreadContext(Settings.EMPTY);
+        when(threadPool.getThreadContext()).thenReturn(threadContext);
         final Client mockedClient = mock(Client.class);
+        when(mockedClient.threadPool()).thenReturn(threadPool);
+
         final JwtRealm mockedJwtRealm = mock(JwtRealm.class);
         final List<ReloadableSecurityComponent> reloadableComponents = List.of(mockedJwtRealm);
         if (failRemoteClusterCredentialsReload) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java
index b313d94a46ce5..7e35297fcb655 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java
@@ -20,7 +20,6 @@
 
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
@@ -47,12 +46,16 @@ public void testDoExecute() {
         final PlainActionFuture<GetServiceAccountResponse> future1 = new PlainActionFuture<>();
         transportGetServiceAccountAction.doExecute(mock(Task.class), request1, future1);
         final GetServiceAccountResponse getServiceAccountResponse1 = future1.actionGet();
-        assertThat(getServiceAccountResponse1.getServiceAccountInfos().length, equalTo(4));
+        assertThat(getServiceAccountResponse1.getServiceAccountInfos().length, equalTo(5));
         assertThat(
-            Arrays.stream(getServiceAccountResponse1.getServiceAccountInfos())
-                .map(ServiceAccountInfo::getPrincipal)
-                .collect(Collectors.toList()),
-            containsInAnyOrder("elastic/enterprise-search-server", "elastic/fleet-server", "elastic/fleet-server-remote", "elastic/kibana")
+            Arrays.stream(getServiceAccountResponse1.getServiceAccountInfos()).map(ServiceAccountInfo::getPrincipal).toList(),
+            containsInAnyOrder(
+                "elastic/auto-ops",
+                "elastic/enterprise-search-server",
+                "elastic/fleet-server",
+                "elastic/fleet-server-remote",
+                "elastic/kibana"
+            )
         );
 
         final GetServiceAccountRequest request2 = new GetServiceAccountRequest("elastic", "fleet-server");
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
index 17bad90415e7c..cb524a48d0ec7 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
@@ -567,7 +567,15 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException
             randomFrom((RoleDescriptor.ApplicationResourcePrivileges[]) null, new RoleDescriptor.ApplicationResourcePrivileges[0]),
             new ConfigurableClusterPrivilege[] {
                 new ConfigurableClusterPrivileges.WriteProfileDataPrivileges(new LinkedHashSet<>(Arrays.asList("", "\""))),
-                new ConfigurableClusterPrivileges.ManageApplicationPrivileges(Set.of("\"")) },
+                new ConfigurableClusterPrivileges.ManageApplicationPrivileges(Set.of("\"")),
+                new ConfigurableClusterPrivileges.ManageRolesPrivilege(
+                    List.of(
+                        new ConfigurableClusterPrivileges.ManageRolesPrivilege.ManageRolesIndexPermissionGroup(
+                            new String[] { "test*" },
+                            new String[] { "read", "write" }
+                        )
+                    )
+                ) },
             new String[] { "\"[a]/" },
             Map.of(),
             Map.of()
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
index 1dce6a038638b..fa6eb307933ec 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
@@ -145,7 +145,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Semaphore;
@@ -230,6 +230,9 @@ public class ApiKeyServiceTests extends ESTestCase {
           "search": [ {"names": ["logs"]} ],
           "replication": [ {"names": ["archive"]} ]
         }""");
+
+    private static final int TEST_THREADPOOL_QUEUE_SIZE = 1000;
+
     private ThreadPool threadPool;
     private Client client;
     private SecurityIndexManager securityIndex;
@@ -245,7 +248,7 @@ public void createThreadPool() {
                     Settings.EMPTY,
                     SECURITY_CRYPTO_THREAD_POOL_NAME,
                     1,
-                    1000,
+                    TEST_THREADPOOL_QUEUE_SIZE,
                     "xpack.security.crypto.thread_pool",
                     EsExecutors.TaskTrackingConfig.DO_NOT_TRACK
                 )
@@ -268,6 +271,90 @@ public void setupMocks() {
         doAnswer(invocation -> Instant.now()).when(clock).instant();
     }
 
+    public void testFloodThreadpool() throws Exception {
+        // We're going to be blocking the security-crypto threadpool so we need a new one for the client
+        ThreadPool clientThreadpool = new TestThreadPool(
+            this.getTestName(),
+            new FixedExecutorBuilder(
+                Settings.EMPTY,
+                this.getTestName(),
+                1,
+                100,
+                "no_settings_used",
+                EsExecutors.TaskTrackingConfig.DO_NOT_TRACK
+            )
+        );
+        try {
+            when(client.threadPool()).thenReturn(clientThreadpool);
+
+            // setup copied from testAuthenticateWithApiKey
+            final Settings settings = Settings.builder().put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), true).build();
+            final ApiKeyService service = createApiKeyService(settings);
+
+            final String id = randomAlphaOfLength(12);
+            final String key = randomAlphaOfLength(16);
+
+            final User user, authUser;
+            if (randomBoolean()) {
+                user = new User("hulk", new String[] { "superuser" }, "Bruce Banner", "hulk@test.com", Map.of(), true);
+                authUser = new User("authenticated_user", "other");
+            } else {
+                user = new User("hulk", new String[] { "superuser" }, "Bruce Banner", "hulk@test.com", Map.of(), true);
+                authUser = null;
+            }
+            final ApiKey.Type type = randomFrom(ApiKey.Type.values());
+            final Map<String, Object> metadata = mockKeyDocument(id, key, user, authUser, false, Duration.ofSeconds(3600), null, type);
+
+            // Block the security crypto threadpool
+            CyclicBarrier barrier = new CyclicBarrier(2);
+            threadPool.executor(SECURITY_CRYPTO_THREAD_POOL_NAME).execute(() -> safeAwait(barrier));
+            // Now fill it up while the one thread is blocked
+            for (int i = 0; i < TEST_THREADPOOL_QUEUE_SIZE; i++) {
+                threadPool.executor(SECURITY_CRYPTO_THREAD_POOL_NAME).execute(() -> {});
+            }
+
+            // Check that it's full
+            for (var stat : threadPool.stats().stats()) {
+                if (stat.name().equals(SECURITY_CRYPTO_THREAD_POOL_NAME)) {
+                    assertThat(stat.queue(), equalTo(TEST_THREADPOOL_QUEUE_SIZE));
+                    assertThat(stat.rejected(), equalTo(0L));
+                }
+            }
+
+            // now try to auth with an API key
+            final AuthenticationResult<User> auth = tryAuthenticate(service, id, key, type);
+            assertThat(auth.getStatus(), is(AuthenticationResult.Status.TERMINATE));
+
+            // Make sure one was rejected and the queue is still full
+            for (var stat : threadPool.stats().stats()) {
+                if (stat.name().equals(SECURITY_CRYPTO_THREAD_POOL_NAME)) {
+                    assertThat(stat.queue(), equalTo(TEST_THREADPOOL_QUEUE_SIZE));
+                    assertThat(stat.rejected(), equalTo(1L));
+                }
+            }
+            ListenableFuture<CachedApiKeyHashResult> cachedValue = service.getApiKeyAuthCache().get(id);
+            assertThat("since the request was rejected, there should be no cache entry for this key", cachedValue, nullValue());
+
+            // unblock the threadpool
+            safeAwait(barrier);
+
+            // wait for the threadpool queue to drain & check that the stats as as expected
+            flushThreadPoolExecutor(threadPool, SECURITY_CRYPTO_THREAD_POOL_NAME);
+            for (var stat : threadPool.stats().stats()) {
+                if (stat.name().equals(SECURITY_CRYPTO_THREAD_POOL_NAME)) {
+                    assertThat(stat.rejected(), equalTo(1L));
+                    assertThat(stat.queue(), equalTo(0));
+                }
+            }
+
+            // try to authenticate again with the same key - if this hangs, check the future caching
+            final AuthenticationResult<User> shouldSucceed = tryAuthenticate(service, id, key, type);
+            assertThat(shouldSucceed.getStatus(), is(AuthenticationResult.Status.SUCCESS));
+        } finally {
+            terminate(clientThreadpool);
+        }
+    }
+
     public void testCreateApiKeyUsesBulkIndexAction() throws Exception {
         final Settings settings = Settings.builder().put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), true).build();
         final ApiKeyService service = createApiKeyService(settings);
@@ -3354,15 +3441,12 @@ public static Authentication createApiKeyAuthentication(
                 Authentication.newApiKeyAuthentication(authenticationResult, "node01"),
                 threadContext
             );
-            final CompletableFuture<Authentication> authFuture = new CompletableFuture<>();
-            securityContext.executeAfterRewritingAuthentication((c) -> {
-                try {
-                    authFuture.complete(authenticationContextSerializer.readFromContext(threadContext));
-                } catch (IOException e) {
-                    throw new RuntimeException(e);
-                }
-            }, version);
-            return authFuture.get();
+            return safeAwait(
+                l -> securityContext.executeAfterRewritingAuthentication(
+                    c -> ActionListener.completeWith(l, () -> authenticationContextSerializer.readFromContext(threadContext)),
+                    version
+                )
+            );
         }
 
         public static Authentication createApiKeyAuthentication(ApiKeyService apiKeyService, Authentication authentication)
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index 85a1dc1aa029d..e1c3b936e5a32 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -55,6 +55,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLog;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -263,6 +264,7 @@ public void init() throws Exception {
         threadPool = new ThreadPool(
             settings,
             MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders(),
             new FixedExecutorBuilder(
                 settings,
                 THREAD_POOL_NAME,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
index adf0b44266260..e53fa83b89617 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
@@ -67,6 +67,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
 import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.ToXContent;
@@ -271,6 +272,7 @@ public static void startThreadPool() throws IOException {
         threadPool = new ThreadPool(
             settings,
             MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders(),
             new FixedExecutorBuilder(
                 settings,
                 TokenService.THREAD_POOL_NAME,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidatorTests.java
index 3732573b2f03d..f1927876eba5f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidatorTests.java
@@ -266,7 +266,7 @@ public void testJwtSignVerifyPassedForAllSupportedAlgorithms() {
             try {
                 helpTestSignatureAlgorithm(signatureAlgorithm, false);
             } catch (Exception e) {
-                fail("signature validation with algorithm [" + signatureAlgorithm + "] should have succeeded");
+                throw new RuntimeException("signature validation with algorithm [" + signatureAlgorithm + "] should have succeeded", e);
             }
         }
         // Fail: "ES256K"
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java
index 466d0e3428d50..6abf6c81b673e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java
@@ -401,59 +401,52 @@ private PortBlockingRunnable(
         public void run() {
             final List<Socket> openedSockets = new ArrayList<>();
             final List<InetAddress> failedAddresses = new ArrayList<>();
-            try {
-                final boolean allSocketsOpened = waitUntil(() -> {
-                    try {
-                        final InetAddress[] allAddresses;
-                        if (serverAddress instanceof Inet4Address) {
-                            allAddresses = NetworkUtils.getAllIPV4Addresses();
-                        } else {
-                            allAddresses = NetworkUtils.getAllIPV6Addresses();
-                        }
-                        final List<InetAddress> inetAddressesToBind = Arrays.stream(allAddresses)
-                            .filter(addr -> openedSockets.stream().noneMatch(s -> addr.equals(s.getLocalAddress())))
-                            .filter(addr -> failedAddresses.contains(addr) == false)
-                            .collect(Collectors.toList());
-                        for (InetAddress localAddress : inetAddressesToBind) {
-                            try {
-                                final Socket socket = openMockSocket(serverAddress, serverPort, localAddress, portToBind);
-                                openedSockets.add(socket);
-                                logger.debug("opened socket [{}]", socket);
-                            } catch (NoRouteToHostException | ConnectException e) {
-                                logger.debug(() -> "marking address [" + localAddress + "] as failed due to:", e);
-                                failedAddresses.add(localAddress);
-                            }
-                        }
-                        if (openedSockets.size() == 0) {
-                            logger.debug("Could not open any sockets from the available addresses");
-                            return false;
+
+            final boolean allSocketsOpened = waitUntil(() -> {
+                try {
+                    final InetAddress[] allAddresses;
+                    if (serverAddress instanceof Inet4Address) {
+                        allAddresses = NetworkUtils.getAllIPV4Addresses();
+                    } else {
+                        allAddresses = NetworkUtils.getAllIPV6Addresses();
+                    }
+                    final List<InetAddress> inetAddressesToBind = Arrays.stream(allAddresses)
+                        .filter(addr -> openedSockets.stream().noneMatch(s -> addr.equals(s.getLocalAddress())))
+                        .filter(addr -> failedAddresses.contains(addr) == false)
+                        .collect(Collectors.toList());
+                    for (InetAddress localAddress : inetAddressesToBind) {
+                        try {
+                            final Socket socket = openMockSocket(serverAddress, serverPort, localAddress, portToBind);
+                            openedSockets.add(socket);
+                            logger.debug("opened socket [{}]", socket);
+                        } catch (NoRouteToHostException | ConnectException e) {
+                            logger.debug(() -> "marking address [" + localAddress + "] as failed due to:", e);
+                            failedAddresses.add(localAddress);
                         }
-                        return true;
-                    } catch (IOException e) {
-                        logger.debug(() -> "caught exception while opening socket on [" + portToBind + "]", e);
+                    }
+                    if (openedSockets.size() == 0) {
+                        logger.debug("Could not open any sockets from the available addresses");
                         return false;
                     }
-                });
-
-                if (allSocketsOpened) {
-                    latch.countDown();
-                } else {
-                    success.set(false);
-                    IOUtils.closeWhileHandlingException(openedSockets);
-                    openedSockets.clear();
-                    latch.countDown();
-                    return;
+                    return true;
+                } catch (IOException e) {
+                    logger.debug(() -> "caught exception while opening socket on [" + portToBind + "]", e);
+                    return false;
                 }
-            } catch (InterruptedException e) {
-                logger.debug(() -> "interrupted while trying to open sockets on [" + portToBind + "]", e);
-                Thread.currentThread().interrupt();
+            });
+
+            if (allSocketsOpened) {
+                latch.countDown();
+            } else {
+                success.set(false);
+                IOUtils.closeWhileHandlingException(openedSockets);
+                openedSockets.clear();
+                latch.countDown();
+                return;
             }
 
             try {
-                closeLatch.await();
-            } catch (InterruptedException e) {
-                logger.debug("caught exception while waiting for close latch", e);
-                Thread.currentThread().interrupt();
+                safeAwait(closeLatch);
             } finally {
                 logger.debug("closing sockets on [{}]", portToBind);
                 IOUtils.closeWhileHandlingException(openedSockets);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectTestCase.java
index be45394b01ec6..a95ecd88f6a8e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectTestCase.java
@@ -6,12 +6,13 @@
  */
 package org.elasticsearch.xpack.security.authc.oidc;
 
+import net.minidev.json.JSONStyle;
+import net.minidev.json.JSONValue;
+import net.minidev.json.reader.JsonWriterI;
+
 import com.nimbusds.jose.JWSAlgorithm;
 import com.nimbusds.jose.JWSHeader;
 import com.nimbusds.jose.crypto.RSASSASigner;
-import com.nimbusds.jose.shaded.json.JSONStyle;
-import com.nimbusds.jose.shaded.json.JSONValue;
-import com.nimbusds.jose.shaded.json.reader.JsonWriterI;
 import com.nimbusds.jwt.JWT;
 import com.nimbusds.jwt.JWTClaimsSet;
 import com.nimbusds.jwt.SignedJWT;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccountsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccountsTests.java
index 756d53285a8f6..21e29469bb02b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccountsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccountsTests.java
@@ -8,18 +8,30 @@
 package org.elasticsearch.xpack.security.authc.service;
 
 import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
+import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction;
+import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.TransportDeleteSnapshotAction;
+import org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.TransportRestoreSnapshotAction;
+import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
 import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
 import org.elasticsearch.action.admin.indices.mapping.put.TransportAutoPutMappingAction;
 import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
 import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
 import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction;
+import org.elasticsearch.action.admin.indices.template.get.GetComponentTemplateAction;
+import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutIndexTemplateAction;
 import org.elasticsearch.action.bulk.TransportBulkAction;
+import org.elasticsearch.action.datastreams.DataStreamsStatsAction;
+import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction;
 import org.elasticsearch.action.delete.TransportDeleteAction;
 import org.elasticsearch.action.get.TransportGetAction;
 import org.elasticsearch.action.get.TransportMultiGetAction;
@@ -52,6 +64,11 @@
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.user.KibanaSystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction;
+import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction;
+import org.elasticsearch.xpack.core.slm.action.GetSLMStatusAction;
+import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleAction;
+import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction;
 import org.elasticsearch.xpack.security.authc.service.ElasticServiceAccounts.ElasticServiceAccount;
 
 import java.util.List;
@@ -67,6 +84,79 @@
 
 public class ElasticServiceAccountsTests extends ESTestCase {
 
+    public void testAutoOpsPrivileges() {
+        final Role role = Role.buildFromRoleDescriptor(
+            ElasticServiceAccounts.ACCOUNTS.get("elastic/auto-ops").roleDescriptor(),
+            new FieldPermissionsCache(Settings.EMPTY),
+            RESTRICTED_INDICES
+        );
+
+        final Authentication authentication = AuthenticationTestHelper.builder().serviceAccount().build();
+        final TransportRequest request = mock(TransportRequest.class);
+
+        // monitor
+        assertThat(role.cluster().check(GetComponentTemplateAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(GetComposableIndexTemplateAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(GetIndexTemplatesAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(TransportClusterHealthAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(TransportNodesStatsAction.TYPE.name(), request, authentication), is(true));
+
+        assertThat(role.cluster().check(ClusterUpdateSettingsAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(TransportPutIndexTemplateAction.TYPE.name(), request, authentication), is(false));
+        assertThat(role.cluster().check(TransportDeleteIndexTemplateAction.TYPE.name(), request, authentication), is(false));
+
+        // read_ilm
+        assertThat(role.cluster().check(GetLifecycleAction.NAME, request, authentication), is(true));
+
+        assertThat(role.cluster().check(ILMActions.STOP.name(), request, authentication), is(false));
+        assertThat(role.cluster().check(ILMActions.PUT.name(), request, authentication), is(false));
+
+        // read_slm
+        assertThat(role.cluster().check(GetSLMStatusAction.NAME, request, authentication), is(true));
+        assertThat(role.cluster().check(GetSnapshotLifecycleAction.NAME, request, authentication), is(true));
+
+        assertThat(role.cluster().check(DeleteSnapshotLifecycleAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(ExecuteSnapshotLifecycleAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(PutSnapshotLifecycleAction.NAME, request, authentication), is(false));
+        assertThat(role.cluster().check(TransportGetSnapshotsAction.TYPE.name(), request, authentication), is(false));
+        assertThat(role.cluster().check(TransportCreateSnapshotAction.TYPE.name(), request, authentication), is(false));
+        assertThat(role.cluster().check(TransportDeleteSnapshotAction.TYPE.name(), request, authentication), is(false));
+        assertThat(role.cluster().check(TransportRestoreSnapshotAction.TYPE.name(), request, authentication), is(false));
+
+        // index monitor
+        List.of(
+            "search-" + randomAlphaOfLengthBetween(1, 20),
+            ".kibana-" + randomAlphaOfLengthBetween(1, 20),
+            ".elastic-analytics-collections",
+            "logs-" + randomAlphaOfLengthBetween(1, 20),
+            "my-index-" + randomAlphaOfLengthBetween(1, 20),
+            ".internal.alerts-default.alerts-default-" + randomAlphaOfLengthBetween(1, 20)
+        ).forEach(index -> {
+            final IndexAbstraction anyIndex = mockIndexAbstraction(index);
+
+            assertThat(role.indices().allowedIndicesMatcher(IndicesStatsAction.NAME).test(anyIndex), is(true));
+            assertThat(role.indices().allowedIndicesMatcher(DataStreamsStatsAction.NAME).test(anyIndex), is(true));
+            assertThat(role.indices().allowedIndicesMatcher(GetAliasesAction.NAME).test(anyIndex), is(true));
+            assertThat(role.indices().allowedIndicesMatcher(GetSettingsAction.NAME).test(anyIndex), is(true));
+            assertThat(role.indices().allowedIndicesMatcher(GetDataStreamLifecycleAction.INSTANCE.name()).test(anyIndex), is(true));
+
+            assertThat(role.indices().allowedIndicesMatcher(AutoCreateAction.NAME).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(TransportDeleteAction.NAME).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(TransportIndicesAliasesAction.NAME).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(TransportBulkAction.NAME).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(TransportGetAction.TYPE.name()).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(TransportMultiGetAction.NAME).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(TransportMultiSearchAction.TYPE.name()).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(TransportUpdateSettingsAction.TYPE.name()).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher(RefreshAction.NAME).test(anyIndex), is(false));
+            assertThat(role.indices().allowedIndicesMatcher("indices:foo").test(anyIndex), is(false));
+        });
+    }
+
     public void testKibanaSystemPrivileges() {
         final RoleDescriptor serviceAccountRoleDescriptor = ElasticServiceAccounts.ACCOUNTS.get("elastic/kibana").roleDescriptor();
         final RoleDescriptor reservedRolesStoreRoleDescriptor = ReservedRolesStore.kibanaSystemRoleDescriptor(KibanaSystemUser.ROLE_NAME);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountServiceTests.java
index c66f3168c7b7d..43fe57dd8b313 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountServiceTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLog;
@@ -82,7 +81,6 @@ public void init() throws UnknownHostException {
         indexServiceAccountTokenStore = mock(IndexServiceAccountTokenStore.class);
         when(fileServiceAccountTokenStore.getTokenSource()).thenReturn(TokenInfo.TokenSource.FILE);
         when(indexServiceAccountTokenStore.getTokenSource()).thenReturn(TokenInfo.TokenSource.INDEX);
-        final Settings.Builder builder = Settings.builder().put("xpack.security.enabled", true);
         client = mock(Client.class);
         when(client.threadPool()).thenReturn(threadPool);
         serviceAccountService = new ServiceAccountService(client, fileServiceAccountTokenStore, indexServiceAccountTokenStore);
@@ -96,11 +94,17 @@ public void stopThreadPool() {
     public void testGetServiceAccountPrincipals() {
         assertThat(
             ServiceAccountService.getServiceAccountPrincipals(),
-            containsInAnyOrder("elastic/enterprise-search-server", "elastic/fleet-server", "elastic/fleet-server-remote", "elastic/kibana")
+            containsInAnyOrder(
+                "elastic/auto-ops",
+                "elastic/enterprise-search-server",
+                "elastic/fleet-server",
+                "elastic/fleet-server-remote",
+                "elastic/kibana"
+            )
         );
     }
 
-    public void testTryParseToken() throws IOException, IllegalAccessException {
+    public void testTryParseToken() throws IOException {
         // Null for null
         assertNull(ServiceAccountService.tryParseToken(null));
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
index 5f878480a7d0d..3be0a17d19253 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
@@ -66,6 +66,7 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.SearchTransportService;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.search.TransportClearScrollAction;
 import org.elasticsearch.action.search.TransportClosePointInTimeAction;
 import org.elasticsearch.action.search.TransportMultiSearchAction;
@@ -3650,7 +3651,7 @@ private static BytesReference createEncodedPIT(Index index) {
         );
         List<SearchPhaseResult> results = new ArrayList<>();
         results.add(testSearchPhaseResult1);
-        return SearchContextId.encode(results, Collections.emptyMap(), TransportVersion.current());
+        return SearchContextId.encode(results, Collections.emptyMap(), TransportVersion.current(), ShardSearchFailure.EMPTY_ARRAY);
     }
 
     private static class RBACAuthorizationInfoRoleMatcher implements ArgumentMatcher<AuthorizationInfo> {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptorTests.java
index 15b187edd1cea..4609767aa7926 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptorTests.java
@@ -48,8 +48,22 @@
 
 public class IndicesAliasesRequestInterceptorTests extends ESTestCase {
 
-    @SuppressWarnings("unchecked")
     public void testInterceptorThrowsWhenFLSDLSEnabled() {
+        checkInterceptorWithDlsFlsConfigured(
+            true,
+            "Alias requests are not allowed for users who have field or document level security enabled on one of the indices"
+        );
+    }
+
+    public void testInterceptorWorksAsNormalWhenFLSDLSDisabled() {
+        checkInterceptorWithDlsFlsConfigured(
+            false,
+            "Adding an alias is not allowed when the alias has more permissions than any of the indices"
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    public void checkInterceptorWithDlsFlsConfigured(boolean dlsFlsFeatureEnabled, String expectedErrorMessage) {
         MockLicenseState licenseState = mock(MockLicenseState.class);
         when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState);
         when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(true);
@@ -89,7 +103,12 @@ public void testInterceptorThrowsWhenFLSDLSEnabled() {
             )
         );
         new SecurityContext(Settings.EMPTY, threadContext).putIndicesAccessControl(accessControl);
-        IndicesAliasesRequestInterceptor interceptor = new IndicesAliasesRequestInterceptor(threadContext, licenseState, auditTrailService);
+        IndicesAliasesRequestInterceptor interceptor = new IndicesAliasesRequestInterceptor(
+            threadContext,
+            licenseState,
+            auditTrailService,
+            dlsFlsFeatureEnabled
+        );
 
         IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
         if (randomBoolean()) {
@@ -112,10 +131,7 @@ public void testInterceptorThrowsWhenFLSDLSEnabled() {
             interceptor.intercept(requestInfo, mockEngine, EmptyAuthorizationInfo.INSTANCE, plainActionFuture);
             plainActionFuture.actionGet();
         });
-        assertEquals(
-            "Alias requests are not allowed for users who have field or document level security enabled on one of the indices",
-            securityException.getMessage()
-        );
+        assertEquals(expectedErrorMessage, securityException.getMessage());
     }
 
     @SuppressWarnings("unchecked")
@@ -136,7 +152,12 @@ public void testInterceptorThrowsWhenTargetHasGreaterPermissions() throws Except
         final String action = TransportIndicesAliasesAction.NAME;
         IndicesAccessControl accessControl = new IndicesAccessControl(true, Collections.emptyMap());
         new SecurityContext(Settings.EMPTY, threadContext).putIndicesAccessControl(accessControl);
-        IndicesAliasesRequestInterceptor interceptor = new IndicesAliasesRequestInterceptor(threadContext, licenseState, auditTrailService);
+        IndicesAliasesRequestInterceptor interceptor = new IndicesAliasesRequestInterceptor(
+            threadContext,
+            licenseState,
+            auditTrailService,
+            false
+        );
 
         final IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
         if (randomBoolean()) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java
index 817d5739b4b9f..68c86a561025a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java
@@ -49,8 +49,22 @@
 
 public class ResizeRequestInterceptorTests extends ESTestCase {
 
-    @SuppressWarnings("unchecked")
     public void testResizeRequestInterceptorThrowsWhenFLSDLSEnabled() {
+        checkResizeWithDlsFlsConfigured(
+            true,
+            "Resize requests are not allowed for users when field or document level security is enabled on the source index"
+        );
+    }
+
+    public void testResizeRequestInterceptorWorksAsNormalWhenFLSDLSDisabled() {
+        checkResizeWithDlsFlsConfigured(
+            false,
+            "Resizing an index is not allowed when the target index has more permissions than the source index"
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    public void checkResizeWithDlsFlsConfigured(boolean dlsFlsFeatureEnabled, String expectedErrorMessage) {
         MockLicenseState licenseState = mock(MockLicenseState.class);
         when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState);
         when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(true);
@@ -89,7 +103,12 @@ public void testResizeRequestInterceptorThrowsWhenFLSDLSEnabled() {
         );
         new SecurityContext(Settings.EMPTY, threadContext).putIndicesAccessControl(accessControl);
 
-        ResizeRequestInterceptor resizeRequestInterceptor = new ResizeRequestInterceptor(threadPool, licenseState, auditTrailService);
+        ResizeRequestInterceptor resizeRequestInterceptor = new ResizeRequestInterceptor(
+            threadPool,
+            licenseState,
+            auditTrailService,
+            dlsFlsFeatureEnabled
+        );
 
         PlainActionFuture<Void> plainActionFuture = new PlainActionFuture<>();
         RequestInfo requestInfo = new RequestInfo(authentication, new ResizeRequest("bar", "foo"), ResizeAction.NAME, null);
@@ -104,10 +123,7 @@ public void testResizeRequestInterceptorThrowsWhenFLSDLSEnabled() {
             resizeRequestInterceptor.intercept(requestInfo, mockEngine, EmptyAuthorizationInfo.INSTANCE, plainActionFuture);
             plainActionFuture.actionGet();
         });
-        assertEquals(
-            "Resize requests are not allowed for users when field or document level security is enabled on the source index",
-            securityException.getMessage()
-        );
+        assertEquals(expectedErrorMessage, securityException.getMessage());
     }
 
     @SuppressWarnings("unchecked")
@@ -126,7 +142,7 @@ public void testResizeRequestInterceptorThrowsWhenTargetHasGreaterPermissions()
             .build();
         IndicesAccessControl accessControl = new IndicesAccessControl(true, Collections.emptyMap());
         new SecurityContext(Settings.EMPTY, threadContext).putIndicesAccessControl(accessControl);
-        ResizeRequestInterceptor resizeRequestInterceptor = new ResizeRequestInterceptor(threadPool, licenseState, auditTrailService);
+        ResizeRequestInterceptor resizeRequestInterceptor = new ResizeRequestInterceptor(threadPool, licenseState, auditTrailService, true);
 
         AuthorizationEngine mockEngine = mock(AuthorizationEngine.class);
         {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java
index 888483613a187..0a1f5f801143d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.node.Node;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.EnrollmentToken;
@@ -85,6 +86,7 @@ public static void startThreadPool() throws IOException {
         threadPool = new ThreadPool(
             settings,
             MeterRegistry.NOOP,
+            new DefaultBuiltInExecutorBuilders(),
             new FixedExecutorBuilder(
                 settings,
                 TokenService.THREAD_POOL_NAME,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java
index d487eab9f7887..9a05230d82ae6 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentType;
@@ -56,7 +57,7 @@ public void setUp() throws Exception {
             .put("node.name", "test-" + getTestName())
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
-        threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java
index 577a8eb9f698e..d88a217cd0949 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentType;
@@ -62,7 +63,7 @@ public void setUp() throws Exception {
             .put("node.name", "test-" + getTestName())
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
-        threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java
index 8bbd051c2fc32..ac472378d4874 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentType;
@@ -54,7 +55,7 @@ public void setUp() throws Exception {
             .put("node.name", "test-" + getTestName())
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
-        threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyActionTests.java
index 2240b72c1a963..d5aa249b1d0f5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyActionTests.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParseException;
@@ -72,7 +73,7 @@ public void setUp() throws Exception {
             .put("node.name", "test-" + getTestName())
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
-        threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HeaderSizeLimitTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HeaderSizeLimitTests.java
index 8c422342c3640..ba7c2e3844521 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HeaderSizeLimitTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HeaderSizeLimitTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteClusterPortSettings;
 import org.elasticsearch.transport.RequestHandlerRegistry;
@@ -78,7 +79,7 @@ public final class SecurityNetty4HeaderSizeLimitTests extends ESTestCase {
 
     @Before
     public void startThreadPool() {
-        threadPool = new ThreadPool(settings, MeterRegistry.NOOP);
+        threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders());
         TaskManager taskManager = new TaskManager(settings, threadPool, Collections.emptySet());
         NetworkService networkService = new NetworkService(Collections.emptyList());
         PageCacheRecycler recycler = new MockPageCacheRecycler(Settings.EMPTY);
diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/audit/logfile/audited_roles.txt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/audit/logfile/audited_roles.txt
index 7b5e24c97d65a..f913c8608960b 100644
--- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/audit/logfile/audited_roles.txt
+++ b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/audit/logfile/audited_roles.txt
@@ -7,6 +7,6 @@ role_descriptor2
 role_descriptor3
 {"cluster":[],"indices":[],"applications":[{"application":"maps","privileges":["{","}","\n","\\","\""],"resources":["raster:*"]},{"application":"maps","privileges":["*:*"],"resources":["noooooo!!\n\n\f\\\\r","{"]}],"run_as":["jack","nich*","//\""],"metadata":{"some meta":42}}
 role_descriptor4
-{"cluster":["manage_ml","grant_api_key","manage_rollup"],"global":{"application":{"manage":{"applications":["a+b+|b+a+"]}},"profile":{}},"indices":[{"names":["/. ? + * | { } [ ] ( ) \" \\/","*"],"privileges":["read","read_cross_cluster"],"field_security":{"grant":["almost","all*"],"except":["denied*"]}}],"applications":[],"run_as":["//+a+\"[a]/"],"metadata":{"?list":["e1","e2","*"],"some other meta":{"r":"t"}}}
+{"cluster":["manage_ml","grant_api_key","manage_rollup"],"global":{"application":{"manage":{"applications":["a+b+|b+a+"]}},"profile":{},"role":{}},"indices":[{"names":["/. ? + * | { } [ ] ( ) \" \\/","*"],"privileges":["read","read_cross_cluster"],"field_security":{"grant":["almost","all*"],"except":["denied*"]}}],"applications":[],"run_as":["//+a+\"[a]/"],"metadata":{"?list":["e1","e2","*"],"some other meta":{"r":"t"}}}
 role_descriptor5
-{"cluster":["all"],"global":{"application":{"manage":{"applications":["\""]}},"profile":{"write":{"applications":["","\""]}}},"indices":[],"applications":[],"run_as":["\"[a]/"]}
+{"cluster":["all"],"global":{"application":{"manage":{"applications":["\""]}},"profile":{"write":{"applications":["","\""]}},"role":{"manage":{"indices":[{"names":["test*"],"privileges":["read","write"]}]}}},"indices":[],"applications":[],"run_as":["\"[a]/"]}
diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java
index 20dedbaa161be..621836cea9f89 100644
--- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java
+++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java
@@ -76,7 +76,7 @@ public List<RestHandler> getRestHandlers(
     @UpdateForV9 // always true in v9 so can be removed
     static boolean serializesWithParentTaskAndTimeouts(TransportVersion transportVersion) {
         return transportVersion.isPatchFrom(TransportVersions.V_8_13_4)
-            || transportVersion.isPatchFrom(TransportVersions.SHUTDOWN_REQUEST_TIMEOUTS_FIX_8_14)
+            || transportVersion.isPatchFrom(TransportVersions.V_8_14_0)
             || transportVersion.onOrAfter(TransportVersions.SHUTDOWN_REQUEST_TIMEOUTS_FIX);
     }
 }
diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeAction.java
index a395833746d34..d66f2cbddd182 100644
--- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeAction.java
+++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeAction.java
@@ -27,8 +27,8 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java
index 33965eca83aee..33a285128e08c 100644
--- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java
+++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java
@@ -32,8 +32,8 @@
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.shutdown.PluginShutdownService;
 import org.elasticsearch.snapshots.SnapshotsInfoService;
 import org.elasticsearch.tasks.CancellableTask;
diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java
index 03c61168761b3..4e1b8c3cf3b9a 100644
--- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java
+++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java
@@ -26,8 +26,8 @@
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/slm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java b/x-pack/plugin/slm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
index abaf9a14aeadb..d42c8ec9655ef 100644
--- a/x-pack/plugin/slm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
+++ b/x-pack/plugin/slm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
@@ -38,6 +38,8 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -108,7 +110,8 @@ public void testFullPolicySnapshot() throws Exception {
         // allow arbitrarily frequent slm snapshots
         disableSLMMinimumIntervalValidation();
 
-        createSnapshotPolicy(policyName, "snap", "*/1 * * * * ?", repoId, indexName, true);
+        var schedule = randomBoolean() ? "*/1 * * * * ?" : "1s";
+        createSnapshotPolicy(policyName, "snap", schedule, repoId, indexName, true);
 
         // Check that the snapshot was actually taken
         assertBusy(() -> {
@@ -176,7 +179,8 @@ public void testPolicyFailure() throws Exception {
         disableSLMMinimumIntervalValidation();
 
         // Create a policy with ignore_unavailable: false and an index that doesn't exist
-        createSnapshotPolicy(policyName, "snap", "*/1 * * * * ?", repoName, indexPattern, false);
+        var schedule = randomBoolean() ? "*/1 * * * * ?" : "1s";
+        createSnapshotPolicy(policyName, "snap", schedule, repoName, indexPattern, false);
 
         assertBusy(() -> {
             // Check that the failure is written to the cluster state
@@ -300,10 +304,11 @@ public void testStartStopStatus() throws Exception {
         });
 
         try {
+            var schedule = randomBoolean() ? "0 0/15 * * * ?" : "15m";
             createSnapshotPolicy(
                 policyName,
                 "snap",
-                "0 0/15 * * * ?",
+                schedule,
                 repoId,
                 indexName,
                 true,
@@ -671,6 +676,36 @@ public void testSnapshotRetentionWithMissingRepo() throws Exception {
         }, 60, TimeUnit.SECONDS);
     }
 
+    @SuppressWarnings("unchecked")
+    public void testGetIntervalSchedule() throws Exception {
+        final String indexName = "index-1";
+        final String policyName = "policy-1";
+        final String repoId = "repo-1";
+
+        initializeRepo(repoId);
+
+        var schedule = "30m";
+        var now = Instant.now();
+        createSnapshotPolicy(policyName, "snap", schedule, repoId, indexName, true);
+
+        assertBusy(() -> {
+            Request getReq = new Request("GET", "/_slm/policy/" + policyName);
+            Response policyMetadata = client().performRequest(getReq);
+            Map<String, Object> policyResponseMap;
+            try (InputStream is = policyMetadata.getEntity().getContent()) {
+                policyResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
+            }
+
+            Map<String, Object> policyMetadataMap = (Map<String, Object>) policyResponseMap.get(policyName);
+            Long nextExecutionMillis = (Long) policyMetadataMap.get("next_execution_millis");
+            assertNotNull(nextExecutionMillis);
+
+            Instant nextExecution = Instant.ofEpochMilli(nextExecutionMillis);
+            assertTrue(nextExecution.isAfter(now.plus(Duration.ofMinutes(29))));
+            assertTrue(nextExecution.isBefore(now.plus(Duration.ofMinutes(31))));
+        });
+    }
+
     public Map<String, Object> getLocation(String path) {
         try {
             Response executeRepsonse = client().performRequest(new Request("GET", path));
diff --git a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java
new file mode 100644
index 0000000000000..d8b1f36c25e54
--- /dev/null
+++ b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java
@@ -0,0 +1,749 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.slm;
+
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
+import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.SnapshotsInProgress;
+import org.elasticsearch.cluster.coordination.Coordinator;
+import org.elasticsearch.cluster.coordination.FollowersChecker;
+import org.elasticsearch.cluster.coordination.LagDetector;
+import org.elasticsearch.cluster.coordination.LeaderChecker;
+import org.elasticsearch.cluster.metadata.RepositoryMetadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.datastreams.DataStreamsPlugin;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
+import org.elasticsearch.indices.recovery.RecoverySettings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.repositories.FinalizeSnapshotContext;
+import org.elasticsearch.repositories.RepositoriesMetrics;
+import org.elasticsearch.repositories.Repository;
+import org.elasticsearch.repositories.SnapshotShardContext;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
+import org.elasticsearch.snapshots.RegisteredPolicySnapshots;
+import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotInfo;
+import org.elasticsearch.snapshots.SnapshotMissingException;
+import org.elasticsearch.snapshots.SnapshotState;
+import org.elasticsearch.snapshots.mockstore.MockRepository;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.disruption.NetworkDisruption;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.TransportSettings;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
+import org.elasticsearch.xpack.core.slm.SnapshotRetentionConfiguration;
+import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction;
+import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction;
+import org.elasticsearch.xpack.ilm.IndexLifecycle;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Test that SLM stats can be lost due to master shutdown,
+ * and then recovered by registering them before snapshotting.
+ */
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class SLMStatDisruptionIT extends AbstractSnapshotIntegTestCase {
+
+    private static final String NEVER_EXECUTE_CRON_SCHEDULE = "* * * 31 FEB ? *";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(
+            MockRepository.Plugin.class,
+            MockTransportService.TestPlugin.class,
+            LocalStateCompositeXPackPlugin.class,
+            IndexLifecycle.class,
+            SnapshotLifecycle.class,
+            DataStreamsPlugin.class,
+            TestDelayedRepoPlugin.class,
+            TestRestartBeforeListenersRepoPlugin.class
+        );
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED, false)
+            .put(DEFAULT_SETTINGS)
+            .build();
+    }
+
+    // copied from AbstractDisruptionTestCase.DEFAULT_SETTINGS
+    public static final Settings DEFAULT_SETTINGS = Settings.builder()
+        .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "5s") // for hitting simulated network failures quickly
+        .put(LeaderChecker.LEADER_CHECK_RETRY_COUNT_SETTING.getKey(), 1) // for hitting simulated network failures quickly
+        .put(FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), "5s") // for hitting simulated network failures quickly
+        .put(FollowersChecker.FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), 1) // for hitting simulated network failures quickly
+        .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "5s") // <-- for hitting simulated network failures quickly
+        .put(LagDetector.CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING.getKey(), "5s") // remove lagging nodes quickly so they can rejoin
+        .put(TransportSettings.CONNECT_TIMEOUT.getKey(), "10s") // Network delay disruption waits for the min between this
+        // value and the time of disruption and does not recover immediately
+        // when disruption is stop. We should make sure we recover faster
+        // then the default of 30s, causing ensureGreen and friends to time out
+        .build();
+
+    public static class TestDelayedRepoPlugin extends Plugin implements RepositoryPlugin {
+
+        // Use static vars since instantiated by plugin system
+        private static final AtomicBoolean doDelay = new AtomicBoolean(true);
+        private static final CountDownLatch delayedRepoLatch = new CountDownLatch(1);
+
+        static void removeDelay() {
+            delayedRepoLatch.countDown();
+        }
+
+        @Override
+        public Map<String, Repository.Factory> getRepositories(
+            Environment env,
+            NamedXContentRegistry namedXContentRegistry,
+            ClusterService clusterService,
+            BigArrays bigArrays,
+            RecoverySettings recoverySettings,
+            RepositoriesMetrics repositoriesMetrics
+        ) {
+            return Map.of(
+                TestDelayedRepo.TYPE,
+                metadata -> new TestDelayedRepo(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings, () -> {
+                    // Only delay the first request
+                    if (doDelay.getAndSet(false)) {
+                        try {
+                            assertTrue(delayedRepoLatch.await(1, TimeUnit.MINUTES));
+                        } catch (InterruptedException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+                })
+            );
+        }
+    }
+
+    static class TestDelayedRepo extends FsRepository {
+        private static final String TYPE = "delayed";
+        private final Runnable delayFn;
+
+        protected TestDelayedRepo(
+            RepositoryMetadata metadata,
+            Environment env,
+            NamedXContentRegistry namedXContentRegistry,
+            ClusterService clusterService,
+            BigArrays bigArrays,
+            RecoverySettings recoverySettings,
+            Runnable delayFn
+        ) {
+            super(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings);
+            this.delayFn = delayFn;
+        }
+
+        @Override
+        protected void snapshotFile(SnapshotShardContext context, BlobStoreIndexShardSnapshot.FileInfo fileInfo) throws IOException {
+            delayFn.run();
+            super.snapshotFile(context, fileInfo);
+        }
+    }
+
+    public static class TestRestartBeforeListenersRepoPlugin extends Plugin implements RepositoryPlugin {
+
+        // Use static vars since instantiated by plugin system
+        private static Runnable onResponse;
+
+        static void setOnResponse(Runnable onResponse) {
+            TestRestartBeforeListenersRepoPlugin.onResponse = onResponse;
+        }
+
+        static void clearOnResponse() {
+            onResponse = () -> {};
+        }
+
+        @Override
+        public Map<String, Repository.Factory> getRepositories(
+            Environment env,
+            NamedXContentRegistry namedXContentRegistry,
+            ClusterService clusterService,
+            BigArrays bigArrays,
+            RecoverySettings recoverySettings,
+            RepositoriesMetrics repositoriesMetrics
+        ) {
+            return Map.of(
+                TestRestartBeforeListenersRepo.TYPE,
+                metadata -> new TestRestartBeforeListenersRepo(
+                    metadata,
+                    env,
+                    namedXContentRegistry,
+                    clusterService,
+                    bigArrays,
+                    recoverySettings,
+                    // add wrapper lambda so can change underlying runnable with static setOnResponse() method
+                    () -> onResponse.run()
+                )
+            );
+        }
+    }
+
+    /**
+     * Repo which forces a runnable to be run after snapshot finalization, but before callbacks
+     * which receive the SnapshotInfo, specifically the SLM callback which will save failure stats.
+     */
+    static class TestRestartBeforeListenersRepo extends FsRepository {
+        private static final String TYPE = "restart_before_listeners";
+
+        private final Runnable beforeResponseRunnable;
+
+        protected TestRestartBeforeListenersRepo(
+            RepositoryMetadata metadata,
+            Environment env,
+            NamedXContentRegistry namedXContentRegistry,
+            ClusterService clusterService,
+            BigArrays bigArrays,
+            RecoverySettings recoverySettings,
+            Runnable beforeResponseRunnable
+        ) {
+            super(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings);
+            this.beforeResponseRunnable = beforeResponseRunnable;
+        }
+
+        @Override
+        public void finalizeSnapshot(FinalizeSnapshotContext fsc) {
+            var newFinalizeContext = new FinalizeSnapshotContext(
+                fsc.updatedShardGenerations(),
+                fsc.repositoryStateId(),
+                fsc.clusterMetadata(),
+                fsc.snapshotInfo(),
+                fsc.repositoryMetaVersion(),
+                fsc,
+                snapshotInfo -> {
+                    // run the passed lambda before calling the usual callback
+                    // this is where the cluster can be restarted before SLM is called back with the snapshotInfo
+                    beforeResponseRunnable.run();
+                    fsc.onDone(snapshotInfo);
+                }
+            );
+            super.finalizeSnapshot(newFinalizeContext);
+        }
+    }
+
+    @Before
+    public void clearRepoFinalizeRunnable() {
+        TestRestartBeforeListenersRepoPlugin.clearOnResponse();
+    }
+
+    /**
+     * Test that if there is a currently running snapshot it is not inferred to be a failure
+     */
+    public void testCurrentlyRunningSnapshotNotRecordedAsFailure() throws Exception {
+        final String idxName = "test-idx";
+        final String repoName = "test-repo";
+        final String policyName = "test-policy";
+
+        internalCluster().startMasterOnlyNodes(1);
+        final String masterNode = internalCluster().getMasterName();
+        final String dataNode = internalCluster().startDataOnlyNode();
+        ensureStableCluster(2);
+
+        createRandomIndex(idxName, dataNode);
+        createRepository(repoName, TestDelayedRepo.TYPE);
+        createSnapshotPolicy(policyName, "snap", NEVER_EXECUTE_CRON_SCHEDULE, repoName, idxName);
+
+        ensureGreen();
+
+        String snapshotA = executePolicy(masterNode, policyName);
+        logger.info("Created snapshot A: " + snapshotA);
+
+        // wait until snapshotA is registered before starting snapshotB
+        assertBusy(() -> assertRegistered(policyName, List.of(snapshotA)), 1, TimeUnit.MINUTES);
+
+        // create another snapshot while A is still running
+        String snapshotB = executePolicy(masterNode, policyName);
+        logger.info("Created snapshot B: " + snapshotB);
+
+        // wait until both snapshots are registered before allowing snapshotA to continue
+        assertBusy(() -> assertRegistered(policyName, List.of(snapshotA, snapshotB)), 1, TimeUnit.MINUTES);
+
+        // remove delay from snapshotA allowing it to finish
+        TestDelayedRepoPlugin.removeDelay();
+
+        waitForSnapshot(repoName, snapshotA);
+        waitForSnapshot(repoName, snapshotB);
+
+        assertBusy(() -> {
+            assertSnapshotSuccess(repoName, snapshotA);
+            assertSnapshotSuccess(repoName, snapshotB);
+            assertMetadata(policyName, 2, 0, 0);
+            assertRegistered(policyName, List.of());
+
+        }, 1, TimeUnit.MINUTES);
+    }
+
+    /**
+     * Test that after successful snapshot registered is empty
+     */
+    public void testSuccessSnapshot() throws Exception {
+        final String idxName = "test-idx";
+        final String repoName = "test-repo";
+        final String policyName = "test-policy";
+
+        internalCluster().startMasterOnlyNodes(1);
+        final String masterNode = internalCluster().getMasterName();
+        final String dataNode = internalCluster().startDataOnlyNode();
+        ensureStableCluster(2);
+
+        createRandomIndex(idxName, dataNode);
+        createRepository(repoName, TestRestartBeforeListenersRepo.TYPE);
+        createSnapshotPolicy(policyName, "snap", NEVER_EXECUTE_CRON_SCHEDULE, repoName, idxName);
+
+        ensureGreen();
+
+        String snapshotName = executePolicy(masterNode, policyName);
+        logger.info("Created snapshot: " + snapshotName);
+
+        waitForSnapshot(repoName, snapshotName);
+
+        assertBusy(() -> {
+            assertSnapshotSuccess(repoName, snapshotName);
+            assertMetadata(policyName, 1, 0, 0);
+            assertRegistered(policyName, List.of());
+        }, 1, TimeUnit.MINUTES);
+    }
+
+    /**
+     * Test that after a failure which fails stats uploads, then a success,
+     * registered snapshot from failure is added to invocationsSinceLastSuccess.
+     */
+    public void testFailSnapshotFailStatsThenSuccessRecoverStats() throws Exception {
+        final String idxName = "test-idx";
+        final String repoName = "test-repo";
+        final String policyName = "test-policy";
+
+        internalCluster().startMasterOnlyNodes(1);
+        final String masterNode = internalCluster().getMasterName();
+        final String dataNode = internalCluster().startDataOnlyNode();
+        ensureStableCluster(2);
+
+        // Add network disruption so snapshot fails with PARTIAL status
+        NetworkDisruption networkDisruption = isolateMasterDisruption(NetworkDisruption.DISCONNECT);
+        internalCluster().setDisruptionScheme(networkDisruption);
+
+        // wire into repo so code can be run on test thread after snapshot finalize, but before SLM is called back
+        var runDuringFinalize = new RunDuringFinalize();
+        TestRestartBeforeListenersRepoPlugin.setOnResponse(runDuringFinalize.finalizeThreadRunnable());
+
+        createRandomIndex(idxName, dataNode);
+        createRepository(repoName, TestRestartBeforeListenersRepo.TYPE);
+        createSnapshotPolicy(policyName, "snap", NEVER_EXECUTE_CRON_SCHEDULE, repoName, idxName);
+
+        ensureGreen();
+
+        networkDisruption.startDisrupting();
+        String snapshotName = executePolicy(masterNode, policyName);
+        logger.info("Created snapshot: " + snapshotName);
+
+        // restart snapshot after snapshot finalize, but before SLM callback called
+        runDuringFinalize.awaitAndRun(() -> {
+            try {
+                internalCluster().restartNode(masterNode);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        assertBusy(() -> {
+            assertSnapshotPartial(repoName, snapshotName);
+            assertMetadata(policyName, 0, 0, 0);
+            assertRegistered(policyName, List.of(snapshotName));
+        }, 1, TimeUnit.MINUTES);
+
+        awaitNoMoreRunningOperations();
+        ensureGreen();
+
+        // Now execute again, and succeed. The failure from the previous run will now be recorded.
+
+        TestRestartBeforeListenersRepoPlugin.clearOnResponse();
+
+        final String snapshotName2 = executePolicy(masterNode, policyName);
+        assertNotEquals(snapshotName, snapshotName2);
+        logger.info("Created snapshot: " + snapshotName2);
+
+        waitForSnapshot(repoName, snapshotName2);
+
+        assertBusy(() -> {
+            assertSnapshotSuccess(repoName, snapshotName2);
+            // Check stats, this time past failure should be accounted for
+            assertMetadata(policyName, 1, 1, 0);
+            assertRegistered(policyName, List.of());
+        }, 1, TimeUnit.MINUTES);
+    }
+
+    /**
+     * Test that after a failure then a failure that successfully sets stats
+     * registeredRuns from failure is added to invocationsSinceLastSuccess.
+     */
+    public void testFailSnapshotFailStatsRecoverStats() throws Exception {
+        final String idxName = "test-idx";
+        final String repoName = "test-repo";
+        final String policyName = "test-policy";
+
+        internalCluster().startMasterOnlyNodes(1);
+        final String masterNode = internalCluster().getMasterName();
+        final String dataNode = internalCluster().startDataOnlyNode();
+        ensureStableCluster(2);
+
+        // Add network disruption so snapshot fails with PARTIAL status
+        NetworkDisruption networkDisruption = isolateMasterDisruption(NetworkDisruption.DISCONNECT);
+        internalCluster().setDisruptionScheme(networkDisruption);
+
+        // wire into repo so code can be run on test thread after snapshot finalize, but before SLM is called back
+        var runDuringFinalize = new RunDuringFinalize();
+        TestRestartBeforeListenersRepoPlugin.setOnResponse(runDuringFinalize.finalizeThreadRunnable());
+
+        createRandomIndex(idxName, dataNode);
+        createRepository(repoName, TestRestartBeforeListenersRepo.TYPE);
+        createSnapshotPolicy(policyName, "snap", NEVER_EXECUTE_CRON_SCHEDULE, repoName, idxName);
+
+        awaitNoMoreRunningOperations();
+        ensureGreen();
+
+        networkDisruption.startDisrupting();
+        String snapshotName = executePolicy(masterNode, policyName);
+        logger.info("Created snapshot: " + snapshotName);
+
+        // restart snapshot after snapshot finalize, but before SLM callback called
+        runDuringFinalize.awaitAndRun(() -> {
+            try {
+                internalCluster().restartNode(masterNode);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        assertBusy(() -> {
+            assertSnapshotPartial(repoName, snapshotName);
+            assertMetadata(policyName, 0, 0, 0);
+            assertRegistered(policyName, List.of(snapshotName));
+        }, 1, TimeUnit.MINUTES);
+
+        awaitNoMoreRunningOperations();
+        ensureGreen();
+
+        // Now execute again, but don't fail the stat upload. The failure from the previous run will now be recorded.
+        var runDuringFinalize2 = new RunDuringFinalize();
+        TestRestartBeforeListenersRepoPlugin.setOnResponse(runDuringFinalize2.finalizeThreadRunnable());
+
+        networkDisruption.startDisrupting();
+        final String snapshotName2 = executePolicy(masterNode, policyName);
+        assertNotEquals(snapshotName, snapshotName2);
+        logger.info("Created snapshot: " + snapshotName2);
+
+        runDuringFinalize2.awaitAndRun(networkDisruption::stopDisrupting);
+
+        assertBusy(() -> {
+            assertSnapshotPartial(repoName, snapshotName2);
+            // Check metadata, this time past failure should be accounted for
+            assertMetadata(policyName, 0, 2, 2);
+            assertRegistered(policyName, List.of());
+        }, 1, TimeUnit.MINUTES);
+    }
+
+    /**
+     * Test that after a failed snapshot with a master restart during stat upload, update of invocationsSinceLastSuccess is lost.
+     */
+    public void testFailedSnapshotFailStats() throws Exception {
+        final String idxName = "test-idx";
+        final String repoName = "test-repo";
+        final String policyName = "test-policy";
+
+        internalCluster().startMasterOnlyNodes(1);
+        final String masterNode = internalCluster().getMasterName();
+        final String dataNode = internalCluster().startDataOnlyNode();
+        ensureStableCluster(2);
+
+        // Add network disruption so snapshot fails with PARTIAL status
+        NetworkDisruption networkDisruption = isolateMasterDisruption(NetworkDisruption.DISCONNECT);
+        internalCluster().setDisruptionScheme(networkDisruption);
+
+        // wire into repo so code can be run on test thread after snapshot finalize, but before SLM is called back
+        var runDuringFinalize = new RunDuringFinalize();
+        TestRestartBeforeListenersRepoPlugin.setOnResponse(runDuringFinalize.finalizeThreadRunnable());
+
+        createRandomIndex(idxName, dataNode);
+        createRepository(repoName, TestRestartBeforeListenersRepo.TYPE);
+        createSnapshotPolicy(policyName, "snap", NEVER_EXECUTE_CRON_SCHEDULE, repoName, idxName);
+
+        ensureGreen();
+
+        networkDisruption.startDisrupting();
+        String snapshotName = executePolicy(masterNode, policyName);
+
+        // restart snapshot after snapshot finalize, but before SLM callback called
+        runDuringFinalize.awaitAndRun(() -> {
+            try {
+                internalCluster().restartNode(masterNode);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        assertBusy(() -> {
+            assertSnapshotPartial(repoName, snapshotName);
+            assertMetadata(policyName, 0, 0, 0);
+        }, 1, TimeUnit.MINUTES);
+    }
+
+    /**
+     * Confirm normal behavior during failure that successfully sets stats in cluster state.
+     */
+    public void testFailedSnapshotSubmitStats() throws Exception {
+        final String idxName = "test-idx";
+        final String repoName = "test-repo";
+        final String policyName = "test-policy";
+
+        internalCluster().startMasterOnlyNodes(1);
+        final String masterNode = internalCluster().getMasterName();
+        final String dataNode = internalCluster().startDataOnlyNode();
+        ensureStableCluster(2);
+
+        // Add network disruption so snapshot fails with PARTIAL status
+        NetworkDisruption networkDisruption = isolateMasterDisruption(NetworkDisruption.DISCONNECT);
+        internalCluster().setDisruptionScheme(networkDisruption);
+
+        // wire into repo so code can be run on test thread after snapshot finalize, but before SLM is called back
+        var runDuringFinalize = new RunDuringFinalize();
+        TestRestartBeforeListenersRepoPlugin.setOnResponse(runDuringFinalize.finalizeThreadRunnable());
+
+        createRandomIndex(idxName, dataNode);
+        createRepository(repoName, TestRestartBeforeListenersRepo.TYPE);
+        createSnapshotPolicy(policyName, "snap", NEVER_EXECUTE_CRON_SCHEDULE, repoName, idxName);
+
+        ensureGreen();
+
+        networkDisruption.startDisrupting();
+        String snapshotName = executePolicy(masterNode, policyName);
+
+        // wait for snapshot to complete and network disruption to stop
+        runDuringFinalize.awaitAndRun(networkDisruption::stopDisrupting);
+
+        assertBusy(() -> {
+            assertSnapshotPartial(repoName, snapshotName);
+            assertMetadata(policyName, 0, 1, 1);
+        }, 1, TimeUnit.MINUTES);
+    }
+
+    private void assertMetadata(String policyName, long taken, long failure, long invocationsSinceLastSuccess) {
+        var snapshotLifecycleMetadata = getSnapshotLifecycleMetadata();
+        var snapshotLifecyclePolicyMetadata = snapshotLifecycleMetadata.getSnapshotConfigurations().get(policyName);
+        assertStats(snapshotLifecycleMetadata, policyName, taken, failure);
+        if (taken > 0) {
+            assertNotNull(snapshotLifecyclePolicyMetadata.getLastSuccess());
+        } else {
+            assertNull(snapshotLifecyclePolicyMetadata.getLastSuccess());
+        }
+        if (failure > 0) {
+            assertNotNull(snapshotLifecyclePolicyMetadata.getLastFailure());
+        } else {
+            assertNull(snapshotLifecyclePolicyMetadata.getLastFailure());
+        }
+        assertEquals(invocationsSinceLastSuccess, snapshotLifecyclePolicyMetadata.getInvocationsSinceLastSuccess());
+    }
+
+    private SnapshotLifecycleMetadata getSnapshotLifecycleMetadata() {
+        final ClusterStateResponse clusterStateResponse = client().admin().cluster().state(new ClusterStateRequest()).actionGet();
+        ClusterState state = clusterStateResponse.getState();
+        return state.metadata().custom(SnapshotLifecycleMetadata.TYPE);
+    }
+
+    private RegisteredPolicySnapshots getRegisteredSnapshots() {
+        final ClusterStateResponse clusterStateResponse = client().admin().cluster().state(new ClusterStateRequest()).actionGet();
+        ClusterState state = clusterStateResponse.getState();
+        return state.metadata().custom(RegisteredPolicySnapshots.TYPE, RegisteredPolicySnapshots.EMPTY);
+    }
+
+    private SnapshotInfo getSnapshotInfo(String repository, String snapshot) {
+        GetSnapshotsResponse snapshotsStatusResponse = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repository)
+            .setSnapshots(snapshot)
+            .get();
+        return snapshotsStatusResponse.getSnapshots().get(0);
+    }
+
+    private SnapshotsStatusResponse getSnapshotStatus(String repo, String snapshotName) {
+        return clusterAdmin().prepareSnapshotStatus(TEST_REQUEST_TIMEOUT, repo).setSnapshots(snapshotName).get();
+    }
+
+    private void assertSnapshotSuccess(String repository, String snapshot) {
+        try {
+            SnapshotInfo snapshotInfo = getSnapshotInfo(repository, snapshot);
+            assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
+            assertEquals(1, snapshotInfo.successfulShards());
+            assertEquals(0, snapshotInfo.failedShards());
+            logger.info("Checked snapshot exists and is state SUCCESS");
+        } catch (SnapshotMissingException e) {
+            fail("expected a snapshot with name " + snapshot + " but it does yet not exist");
+        }
+    }
+
+    private void assertSnapshotPartial(String repository, String snapshot) {
+        try {
+            SnapshotInfo snapshotInfo = getSnapshotInfo(repository, snapshot);
+            assertEquals(SnapshotState.PARTIAL, snapshotInfo.state());
+            assertEquals(0, snapshotInfo.successfulShards());
+            assertEquals(1, snapshotInfo.failedShards());
+            logger.info("Checked snapshot exists and is state PARTIAL");
+        } catch (SnapshotMissingException e) {
+            fail("expected a snapshot with name " + snapshot + " but it does yet not exist");
+        }
+    }
+
+    private void assertStats(SnapshotLifecycleMetadata snapshotLifecycleMetadata, String policyName, long taken, long failed) {
+        var stats = snapshotLifecycleMetadata.getStats().getMetrics().get(policyName);
+        logger.info("stats: " + stats);
+        if (taken == 0 && failed == 0) {
+            assertTrue(stats == null || (stats.getSnapshotTakenCount() == 0 && stats.getSnapshotFailedCount() == 0));
+        } else {
+            assertNotNull(stats);
+            assertEquals(taken, stats.getSnapshotTakenCount());
+            assertEquals(failed, stats.getSnapshotFailedCount());
+        }
+    }
+
+    private void assertRegistered(String policyName, List<String> expected) {
+        var registered = getRegisteredSnapshots();
+        var policySnaps = registered.getSnapshotsByPolicy(policyName).stream().map(SnapshotId::getName).toList();
+        assertEquals(expected, policySnaps);
+    }
+
+    private void createRandomIndex(String idxName, String dataNodeName) throws InterruptedException {
+        Settings settings = indexSettings(1, 0).put("index.routing.allocation.require._name", dataNodeName).build();
+        createIndex(idxName, settings);
+
+        logger.info("--> indexing some data");
+        final int numdocs = randomIntBetween(10, 100);
+        IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs];
+        for (int i = 0; i < builders.length; i++) {
+            builders[i] = prepareIndex(idxName).setId(Integer.toString(i)).setSource("field1", "bar " + i);
+        }
+        indexRandom(true, builders);
+        indicesAdmin().refresh(new RefreshRequest(idxName)).actionGet();
+    }
+
+    private void createSnapshotPolicy(String policyName, String snapshotNamePattern, String schedule, String repoId, String indexPattern) {
+        Map<String, Object> snapConfig = new HashMap<>();
+        snapConfig.put("indices", Collections.singletonList(indexPattern));
+        snapConfig.put("ignore_unavailable", false);
+        snapConfig.put("partial", true);
+
+        SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(
+            policyName,
+            snapshotNamePattern,
+            schedule,
+            repoId,
+            snapConfig,
+            SnapshotRetentionConfiguration.EMPTY
+        );
+
+        PutSnapshotLifecycleAction.Request putLifecycle = new PutSnapshotLifecycleAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            policyName,
+            policy
+        );
+        try {
+            client().execute(PutSnapshotLifecycleAction.INSTANCE, putLifecycle).get();
+        } catch (Exception e) {
+            logger.error("failed to create slm policy", e);
+            fail("failed to create policy " + policy + " got: " + e);
+        }
+    }
+
+    /**
+     * Execute the given policy and return the generated snapshot name
+     */
+    private String executePolicy(String node, String policyId) throws ExecutionException, InterruptedException {
+        ExecuteSnapshotLifecycleAction.Request executeReq = new ExecuteSnapshotLifecycleAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            policyId
+        );
+        ExecuteSnapshotLifecycleAction.Response resp = client(node).execute(ExecuteSnapshotLifecycleAction.INSTANCE, executeReq).get();
+        return resp.getSnapshotName();
+    }
+
+    private void waitForSnapshot(String repo, String snapshotName) throws Exception {
+        assertBusy(() -> {
+            try {
+                SnapshotsStatusResponse s = getSnapshotStatus(repo, snapshotName);
+                assertThat("expected a snapshot but none were returned", s.getSnapshots().size(), equalTo(1));
+                SnapshotStatus status = s.getSnapshots().get(0);
+                logger.info("--> waiting for snapshot {} to be completed, got: {}", snapshotName, status.getState());
+                assertThat(status.getState(), equalTo(SnapshotsInProgress.State.SUCCESS));
+            } catch (SnapshotMissingException e) {
+                logger.error("expected a snapshot but it was missing", e);
+                fail("expected a snapshot with name " + snapshotName + " but it does not exist");
+            }
+        });
+    }
+
+    /**
+     * The purpose of this class is to allow a cluster restart to occur after a snapshot has been finalized, but before the SLM callback
+     * which processes snapshotInfo has been called. TestRestartBeforeListenersRepo allows a runnable to be called at this time. But,
+     * internalCluster().restartNode() cannot be called on the thread running finalize. It is likely missing necessary context, or simply
+     * cannot be called by a thread on the master node. This helper class uses latches to run a callback on the test thread while the
+     * master node is waiting between finalizing the snapshot and before calling the SLM callback.
+     */
+    static class RunDuringFinalize {
+        private final CountDownLatch latch1 = new CountDownLatch(1);
+        private final CountDownLatch latch2 = new CountDownLatch(1);
+
+        Runnable finalizeThreadRunnable() {
+            return () -> {
+                latch1.countDown();
+                try {
+                    // While this is waiting the runnable called in awaitAndRan will be called by another thread.
+                    // The runnable is usually a cluster restart. During a restart, 10s is waited before this waiting thread is preempted.
+                    // To keep the tests fast, this wait is kept to 1s. This is actually a race condition which could cause a test to fail,
+                    // as the SLM callback could be called. But, after 1s, the restart should have started so this is unlikely.
+                    latch2.await(1, TimeUnit.SECONDS);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            };
+        }
+
+        void awaitAndRun(Runnable runnable) throws InterruptedException {
+            assertTrue(latch1.await(1, TimeUnit.MINUTES));
+            // this is where the cluster restart occurs
+            runnable.run();
+            latch2.countDown();
+        }
+    }
+}
diff --git a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatsImmutableIT.java b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatsImmutableIT.java
new file mode 100644
index 0000000000000..352647e86899c
--- /dev/null
+++ b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatsImmutableIT.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.slm;
+
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.datastreams.DataStreamsPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
+import org.elasticsearch.xpack.core.slm.SnapshotRetentionConfiguration;
+import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction;
+import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleStatsAction;
+import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction;
+import org.elasticsearch.xpack.ilm.IndexLifecycle;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 2)
+public class SLMStatsImmutableIT extends AbstractSnapshotIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(LocalStateCompositeXPackPlugin.class, IndexLifecycle.class, SnapshotLifecycle.class, DataStreamsPlugin.class);
+    }
+
+    private static final String NEVER_EXECUTE_CRON_SCHEDULE = "* * * 31 FEB ? *";
+
+    public void testSnapshotLifeCycleMetadataEmptyNotChanged() throws Exception {
+        final String policyId = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        final String idxName = "test-idx";
+        final String repoName = "test-repo";
+        final String snapshot = "snap";
+
+        createRepository(repoName, "fs");
+
+        createSnapshotPolicy(policyId, snapshot, NEVER_EXECUTE_CRON_SCHEDULE, repoName, idxName);
+        assertEquals(Map.of(), SnapshotLifecycleMetadata.EMPTY.getStats().getMetrics());
+
+        executePolicy(policyId);
+
+        assertBusy(() -> {
+            GetSnapshotLifecycleStatsAction.Response policyStats = getPolicyStats();
+            assertNotNull(policyStats.getSlmStats());
+            assertTrue(policyStats.getSlmStats().getMetrics().containsKey(policyId));
+        });
+
+        assertEquals(Map.of(), SnapshotLifecycleMetadata.EMPTY.getStats().getMetrics());
+        disableRepoConsistencyCheck("nothing stored in repo");
+    }
+
+    private GetSnapshotLifecycleStatsAction.Response getPolicyStats() {
+        try {
+            final var req = new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
+            return client().execute(GetSnapshotLifecycleStatsAction.INSTANCE, req).get();
+        } catch (Exception e) {
+            fail("failed to get stats");
+        }
+        return null;
+    }
+
+    private void createSnapshotPolicy(String policyName, String snapshotNamePattern, String schedule, String repoId, String indexPattern) {
+        logger.info("creating snapshot lifecycle policy: " + policyName);
+        Map<String, Object> snapConfig = new HashMap<>();
+        snapConfig.put("indices", Collections.singletonList(indexPattern));
+        snapConfig.put("ignore_unavailable", false);
+        snapConfig.put("partial", true);
+
+        SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(
+            policyName,
+            snapshotNamePattern,
+            schedule,
+            repoId,
+            snapConfig,
+            SnapshotRetentionConfiguration.EMPTY
+        );
+
+        PutSnapshotLifecycleAction.Request putLifecycle = new PutSnapshotLifecycleAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            policyName,
+            policy
+        );
+        try {
+            client().execute(PutSnapshotLifecycleAction.INSTANCE, putLifecycle).get();
+        } catch (Exception e) {
+            logger.error("failed to create slm policy", e);
+            fail("failed to create policy " + policy + " got: " + e);
+        }
+    }
+
+    private String executePolicy(String policyId) throws ExecutionException, InterruptedException {
+        ExecuteSnapshotLifecycleAction.Request executeReq = new ExecuteSnapshotLifecycleAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            policyId
+        );
+        ExecuteSnapshotLifecycleAction.Response resp = client().execute(ExecuteSnapshotLifecycleAction.INSTANCE, executeReq).get();
+        return resp.getSnapshotName();
+    }
+}
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SLMInfoTransportAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SLMInfoTransportAction.java
index afe13a97e388b..e9ca1e66021a4 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SLMInfoTransportAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SLMInfoTransportAction.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.slm;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SLMUsageTransportAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SLMUsageTransportAction.java
index a54dfdd8f1714..76bcb95b7f740 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SLMUsageTransportAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SLMUsageTransportAction.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java
index cb0344dd70ad5..192807d667abb 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycle.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.health.HealthIndicatorService;
 import org.elasticsearch.license.XPackLicenseState;
@@ -32,6 +33,7 @@
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.snapshots.RegisteredPolicySnapshots;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
@@ -91,6 +93,7 @@ public class SnapshotLifecycle extends Plugin implements ActionPlugin, HealthPlu
     private final SetOnce<SnapshotRetentionService> snapshotRetentionService = new SetOnce<>();
     private final SetOnce<SnapshotHistoryStore> snapshotHistoryStore = new SetOnce<>();
     private final SetOnce<SlmHealthIndicatorService> slmHealthIndicatorService = new SetOnce<>();
+    private final SetOnce<FeatureService> featureService = new SetOnce<>();
     private final Settings settings;
 
     public SnapshotLifecycle(Settings settings) {
@@ -123,7 +126,7 @@ public Collection<?> createComponents(PluginServices services) {
         ClusterService clusterService = services.clusterService();
         ThreadPool threadPool = services.threadPool();
         final List<Object> components = new ArrayList<>();
-
+        featureService.set(services.featureService());
         SnapshotLifecycleTemplateRegistry templateRegistry = new SnapshotLifecycleTemplateRegistry(
             settings,
             clusterService,
@@ -169,6 +172,11 @@ private static List<NamedXContentRegistry.Entry> xContentEntries() {
                 Metadata.Custom.class,
                 new ParseField(SnapshotLifecycleMetadata.TYPE),
                 parser -> SnapshotLifecycleMetadata.PARSER.parse(parser, null)
+            ),
+            new NamedXContentRegistry.Entry(
+                Metadata.Custom.class,
+                new ParseField(RegisteredPolicySnapshots.TYPE),
+                RegisteredPolicySnapshots::parse
             )
         );
     }
@@ -230,7 +238,7 @@ public List<RestHandler> getRestHandlers(
     }
 
     List<ReservedClusterStateHandler<?>> reservedClusterStateHandlers() {
-        return List.of(new ReservedSnapshotAction());
+        return List.of(new ReservedSnapshotAction(featureService.get()));
     }
 
     @Override
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleFeatures.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleFeatures.java
index f3dfe4fb26f65..96b962f70a1b6 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleFeatures.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleFeatures.java
@@ -13,8 +13,14 @@
 import org.elasticsearch.xpack.slm.history.SnapshotLifecycleTemplateRegistry;
 
 import java.util.Map;
+import java.util.Set;
 
 public class SnapshotLifecycleFeatures implements FeatureSpecification {
+    @Override
+    public Set<NodeFeature> getFeatures() {
+        return Set.of(SnapshotLifecycleService.INTERVAL_SCHEDULE);
+    }
+
     @Override
     public Map<NodeFeature, Version> getHistoricalFeatures() {
         return Map.of(SnapshotLifecycleTemplateRegistry.MANAGED_BY_DATA_STREAM_LIFECYCLE, Version.V_8_12_0);
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java
index 6d77926149334..b93f90de73f05 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java
@@ -20,10 +20,11 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
 import org.elasticsearch.xpack.core.ilm.OperationModeUpdateTask;
-import org.elasticsearch.xpack.core.scheduler.CronSchedule;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata;
@@ -45,7 +46,7 @@
  * task according to the policy's schedule.
  */
 public class SnapshotLifecycleService implements Closeable, ClusterStateListener {
-
+    public static final NodeFeature INTERVAL_SCHEDULE = new NodeFeature("slm.interval_schedule");
     private static final Logger logger = LogManager.getLogger(SnapshotLifecycleService.class);
     private static final String JOB_PATTERN_SUFFIX = "-\\d+$";
 
@@ -193,15 +194,13 @@ public void maybeScheduleSnapshot(final SnapshotLifecyclePolicyMetadata snapshot
         // is identical to an existing job (meaning the version has not changed) then this does
         // not reschedule it.
         scheduledTasks.computeIfAbsent(jobId, id -> {
-            final SchedulerEngine.Job job = new SchedulerEngine.Job(
-                jobId,
-                new CronSchedule(snapshotLifecyclePolicy.getPolicy().getSchedule())
-            );
             if (existingJobsFoundAndCancelled) {
                 logger.info("rescheduling updated snapshot lifecycle job [{}]", jobId);
             } else {
                 logger.info("scheduling snapshot lifecycle job [{}]", jobId);
             }
+
+            final SchedulerEngine.Job job = snapshotLifecyclePolicy.buildSchedulerJob(jobId);
             scheduler.add(job);
             return job;
         });
@@ -249,7 +248,7 @@ public static void validateRepositoryExists(final String repository, final Clust
      */
     public static void validateMinimumInterval(final SnapshotLifecyclePolicy lifecycle, final ClusterState state) {
         TimeValue minimum = LifecycleSettings.SLM_MINIMUM_INTERVAL_SETTING.get(state.metadata().settings());
-        TimeValue next = lifecycle.calculateNextInterval();
+        TimeValue next = lifecycle.calculateNextInterval(Clock.systemUTC());
         if (next.duration() > 0 && minimum.duration() > 0 && next.millis() < minimum.millis()) {
             throw new IllegalArgumentException(
                 "invalid schedule ["
@@ -262,6 +261,18 @@ public static void validateMinimumInterval(final SnapshotLifecyclePolicy lifecyc
         }
     }
 
+    /**
+     * Validate that interval schedule feature is not supported by all nodes
+     * @throws IllegalArgumentException if is interval expression but interval schedule not supported
+     */
+    public static void validateIntervalScheduleSupport(String schedule, FeatureService featureService, ClusterState state) {
+        if (SnapshotLifecyclePolicy.isIntervalSchedule(schedule) && featureService.clusterHasFeature(state, INTERVAL_SCHEDULE) == false) {
+            throw new IllegalArgumentException(
+                "Unable to use slm interval schedules in mixed-clusters with nodes that do not support feature " + INTERVAL_SCHEDULE.id()
+            );
+        }
+    }
+
     @Override
     public void close() {
         if (this.running.compareAndSet(true, false)) {
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java
index 028633a480314..d49f32869f28a 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java
@@ -16,13 +16,17 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.scheduler.SchedulerEngine;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.snapshots.RegisteredPolicySnapshots;
+import org.elasticsearch.snapshots.RegisteredPolicySnapshots.PolicySnapshot;
 import org.elasticsearch.snapshots.SnapshotException;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xpack.core.ClientHelper;
@@ -36,9 +40,14 @@
 
 import java.io.IOException;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
@@ -59,21 +68,21 @@ public SnapshotLifecycleTask(final Client client, final ClusterService clusterSe
 
     @Override
     public void triggered(SchedulerEngine.Event event) {
-        logger.debug("snapshot lifecycle policy task triggered from job [{}]", event.getJobName());
+        logger.debug("snapshot lifecycle policy task triggered from job [{}]", event.jobName());
 
-        final Optional<String> snapshotName = maybeTakeSnapshot(event.getJobName(), client, clusterService, historyStore);
+        final Optional<String> snapshotName = maybeTakeSnapshot(event.jobName(), client, clusterService, historyStore);
 
         // Would be cleaner if we could use Optional#ifPresentOrElse
         snapshotName.ifPresent(
             name -> logger.info(
                 "snapshot lifecycle policy job [{}] issued new snapshot creation for [{}] successfully",
-                event.getJobName(),
+                event.jobName(),
                 name
             )
         );
 
         if (snapshotName.isPresent() == false) {
-            logger.warn("snapshot lifecycle policy for job [{}] no longer exists, snapshot not created", event.getJobName());
+            logger.warn("snapshot lifecycle policy for job [{}] no longer exists, snapshot not created", event.jobName());
         }
     }
 
@@ -93,6 +102,8 @@ public static Optional<String> maybeTakeSnapshot(
         String snapshotName = maybeMetadata.map(policyMetadata -> {
             // don't time out on this request to not produce failed SLM runs in case of a temporarily slow master node
             CreateSnapshotRequest request = policyMetadata.getPolicy().toRequest(TimeValue.MAX_VALUE);
+            final SnapshotId snapshotId = new SnapshotId(request.snapshot(), request.uuid());
+
             final LifecyclePolicySecurityClient clientWithHeaders = new LifecyclePolicySecurityClient(
                 client,
                 ClientHelper.INDEX_LIFECYCLE_ORIGIN,
@@ -120,7 +131,7 @@ public void onResponse(CreateSnapshotResponse createSnapshotResponse) {
                         submitUnbatchedTask(
                             clusterService,
                             "slm-record-success-" + policyMetadata.getPolicy().getId(),
-                            WriteJobStatus.success(policyMetadata.getPolicy().getId(), request.snapshot(), snapshotStartTime, timestamp)
+                            WriteJobStatus.success(policyMetadata.getPolicy().getId(), snapshotId, snapshotStartTime, timestamp)
                         );
                         historyStore.putAsync(
                             SnapshotHistoryItem.creationSuccessRecord(timestamp, policyMetadata.getPolicy(), request.snapshot())
@@ -145,7 +156,7 @@ public void onFailure(Exception e) {
                     submitUnbatchedTask(
                         clusterService,
                         "slm-record-failure-" + policyMetadata.getPolicy().getId(),
-                        WriteJobStatus.failure(policyMetadata.getPolicy().getId(), request.snapshot(), timestamp, e)
+                        WriteJobStatus.failure(policyMetadata.getPolicy().getId(), snapshotId, timestamp, e)
                     );
                     final SnapshotHistoryItem failureRecord;
                     try {
@@ -157,7 +168,8 @@ public void onFailure(Exception e) {
                         );
                         historyStore.putAsync(failureRecord);
                     } catch (IOException ex) {
-                        // This shouldn't happen unless there's an issue with serializing the original exception, which shouldn't happen
+                        // This shouldn't happen unless there's an issue with serializing the original exception, which
+                        // shouldn't happen
                         logger.error(
                             () -> format(
                                 "failed to record snapshot creation failure for snapshot lifecycle policy [%s]",
@@ -204,53 +216,65 @@ public static String exceptionToString(Exception ex) {
         }, ToXContent.EMPTY_PARAMS);
     }
 
+    static Set<SnapshotId> currentlyRunningSnapshots(ClusterState clusterState) {
+        final SnapshotsInProgress snapshots = clusterState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
+        final Set<SnapshotId> currentlyRunning = new HashSet<>();
+        for (final List<SnapshotsInProgress.Entry> entriesForRepo : snapshots.entriesByRepo()) {
+            for (SnapshotsInProgress.Entry entry : entriesForRepo) {
+                currentlyRunning.add(entry.snapshot().getSnapshotId());
+            }
+        }
+        return currentlyRunning;
+    }
+
+    static SnapshotInvocationRecord buildFailedSnapshotRecord(SnapshotId snapshot) {
+        return new SnapshotInvocationRecord(
+            snapshot.getName(),
+            null,
+            Instant.now().toEpochMilli(),
+            String.format(Locale.ROOT, "found registered snapshot [%s] which is no longer running, assuming failed.", snapshot.getName())
+        );
+    }
+
     /**
      * A cluster state update task to write the result of a snapshot job to the cluster metadata for the associated policy.
      */
-    private static class WriteJobStatus extends ClusterStateUpdateTask {
+    static class WriteJobStatus extends ClusterStateUpdateTask {
 
         private final String policyName;
-        private final String snapshotName;
+        private final SnapshotId snapshotId;
         private final long snapshotStartTime;
         private final long snapshotFinishTime;
         private final Optional<Exception> exception;
 
         private WriteJobStatus(
             String policyName,
-            String snapshotName,
+            SnapshotId snapshotId,
             long snapshotStartTime,
             long snapshotFinishTime,
             Optional<Exception> exception
         ) {
             this.policyName = policyName;
-            this.snapshotName = snapshotName;
+            this.snapshotId = snapshotId;
             this.exception = exception;
             this.snapshotStartTime = snapshotStartTime;
             this.snapshotFinishTime = snapshotFinishTime;
         }
 
-        static WriteJobStatus success(String policyId, String snapshotName, long snapshotStartTime, long snapshotFinishTime) {
-            return new WriteJobStatus(policyId, snapshotName, snapshotStartTime, snapshotFinishTime, Optional.empty());
+        static WriteJobStatus success(String policyId, SnapshotId snapshotId, long snapshotStartTime, long snapshotFinishTime) {
+            return new WriteJobStatus(policyId, snapshotId, snapshotStartTime, snapshotFinishTime, Optional.empty());
         }
 
-        static WriteJobStatus failure(String policyId, String snapshotName, long timestamp, Exception exception) {
-            return new WriteJobStatus(policyId, snapshotName, timestamp, timestamp, Optional.of(exception));
+        static WriteJobStatus failure(String policyId, SnapshotId snapshotId, long timestamp, Exception exception) {
+            return new WriteJobStatus(policyId, snapshotId, timestamp, timestamp, Optional.of(exception));
         }
 
         @Override
         public ClusterState execute(ClusterState currentState) throws Exception {
-            SnapshotLifecycleMetadata snapMeta = currentState.metadata().custom(SnapshotLifecycleMetadata.TYPE);
-
-            assert snapMeta != null : "this should never be called while the snapshot lifecycle cluster metadata is null";
-            if (snapMeta == null) {
-                logger.error(
-                    "failed to record snapshot [{}] for snapshot [{}] in policy [{}]: snapshot lifecycle metadata is null",
-                    exception.isPresent() ? "failure" : "success",
-                    snapshotName,
-                    policyName
-                );
-                return currentState;
-            }
+            SnapshotLifecycleMetadata snapMeta = currentState.metadata()
+                .custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
+            RegisteredPolicySnapshots registeredSnapshots = currentState.metadata()
+                .custom(RegisteredPolicySnapshots.TYPE, RegisteredPolicySnapshots.EMPTY);
 
             Map<String, SnapshotLifecyclePolicyMetadata> snapLifecycles = new HashMap<>(snapMeta.getSnapshotConfigurations());
             SnapshotLifecyclePolicyMetadata policyMetadata = snapLifecycles.get(policyName);
@@ -258,29 +282,62 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                 logger.warn(
                     "failed to record snapshot [{}] for snapshot [{}] in policy [{}]: policy not found",
                     exception.isPresent() ? "failure" : "success",
-                    snapshotName,
+                    snapshotId.getName(),
                     policyName
                 );
                 return currentState;
             }
 
-            SnapshotLifecyclePolicyMetadata.Builder newPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder(policyMetadata);
-            final SnapshotLifecycleStats stats = snapMeta.getStats();
+            final SnapshotLifecyclePolicyMetadata.Builder newPolicyMetadata = SnapshotLifecyclePolicyMetadata.builder(policyMetadata);
+            SnapshotLifecycleStats newStats = snapMeta.getStats();
+
+            if (registeredSnapshots.contains(snapshotId) == false) {
+                logger.warn(
+                    "Snapshot [{}] not found in registered set after snapshot completion. This means snapshot was"
+                        + " recorded as a failure by another snapshot's cleanup run.",
+                    snapshotId.getName()
+                );
+            }
 
+            final Set<SnapshotId> runningSnapshots = currentlyRunningSnapshots(currentState);
+            final List<PolicySnapshot> newRegistered = new ArrayList<>();
+            for (PolicySnapshot snapshot : registeredSnapshots.getSnapshots()) {
+                if (snapshot.getSnapshotId().equals(snapshotId) == false) {
+                    if (snapshot.getPolicy().equals(policyName)) {
+                        if (runningSnapshots.contains(snapshot.getSnapshotId())) {
+                            // Snapshot is for this policy and is still running so keep it in registered set
+                            newRegistered.add(snapshot);
+                        } else {
+                            // Snapshot is for this policy but is not running so infer failure, update stats accordingly,
+                            // and remove from registered set
+                            newStats = newStats.withFailedIncremented(policyName);
+                            newPolicyMetadata.incrementInvocationsSinceLastSuccess()
+                                .setLastFailure(buildFailedSnapshotRecord(snapshot.getSnapshotId()));
+                        }
+                    } else if (snapLifecycles.containsKey(snapshot.getPolicy())) {
+                        // Snapshot is for another policy so keep in the registered set and that policy deal with it
+                        newRegistered.add(snapshot);
+                    }
+                }
+            }
+
+            // Add stats from the just completed snapshot execution
             if (exception.isPresent()) {
-                stats.snapshotFailed(policyName);
+                newStats = newStats.withFailedIncremented(policyName);
                 newPolicyMetadata.setLastFailure(
                     new SnapshotInvocationRecord(
-                        snapshotName,
+                        snapshotId.getName(),
                         null,
                         snapshotFinishTime,
                         exception.map(SnapshotLifecycleTask::exceptionToString).orElse(null)
                     )
                 );
-                newPolicyMetadata.setInvocationsSinceLastSuccess(policyMetadata.getInvocationsSinceLastSuccess() + 1L);
+                newPolicyMetadata.incrementInvocationsSinceLastSuccess();
             } else {
-                stats.snapshotTaken(policyName);
-                newPolicyMetadata.setLastSuccess(new SnapshotInvocationRecord(snapshotName, snapshotStartTime, snapshotFinishTime, null));
+                newStats = newStats.withTakenIncremented(policyName);
+                newPolicyMetadata.setLastSuccess(
+                    new SnapshotInvocationRecord(snapshotId.getName(), snapshotStartTime, snapshotFinishTime, null)
+                );
                 newPolicyMetadata.setInvocationsSinceLastSuccess(0L);
             }
 
@@ -288,12 +345,13 @@ public ClusterState execute(ClusterState currentState) throws Exception {
             SnapshotLifecycleMetadata lifecycleMetadata = new SnapshotLifecycleMetadata(
                 snapLifecycles,
                 currentSLMMode(currentState),
-                stats
+                newStats
             );
-            Metadata currentMeta = currentState.metadata();
-            return ClusterState.builder(currentState)
-                .metadata(Metadata.builder(currentMeta).putCustom(SnapshotLifecycleMetadata.TYPE, lifecycleMetadata))
+            Metadata newMeta = Metadata.builder(currentState.metadata())
+                .putCustom(SnapshotLifecycleMetadata.TYPE, lifecycleMetadata)
+                .putCustom(RegisteredPolicySnapshots.TYPE, new RegisteredPolicySnapshots(newRegistered))
                 .build();
+            return ClusterState.builder(currentState).metadata(newMeta).build();
         }
 
         @Override
@@ -301,7 +359,7 @@ public void onFailure(Exception e) {
             logger.error(
                 "failed to record snapshot policy execution status [{}] for snapshot [{}] in policy [{}]: {}",
                 exception.isPresent() ? "failure" : "success",
-                snapshotName,
+                snapshotId.getName(),
                 policyName,
                 e
             );
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java
index fea84e1a032dd..678e6941599c9 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java
@@ -41,6 +41,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.LongSupplier;
 import java.util.stream.Collectors;
@@ -87,33 +88,33 @@ public SnapshotRetentionTask(
 
     @Override
     public void triggered(SchedulerEngine.Event event) {
-        assert event.getJobName().equals(SnapshotRetentionService.SLM_RETENTION_JOB_ID)
-            || event.getJobName().equals(SnapshotRetentionService.SLM_RETENTION_MANUAL_JOB_ID)
+        assert event.jobName().equals(SnapshotRetentionService.SLM_RETENTION_JOB_ID)
+            || event.jobName().equals(SnapshotRetentionService.SLM_RETENTION_MANUAL_JOB_ID)
             : "expected id to be "
                 + SnapshotRetentionService.SLM_RETENTION_JOB_ID
                 + " or "
                 + SnapshotRetentionService.SLM_RETENTION_MANUAL_JOB_ID
                 + " but it was "
-                + event.getJobName();
+                + event.jobName();
 
         final ClusterState state = clusterService.state();
 
         // Skip running retention if SLM is disabled, however, even if it's
         // disabled we allow manual running.
         if (SnapshotLifecycleService.slmStoppedOrStopping(state)
-            && event.getJobName().equals(SnapshotRetentionService.SLM_RETENTION_MANUAL_JOB_ID) == false) {
+            && event.jobName().equals(SnapshotRetentionService.SLM_RETENTION_MANUAL_JOB_ID) == false) {
             logger.debug("skipping SLM retention as SLM is currently stopped or stopping");
             return;
         }
 
-        final SnapshotLifecycleStats slmStats = new SnapshotLifecycleStats();
+        AtomicReference<SnapshotLifecycleStats> slmStats = new AtomicReference<>(new SnapshotLifecycleStats());
 
         // Defined here so it can be re-used without having to repeat it
         final Consumer<Exception> failureHandler = e -> {
             try {
                 logger.error("error during snapshot retention task", e);
-                slmStats.retentionFailed();
-                updateStateWithStats(slmStats);
+                slmStats.getAndUpdate(SnapshotLifecycleStats::withRetentionFailedIncremented);
+                updateStateWithStats(slmStats.get());
             } finally {
                 logger.info("SLM retention snapshot cleanup task completed with error");
             }
@@ -121,7 +122,9 @@ public void triggered(SchedulerEngine.Event event) {
 
         try {
             logger.info("starting SLM retention snapshot cleanup task");
-            slmStats.retentionRun();
+
+            slmStats.getAndUpdate(SnapshotLifecycleStats::withRetentionRunIncremented);
+
             // Find all SLM policies that have retention enabled
             final Map<String, SnapshotLifecyclePolicy> policiesWithRetention = getAllPoliciesWithRetentionEnabled(state);
             logger.trace("policies with retention enabled: {}", policiesWithRetention.keySet());
@@ -150,7 +153,7 @@ public void onResponse(Map<String, List<Tuple<SnapshotId, String>>> snapshotsToB
 
                     // Finally, delete the snapshots that need to be deleted
                     deleteSnapshots(snapshotsToBeDeleted, slmStats, ActionListener.running(() -> {
-                        updateStateWithStats(slmStats);
+                        updateStateWithStats(slmStats.get());
                         logger.info("SLM retention snapshot cleanup task complete");
                     }));
                 }
@@ -192,7 +195,7 @@ void getSnapshotsEligibleForDeletion(
 
     void deleteSnapshots(
         Map<String, List<Tuple<SnapshotId, String>>> snapshotsToDelete,
-        SnapshotLifecycleStats slmStats,
+        AtomicReference<SnapshotLifecycleStats> slmStats,
         ActionListener<Void> listener
     ) {
         int count = snapshotsToDelete.values().stream().mapToInt(List::size).sum();
@@ -211,7 +214,7 @@ void deleteSnapshots(
             ActionListener.runAfter(listener, () -> {
                 TimeValue totalElapsedTime = TimeValue.timeValueNanos(nowNanoSupplier.getAsLong() - startTime);
                 logger.debug("total elapsed time for deletion of [{}] snapshots: {}", deleted, totalElapsedTime);
-                slmStats.deletionTime(totalElapsedTime);
+                slmStats.getAndUpdate(s -> s.withDeletionTimeUpdated(totalElapsedTime));
             })
         );
         for (Map.Entry<String, List<Tuple<SnapshotId, String>>> entry : snapshotsToDelete.entrySet()) {
@@ -224,7 +227,7 @@ void deleteSnapshots(
     }
 
     private void deleteSnapshots(
-        SnapshotLifecycleStats slmStats,
+        AtomicReference<SnapshotLifecycleStats> slmStats,
         AtomicInteger deleted,
         AtomicInteger failed,
         String repo,
@@ -302,7 +305,7 @@ void deleteSnapshot(
         String slmPolicy,
         String repo,
         SnapshotId snapshot,
-        SnapshotLifecycleStats slmStats,
+        AtomicReference<SnapshotLifecycleStats> slmStats,
         ActionListener<AcknowledgedResponse> listener
     ) {
         logger.info("[{}] snapshot retention deleting snapshot [{}]", repo, snapshot);
@@ -311,12 +314,12 @@ void deleteSnapshot(
             .cluster()
             .prepareDeleteSnapshot(TimeValue.MAX_VALUE, repo, snapshot.getName())
             .execute(ActionListener.wrap(acknowledgedResponse -> {
-                slmStats.snapshotDeleted(slmPolicy);
+                slmStats.getAndUpdate(s -> s.withDeletedIncremented(slmPolicy));
                 listener.onResponse(acknowledgedResponse);
             }, e -> {
                 try {
                     logger.warn(() -> format("[%s] failed to delete snapshot [%s] for retention", repo, snapshot), e);
-                    slmStats.snapshotDeleteFailure(slmPolicy);
+                    slmStats.getAndUpdate(s -> s.withDeleteFailureIncremented(slmPolicy));
                 } finally {
                     listener.onFailure(e);
                 }
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/TransportSLMGetExpiredSnapshotsAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/TransportSLMGetExpiredSnapshotsAction.java
index cf3a114fc5803..a033d357ec034 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/TransportSLMGetExpiredSnapshotsAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/TransportSLMGetExpiredSnapshotsAction.java
@@ -21,10 +21,11 @@
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryData;
@@ -70,7 +71,7 @@ public TransportSLMGetExpiredSnapshotsAction(
         RepositoriesService repositoriesService,
         ActionFilters actionFilters
     ) {
-        super(INSTANCE.name(), actionFilters, transportService.getTaskManager());
+        super(INSTANCE.name(), actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
         this.repositoriesService = repositoriesService;
         this.retentionExecutor = transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT);
     }
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java
index f14edd89b826d..192b03aa385d5 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.xcontent.XContentParser;
@@ -41,7 +42,11 @@ public class ReservedSnapshotAction implements ReservedClusterStateHandler<List<
 
     public static final String NAME = "slm";
 
-    public ReservedSnapshotAction() {}
+    private final FeatureService featureService;
+
+    public ReservedSnapshotAction(FeatureService featureService) {
+        this.featureService = featureService;
+    }
 
     @Override
     public String name() {
@@ -63,6 +68,7 @@ private Collection<PutSnapshotLifecycleAction.Request> prepare(List<SnapshotLife
             );
             try {
                 validate(request);
+                SnapshotLifecycleService.validateIntervalScheduleSupport(request.getLifecycle().getSchedule(), featureService, state);
                 SnapshotLifecycleService.validateRepositoryExists(request.getLifecycle().getRepository(), state);
                 SnapshotLifecycleService.validateMinimumInterval(request.getLifecycle(), state);
                 result.add(request);
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java
index 062954e40d82c..7fc985b1b2bb8 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java
@@ -20,9 +20,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotLifecycleAction.java
index 6eecba174a8a3..1933430499474 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotLifecycleAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotLifecycleAction.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotRetentionAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotRetentionAction.java
index 745f8af4b98a0..bfff45f27617b 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotRetentionAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotRetentionAction.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java
index 747cc45749c4f..ea9e8da453de6 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java
index 8cecf9c0f76d1..00dc44318846b 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleStatsAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleStatsAction.java
index 2cd8ac3012568..689621f514c0d 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleStatsAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSnapshotLifecycleStatsAction.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java
index 9b3f261976125..f14afefe6f47a 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java
@@ -21,10 +21,11 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -49,6 +50,7 @@ public class TransportPutSnapshotLifecycleAction extends TransportMasterNodeActi
     AcknowledgedResponse> {
 
     private static final Logger logger = LogManager.getLogger(TransportPutSnapshotLifecycleAction.class);
+    private final FeatureService featureService;
 
     @Inject
     public TransportPutSnapshotLifecycleAction(
@@ -56,7 +58,8 @@ public TransportPutSnapshotLifecycleAction(
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        FeatureService featureService
     ) {
         super(
             PutSnapshotLifecycleAction.NAME,
@@ -69,6 +72,7 @@ public TransportPutSnapshotLifecycleAction(
             AcknowledgedResponse::readFrom,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
+        this.featureService = featureService;
     }
 
     @Override
@@ -78,8 +82,8 @@ protected void masterOperation(
         final ClusterState state,
         final ActionListener<AcknowledgedResponse> listener
     ) {
+        SnapshotLifecycleService.validateIntervalScheduleSupport(request.getLifecycle().getSchedule(), featureService, state);
         SnapshotLifecycleService.validateRepositoryExists(request.getLifecycle().getRepository(), state);
-
         SnapshotLifecycleService.validateMinimumInterval(request.getLifecycle(), state);
 
         // headers from the thread context stored by the AuthenticationService to be shared between the
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStartSLMAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStartSLMAction.java
index 48cd7ff0309bd..602a09bce6df5 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStartSLMAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStartSLMAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStopSLMAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStopSLMAction.java
index 5f0eaa6cb90cb..5c93de22fd7f3 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStopSLMAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportStopSLMAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java
index 9e5265d91dc75..b7674a2d60bff 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecyclePolicyTests.java
@@ -19,11 +19,17 @@
 import org.elasticsearch.xpack.core.slm.SnapshotRetentionConfiguration;
 
 import java.io.IOException;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.temporal.ChronoUnit;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
 import static org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadataTests.randomSnapshotLifecyclePolicy;
+import static org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadataTests.randomTimeValueString;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
@@ -34,10 +40,11 @@ public class SnapshotLifecyclePolicyTests extends AbstractXContentSerializingTes
     private String id;
 
     public void testToRequest() {
+        var schedule = randomBoolean() ? "0 1 2 3 4 ? 2099" : "30m";
         SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy(
             "id",
             "name",
-            "0 1 2 3 4 ? 2099",
+            schedule,
             "repo",
             Collections.emptyMap(),
             SnapshotRetentionConfiguration.EMPTY
@@ -47,13 +54,13 @@ public void testToRequest() {
             Collections.singletonMap("policy", "id")
         );
 
-        p = new SnapshotLifecyclePolicy("id", "name", "0 1 2 3 4 ? 2099", "repo", null, null);
+        p = new SnapshotLifecyclePolicy("id", "name", schedule, "repo", null, null);
         request = p.toRequest(TEST_REQUEST_TIMEOUT);
-        expected.waitForCompletion(true).snapshot(request.snapshot()).repository("repo");
+        expected.waitForCompletion(true).snapshot(request.snapshot()).repository("repo").uuid(request.uuid());
         assertEquals(expected, request);
     }
 
-    public void testNextExecutionTime() {
+    public void testNextExecutionTimeSchedule() {
         SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy(
             "id",
             "name",
@@ -62,10 +69,100 @@ public void testNextExecutionTime() {
             Collections.emptyMap(),
             SnapshotRetentionConfiguration.EMPTY
         );
-        assertThat(p.calculateNextExecution(), equalTo(4078864860000L));
+        assertThat(p.calculateNextExecution(-1, Clock.systemUTC()), equalTo(4078864860000L));
     }
 
-    public void testCalculateNextInterval() {
+    public void testNextExecutionTimeInterval() {
+        SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy(
+            "id",
+            "name",
+            "30m",
+            "repo",
+            Collections.emptyMap(),
+            SnapshotRetentionConfiguration.EMPTY
+        );
+
+        {
+            // current time is exactly modified time
+            Instant modifiedTime = Instant.parse("2024-07-17T00:00:00.000Z").truncatedTo(ChronoUnit.SECONDS);
+            Instant currentTime = modifiedTime;
+            Instant expected = Instant.parse("2024-07-17T00:30:00.000Z").truncatedTo(ChronoUnit.SECONDS);
+            assertThat(p.calculateNextExecution(modifiedTime.toEpochMilli(), fixedClock(currentTime)), equalTo(expected.toEpochMilli()));
+        }
+
+        {
+            // current time is half an interval past modified time
+            Instant modifiedTime = Instant.parse("2024-07-17T00:00:00.000Z").truncatedTo(ChronoUnit.SECONDS);
+            Instant currentTime = modifiedTime.plus(Duration.ofMinutes(15));
+            Instant expected = Instant.parse("2024-07-17T00:30:00.000Z").truncatedTo(ChronoUnit.SECONDS);
+            assertThat(p.calculateNextExecution(modifiedTime.toEpochMilli(), fixedClock(currentTime)), equalTo(expected.toEpochMilli()));
+        }
+
+        {
+            // current time is a full day (24 intervals) ahead of modified time
+            Instant modifiedTime = Instant.parse("2024-07-17T00:00:00.000Z").truncatedTo(ChronoUnit.SECONDS);
+            Instant currentTime = modifiedTime.plus(Duration.ofDays(1));
+            Instant expected = Instant.parse("2024-07-18T00:30:00.000Z").truncatedTo(ChronoUnit.SECONDS);
+            assertThat(p.calculateNextExecution(modifiedTime.toEpochMilli(), fixedClock(currentTime)), equalTo(expected.toEpochMilli()));
+        }
+
+        {
+            // current time before modified time
+            Instant modifiedTime = Instant.parse("2024-07-17T00:00:00.000Z").truncatedTo(ChronoUnit.SECONDS);
+            Instant currentTime = modifiedTime.minus(Duration.ofHours(1));
+            expectThrows(AssertionError.class, () -> p.calculateNextExecution(modifiedTime.toEpochMilli(), fixedClock(currentTime)));
+        }
+
+        {
+            // current time is every minute of a day
+            Instant modifiedTime = Instant.parse("2024-07-17T00:00:00.000Z").truncatedTo(ChronoUnit.SECONDS);
+            Instant currentTime = modifiedTime;
+            Instant expectedTime = modifiedTime.plus(Duration.ofMinutes(30));
+
+            for (; currentTime.isBefore(modifiedTime.plus(Duration.ofDays(1))); currentTime = currentTime.plus(Duration.ofMinutes(1))) {
+                if (currentTime.equals(expectedTime)) {
+                    expectedTime = expectedTime.plus(Duration.ofMinutes(30));
+                }
+                assertThat(
+                    p.calculateNextExecution(modifiedTime.toEpochMilli(), fixedClock(currentTime)),
+                    equalTo(expectedTime.toEpochMilli())
+                );
+            }
+        }
+    }
+
+    private static Clock fixedClock(Instant instant) {
+        return Clock.fixed(instant, ZoneOffset.UTC);
+    }
+
+    public void testCalculateNextIntervalInterval() {
+
+        {
+            SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy(
+                "id",
+                "name",
+                "30m",
+                "repo",
+                Collections.emptyMap(),
+                SnapshotRetentionConfiguration.EMPTY
+            );
+            assertThat(p.calculateNextInterval(Clock.systemUTC()), equalTo(TimeValue.timeValueMinutes(30)));
+        }
+        {
+            String schedule = randomTimeValueString();
+            SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy(
+                "id",
+                "name",
+                schedule,
+                "repo",
+                Collections.emptyMap(),
+                SnapshotRetentionConfiguration.EMPTY
+            );
+            assertThat(p.calculateNextInterval(Clock.systemUTC()), equalTo(TimeValue.parseTimeValue(schedule, "schedule")));
+        }
+    }
+
+    public void testCalculateNextIntervalSchedule() {
         {
             SnapshotLifecyclePolicy p = new SnapshotLifecyclePolicy(
                 "id",
@@ -75,7 +172,7 @@ public void testCalculateNextInterval() {
                 Collections.emptyMap(),
                 SnapshotRetentionConfiguration.EMPTY
             );
-            assertThat(p.calculateNextInterval(), equalTo(TimeValue.timeValueMinutes(5)));
+            assertThat(p.calculateNextInterval(Clock.systemUTC()), equalTo(TimeValue.timeValueMinutes(5)));
         }
 
         {
@@ -87,7 +184,7 @@ public void testCalculateNextInterval() {
                 Collections.emptyMap(),
                 SnapshotRetentionConfiguration.EMPTY
             );
-            assertThat(p.calculateNextInterval(), equalTo(TimeValue.MINUS_ONE));
+            assertThat(p.calculateNextInterval(Clock.systemUTC()), equalTo(TimeValue.MINUS_ONE));
         }
 
         {
@@ -99,7 +196,7 @@ public void testCalculateNextInterval() {
                 Collections.emptyMap(),
                 SnapshotRetentionConfiguration.EMPTY
             );
-            assertThat(p.calculateNextInterval(), equalTo(TimeValue.MINUS_ONE));
+            assertThat(p.calculateNextInterval(Clock.systemUTC()), equalTo(TimeValue.MINUS_ONE));
         }
     }
 
@@ -123,7 +220,7 @@ public void testValidation() {
                         + " the following characters "
                         + Strings.INVALID_FILENAME_CHARS,
                     "invalid repository name [  ]: cannot be empty",
-                    "invalid schedule: invalid cron expression [* * * * * L]"
+                    "invalid schedule [* * * * * L]: must be a valid cron expression or time unit"
                 )
             );
         }
@@ -149,6 +246,34 @@ public void testValidation() {
             );
         }
 
+        {
+            SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(
+                "my_policy",
+                "my_snap",
+                "0d",
+                "repo",
+                Collections.emptyMap(),
+                SnapshotRetentionConfiguration.EMPTY
+            );
+
+            ValidationException e = policy.validate();
+            assertThat(e.validationErrors(), containsInAnyOrder("invalid schedule [0d]: time unit must be at least 1 millisecond"));
+        }
+
+        {
+            SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(
+                "my_policy",
+                "my_snap",
+                "999micros",
+                "repo",
+                Collections.emptyMap(),
+                SnapshotRetentionConfiguration.EMPTY
+            );
+
+            ValidationException e = policy.validate();
+            assertThat(e.validationErrors(), containsInAnyOrder("invalid schedule [999micros]: time unit must be at least 1 millisecond"));
+        }
+
         {
             SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(
                 "my_policy",
@@ -161,6 +286,33 @@ public void testValidation() {
             ValidationException e = policy.validate();
             assertThat(e, nullValue());
         }
+
+        {
+            SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(
+                "my_policy",
+                "my_snap",
+                "30m",
+                "repo",
+                Collections.emptyMap(),
+                SnapshotRetentionConfiguration.EMPTY
+            );
+            ValidationException e = policy.validate();
+            assertThat(e, nullValue());
+        }
+
+        {
+            SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(
+                "my_policy",
+                "my_snap",
+                "1ms",
+                "repo",
+                Collections.emptyMap(),
+                SnapshotRetentionConfiguration.EMPTY
+            );
+
+            ValidationException e = policy.validate();
+            assertThat(e, nullValue());
+        }
     }
 
     public void testMetadataValidation() {
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java
index 5b59ac9efc0ab..36887681f5575 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleServiceTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -37,6 +38,7 @@
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadataTests;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats;
 import org.elasticsearch.xpack.core.slm.SnapshotRetentionConfiguration;
 import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
@@ -48,6 +50,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -108,7 +111,7 @@ public void testRepositoryExistenceForMissingRepo() {
     public void testNothingScheduledWhenNotRunning() throws InterruptedException {
         ClockMock clock = new ClockMock();
         SnapshotLifecyclePolicyMetadata initialPolicy = SnapshotLifecyclePolicyMetadata.builder()
-            .setPolicy(createPolicy("initial", "*/1 * * * * ?"))
+            .setPolicy(createPolicy("initial", randomBoolean() ? "*/1 * * * * ?" : "1s"))
             .setHeaders(Collections.emptyMap())
             .setVersion(1)
             .setModifiedDate(1)
@@ -133,7 +136,7 @@ public void testNothingScheduledWhenNotRunning() throws InterruptedException {
             sls.init();
 
             SnapshotLifecyclePolicyMetadata newPolicy = SnapshotLifecyclePolicyMetadata.builder()
-                .setPolicy(createPolicy("foo", "*/1 * * * * ?"))
+                .setPolicy(createPolicy("foo", randomBoolean() ? "*/1 * * * * ?" : "1s"))
                 .setHeaders(Collections.emptyMap())
                 .setVersion(2)
                 .setModifiedDate(2)
@@ -211,7 +214,7 @@ public void testPolicyCRUD() throws Exception {
             Map<String, SnapshotLifecyclePolicyMetadata> policies = new HashMap<>();
 
             SnapshotLifecyclePolicyMetadata policy = SnapshotLifecyclePolicyMetadata.builder()
-                .setPolicy(createPolicy("foo", "*/1 * * * * ?"))
+                .setPolicy(createPolicy("foo", randomBoolean() ? "*/1 * * * * ?" : "1s"))
                 .setHeaders(Collections.emptyMap())
                 .setModifiedDate(1)
                 .build();
@@ -240,7 +243,7 @@ public void testPolicyCRUD() throws Exception {
             int currentCount = triggerCount.get();
             previousState = state;
             SnapshotLifecyclePolicyMetadata newPolicy = SnapshotLifecyclePolicyMetadata.builder()
-                .setPolicy(createPolicy("foo", "*/1 * * * * ?"))
+                .setPolicy(createPolicy("foo", randomBoolean() ? "*/1 * * * * ?" : "1s"))
                 .setHeaders(Collections.emptyMap())
                 .setVersion(2)
                 .setModifiedDate(2)
@@ -253,7 +256,7 @@ public void testPolicyCRUD() throws Exception {
 
             CopyOnWriteArrayList<String> triggeredJobs = new CopyOnWriteArrayList<>();
             trigger.set(e -> {
-                triggeredJobs.add(e.getJobName());
+                triggeredJobs.add(e.jobName());
                 triggerCount.incrementAndGet();
             });
             clock.fastForwardSeconds(1);
@@ -283,7 +286,7 @@ public void testPolicyCRUD() throws Exception {
 
             // When the service is no longer master, all jobs should be automatically cancelled
             policy = SnapshotLifecyclePolicyMetadata.builder()
-                .setPolicy(createPolicy("foo", "*/1 * * * * ?"))
+                .setPolicy(createPolicy("foo", randomBoolean() ? "*/1 * * * * ?" : "1s"))
                 .setHeaders(Collections.emptyMap())
                 .setVersion(3)
                 .setModifiedDate(1)
@@ -343,7 +346,7 @@ public void testPolicyNamesEndingInNumbers() throws Exception {
             Map<String, SnapshotLifecyclePolicyMetadata> policies = new HashMap<>();
 
             SnapshotLifecyclePolicyMetadata policy = SnapshotLifecyclePolicyMetadata.builder()
-                .setPolicy(createPolicy("foo-2", "30 * * * * ?"))
+                .setPolicy(createPolicy("foo-2", randomBoolean() ? "30 * * * * ?" : "30s"))
                 .setHeaders(Collections.emptyMap())
                 .setVersion(1)
                 .setModifiedDate(1)
@@ -358,7 +361,7 @@ public void testPolicyNamesEndingInNumbers() throws Exception {
             assertThat(sls.getScheduler().scheduledJobIds(), equalTo(Collections.singleton("foo-2-1")));
 
             SnapshotLifecyclePolicyMetadata secondPolicy = SnapshotLifecyclePolicyMetadata.builder()
-                .setPolicy(createPolicy("foo-1", "45 * * * * ?"))
+                .setPolicy(createPolicy("foo-1", randomBoolean() ? "45 * * * * ?" : "45s"))
                 .setHeaders(Collections.emptyMap())
                 .setVersion(2)
                 .setModifiedDate(1)
@@ -410,33 +413,70 @@ public void testValidateMinimumInterval() {
             )
             .build();
 
-        for (String schedule : List.of("0 0/15 * * * ?", "0 0 1 * * ?", "0 0 0 1 1 ? 2099" /* once */, "* * * 31 FEB ? *" /* never */)) {
-            SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", schedule), defaultState);
-            SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", schedule), validationOneMinuteState);
-            SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", schedule), validationDisabledState);
+        { // using chron schedule
+            for (String schedule : List.of(
+                "0 0/15 * * * ?",
+                "0 0 1 * * ?",
+                "0 0 0 1 1 ? 2099" /* once */,
+                "* * * 31 FEB ? *" /* never */
+            )) {
+                SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", schedule), defaultState);
+                SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", schedule), validationOneMinuteState);
+                SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", schedule), validationDisabledState);
+            }
+
+            IllegalArgumentException e;
+
+            e = expectThrows(
+                IllegalArgumentException.class,
+                () -> SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "0 0/1 * * * ?"), defaultState)
+            );
+            assertThat(
+                e.getMessage(),
+                equalTo("invalid schedule [0 0/1 * * * ?]: " + "schedule would be too frequent, executing more than every [15m]")
+            );
+            SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "0 0/1 * * * ?"), validationOneMinuteState);
+
+            e = expectThrows(
+                IllegalArgumentException.class,
+                () -> SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "0/30 0/1 * * * ?"), validationOneMinuteState)
+            );
+            assertThat(
+                e.getMessage(),
+                equalTo("invalid schedule [0/30 0/1 * * * ?]: " + "schedule would be too frequent, executing more than every [1m]")
+            );
+            SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "0/30 0/1 * * * ?"), validationDisabledState);
         }
 
-        IllegalArgumentException e;
+        { // using time value
+            for (String interval : List.of("15m", "1h", "1d")) {
+                SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", interval), defaultState);
+                SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", interval), validationOneMinuteState);
+                SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", interval), validationDisabledState);
+            }
 
-        e = expectThrows(
-            IllegalArgumentException.class,
-            () -> SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "0 0/1 * * * ?"), defaultState)
-        );
-        assertThat(
-            e.getMessage(),
-            equalTo("invalid schedule [0 0/1 * * * ?]: " + "schedule would be too frequent, executing more than every [15m]")
-        );
-        SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "0 0/1 * * * ?"), validationOneMinuteState);
+            IllegalArgumentException e;
 
-        e = expectThrows(
-            IllegalArgumentException.class,
-            () -> SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "0/30 0/1 * * * ?"), validationOneMinuteState)
-        );
-        assertThat(
-            e.getMessage(),
-            equalTo("invalid schedule [0/30 0/1 * * * ?]: " + "schedule would be too frequent, executing more than every [1m]")
-        );
-        SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "0/30 0/1 * * * ?"), validationDisabledState);
+            e = expectThrows(
+                IllegalArgumentException.class,
+                () -> SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "1m"), defaultState)
+            );
+            assertThat(
+                e.getMessage(),
+                equalTo("invalid schedule [1m]: " + "schedule would be too frequent, executing more than every [15m]")
+            );
+            SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "1m"), validationOneMinuteState);
+
+            e = expectThrows(
+                IllegalArgumentException.class,
+                () -> SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "30s"), validationOneMinuteState)
+            );
+            assertThat(
+                e.getMessage(),
+                equalTo("invalid schedule [30s]: " + "schedule would be too frequent, executing more than every [1m]")
+            );
+            SnapshotLifecycleService.validateMinimumInterval(createPolicy("foo-1", "30s"), validationDisabledState);
+        }
     }
 
     public void testStoppedPriority() {
@@ -485,6 +525,41 @@ public void submitUnbatchedStateUpdateTask(String source, ClusterStateUpdateTask
         }
     }
 
+    public void testValidateIntervalScheduleSupport() {
+        var featureService = new FeatureService(List.of(new SnapshotLifecycleFeatures()));
+        {
+            ClusterState state = ClusterState.builder(new ClusterName("cluster"))
+                .nodeFeatures(Map.of("a", Set.of(), "b", Set.of(SnapshotLifecycleService.INTERVAL_SCHEDULE.id())))
+                .build();
+
+            IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> SnapshotLifecycleService.validateIntervalScheduleSupport("30d", featureService, state)
+            );
+            assertThat(e.getMessage(), containsString("Unable to use slm interval schedules"));
+        }
+        {
+            ClusterState state = ClusterState.builder(new ClusterName("cluster"))
+                .nodeFeatures(Map.of("a", Set.of(SnapshotLifecycleService.INTERVAL_SCHEDULE.id())))
+                .build();
+            try {
+                SnapshotLifecycleService.validateIntervalScheduleSupport("30d", featureService, state);
+            } catch (Exception e) {
+                fail("interval schedule is supported by version and should not fail");
+            }
+        }
+        {
+            ClusterState state = ClusterState.builder(new ClusterName("cluster"))
+                .nodeFeatures(Map.of("a", Set.of(), "b", Set.of(SnapshotLifecycleService.INTERVAL_SCHEDULE.id())))
+                .build();
+            try {
+                SnapshotLifecycleService.validateIntervalScheduleSupport("*/1 * * * * ?", featureService, state);
+            } catch (Exception e) {
+                fail("cron schedule does not need feature check and should not fail");
+            }
+        }
+    }
+
     class FakeSnapshotTask extends SnapshotLifecycleTask {
         private final Consumer<SchedulerEngine.Event> onTriggered;
 
@@ -515,7 +590,7 @@ public ClusterState createState(SnapshotLifecycleMetadata snapMeta, boolean loca
     }
 
     public static SnapshotLifecyclePolicy createPolicy(String id) {
-        return createPolicy(id, randomSchedule());
+        return createPolicy(id, SnapshotLifecyclePolicyMetadataTests.randomSchedule());
     }
 
     public static SnapshotLifecyclePolicy createPolicy(String id, String schedule) {
@@ -534,8 +609,4 @@ public static SnapshotLifecyclePolicy createPolicy(String id, String schedule) {
             SnapshotRetentionConfiguration.EMPTY
         );
     }
-
-    public static String randomSchedule() {
-        return randomIntBetween(0, 59) + " " + randomIntBetween(0, 59) + " " + randomIntBetween(0, 12) + " * * ?";
-    }
 }
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java
index 2c698a0383add..0b273c307cf47 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java
@@ -19,7 +19,10 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.scheduler.SchedulerEngine;
@@ -27,12 +30,15 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.snapshots.RegisteredPolicySnapshots;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotInfoUtils;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
+import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
@@ -48,9 +54,11 @@
 import org.elasticsearch.xpack.slm.history.SnapshotHistoryStore;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -129,7 +137,7 @@ public void testSkipCreatingSnapshotWhenJobDoesNotMatch() {
         threadPool.shutdownNow();
     }
 
-    public void testCreateSnapshotOnTrigger() {
+    public void testCreateSnapshotOnTrigger() throws Exception {
         final String id = randomAlphaOfLength(4);
         final SnapshotLifecyclePolicyMetadata slpm = makePolicyMeta(id);
         final SnapshotLifecycleMetadata meta = new SnapshotLifecycleMetadata(
@@ -140,6 +148,12 @@ public void testCreateSnapshotOnTrigger() {
 
         final ClusterState state = ClusterState.builder(new ClusterName("test"))
             .metadata(Metadata.builder().putCustom(SnapshotLifecycleMetadata.TYPE, meta).build())
+            .nodes(
+                DiscoveryNodes.builder()
+                    .add(DiscoveryNodeUtils.builder("nodeId").name("nodeId").build())
+                    .localNodeId("nodeId")
+                    .masterNodeId("nodeId")
+            )
             .build();
 
         final ThreadPool threadPool = new TestThreadPool("test");
@@ -220,11 +234,13 @@ public void testCreateSnapshotOnTrigger() {
                 new SchedulerEngine.Event(SnapshotLifecycleService.getJobId(slpm), System.currentTimeMillis(), System.currentTimeMillis())
             );
 
-            assertTrue("snapshot should be triggered once", clientCalled.get());
-            assertTrue("history store should be called once", historyStoreCalled.get());
+            assertBusy(() -> {
+                assertTrue("snapshot should be triggered once", clientCalled.get());
+                assertTrue("history store should be called once", historyStoreCalled.get());
+            });
+        } finally {
+            threadPool.shutdownNow();
         }
-
-        threadPool.shutdownNow();
     }
 
     public void testPartialFailureSnapshot() throws Exception {
@@ -238,6 +254,12 @@ public void testPartialFailureSnapshot() throws Exception {
 
         final ClusterState state = ClusterState.builder(new ClusterName("test"))
             .metadata(Metadata.builder().putCustom(SnapshotLifecycleMetadata.TYPE, meta).build())
+            .nodes(
+                DiscoveryNodes.builder()
+                    .add(DiscoveryNodeUtils.builder("nodeId").name("nodeId").build())
+                    .localNodeId("nodeId")
+                    .masterNodeId("nodeId")
+            )
             .build();
 
         final ThreadPool threadPool = new TestThreadPool("test");
@@ -307,11 +329,252 @@ public void testPartialFailureSnapshot() throws Exception {
                 new SchedulerEngine.Event(SnapshotLifecycleService.getJobId(slpm), System.currentTimeMillis(), System.currentTimeMillis())
             );
 
-            assertTrue("snapshot should be triggered once", clientCalled.get());
-            assertTrue("history store should be called once", historyStoreCalled.get());
+            assertBusy(() -> {
+                assertTrue("snapshot should be triggered once", clientCalled.get());
+                assertTrue("history store should be called once", historyStoreCalled.get());
+            });
+        } finally {
+            threadPool.shutdownNow();
         }
+    }
 
-        threadPool.shutdownNow();
+    public void testDeletedPoliciesHaveRegisteredRemoved() throws Exception {
+        final String policyId = randomAlphaOfLength(10);
+        final SnapshotId initiatingSnap = randSnapshotId();
+
+        final String deletedPolicy = randomAlphaOfLength(10);
+        final SnapshotId snapForDeletedPolicy = randSnapshotId();
+
+        SnapshotLifecycleTask.WriteJobStatus writeJobStatus = randomBoolean()
+            ? SnapshotLifecycleTask.WriteJobStatus.success(policyId, initiatingSnap, randomLong(), randomLong())
+            : SnapshotLifecycleTask.WriteJobStatus.failure(policyId, initiatingSnap, randomLong(), new RuntimeException());
+
+        // deletedPolicy is no longer defined
+        var definedSlmPolicies = List.of(policyId);
+        var registeredSnapshots = Map.of(policyId, List.of(initiatingSnap), deletedPolicy, List.of(snapForDeletedPolicy));
+        // behavior is same whether initiatingSnap still in progress
+        var inProgress = Map.of(policyId, randomBoolean() ? List.of(initiatingSnap) : List.<SnapshotId>of());
+        ClusterState clusterState = buildClusterState(definedSlmPolicies, registeredSnapshots, inProgress);
+
+        ClusterState newClusterState = writeJobStatus.execute(clusterState);
+        RegisteredPolicySnapshots newRegisteredPolicySnapshots = newClusterState.metadata().custom(RegisteredPolicySnapshots.TYPE);
+
+        assertEquals(List.of(), newRegisteredPolicySnapshots.getSnapshots());
+    }
+
+    public void testOtherDefinedPoliciesUneffected() throws Exception {
+        final String policyId = randomAlphaOfLength(10);
+        final SnapshotId initiatingSnap = randSnapshotId();
+
+        final String otherPolicy = randomAlphaOfLength(10);
+        final SnapshotId otherSnapRunning = randSnapshotId();
+        final SnapshotId otherSnapNotRunning = randSnapshotId();
+
+        SnapshotLifecycleTask.WriteJobStatus writeJobStatus = randomBoolean()
+            ? SnapshotLifecycleTask.WriteJobStatus.success(policyId, initiatingSnap, randomLong(), randomLong())
+            : SnapshotLifecycleTask.WriteJobStatus.failure(policyId, initiatingSnap, randomLong(), new RuntimeException());
+
+        var definedSlmPolicies = List.of(policyId, otherPolicy);
+        var registeredSnapshots = Map.of(policyId, List.of(initiatingSnap), otherPolicy, List.of(otherSnapRunning, otherSnapNotRunning));
+        var inProgress = Map.of(policyId, List.<SnapshotId>of(), otherPolicy, List.of(otherSnapRunning));
+        ClusterState clusterState = buildClusterState(definedSlmPolicies, registeredSnapshots, inProgress);
+
+        ClusterState newClusterState = writeJobStatus.execute(clusterState);
+        RegisteredPolicySnapshots newRegisteredPolicySnapshots = newClusterState.metadata().custom(RegisteredPolicySnapshots.TYPE);
+
+        assertEquals(List.of(otherSnapRunning, otherSnapNotRunning), newRegisteredPolicySnapshots.getSnapshotsByPolicy(otherPolicy));
+        assertEquals(List.of(), newRegisteredPolicySnapshots.getSnapshotsByPolicy(policyId));
+    }
+
+    public void testInitiatingSnapRemovedButStillRunningRemains() throws Exception {
+        final String policyId = randomAlphaOfLength(10);
+        final SnapshotId initiatingSnap = randSnapshotId();
+
+        SnapshotLifecycleTask.WriteJobStatus writeJobStatus = randomBoolean()
+            ? SnapshotLifecycleTask.WriteJobStatus.success(policyId, initiatingSnap, randomLong(), randomLong())
+            : SnapshotLifecycleTask.WriteJobStatus.failure(policyId, initiatingSnap, randomLong(), new RuntimeException());
+
+        final SnapshotId stillRunning = randSnapshotId();
+
+        var definedSlmPolicies = List.of(policyId);
+        var registeredSnapshots = Map.of(policyId, List.of(stillRunning, initiatingSnap));
+        // behavior is same whether initiatingSnap still in progress
+        var inProgress = Map.of(policyId, randomBoolean() ? List.of(stillRunning, initiatingSnap) : List.of(stillRunning));
+        ClusterState clusterState = buildClusterState(definedSlmPolicies, registeredSnapshots, inProgress);
+
+        ClusterState newClusterState = writeJobStatus.execute(clusterState);
+        RegisteredPolicySnapshots newRegisteredPolicySnapshots = newClusterState.metadata().custom(RegisteredPolicySnapshots.TYPE);
+
+        assertEquals(List.of(stillRunning), newRegisteredPolicySnapshots.getSnapshotsByPolicy(policyId));
+    }
+
+    public void testInferFailureInitiatedBySuccess() throws Exception {
+        final String policyId = randomAlphaOfLength(10);
+        final SnapshotId initiatingSnapshot = randSnapshotId();
+        final SnapshotId previousFailedSnapshot = randSnapshotId();
+        // currently running snapshots
+        final SnapshotId stillRunning = randSnapshotId();
+
+        var definedSlmPolicies = List.of(policyId);
+        var registeredSnapshots = Map.of(policyId, List.of(stillRunning, previousFailedSnapshot));
+        var inProgress = Map.of(policyId, List.of(stillRunning));
+        ClusterState clusterState = buildClusterState(definedSlmPolicies, registeredSnapshots, inProgress);
+
+        var writeJobTask = SnapshotLifecycleTask.WriteJobStatus.success(policyId, initiatingSnapshot, randomLong(), randomLong());
+
+        ClusterState newClusterState = writeJobTask.execute(clusterState);
+
+        // previous failure is now recorded in stats and metadata
+        SnapshotLifecycleMetadata newSlmMetadata = newClusterState.metadata().custom(SnapshotLifecycleMetadata.TYPE);
+        SnapshotLifecycleStats newStats = newSlmMetadata.getStats();
+        SnapshotLifecycleStats.SnapshotPolicyStats snapshotPolicyStats = newStats.getMetrics().get(policyId);
+        assertEquals(1, snapshotPolicyStats.getSnapshotFailedCount());
+        assertEquals(1, snapshotPolicyStats.getSnapshotTakenCount());
+
+        SnapshotLifecyclePolicyMetadata newSlmPolicyMetadata = newSlmMetadata.getSnapshotConfigurations().get(policyId);
+        assertEquals(previousFailedSnapshot.getName(), newSlmPolicyMetadata.getLastFailure().getSnapshotName());
+        assertEquals(initiatingSnapshot.getName(), newSlmPolicyMetadata.getLastSuccess().getSnapshotName());
+        assertEquals(0, newSlmPolicyMetadata.getInvocationsSinceLastSuccess());
+
+        // failed snapshot no longer in registeredSnapshot set
+        RegisteredPolicySnapshots newRegisteredPolicySnapshots = newClusterState.metadata().custom(RegisteredPolicySnapshots.TYPE);
+        List<SnapshotId> newRegisteredSnapIds = newRegisteredPolicySnapshots.getSnapshotsByPolicy(policyId);
+        assertEquals(List.of(stillRunning), newRegisteredSnapIds);
+    }
+
+    public void testInferFailureInitiatedByFailure() throws Exception {
+        final String policyId = randomAlphaOfLength(10);
+        final SnapshotId initiatingSnapshot = randSnapshotId();
+        final SnapshotId previousFailedSnapshot = randSnapshotId();
+        final SnapshotId stillRunning = randSnapshotId();
+
+        var definedSlmPolicies = List.of(policyId);
+        var registeredSnapshots = Map.of(policyId, List.of(stillRunning, previousFailedSnapshot));
+        var inProgress = Map.of(policyId, List.of(stillRunning));
+        ClusterState clusterState = buildClusterState(definedSlmPolicies, registeredSnapshots, inProgress);
+
+        var writeJobTask = SnapshotLifecycleTask.WriteJobStatus.failure(policyId, initiatingSnapshot, randomLong(), new RuntimeException());
+
+        ClusterState newClusterState = writeJobTask.execute(clusterState);
+
+        // previous failure is now recorded in stats and metadata
+        SnapshotLifecycleMetadata newSlmMetadata = newClusterState.metadata().custom(SnapshotLifecycleMetadata.TYPE);
+        SnapshotLifecycleStats newStats = newSlmMetadata.getStats();
+        SnapshotLifecycleStats.SnapshotPolicyStats snapshotPolicyStats = newStats.getMetrics().get(policyId);
+        assertEquals(2, snapshotPolicyStats.getSnapshotFailedCount());
+        assertEquals(0, snapshotPolicyStats.getSnapshotTakenCount());
+
+        SnapshotLifecyclePolicyMetadata newSlmPolicyMetadata = newSlmMetadata.getSnapshotConfigurations().get(policyId);
+        assertEquals(initiatingSnapshot.getName(), newSlmPolicyMetadata.getLastFailure().getSnapshotName());
+        assertNull(newSlmPolicyMetadata.getLastSuccess());
+        assertEquals(2, newSlmPolicyMetadata.getInvocationsSinceLastSuccess());
+
+        // failed snapshot no longer in registeredSnapshot set
+        RegisteredPolicySnapshots newRegisteredPolicySnapshots = newClusterState.metadata().custom(RegisteredPolicySnapshots.TYPE);
+        List<SnapshotId> newRegisteredSnapIds = newRegisteredPolicySnapshots.getSnapshotsByPolicy(policyId);
+        assertEquals(List.of(stillRunning), newRegisteredSnapIds);
+    }
+
+    public void testGetCurrentlyRunningSnapshots() {
+        final SnapshotId snapshot1 = randSnapshotId();
+        final SnapshotId snapshot2 = randSnapshotId();
+        final SnapshotId snapshot3 = randSnapshotId();
+        final SnapshotId snapshot4 = randSnapshotId();
+
+        final String repo1 = randomAlphaOfLength(10);
+        final String repo2 = randomAlphaOfLength(10);
+
+        final var snapshotsInProgress = SnapshotsInProgress.EMPTY.withUpdatedEntriesForRepo(
+            repo1,
+            List.of(
+                makeSnapshotInProgress(repo1, "some-policy", snapshot1),
+                makeSnapshotInProgress(repo1, "some-policy", snapshot2),
+                makeSnapshotInProgress(repo1, "other-policy", snapshot3)
+            )
+        ).withUpdatedEntriesForRepo(repo2, List.of(makeSnapshotInProgress(repo2, "other-policy", snapshot4)));
+
+        final ClusterState clusterState = ClusterState.builder(new ClusterName("cluster"))
+            .putCustom(SnapshotsInProgress.TYPE, snapshotsInProgress)
+            .build();
+
+        Set<SnapshotId> currentlyRunning = SnapshotLifecycleTask.currentlyRunningSnapshots(clusterState);
+        assertEquals(currentlyRunning, Set.of(snapshot1, snapshot2, snapshot3, snapshot4));
+    }
+
+    private static SnapshotId randSnapshotId() {
+        return new SnapshotId(randomAlphaOfLength(10), randomUUID());
+    }
+
+    private static ClusterState buildClusterState(
+        List<String> slmPolicies,
+        Map<String, List<SnapshotId>> registeredSnaps,
+        Map<String, List<SnapshotId>> inProgress
+    ) {
+        final String repo = randomAlphaOfLength(10);
+        List<SnapshotsInProgress.Entry> inProgressEntries = new ArrayList<>();
+        for (String policy : inProgress.keySet()) {
+            for (SnapshotId snap : inProgress.get(policy)) {
+                inProgressEntries.add(makeSnapshotInProgress(repo, policy, snap));
+            }
+        }
+
+        final List<RegisteredPolicySnapshots.PolicySnapshot> policySnapshots = new ArrayList<>();
+        for (Map.Entry<String, List<SnapshotId>> policySnaps : registeredSnaps.entrySet()) {
+            for (SnapshotId snapshotId : policySnaps.getValue()) {
+                policySnapshots.add(new RegisteredPolicySnapshots.PolicySnapshot(policySnaps.getKey(), snapshotId));
+            }
+        }
+
+        final ClusterState clusterState = ClusterState.builder(new ClusterName("cluster"))
+            .putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY.withUpdatedEntriesForRepo(repo, inProgressEntries))
+            .metadata(
+                Metadata.builder()
+                    .putCustom(SnapshotLifecycleMetadata.TYPE, makeSnapMeta(slmPolicies))
+                    .putCustom(RegisteredPolicySnapshots.TYPE, new RegisteredPolicySnapshots(policySnapshots))
+            )
+            .build();
+
+        return clusterState;
+    }
+
+    private static SnapshotLifecycleMetadata makeSnapMeta(List<String> policies) {
+        Map<String, SnapshotLifecyclePolicyMetadata> slmMeta = new HashMap<>();
+
+        for (String policy : policies) {
+            SnapshotLifecyclePolicyMetadata slmPolicyMeta = SnapshotLifecyclePolicyMetadata.builder()
+                .setModifiedDate(randomLong())
+                .setPolicy(new SnapshotLifecyclePolicy(policy, "snap", "", "repo-name", null, null))
+                .build();
+            slmMeta.put(policy, slmPolicyMeta);
+        }
+
+        SnapshotLifecycleStats stats = new SnapshotLifecycleStats(
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            new HashMap<>()
+        );
+        return new SnapshotLifecycleMetadata(slmMeta, OperationMode.RUNNING, stats);
+    }
+
+    private static SnapshotsInProgress.Entry makeSnapshotInProgress(String repo, String policyId, SnapshotId snapshotId) {
+        final Map<String, Object> metadata = Map.of(SnapshotsService.POLICY_ID_METADATA_FIELD, policyId);
+        return SnapshotsInProgress.Entry.snapshot(
+            new Snapshot(repo, snapshotId),
+            randomBoolean(),
+            randomBoolean(),
+            SnapshotsInProgress.State.SUCCESS,
+            Map.of(),
+            List.of(),
+            List.of(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            Map.of(),
+            null,
+            metadata,
+            IndexVersion.current()
+        );
     }
 
     /**
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionServiceTests.java
index dbb22f8dd49d8..877aa0ddb7342 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionServiceTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionServiceTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadataTests;
 import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
 import org.elasticsearch.xpack.slm.history.SnapshotHistoryStore;
 
@@ -52,7 +53,7 @@ public void testJobsAreScheduled() throws InterruptedException {
             assertThat(service.getScheduler().jobCount(), equalTo(0));
 
             service.onMaster();
-            service.setUpdateSchedule(SnapshotLifecycleServiceTests.randomSchedule());
+            service.setUpdateSchedule(SnapshotLifecyclePolicyMetadataTests.randomCronSchedule());
             assertThat(service.getScheduler().scheduledJobIds(), containsInAnyOrder(SnapshotRetentionService.SLM_RETENTION_JOB_ID));
 
             service.offMaster();
@@ -81,7 +82,7 @@ public void testManualTriggering() throws InterruptedException {
         try (
             ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool, clusterSettings);
             SnapshotRetentionService service = new SnapshotRetentionService(Settings.EMPTY, () -> new FakeRetentionTask(event -> {
-                assertThat(event.getJobName(), equalTo(SnapshotRetentionService.SLM_RETENTION_MANUAL_JOB_ID));
+                assertThat(event.jobName(), equalTo(SnapshotRetentionService.SLM_RETENTION_MANUAL_JOB_ID));
                 invoked.incrementAndGet();
             }), clock)
         ) {
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java
index 1384cd4499624..184d207ee173d 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java
@@ -44,7 +44,6 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -351,27 +350,25 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
                 )
             );
 
+            AtomicReference<SnapshotLifecycleStats> slmStats = new AtomicReference<>(new SnapshotLifecycleStats());
             AtomicBoolean onFailureCalled = new AtomicBoolean(false);
-            task.deleteSnapshot(
-                "policy",
-                "foo",
-                new SnapshotId("name", "uuid"),
-                new SnapshotLifecycleStats(0, 0, 0, 0, new HashMap<>()),
-                new ActionListener<>() {
-                    @Override
-                    public void onResponse(AcknowledgedResponse acknowledgedResponse) {
-                        logger.info("--> forcing failure");
-                        throw new ElasticsearchException("forced failure");
-                    }
+            task.deleteSnapshot("policy", "foo", new SnapshotId("name", "uuid"), slmStats, new ActionListener<>() {
+                @Override
+                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                    logger.info("--> forcing failure");
+                    throw new ElasticsearchException("forced failure");
+                }
 
-                    @Override
-                    public void onFailure(Exception e) {
-                        onFailureCalled.set(true);
-                    }
+                @Override
+                public void onFailure(Exception e) {
+                    onFailureCalled.set(true);
                 }
-            );
+            });
 
             assertThat(onFailureCalled.get(), equalTo(true));
+
+            var expectedPolicyStats = Map.of(policyId, new SnapshotLifecycleStats.SnapshotPolicyStats(policyId, 0, 0, 1, 1));
+            assertThat(slmStats.get(), equalTo(new SnapshotLifecycleStats(0, 0, 0, 0, expectedPolicyStats)));
         } finally {
             threadPool.shutdownNow();
             threadPool.awaitTermination(10, TimeUnit.SECONDS);
@@ -544,7 +541,7 @@ void deleteSnapshot(
             String policyId,
             String repo,
             SnapshotId snapshot,
-            SnapshotLifecycleStats slmStats,
+            AtomicReference<SnapshotLifecycleStats> slmStats,
             ActionListener<AcknowledgedResponse> listener
         ) {
             deleteRunner.apply(policyId, repo, snapshot, slmStats, listener);
@@ -557,7 +554,7 @@ void apply(
             String policyId,
             String repo,
             SnapshotId snapshot,
-            SnapshotLifecycleStats slmStats,
+            AtomicReference<SnapshotLifecycleStats> slmStats,
             ActionListener<AcknowledgedResponse> listener
         );
     }
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
index 71346ebc495d4..0fcc4b8007c6d 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
@@ -79,17 +80,17 @@ private TransformState processJSON(ReservedSnapshotAction action, TransformState
     }
 
     public void testDependencies() {
-        var action = new ReservedSnapshotAction();
+        var action = new ReservedSnapshotAction(mock(FeatureService.class));
         assertThat(action.optionalDependencies(), contains(ReservedRepositoryAction.NAME));
     }
 
-    public void testValidationFails() {
+    public void testValidationFailsNeitherScheduleOrInterval() {
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         final ClusterName clusterName = new ClusterName("elasticsearch");
 
         ClusterState state = ClusterState.builder(clusterName).build();
-        ReservedSnapshotAction action = new ReservedSnapshotAction();
+        ReservedSnapshotAction action = new ReservedSnapshotAction(mock(FeatureService.class));
         TransformState prevState = new TransformState(state, Set.of());
 
         String badPolicyJSON = """
@@ -117,6 +118,56 @@ public void testValidationFails() {
         );
     }
 
+    public void testIntervalScheduleSupportValidation() {
+        Client client = mock(Client.class);
+        when(client.settings()).thenReturn(Settings.EMPTY);
+        final ClusterName clusterName = new ClusterName("elasticsearch");
+        List<RepositoryMetadata> repositoriesMetadata = List.of(new RepositoryMetadata("repo", "fs", Settings.EMPTY));
+        Metadata.Builder mdBuilder = Metadata.builder();
+        mdBuilder.putCustom(RepositoriesMetadata.TYPE, new RepositoriesMetadata(repositoriesMetadata));
+        ClusterState state = ClusterState.builder(clusterName).metadata(mdBuilder).build();
+        TransformState prevState = new TransformState(state, Set.of());
+        String goodPolicyJSON = """
+            {
+               "daily-snapshots": {
+                 "schedule": "30d",
+                 "name": "<production-snap-{now/d}>",
+                 "repository": "repo",
+                 "config": {
+                   "indices": ["foo-*", "important"],
+                   "ignore_unavailable": true,
+                   "include_global_state": false
+                 },
+                 "retention": {
+                   "expire_after": "30d",
+                   "min_count": 1,
+                   "max_count": 50
+                 }
+               }
+            }
+            """;
+
+        {
+            FeatureService featureService = mock(FeatureService.class);
+            when(featureService.clusterHasFeature(any(), any())).thenReturn(false);
+            ReservedSnapshotAction action = new ReservedSnapshotAction(featureService);
+            assertThat(
+                expectThrows(IllegalArgumentException.class, () -> processJSON(action, prevState, goodPolicyJSON)).getMessage(),
+                is("Error on validating SLM requests")
+            );
+        }
+        {
+            FeatureService featureService = mock(FeatureService.class);
+            when(featureService.clusterHasFeature(any(), any())).thenReturn(true);
+            ReservedSnapshotAction action = new ReservedSnapshotAction(featureService);
+            try {
+                processJSON(action, prevState, goodPolicyJSON);
+            } catch (Exception e) {
+                fail("interval schedule with interval feature should pass validation");
+            }
+        }
+    }
+
     public void testActionAddRemove() throws Exception {
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
@@ -128,7 +179,7 @@ public void testActionAddRemove() throws Exception {
         mdBuilder.putCustom(RepositoriesMetadata.TYPE, new RepositoriesMetadata(repositoriesMetadata));
         ClusterState state = ClusterState.builder(clusterName).metadata(mdBuilder).build();
 
-        ReservedSnapshotAction action = new ReservedSnapshotAction();
+        ReservedSnapshotAction action = new ReservedSnapshotAction(mock(FeatureService.class));
 
         String emptyJSON = "";
 
@@ -362,7 +413,7 @@ public void testOperatorControllerFromJSONContent() throws IOException {
             null,
             List.of(
                 new ReservedClusterSettingsAction(clusterSettings),
-                new ReservedSnapshotAction(),
+                new ReservedSnapshotAction(mock(FeatureService.class)),
                 new ReservedRepositoryAction(repositoriesService)
             )
         );
@@ -396,7 +447,8 @@ public void testPutSLMReservedStateHandler() throws Exception {
             mock(ClusterService.class),
             threadPool,
             mock(ActionFilters.class),
-            mock(IndexNameExpressionResolver.class)
+            mock(IndexNameExpressionResolver.class),
+            mock(FeatureService.class)
         );
         assertThat(putAction.reservedStateHandlerName().get(), equalTo(ReservedSnapshotAction.NAME));
 
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryStoreTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryStoreTests.java
index 750fdd40c12d6..211afe8e55a15 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryStoreTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryStoreTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadataTests;
 import org.junit.After;
 import org.junit.Before;
 
@@ -194,10 +195,14 @@ public static SnapshotLifecyclePolicy randomSnapshotLifecyclePolicy(String id) {
                 config.put(randomAlphaOfLength(4), randomAlphaOfLength(4));
             }
         }
-        return new SnapshotLifecyclePolicy(id, randomAlphaOfLength(4), randomSchedule(), randomAlphaOfLength(4), config, null);
-    }
 
-    private static String randomSchedule() {
-        return randomIntBetween(0, 59) + " " + randomIntBetween(0, 59) + " " + randomIntBetween(0, 12) + " * * ?";
+        return new SnapshotLifecyclePolicy(
+            id,
+            randomAlphaOfLength(4),
+            SnapshotLifecyclePolicyMetadataTests.randomSchedule(),
+            randomAlphaOfLength(4),
+            config,
+            null
+        );
     }
 }
diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/AzureSnapshotBasedRecoveryIT.java b/x-pack/plugin/snapshot-based-recoveries/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/AzureSnapshotBasedRecoveryIT.java
index 233851919ebe4..591d4582d5905 100644
--- a/x-pack/plugin/snapshot-based-recoveries/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/AzureSnapshotBasedRecoveryIT.java
+++ b/x-pack/plugin/snapshot-based-recoveries/qa/azure/src/javaRestTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/AzureSnapshotBasedRecoveryIT.java
@@ -11,6 +11,8 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestTrustStore;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
@@ -21,12 +23,26 @@
 
 public class AzureSnapshotBasedRecoveryIT extends AbstractSnapshotBasedRecoveryRestTestCase {
     private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.azure.fixture", "true"));
+    private static final boolean USE_HTTPS_FIXTURE = USE_FIXTURE && ESTestCase.inFipsJvm() == false;
+    // TODO when https://github.com/elastic/elasticsearch/issues/111532 addressed, use a HTTPS fixture in FIPS mode too
+
     private static final String AZURE_TEST_ACCOUNT = System.getProperty("test.azure.account");
     private static final String AZURE_TEST_CONTAINER = System.getProperty("test.azure.container");
     private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key");
     private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token");
 
-    private static AzureHttpFixture fixture = new AzureHttpFixture(USE_FIXTURE, AZURE_TEST_ACCOUNT, AZURE_TEST_CONTAINER);
+    private static AzureHttpFixture fixture = new AzureHttpFixture(
+        USE_HTTPS_FIXTURE ? AzureHttpFixture.Protocol.HTTPS : USE_FIXTURE ? AzureHttpFixture.Protocol.HTTP : AzureHttpFixture.Protocol.NONE,
+        AZURE_TEST_ACCOUNT,
+        AZURE_TEST_CONTAINER,
+        System.getProperty("test.azure.tenant_id"),
+        System.getProperty("test.azure.client_id"),
+        AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_TEST_ACCOUNT)
+    );
+
+    private static TestTrustStore trustStore = new TestTrustStore(
+        () -> AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem")
+    );
 
     private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .nodes(3)
@@ -49,10 +65,12 @@ public class AzureSnapshotBasedRecoveryIT extends AbstractSnapshotBasedRecoveryR
             s -> USE_FIXTURE
         )
         .setting("xpack.license.self_generated.type", "trial")
+        .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_HTTPS_FIXTURE)
+        .systemProperty("javax.net.ssl.trustStoreType", () -> "jks", s -> USE_HTTPS_FIXTURE)
         .build();
 
-    @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
+    @ClassRule(order = 1)
+    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster);
 
     @Override
     protected String getTestRestCluster() {
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle
index ea7bfdd024d95..e304b2ff5c263 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle
@@ -7,6 +7,7 @@
 
 
 import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.RestIntegTestTask
 
 apply plugin: 'elasticsearch.internal-java-rest-test'
 
@@ -23,6 +24,8 @@ String azureKey = System.getenv("azure_storage_key")
 String azureContainer = System.getenv("azure_storage_container")
 String azureBasePath = System.getenv("azure_storage_base_path")
 String azureSasToken = System.getenv("azure_storage_sas_token")
+String azureTenantId = System.getenv("azure_storage_tenant_id")
+String azureClientId = System.getenv("azure_storage_client_id")
 
 if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
   azureAccount = 'azure_integration_test_account'
@@ -30,6 +33,8 @@ if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSas
   azureContainer = 'container'
   azureBasePath = ''
   azureSasToken = ''
+  azureTenantId = ''
+  azureClientId = ''
   useFixture = true
 }
 
@@ -39,9 +44,46 @@ tasks.named("javaRestTest") {
   systemProperty 'test.azure.container', azureContainer
   systemProperty 'test.azure.key', azureKey
   systemProperty 'test.azure.sas_token', azureSasToken
+  systemProperty 'test.azure.tenant_id', azureTenantId
+  systemProperty 'test.azure.client_id', azureClientId
   nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repository_test_kit_tests_" + BuildParams.testSeed
 }
 
 tasks.register("azureThirdPartyTest") {
   dependsOn "javaRestTest"
 }
+
+tasks.register("managedIdentityJavaRestTest", RestIntegTestTask) {
+  testClassesDirs = sourceSets.javaRestTest.output.classesDirs
+  classpath = sourceSets.javaRestTest.runtimeClasspath
+  systemProperty 'test.azure.fixture', Boolean.toString(useFixture)
+  systemProperty 'test.azure.account', azureAccount
+  systemProperty 'test.azure.container', azureContainer
+  // omitting key and sas_token so that we use a bearer token from the metadata service
+  // omitting client_id and tenant_id so that we use a bearer token from the metadata service, not from workload identity
+  nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repository_test_kit_tests_" + BuildParams.testSeed
+}
+
+tasks.register("workloadIdentityJavaRestTest", RestIntegTestTask) {
+  testClassesDirs = sourceSets.javaRestTest.output.classesDirs
+  classpath = sourceSets.javaRestTest.runtimeClasspath
+  systemProperty 'test.azure.fixture', Boolean.toString(useFixture)
+  systemProperty 'test.azure.account', azureAccount
+  systemProperty 'test.azure.container', azureContainer
+  // random uuids to satisfy format requirements -- the actual values are not important
+  systemProperty 'test.azure.tenant_id', azureTenantId ?: "583d4f71-148a-4163-bad5-2311e13c60dc"
+  systemProperty 'test.azure.client_id', azureClientId ?: "86dd1b33-96c1-4a2e-92ac-b844404fc691"
+  // omitting key and sas_token so that we use a bearer token from workload identity
+  nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repository_test_kit_tests_" + BuildParams.testSeed
+}
+
+if (BuildParams.inFipsJvm) {
+  // Cannot override the trust store in FIPS mode, and these tasks require a HTTPS fixture
+  tasks.named("managedIdentityJavaRestTest").configure { enabled = false }
+  tasks.named("workloadIdentityJavaRestTest").configure { enabled = false }
+}
+
+tasks.named("check") {
+  dependsOn("managedIdentityJavaRestTest")
+  dependsOn("workloadIdentityJavaRestTest")
+}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/AzureSnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/AzureSnapshotRepoTestKitIT.java
deleted file mode 100644
index bae37bd43b32b..0000000000000
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/AzureSnapshotRepoTestKitIT.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.repositories.blobstore.testkit;
-
-import fixture.azure.AzureHttpFixture;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.Booleans;
-import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.junit.ClassRule;
-import org.junit.rules.RuleChain;
-import org.junit.rules.TestRule;
-
-import static org.hamcrest.Matchers.blankOrNullString;
-import static org.hamcrest.Matchers.not;
-
-public class AzureSnapshotRepoTestKitIT extends AbstractSnapshotRepoTestKitRestTestCase {
-    private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.azure.fixture", "true"));
-    private static final String AZURE_TEST_ACCOUNT = System.getProperty("test.azure.account");
-    private static final String AZURE_TEST_CONTAINER = System.getProperty("test.azure.container");
-    private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key");
-    private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token");
-
-    private static AzureHttpFixture fixture = new AzureHttpFixture(USE_FIXTURE, AZURE_TEST_ACCOUNT, AZURE_TEST_CONTAINER);
-
-    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .module("repository-azure")
-        .module("snapshot-repo-test-kit")
-        .keystore("azure.client.repository_test_kit.account", AZURE_TEST_ACCOUNT)
-        .keystore(
-            "azure.client.repository_test_kit.key",
-            () -> AZURE_TEST_KEY,
-            s -> AZURE_TEST_KEY != null && AZURE_TEST_KEY.isEmpty() == false
-        )
-        .keystore(
-            "azure.client.repository_test_kit.sas_token",
-            () -> AZURE_TEST_SASTOKEN,
-            s -> AZURE_TEST_SASTOKEN != null && AZURE_TEST_SASTOKEN.isEmpty() == false
-        )
-        .setting(
-            "azure.client.repository_test_kit.endpoint_suffix",
-            () -> "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + fixture.getAddress(),
-            s -> USE_FIXTURE
-        )
-        .apply(c -> {
-            if (USE_FIXTURE) {
-                // test fixture does not support CAS yet; TODO fix this
-                c.systemProperty("test.repository_test_kit.skip_cas", "true");
-            }
-        })
-        .build();
-
-    @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
-
-    @Override
-    protected String getTestRestCluster() {
-        return cluster.getHttpAddresses();
-    }
-
-    @Override
-    protected String repositoryType() {
-        return "azure";
-    }
-
-    @Override
-    protected Settings repositorySettings() {
-        final String container = System.getProperty("test.azure.container");
-        assertThat(container, not(blankOrNullString()));
-
-        final String basePath = System.getProperty("test.azure.base_path");
-        assertThat(basePath, not(blankOrNullString()));
-
-        return Settings.builder().put("client", "repository_test_kit").put("container", container).put("base_path", basePath).build();
-    }
-}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/AzureRepositoryAnalysisRestIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/AzureRepositoryAnalysisRestIT.java
new file mode 100644
index 0000000000000..a9b8fe51c01cc
--- /dev/null
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/AzureRepositoryAnalysisRestIT.java
@@ -0,0 +1,159 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
+
+import fixture.azure.AzureHttpFixture;
+
+import org.apache.http.client.methods.HttpGet;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestTrustStore;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.test.rest.ObjectPath;
+import org.hamcrest.Matcher;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Predicate;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+
+public class AzureRepositoryAnalysisRestIT extends AbstractRepositoryAnalysisRestTestCase {
+    private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.azure.fixture", "true"));
+    private static final boolean USE_HTTPS_FIXTURE = USE_FIXTURE && ESTestCase.inFipsJvm() == false;
+    // TODO when https://github.com/elastic/elasticsearch/issues/111532 addressed, use a HTTPS fixture in FIPS mode too
+
+    private static final String AZURE_TEST_ACCOUNT = System.getProperty("test.azure.account");
+    private static final String AZURE_TEST_CONTAINER = System.getProperty("test.azure.container");
+    private static final String AZURE_TEST_KEY = System.getProperty("test.azure.key");
+    private static final String AZURE_TEST_SASTOKEN = System.getProperty("test.azure.sas_token");
+    private static final String AZURE_TEST_TENANT_ID = System.getProperty("test.azure.tenant_id");
+    private static final String AZURE_TEST_CLIENT_ID = System.getProperty("test.azure.client_id");
+    private static final AzureHttpFixture fixture = new AzureHttpFixture(
+        USE_HTTPS_FIXTURE ? AzureHttpFixture.Protocol.HTTPS : USE_FIXTURE ? AzureHttpFixture.Protocol.HTTP : AzureHttpFixture.Protocol.NONE,
+        AZURE_TEST_ACCOUNT,
+        AZURE_TEST_CONTAINER,
+        AZURE_TEST_TENANT_ID,
+        AZURE_TEST_CLIENT_ID,
+        decideAuthHeaderPredicate()
+    );
+
+    private static Predicate<String> decideAuthHeaderPredicate() {
+        if (Strings.hasText(AZURE_TEST_KEY) || Strings.hasText(AZURE_TEST_SASTOKEN)) {
+            return AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_TEST_ACCOUNT);
+        } else if (Strings.hasText(AZURE_TEST_TENANT_ID) && Strings.hasText(AZURE_TEST_CLIENT_ID)) {
+            return AzureHttpFixture.WORK_IDENTITY_BEARER_TOKEN_PREDICATE;
+        } else if (Strings.hasText(AZURE_TEST_TENANT_ID) || Strings.hasText(AZURE_TEST_CLIENT_ID)) {
+            fail(null, "Both [test.azure.tenant_id] and [test.azure.client_id] must be set if either is set");
+        }
+        return AzureHttpFixture.MANAGED_IDENTITY_BEARER_TOKEN_PREDICATE;
+    }
+
+    private static final TestTrustStore trustStore = new TestTrustStore(
+        () -> AzureHttpFixture.class.getResourceAsStream("azure-http-fixture.pem")
+    );
+
+    private static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-azure")
+        .module("snapshot-repo-test-kit")
+        .keystore("azure.client.repository_test_kit.account", AZURE_TEST_ACCOUNT)
+        .keystore("azure.client.repository_test_kit.key", () -> AZURE_TEST_KEY, s -> Strings.hasText(AZURE_TEST_KEY))
+        .keystore("azure.client.repository_test_kit.sas_token", () -> AZURE_TEST_SASTOKEN, s -> Strings.hasText(AZURE_TEST_SASTOKEN))
+        .setting(
+            "azure.client.repository_test_kit.endpoint_suffix",
+            () -> "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + fixture.getAddress(),
+            s -> USE_FIXTURE
+        )
+        .apply(c -> {
+            if (USE_FIXTURE) {
+                // test fixture does not support CAS yet; TODO fix this
+                c.systemProperty("test.repository_test_kit.skip_cas", "true");
+            }
+        })
+        .systemProperty(
+            "tests.azure.credentials.disable_instance_discovery",
+            () -> "true",
+            s -> USE_HTTPS_FIXTURE && Strings.hasText(AZURE_TEST_CLIENT_ID) && Strings.hasText(AZURE_TEST_TENANT_ID)
+        )
+        .systemProperty("AZURE_POD_IDENTITY_AUTHORITY_HOST", fixture::getMetadataAddress, s -> USE_FIXTURE)
+        .systemProperty("AZURE_AUTHORITY_HOST", fixture::getOAuthTokenServiceAddress, s -> USE_HTTPS_FIXTURE)
+        .systemProperty("AZURE_CLIENT_ID", () -> AZURE_TEST_CLIENT_ID, s -> Strings.hasText(AZURE_TEST_CLIENT_ID))
+        .systemProperty("AZURE_TENANT_ID", () -> AZURE_TEST_TENANT_ID, s -> Strings.hasText(AZURE_TEST_TENANT_ID))
+        .configFile("storage-azure/azure-federated-token", Resource.fromString(fixture.getFederatedToken()))
+        .environment(
+            nodeSpec -> USE_HTTPS_FIXTURE && Strings.hasText(AZURE_TEST_CLIENT_ID) && Strings.hasText(AZURE_TEST_TENANT_ID)
+                ? Map.of("AZURE_FEDERATED_TOKEN_FILE", "${ES_PATH_CONF}/storage-azure/azure-federated-token")
+                : Map.of()
+        )
+        .systemProperty("javax.net.ssl.trustStore", () -> trustStore.getTrustStorePath().toString(), s -> USE_HTTPS_FIXTURE)
+        .systemProperty("javax.net.ssl.trustStoreType", () -> "jks", s -> USE_HTTPS_FIXTURE)
+        .build();
+
+    @ClassRule(order = 1)
+    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(trustStore).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected String repositoryType() {
+        return "azure";
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        final String container = System.getProperty("test.azure.container");
+        assertThat(container, not(blankOrNullString()));
+
+        final String basePath = System.getProperty("test.azure.base_path");
+        assertThat(basePath, not(blankOrNullString()));
+
+        return Settings.builder().put("client", "repository_test_kit").put("container", container).put("base_path", basePath).build();
+    }
+
+    public void testClusterStats() throws IOException {
+        registerRepository(randomIdentifier(), repositoryType(), true, repositorySettings());
+
+        final var request = new Request(HttpGet.METHOD_NAME, "/_cluster/stats");
+        final var response = client().performRequest(request);
+        assertOK(response);
+
+        final var objectPath = ObjectPath.createFromResponse(response);
+        assertThat(objectPath.evaluate("repositories.azure.count"), isSetIff(true));
+        assertThat(objectPath.evaluate("repositories.azure.read_write"), isSetIff(true));
+
+        assertThat(objectPath.evaluate("repositories.azure.uses_key_credentials"), isSetIff(Strings.hasText(AZURE_TEST_KEY)));
+        assertThat(objectPath.evaluate("repositories.azure.uses_sas_token"), isSetIff(Strings.hasText(AZURE_TEST_SASTOKEN)));
+        assertThat(
+            objectPath.evaluate("repositories.azure.uses_default_credentials"),
+            isSetIff((Strings.hasText(AZURE_TEST_SASTOKEN) || Strings.hasText(AZURE_TEST_KEY)) == false)
+        );
+        assertThat(
+            objectPath.evaluate("repositories.azure.uses_managed_identity"),
+            isSetIff(
+                (Strings.hasText(AZURE_TEST_SASTOKEN) || Strings.hasText(AZURE_TEST_KEY) || Strings.hasText(AZURE_TEST_CLIENT_ID)) == false
+            )
+        );
+        assertThat(objectPath.evaluate("repositories.azure.uses_workload_identity"), isSetIff(Strings.hasText(AZURE_TEST_CLIENT_ID)));
+    }
+
+    private static Matcher<Integer> isSetIff(boolean predicate) {
+        return predicate ? equalTo(1) : nullValue(Integer.class);
+    }
+}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/GCSSnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/GCSSnapshotRepoTestKitIT.java
deleted file mode 100644
index 95b6f4aed5221..0000000000000
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/GCSSnapshotRepoTestKitIT.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.repositories.blobstore.testkit;
-
-import fixture.gcs.GoogleCloudStorageHttpFixture;
-import fixture.gcs.TestUtils;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.Booleans;
-import org.elasticsearch.core.PathUtils;
-import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.cluster.util.resource.Resource;
-import org.junit.ClassRule;
-import org.junit.rules.RuleChain;
-import org.junit.rules.TestRule;
-
-import java.nio.charset.StandardCharsets;
-
-import static org.hamcrest.Matchers.blankOrNullString;
-import static org.hamcrest.Matchers.not;
-
-public class GCSSnapshotRepoTestKitIT extends AbstractSnapshotRepoTestKitRestTestCase {
-    private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.google.fixture", "true"));
-
-    private static GoogleCloudStorageHttpFixture fixture = new GoogleCloudStorageHttpFixture(USE_FIXTURE, "bucket", "o/oauth2/token");
-
-    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .module("repository-gcs")
-        .module("snapshot-repo-test-kit")
-        .setting("gcs.client.repository_test_kit.endpoint", () -> fixture.getAddress(), s -> USE_FIXTURE)
-        .setting("gcs.client.repository_test_kit.token_uri", () -> fixture.getAddress() + "/o/oauth2/token", s -> USE_FIXTURE)
-        .apply(c -> {
-            if (USE_FIXTURE) {
-                c.keystore(
-                    "gcs.client.repository_test_kit.credentials_file",
-                    Resource.fromString(() -> new String(TestUtils.createServiceAccount(random()), StandardCharsets.UTF_8))
-                );
-            } else {
-                c.keystore(
-                    "gcs.client.repository_test_kit.credentials_file",
-                    Resource.fromFile(PathUtils.get(System.getProperty("test.google.account")))
-                );
-            }
-        })
-        .apply(c -> {
-            if (USE_FIXTURE) {
-                // test fixture does not support CAS yet; TODO fix this
-                c.systemProperty("test.repository_test_kit.skip_cas", "true");
-            }
-        })
-        .build();
-
-    @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
-
-    @Override
-    protected String getTestRestCluster() {
-        return cluster.getHttpAddresses();
-    }
-
-    @Override
-    protected String repositoryType() {
-        return "gcs";
-    }
-
-    @Override
-    protected Settings repositorySettings() {
-        final String bucket = System.getProperty("test.gcs.bucket");
-        assertThat(bucket, not(blankOrNullString()));
-
-        final String basePath = System.getProperty("test.gcs.base_path");
-        assertThat(basePath, not(blankOrNullString()));
-
-        return Settings.builder().put("client", "repository_test_kit").put("bucket", bucket).put("base_path", basePath).build();
-    }
-}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java
new file mode 100644
index 0000000000000..7f7540d138825
--- /dev/null
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GCSRepositoryAnalysisRestIT.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
+
+import fixture.gcs.GoogleCloudStorageHttpFixture;
+import fixture.gcs.TestUtils;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.not;
+
+public class GCSRepositoryAnalysisRestIT extends AbstractRepositoryAnalysisRestTestCase {
+    private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.google.fixture", "true"));
+
+    private static GoogleCloudStorageHttpFixture fixture = new GoogleCloudStorageHttpFixture(USE_FIXTURE, "bucket", "o/oauth2/token");
+
+    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-gcs")
+        .module("snapshot-repo-test-kit")
+        .setting("gcs.client.repository_test_kit.endpoint", () -> fixture.getAddress(), s -> USE_FIXTURE)
+        .setting("gcs.client.repository_test_kit.token_uri", () -> fixture.getAddress() + "/o/oauth2/token", s -> USE_FIXTURE)
+        .apply(c -> {
+            if (USE_FIXTURE) {
+                c.keystore(
+                    "gcs.client.repository_test_kit.credentials_file",
+                    Resource.fromString(() -> new String(TestUtils.createServiceAccount(random()), StandardCharsets.UTF_8))
+                );
+            } else {
+                c.keystore(
+                    "gcs.client.repository_test_kit.credentials_file",
+                    Resource.fromFile(PathUtils.get(System.getProperty("test.google.account")))
+                );
+            }
+        })
+        .apply(c -> {
+            if (USE_FIXTURE) {
+                // test fixture does not support CAS yet; TODO fix this
+                c.systemProperty("test.repository_test_kit.skip_cas", "true");
+            }
+        })
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected String repositoryType() {
+        return "gcs";
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        final String bucket = System.getProperty("test.gcs.bucket");
+        assertThat(bucket, not(blankOrNullString()));
+
+        final String basePath = System.getProperty("test.gcs.base_path");
+        assertThat(basePath, not(blankOrNullString()));
+
+        return Settings.builder().put("client", "repository_test_kit").put("bucket", bucket).put("base_path", basePath).build();
+    }
+}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/AbstractHdfsSnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/AbstractHdfsSnapshotRepoTestKitIT.java
deleted file mode 100644
index 2810c4801e8dd..0000000000000
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/AbstractHdfsSnapshotRepoTestKitIT.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.repositories.blobstore.testkit;
-
-import org.elasticsearch.common.settings.Settings;
-
-import static org.hamcrest.Matchers.blankOrNullString;
-import static org.hamcrest.Matchers.not;
-
-public abstract class AbstractHdfsSnapshotRepoTestKitIT extends AbstractSnapshotRepoTestKitRestTestCase {
-
-    @Override
-    protected String repositoryType() {
-        return "hdfs";
-    }
-
-    @Override
-    protected Settings repositorySettings() {
-        final String uri = "hdfs://localhost:" + getHdfsPort();
-        // final String uri = System.getProperty("test.hdfs.uri");
-        assertThat(uri, not(blankOrNullString()));
-
-        final String path = getRepositoryPath();
-        assertThat(path, not(blankOrNullString()));
-        Settings.Builder repositorySettings = Settings.builder().put("client", "repository_test_kit").put("uri", uri).put("path", path);
-        return repositorySettings.build();
-    }
-
-    protected abstract String getRepositoryPath();
-
-    protected abstract int getHdfsPort();
-
-}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/HdfsSnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/HdfsSnapshotRepoTestKitIT.java
deleted file mode 100644
index e9787ecdce854..0000000000000
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/HdfsSnapshotRepoTestKitIT.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.repositories.blobstore.testkit;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
-import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.cluster.local.distribution.DistributionType;
-import org.elasticsearch.test.fixtures.hdfs.HdfsClientThreadLeakFilter;
-import org.elasticsearch.test.fixtures.hdfs.HdfsFixture;
-import org.junit.ClassRule;
-import org.junit.rules.RuleChain;
-import org.junit.rules.TestRule;
-
-@ThreadLeakFilters(filters = { HdfsClientThreadLeakFilter.class })
-public class HdfsSnapshotRepoTestKitIT extends AbstractHdfsSnapshotRepoTestKitIT {
-
-    public static HdfsFixture hdfsFixture = new HdfsFixture();
-
-    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .distribution(DistributionType.DEFAULT)
-        .plugin("repository-hdfs")
-        .setting("xpack.license.self_generated.type", "trial")
-        .setting("xpack.security.enabled", "false")
-        .build();
-
-    @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(hdfsFixture).around(cluster);
-
-    @Override
-    protected String getTestRestCluster() {
-        return cluster.getHttpAddresses();
-    }
-
-    @Override
-    protected String getRepositoryPath() {
-        return "/user/elasticsearch/test/repository_test_kit/simple";
-    }
-
-    @Override
-    protected int getHdfsPort() {
-        return hdfsFixture.getPort();
-    }
-}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/SecureHdfsSnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/SecureHdfsSnapshotRepoTestKitIT.java
deleted file mode 100644
index 6d599e41e3b9f..0000000000000
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/SecureHdfsSnapshotRepoTestKitIT.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.repositories.blobstore.testkit;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.cluster.local.distribution.DistributionType;
-import org.elasticsearch.test.cluster.util.resource.Resource;
-import org.elasticsearch.test.fixtures.hdfs.HdfsClientThreadLeakFilter;
-import org.elasticsearch.test.fixtures.hdfs.HdfsFixture;
-import org.elasticsearch.test.fixtures.krb5kdc.Krb5kDcContainer;
-import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
-import org.junit.ClassRule;
-import org.junit.rules.RuleChain;
-import org.junit.rules.TestRule;
-
-@ThreadLeakFilters(filters = { HdfsClientThreadLeakFilter.class, TestContainersThreadFilter.class })
-public class SecureHdfsSnapshotRepoTestKitIT extends AbstractHdfsSnapshotRepoTestKitIT {
-
-    public static Krb5kDcContainer krb5Fixture = new Krb5kDcContainer();
-
-    public static HdfsFixture hdfsFixture = new HdfsFixture().withKerberos(() -> krb5Fixture.getPrincipal(), () -> krb5Fixture.getKeytab());
-
-    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .distribution(DistributionType.DEFAULT)
-        .plugin("repository-hdfs")
-        .setting("xpack.license.self_generated.type", "trial")
-        .setting("xpack.security.enabled", "false")
-        .systemProperty("java.security.krb5.conf", () -> krb5Fixture.getConfPath().toString())
-        .configFile("repository-hdfs/krb5.conf", Resource.fromString(() -> krb5Fixture.getConf()))
-        .configFile("repository-hdfs/krb5.keytab", Resource.fromFile(() -> krb5Fixture.getEsKeytab()))
-        .build();
-
-    @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(krb5Fixture).around(hdfsFixture).around(cluster);
-
-    @Override
-    protected String getTestRestCluster() {
-        return cluster.getHttpAddresses();
-    }
-
-    @Override
-    protected int getHdfsPort() {
-        return hdfsFixture.getPort();
-    }
-
-    @Override
-    protected String getRepositoryPath() {
-        return "/user/elasticsearch/test/repository_test_kit/secure";
-    }
-
-    @Override
-    protected Settings repositorySettings() {
-        return Settings.builder().put(super.repositorySettings()).put("security.principal", "elasticsearch@BUILD.ELASTIC.CO").build();
-    }
-}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/AbstractHdfsRepositoryAnalysisRestIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/AbstractHdfsRepositoryAnalysisRestIT.java
new file mode 100644
index 0000000000000..2aec22476d6cc
--- /dev/null
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/AbstractHdfsRepositoryAnalysisRestIT.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
+
+import org.elasticsearch.common.settings.Settings;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.not;
+
+public abstract class AbstractHdfsRepositoryAnalysisRestIT extends AbstractRepositoryAnalysisRestTestCase {
+
+    @Override
+    protected String repositoryType() {
+        return "hdfs";
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        final String uri = "hdfs://localhost:" + getHdfsPort();
+        // final String uri = System.getProperty("test.hdfs.uri");
+        assertThat(uri, not(blankOrNullString()));
+
+        final String path = getRepositoryPath();
+        assertThat(path, not(blankOrNullString()));
+        Settings.Builder repositorySettings = Settings.builder().put("client", "repository_test_kit").put("uri", uri).put("path", path);
+        return repositorySettings.build();
+    }
+
+    protected abstract String getRepositoryPath();
+
+    protected abstract int getHdfsPort();
+
+}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/HdfsRepositoryAnalysisRestIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/HdfsRepositoryAnalysisRestIT.java
new file mode 100644
index 0000000000000..d60497949ff61
--- /dev/null
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/HdfsRepositoryAnalysisRestIT.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.fixtures.hdfs.HdfsClientThreadLeakFilter;
+import org.elasticsearch.test.fixtures.hdfs.HdfsFixture;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+@ThreadLeakFilters(filters = { HdfsClientThreadLeakFilter.class })
+public class HdfsRepositoryAnalysisRestIT extends AbstractHdfsRepositoryAnalysisRestIT {
+
+    public static HdfsFixture hdfsFixture = new HdfsFixture();
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .plugin("repository-hdfs")
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("xpack.security.enabled", "false")
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(hdfsFixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected String getRepositoryPath() {
+        return "/user/elasticsearch/test/repository_test_kit/simple";
+    }
+
+    @Override
+    protected int getHdfsPort() {
+        return hdfsFixture.getPort();
+    }
+}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/SecureHdfsRepositoryAnalysisRestIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/SecureHdfsRepositoryAnalysisRestIT.java
new file mode 100644
index 0000000000000..dd388c0a79776
--- /dev/null
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/SecureHdfsRepositoryAnalysisRestIT.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.test.fixtures.hdfs.HdfsClientThreadLeakFilter;
+import org.elasticsearch.test.fixtures.hdfs.HdfsFixture;
+import org.elasticsearch.test.fixtures.krb5kdc.Krb5kDcContainer;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+@ThreadLeakFilters(filters = { HdfsClientThreadLeakFilter.class, TestContainersThreadFilter.class })
+public class SecureHdfsRepositoryAnalysisRestIT extends AbstractHdfsRepositoryAnalysisRestIT {
+
+    public static Krb5kDcContainer krb5Fixture = new Krb5kDcContainer();
+
+    public static HdfsFixture hdfsFixture = new HdfsFixture().withKerberos(() -> krb5Fixture.getPrincipal(), () -> krb5Fixture.getKeytab());
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .plugin("repository-hdfs")
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("xpack.security.enabled", "false")
+        .systemProperty("java.security.krb5.conf", () -> krb5Fixture.getConfPath().toString())
+        .configFile("repository-hdfs/krb5.conf", Resource.fromString(() -> krb5Fixture.getConf()))
+        .configFile("repository-hdfs/krb5.keytab", Resource.fromFile(() -> krb5Fixture.getEsKeytab()))
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(krb5Fixture).around(hdfsFixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected int getHdfsPort() {
+        return hdfsFixture.getPort();
+    }
+
+    @Override
+    protected String getRepositoryPath() {
+        return "/user/elasticsearch/test/repository_test_kit/secure";
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        return Settings.builder().put(super.repositorySettings()).put("security.principal", "elasticsearch@BUILD.ELASTIC.CO").build();
+    }
+}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/minio/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/MinioSnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/minio/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/MinioSnapshotRepoTestKitIT.java
deleted file mode 100644
index 3e58a8d89ff31..0000000000000
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/minio/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/MinioSnapshotRepoTestKitIT.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.repositories.blobstore.testkit;
-
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.cluster.local.distribution.DistributionType;
-import org.elasticsearch.test.fixtures.minio.MinioTestContainer;
-import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
-import org.junit.ClassRule;
-import org.junit.rules.RuleChain;
-import org.junit.rules.TestRule;
-
-@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
-public class MinioSnapshotRepoTestKitIT extends AbstractSnapshotRepoTestKitRestTestCase {
-
-    public static final MinioTestContainer minioFixture = new MinioTestContainer();
-
-    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .distribution(DistributionType.DEFAULT)
-        .keystore("s3.client.repository_test_kit.access_key", "s3_test_access_key")
-        .keystore("s3.client.repository_test_kit.secret_key", "s3_test_secret_key")
-        .setting("s3.client.repository_test_kit.protocol", () -> "http")
-        .setting("s3.client.repository_test_kit.endpoint", minioFixture::getAddress)
-        .setting("xpack.security.enabled", "false")
-        // Additional tracing related to investigation into https://github.com/elastic/elasticsearch/issues/102294
-        .setting("xpack.ml.enabled", "false")
-        .build();
-
-    @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(minioFixture).around(cluster);
-
-    @Override
-    protected String repositoryType() {
-        return "s3";
-    }
-
-    @Override
-    protected String getTestRestCluster() {
-        return cluster.getHttpAddresses();
-    }
-
-    @Override
-    protected Settings repositorySettings() {
-        final String bucket = "bucket";
-        final String basePath = "repository_test_kit_tests";
-        return Settings.builder().put("client", "repository_test_kit").put("bucket", bucket).put("base_path", basePath).build();
-    }
-}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/minio/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/MinioRepositoryAnalysisRestIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/minio/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/MinioRepositoryAnalysisRestIT.java
new file mode 100644
index 0000000000000..b0068bd7bfdaf
--- /dev/null
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/minio/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/MinioRepositoryAnalysisRestIT.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.fixtures.minio.MinioTestContainer;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
+public class MinioRepositoryAnalysisRestIT extends AbstractRepositoryAnalysisRestTestCase {
+
+    public static final MinioTestContainer minioFixture = new MinioTestContainer();
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .keystore("s3.client.repository_test_kit.access_key", "s3_test_access_key")
+        .keystore("s3.client.repository_test_kit.secret_key", "s3_test_secret_key")
+        .setting("s3.client.repository_test_kit.protocol", () -> "http")
+        .setting("s3.client.repository_test_kit.endpoint", minioFixture::getAddress)
+        .setting("xpack.security.enabled", "false")
+        // Additional tracing related to investigation into https://github.com/elastic/elasticsearch/issues/102294
+        .setting("xpack.ml.enabled", "false")
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(minioFixture).around(cluster);
+
+    @Override
+    protected String repositoryType() {
+        return "s3";
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        final String bucket = "bucket";
+        final String basePath = "repository_test_kit_tests";
+        return Settings.builder().put("client", "repository_test_kit").put("bucket", bucket).put("base_path", basePath).build();
+    }
+}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle
index 17df249b08cf6..8a5dbca7dd0b2 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle
@@ -21,7 +21,7 @@ testClusters.matching { it.name == "yamlRestTest" }.configureEach {
 }
 
 tasks.named('yamlRestTestTestingConventions').configure {
-  baseClass 'org.elasticsearch.repositories.blobstore.testkit.AbstractSnapshotRepoTestKitRestTestCase'
+  baseClass 'org.elasticsearch.repositories.blobstore.testkit.analyze.AbstractRepositoryAnalysisRestTestCase'
   baseClass 'org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase'
 }
 
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/FsRepositoryAnalysisRestIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/FsRepositoryAnalysisRestIT.java
new file mode 100644
index 0000000000000..7151b6e80a4d5
--- /dev/null
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/FsRepositoryAnalysisRestIT.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.fs.FsRepository;
+
+public class FsRepositoryAnalysisRestIT extends AbstractRepositoryAnalysisRestTestCase {
+
+    @Override
+    protected String repositoryType() {
+        return FsRepository.TYPE;
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        return Settings.builder().put("location", System.getProperty("tests.path.repo")).build();
+    }
+}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/java/org/elasticsearch/repositories/blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/java/org/elasticsearch/repositories/blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java
deleted file mode 100644
index 77dfb3902805a..0000000000000
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/java/org/elasticsearch/repositories/blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.repositories.blobstore.testkit.rest;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.repositories.blobstore.testkit.AbstractSnapshotRepoTestKitRestTestCase;
-import org.elasticsearch.repositories.fs.FsRepository;
-
-public class FsSnapshotRepoTestKitIT extends AbstractSnapshotRepoTestKitRestTestCase {
-
-    @Override
-    protected String repositoryType() {
-        return FsRepository.TYPE;
-    }
-
-    @Override
-    protected Settings repositorySettings() {
-        return Settings.builder().put("location", System.getProperty("tests.path.repo")).build();
-    }
-}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/S3SnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/S3SnapshotRepoTestKitIT.java
deleted file mode 100644
index c38bd1204189f..0000000000000
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/S3SnapshotRepoTestKitIT.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.repositories.blobstore.testkit;
-
-import fixture.s3.S3HttpFixture;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.cluster.ElasticsearchCluster;
-import org.elasticsearch.test.cluster.local.distribution.DistributionType;
-import org.junit.ClassRule;
-import org.junit.rules.RuleChain;
-import org.junit.rules.TestRule;
-
-import static org.hamcrest.Matchers.blankOrNullString;
-import static org.hamcrest.Matchers.not;
-
-public class S3SnapshotRepoTestKitIT extends AbstractSnapshotRepoTestKitRestTestCase {
-
-    static final boolean USE_FIXTURE = Boolean.parseBoolean(System.getProperty("tests.use.fixture", "true"));
-
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture(USE_FIXTURE);
-
-    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .distribution(DistributionType.DEFAULT)
-        .keystore("s3.client.repo_test_kit.access_key", System.getProperty("s3AccessKey"))
-        .keystore("s3.client.repo_test_kit.secret_key", System.getProperty("s3SecretKey"))
-        .setting("s3.client.repo_test_kit.protocol", () -> "http", (n) -> USE_FIXTURE)
-        .setting("s3.client.repo_test_kit.endpoint", s3Fixture::getAddress, (n) -> USE_FIXTURE)
-        .setting("xpack.security.enabled", "false")
-        // Additional tracing related to investigation into https://github.com/elastic/elasticsearch/issues/102294
-        .setting("logger.org.elasticsearch.repositories.s3", "TRACE")
-        .setting("logger.org.elasticsearch.repositories.blobstore.testkit", "TRACE")
-        .setting("logger.com.amazonaws.request", "DEBUG")
-        .setting("logger.org.apache.http.wire", "DEBUG")
-        // Necessary to permit setting the above two restricted loggers to DEBUG
-        .jvmArg("-Des.insecure_network_trace_enabled=true")
-        .build();
-
-    @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster);
-
-    @Override
-    protected String getTestRestCluster() {
-        return cluster.getHttpAddresses();
-    }
-
-    @Override
-    protected String repositoryType() {
-        return "s3";
-    }
-
-    @Override
-    protected Settings repositorySettings() {
-        final String bucket = System.getProperty("test.s3.bucket");
-        assertThat(bucket, not(blankOrNullString()));
-
-        final String basePath = System.getProperty("test.s3.base_path");
-        assertThat(basePath, not(blankOrNullString()));
-
-        return Settings.builder()
-            .put("client", "repo_test_kit")
-            .put("bucket", bucket)
-            .put("base_path", basePath)
-            .put("delete_objects_max_size", between(1, 1000))
-            .build();
-    }
-}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/S3RepositoryAnalysisRestIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/S3RepositoryAnalysisRestIT.java
new file mode 100644
index 0000000000000..8986cf1059191
--- /dev/null
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/S3RepositoryAnalysisRestIT.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
+
+import fixture.s3.S3HttpFixture;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.not;
+
+public class S3RepositoryAnalysisRestIT extends AbstractRepositoryAnalysisRestTestCase {
+
+    static final boolean USE_FIXTURE = Boolean.parseBoolean(System.getProperty("tests.use.fixture", "true"));
+
+    public static final S3HttpFixture s3Fixture = new S3HttpFixture(USE_FIXTURE);
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .keystore("s3.client.repo_test_kit.access_key", System.getProperty("s3AccessKey"))
+        .keystore("s3.client.repo_test_kit.secret_key", System.getProperty("s3SecretKey"))
+        .setting("s3.client.repo_test_kit.protocol", () -> "http", (n) -> USE_FIXTURE)
+        .setting("s3.client.repo_test_kit.endpoint", s3Fixture::getAddress, (n) -> USE_FIXTURE)
+        .setting("xpack.security.enabled", "false")
+        // Additional tracing related to investigation into https://github.com/elastic/elasticsearch/issues/102294
+        .setting("logger.org.elasticsearch.repositories.s3", "TRACE")
+        .setting("logger.org.elasticsearch.repositories.blobstore.testkit", "TRACE")
+        .setting("logger.com.amazonaws.request", "DEBUG")
+        .setting("logger.org.apache.http.wire", "DEBUG")
+        // Necessary to permit setting the above two restricted loggers to DEBUG
+        .jvmArg("-Des.insecure_network_trace_enabled=true")
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected String repositoryType() {
+        return "s3";
+    }
+
+    @Override
+    protected Settings repositorySettings() {
+        final String bucket = System.getProperty("test.s3.bucket");
+        assertThat(bucket, not(blankOrNullString()));
+
+        final String basePath = System.getProperty("test.s3.base_path");
+        assertThat(basePath, not(blankOrNullString()));
+
+        return Settings.builder()
+            .put("client", "repo_test_kit")
+            .put("bucket", bucket)
+            .put("base_path", basePath)
+            .put("delete_objects_max_size", between(1, 1000))
+            .build();
+    }
+}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/BytesRegister.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/BytesRegister.java
similarity index 93%
rename from x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/BytesRegister.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/BytesRegister.java
index 4303fff673359..3f5e406ac797b 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/BytesRegister.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/BytesRegister.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisFailureIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java
similarity index 86%
rename from x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisFailureIT.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java
index 2ca5685c83db3..e61f883abd60f 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisFailureIT.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@@ -39,6 +40,7 @@
 import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.repositories.RepositoryVerificationException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
+import org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKit;
 import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
@@ -57,14 +59,16 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.repositories.blobstore.testkit.ContendedRegisterAnalyzeAction.bytesFromLong;
-import static org.elasticsearch.repositories.blobstore.testkit.ContendedRegisterAnalyzeAction.longFromBytes;
+import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING;
+import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.MAX_RESTORE_BYTES_PER_SEC;
+import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.MAX_SNAPSHOT_BYTES_PER_SEC;
+import static org.elasticsearch.repositories.blobstore.testkit.analyze.ContendedRegisterAnalyzeAction.bytesFromLong;
+import static org.elasticsearch.repositories.blobstore.testkit.analyze.ContendedRegisterAnalyzeAction.longFromBytes;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.anyOf;
@@ -86,9 +90,26 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return List.of(TestPlugin.class, LocalStateCompositeXPackPlugin.class, SnapshotRepositoryTestKit.class);
     }
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            // no throttling, so that even analyses which run to completion do not take too long
+            .put(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), ByteSizeValue.ZERO)
+            .build();
+    }
+
     @Before
     public void createBlobStore() {
-        createRepositoryNoVerify("test-repo", TestPlugin.DISRUPTABLE_REPO_TYPE);
+        createRepository(
+            "test-repo",
+            TestPlugin.DISRUPTABLE_REPO_TYPE,
+            randomRepositorySettings()
+                // no throttling, so that even analyses which run to completion do not take too long
+                .put(MAX_SNAPSHOT_BYTES_PER_SEC.getKey(), ByteSizeValue.ZERO)
+                .put(MAX_RESTORE_BYTES_PER_SEC.getKey(), ByteSizeValue.ZERO),
+            false
+        );
 
         blobStore = new DisruptableBlobStore();
         for (final RepositoriesService repositoriesService : internalCluster().getInstances(RepositoriesService.class)) {
@@ -104,8 +125,7 @@ public void testSuccess() {
         final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("test-repo");
         request.blobCount(1);
         request.maxBlobSize(ByteSizeValue.ofBytes(10L));
-
-        analyseRepository(request);
+        safeAwait((ActionListener<RepositoryAnalyzeAction.Response> l) -> analyseRepository(request, l));
     }
 
     public void testFailsOnReadError() {
@@ -124,7 +144,8 @@ public byte[] onRead(byte[] actualContents, long position, long length) throws I
             }
         });
 
-        final Exception exception = expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        final Exception exception = analyseRepositoryExpectFailure(request);
+        assertAnalysisFailureMessage(exception.getMessage());
         final IOException ioException = (IOException) ExceptionsHelper.unwrap(exception, IOException.class);
         assert ioException != null : exception;
         assertThat(ioException.getMessage(), equalTo("simulated"));
@@ -148,7 +169,7 @@ public byte[] onRead(byte[] actualContents, long position, long length) {
             }
         });
 
-        expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        assertAnalysisFailureMessage(analyseRepositoryExpectFailure(request).getMessage());
     }
 
     public void testFailsOnChecksumMismatch() {
@@ -179,7 +200,7 @@ public byte[] onRead(byte[] actualContents, long position, long length) {
             }
         });
 
-        expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        assertAnalysisFailureMessage(analyseRepositoryExpectFailure(request).getMessage());
     }
 
     public void testFailsOnWriteException() {
@@ -200,7 +221,8 @@ public void onWrite() throws IOException {
 
         });
 
-        final Exception exception = expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        final Exception exception = analyseRepositoryExpectFailure(request);
+        assertAnalysisFailureMessage(exception.getMessage());
         final IOException ioException = (IOException) ExceptionsHelper.unwrap(exception, IOException.class);
         assert ioException != null : exception;
         assertThat(ioException.getMessage(), equalTo("simulated"));
@@ -222,7 +244,7 @@ public Map<String, BlobMetadata> onList(Map<String, BlobMetadata> actualListing)
 
         });
 
-        expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        assertAnalysisFailureMessage(analyseRepositoryExpectFailure(request).getMessage());
     }
 
     public void testFailsOnListingException() {
@@ -242,7 +264,7 @@ public Map<String, BlobMetadata> onList(Map<String, BlobMetadata> actualListing)
             }
         });
 
-        expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        assertAnalysisFailureMessage(analyseRepositoryExpectFailure(request).getMessage());
     }
 
     public void testFailsOnDeleteException() {
@@ -257,7 +279,7 @@ public void onDelete() throws IOException {
             }
         });
 
-        expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        assertAnalysisFailureMessage(analyseRepositoryExpectFailure(request).getMessage());
     }
 
     public void testFailsOnIncompleteDelete() {
@@ -285,7 +307,7 @@ public Map<String, BlobMetadata> onList(Map<String, BlobMetadata> actualListing)
             }
         });
 
-        expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        assertAnalysisFailureMessage(analyseRepositoryExpectFailure(request).getMessage());
     }
 
     public void testFailsIfBlobCreatedOnAbort() {
@@ -302,12 +324,21 @@ public boolean createBlobOnAbort() {
             }
         });
 
-        try {
-            analyseRepository(request);
-            assertFalse(writeWasAborted.get());
-        } catch (RepositoryVerificationException e) {
-            assertTrue(writeWasAborted.get());
-        }
+        safeAwait((ActionListener<RepositoryAnalyzeAction.Response> l) -> analyseRepository(request, l.delegateResponse((ll, e) -> {
+            if (ExceptionsHelper.unwrapCause(e) instanceof RepositoryVerificationException repositoryVerificationException) {
+                assertAnalysisFailureMessage(repositoryVerificationException.getMessage());
+                assertTrue(
+                    "did not abort write, so why did the verification fail?",
+                    // clear flag for final assertion
+                    writeWasAborted.compareAndSet(true, false)
+                );
+                ll.onResponse(null);
+            } else {
+                ll.onFailure(e);
+            }
+        })));
+
+        assertFalse(writeWasAborted.get());
     }
 
     public void testFailsIfRegisterIncorrect() {
@@ -324,7 +355,7 @@ public BytesReference onContendedCompareAndExchange(BytesRegister register, Byte
                 return register.compareAndExchange(expected, updated);
             }
         });
-        expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        assertAnalysisFailureMessage(analyseRepositoryExpectFailure(request).getMessage());
     }
 
     public void testFailsIfRegisterHoldsSpuriousValue() {
@@ -354,14 +385,22 @@ public BytesReference onContendedCompareAndExchange(BytesRegister register, Byte
                 return register.compareAndExchange(expected, updated);
             }
         });
-        try {
-            analyseRepository(request);
-            assertFalse(sawSpuriousValue.get());
-        } catch (RepositoryVerificationException e) {
-            if (sawSpuriousValue.get() == false) {
-                fail(e, "did not see spurious value, so why did the verification fail?");
+
+        safeAwait((ActionListener<RepositoryAnalyzeAction.Response> l) -> analyseRepository(request, l.delegateResponse((ll, e) -> {
+            if (ExceptionsHelper.unwrapCause(e) instanceof RepositoryVerificationException repositoryVerificationException) {
+                assertAnalysisFailureMessage(repositoryVerificationException.getMessage());
+                assertTrue(
+                    "did not see spurious value, so why did the verification fail?",
+                    // clear flag for final assertion
+                    sawSpuriousValue.compareAndSet(true, false)
+                );
+                ll.onResponse(null);
+            } else {
+                ll.onFailure(e);
             }
-        }
+        })));
+
+        assertFalse(sawSpuriousValue.get());
     }
 
     private static void assertAnalysisFailureMessage(String message) {
@@ -386,7 +425,7 @@ public boolean compareAndExchangeReturnsWitness(String key) {
                 return isContendedRegisterKey(key) == false;
             }
         });
-        final var exception = expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        final var exception = analyseRepositoryExpectFailure(request);
         assertThat(
             exception.getMessage(),
             allOf(
@@ -408,7 +447,7 @@ public boolean compareAndExchangeReturnsWitness(String key) {
                 return false;
             }
         });
-        final var exception = expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        final var exception = analyseRepositoryExpectFailure(request);
         assertAnalysisFailureMessage(exception.getMessage());
         assertThat(
             asInstanceOf(RepositoryVerificationException.class, ExceptionsHelper.unwrapCause(exception.getCause())).getMessage(),
@@ -424,7 +463,7 @@ public boolean acceptsEmptyRegister() {
                 return false;
             }
         });
-        final var exception = expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request));
+        final var exception = analyseRepositoryExpectFailure(request);
         assertAnalysisFailureMessage(exception.getMessage());
         final var cause = ExceptionsHelper.unwrapCause(exception.getCause());
         if (cause instanceof IOException ioException) {
@@ -440,8 +479,18 @@ public boolean acceptsEmptyRegister() {
         }
     }
 
-    private void analyseRepository(RepositoryAnalyzeAction.Request request) {
-        client().execute(RepositoryAnalyzeAction.INSTANCE, request).actionGet(5L, TimeUnit.MINUTES);
+    private RepositoryVerificationException analyseRepositoryExpectFailure(RepositoryAnalyzeAction.Request request) {
+        return asInstanceOf(
+            RepositoryVerificationException.class,
+            ExceptionsHelper.unwrapCause(safeAwaitFailure(RepositoryAnalyzeAction.Response.class, l -> analyseRepository(request, l)))
+        );
+    }
+
+    private void analyseRepository(RepositoryAnalyzeAction.Request request, ActionListener<RepositoryAnalyzeAction.Response> listener) {
+        client().execute(RepositoryAnalyzeAction.INSTANCE, request, listener.delegateFailureAndWrap((l, response) -> {
+            RepositoryAnalysisSuccessIT.assertNoThrottling(response);
+            l.onResponse(response);
+        }));
     }
 
     private static void assertPurpose(OperationPurpose purpose) {
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisSuccessIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java
similarity index 91%
rename from x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisSuccessIT.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java
index 47b44f41f72d2..bb452ad2a64ce 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisSuccessIT.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java
@@ -5,11 +5,12 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
@@ -26,7 +27,6 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.plugins.Plugin;
@@ -36,8 +36,10 @@
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
+import org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKit;
 import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.junit.Before;
 
@@ -51,15 +53,18 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.repositories.blobstore.testkit.ContendedRegisterAnalyzeAction.longFromBytes;
-import static org.elasticsearch.repositories.blobstore.testkit.RepositoryAnalysisFailureIT.isContendedRegisterKey;
+import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING;
+import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.MAX_RESTORE_BYTES_PER_SEC;
+import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.MAX_SNAPSHOT_BYTES_PER_SEC;
+import static org.elasticsearch.repositories.blobstore.testkit.analyze.ContendedRegisterAnalyzeAction.longFromBytes;
+import static org.elasticsearch.repositories.blobstore.testkit.analyze.RepositoryAnalysisFailureIT.isContendedRegisterKey;
+import static org.elasticsearch.test.XContentTestUtils.convertToMap;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
@@ -83,9 +88,21 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return List.of(TestPlugin.class, LocalStateCompositeXPackPlugin.class, SnapshotRepositoryTestKit.class);
     }
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), ByteSizeValue.ZERO)
+            .build();
+    }
+
     public void testRepositoryAnalysis() {
 
-        final Settings.Builder settings = Settings.builder();
+        final Settings.Builder settings = Settings.builder()
+            // no throttling
+            .put(MAX_SNAPSHOT_BYTES_PER_SEC.getKey(), ByteSizeValue.ZERO)
+            .put(MAX_RESTORE_BYTES_PER_SEC.getKey(), ByteSizeValue.ZERO);
+
         if (randomBoolean()) {
             settings.put(BASE_PATH_SETTING_KEY, randomAlphaOfLength(10));
         }
@@ -136,11 +153,22 @@ public void testRepositoryAnalysis() {
             blobStore.setMaxTotalBlobSize(request.getMaxTotalDataSize().getBytes());
         }
 
-        request.timeout(TimeValue.timeValueSeconds(20));
-
-        client().execute(RepositoryAnalyzeAction.INSTANCE, request).actionGet(30L, TimeUnit.SECONDS);
+        request.timeout(SAFE_AWAIT_TIMEOUT);
+        final RepositoryAnalyzeAction.Response response = safeAwait(l -> client().execute(RepositoryAnalyzeAction.INSTANCE, request, l));
 
         assertThat(blobStore.currentPath, nullValue());
+
+        assertNoThrottling(response);
+    }
+
+    static void assertNoThrottling(RepositoryAnalyzeAction.Response response) {
+        try {
+            final var responseMap = convertToMap(response);
+            assertEquals(Strings.toString(response), 0, (int) ObjectPath.eval("summary.write.total_throttled_nanos", responseMap));
+            assertEquals(Strings.toString(response), 0, (int) ObjectPath.eval("summary.read.total_throttled_nanos", responseMap));
+        } catch (IOException e) {
+            fail(e);
+        }
     }
 
     public static class TestPlugin extends Plugin implements RepositoryPlugin {
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java
index 124174a2a025b..04d59906e6db3 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java
@@ -20,6 +20,8 @@
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.blobstore.testkit.analyze.RepositoryAnalyzeAction;
+import org.elasticsearch.repositories.blobstore.testkit.analyze.RestRepositoryAnalyzeAction;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -51,7 +53,8 @@ public List<RestHandler> getRestHandlers(
         return List.of(new RestRepositoryAnalyzeAction());
     }
 
-    static void humanReadableNanos(XContentBuilder builder, String rawFieldName, String readableFieldName, long nanos) throws IOException {
+    public static void humanReadableNanos(XContentBuilder builder, String rawFieldName, String readableFieldName, long nanos)
+        throws IOException {
         assert rawFieldName.equals(readableFieldName) == false : rawFieldName + " vs " + readableFieldName;
 
         if (builder.humanReadable()) {
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/BlobAnalyzeAction.java
similarity index 99%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/BlobAnalyzeAction.java
index aa0cf3e3cfc1b..6007968d7cb4d 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/BlobAnalyzeAction.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobWriteAbortedException.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/BlobWriteAbortedException.java
similarity index 85%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobWriteAbortedException.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/BlobWriteAbortedException.java
index 11c73993a3e6e..8a7bbb7255c5a 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobWriteAbortedException.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/BlobWriteAbortedException.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 public class BlobWriteAbortedException extends RuntimeException {
     public BlobWriteAbortedException() {
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/ContendedRegisterAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/ContendedRegisterAnalyzeAction.java
similarity index 99%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/ContendedRegisterAnalyzeAction.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/ContendedRegisterAnalyzeAction.java
index 40cb4a45a0339..f527a46371641 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/ContendedRegisterAnalyzeAction.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/ContendedRegisterAnalyzeAction.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GetBlobChecksumAction.java
similarity index 99%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GetBlobChecksumAction.java
index f706ff79bf073..816f9e860a33a 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/GetBlobChecksumAction.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContent.java
similarity index 98%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContent.java
index aa9125f214f58..d5061b303f93d 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContent.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.elasticsearch.repositories.RepositoryVerificationException;
 
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentBytesReference.java
similarity index 97%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentBytesReference.java
index 44627000a2de9..eee40992cb0d7 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentBytesReference.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStream.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentStream.java
similarity index 97%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStream.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentStream.java
index c6163a7ffd82d..15fa370c5fe0a 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStream.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentStream.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import java.io.InputStream;
 
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalyzeAction.java
similarity index 99%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalyzeAction.java
index 494d1d3fedcd9..5ced0176a4f81 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalyzeAction.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -36,7 +36,6 @@
 import org.elasticsearch.common.blobstore.OptionalBytesReference;
 import org.elasticsearch.common.blobstore.support.BlobMetadata;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -46,6 +45,7 @@
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryVerificationException;
@@ -83,10 +83,10 @@
 import java.util.stream.IntStream;
 
 import static org.elasticsearch.core.Strings.format;
-import static org.elasticsearch.repositories.blobstore.testkit.BlobAnalyzeAction.MAX_ATOMIC_WRITE_SIZE;
-import static org.elasticsearch.repositories.blobstore.testkit.ContendedRegisterAnalyzeAction.bytesFromLong;
-import static org.elasticsearch.repositories.blobstore.testkit.ContendedRegisterAnalyzeAction.longFromBytes;
 import static org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKit.humanReadableNanos;
+import static org.elasticsearch.repositories.blobstore.testkit.analyze.BlobAnalyzeAction.MAX_ATOMIC_WRITE_SIZE;
+import static org.elasticsearch.repositories.blobstore.testkit.analyze.ContendedRegisterAnalyzeAction.bytesFromLong;
+import static org.elasticsearch.repositories.blobstore.testkit.analyze.ContendedRegisterAnalyzeAction.longFromBytes;
 
 /**
  * Action which distributes a bunch of {@link BlobAnalyzeAction}s over the nodes in the cluster, with limited concurrency, and collects
@@ -150,7 +150,7 @@ protected void doExecute(Task task, Request request, ActionListener<Response> li
                 request,
                 state.nodes(),
                 state.getMinTransportVersion(),
-                threadPool::relativeTimeInMillis,
+                threadPool.relativeTimeInMillisSupplier(),
                 listener
             ).run();
             return;
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryPerformanceSummary.java
similarity index 98%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryPerformanceSummary.java
index 3ee8805480023..c2625285a8912 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryPerformanceSummary.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RestRepositoryAnalyzeAction.java
similarity index 98%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyzeAction.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RestRepositoryAnalyzeAction.java
index 2a549db8b3255..b0f6b01936ffa 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyzeAction.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RestRepositoryAnalyzeAction.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/UncontendedRegisterAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/UncontendedRegisterAnalyzeAction.java
similarity index 96%
rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/UncontendedRegisterAnalyzeAction.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/UncontendedRegisterAnalyzeAction.java
index 1986b47e3188c..23c25e466b917 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/UncontendedRegisterAnalyzeAction.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/analyze/UncontendedRegisterAnalyzeAction.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -38,8 +38,8 @@
 import java.io.IOException;
 import java.util.Map;
 
-import static org.elasticsearch.repositories.blobstore.testkit.ContendedRegisterAnalyzeAction.bytesFromLong;
-import static org.elasticsearch.repositories.blobstore.testkit.ContendedRegisterAnalyzeAction.longFromBytes;
+import static org.elasticsearch.repositories.blobstore.testkit.analyze.ContendedRegisterAnalyzeAction.bytesFromLong;
+import static org.elasticsearch.repositories.blobstore.testkit.analyze.ContendedRegisterAnalyzeAction.longFromBytes;
 
 class UncontendedRegisterAnalyzeAction extends HandledTransportAction<UncontendedRegisterAnalyzeAction.Request, ActionResponse.Empty> {
 
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java
deleted file mode 100644
index 3af8c118803a7..0000000000000
--- a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.repositories.blobstore.testkit;
-
-import org.apache.http.client.methods.HttpPost;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.rest.ESRestTestCase;
-
-public abstract class AbstractSnapshotRepoTestKitRestTestCase extends ESRestTestCase {
-
-    protected abstract String repositoryType();
-
-    protected abstract Settings repositorySettings();
-
-    public void testRepositoryAnalysis() throws Exception {
-        final String repositoryType = repositoryType();
-        final Settings repositorySettings = repositorySettings();
-
-        final String repository = "repository";
-        logger.info("creating repository [{}] of type [{}]", repository, repositoryType);
-        registerRepository(repository, repositoryType, true, repositorySettings);
-
-        final Request request = new Request(HttpPost.METHOD_NAME, "/_snapshot/" + repository + "/_analyze");
-        request.addParameter("blob_count", "10");
-        request.addParameter("concurrency", "4");
-        request.addParameter("max_blob_size", "1mb");
-        request.addParameter("timeout", "120s");
-        request.addParameter("seed", Long.toString(randomLong()));
-        assertOK(client().performRequest(request));
-    }
-
-}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/AbstractRepositoryAnalysisRestTestCase.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/AbstractRepositoryAnalysisRestTestCase.java
new file mode 100644
index 0000000000000..2c96003f7e3d3
--- /dev/null
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/AbstractRepositoryAnalysisRestTestCase.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
+
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+public abstract class AbstractRepositoryAnalysisRestTestCase extends ESRestTestCase {
+
+    protected abstract String repositoryType();
+
+    protected abstract Settings repositorySettings();
+
+    public void testRepositoryAnalysis() throws Exception {
+        final String repositoryType = repositoryType();
+        final Settings repositorySettings = repositorySettings();
+
+        final String repository = "repository";
+        logger.info("creating repository [{}] of type [{}]", repository, repositoryType);
+        registerRepository(repository, repositoryType, true, repositorySettings);
+
+        final Request request = new Request(HttpPost.METHOD_NAME, "/_snapshot/" + repository + "/_analyze");
+        request.addParameter("blob_count", "10");
+        request.addParameter("concurrency", "4");
+        request.addParameter("max_blob_size", "1mb");
+        request.addParameter("timeout", "120s");
+        request.addParameter("seed", Long.toString(randomLong()));
+        assertOK(client().performRequest(request));
+    }
+
+}
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentBytesReferenceTests.java
similarity index 91%
rename from x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentBytesReferenceTests.java
index c85b634083faf..29a6253c031d8 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentBytesReferenceTests.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.elasticsearch.test.ESTestCase;
 
@@ -13,7 +13,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.zip.CRC32;
 
-import static org.elasticsearch.repositories.blobstore.testkit.RandomBlobContent.BUFFER_SIZE;
+import static org.elasticsearch.repositories.blobstore.testkit.analyze.RandomBlobContent.BUFFER_SIZE;
 import static org.hamcrest.Matchers.equalTo;
 
 public class RandomBlobContentBytesReferenceTests extends ESTestCase {
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStreamTests.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentStreamTests.java
similarity index 97%
rename from x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStreamTests.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentStreamTests.java
index 6c353e0937a33..1854d98f7ec79 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStreamTests.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RandomBlobContentStreamTests.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.elasticsearch.repositories.RepositoryVerificationException;
 import org.elasticsearch.test.ESTestCase;
@@ -14,7 +14,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.zip.CRC32;
 
-import static org.elasticsearch.repositories.blobstore.testkit.RandomBlobContent.BUFFER_SIZE;
+import static org.elasticsearch.repositories.blobstore.testkit.analyze.RandomBlobContent.BUFFER_SIZE;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeActionTests.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalyzeActionTests.java
similarity index 98%
rename from x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeActionTests.java
rename to x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalyzeActionTests.java
index 0d1bdc86002b4..44770e68d714b 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeActionTests.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalyzeActionTests.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.repositories.blobstore.testkit;
+package org.elasticsearch.repositories.blobstore.testkit.analyze;
 
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.test.ESTestCase;
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverPointTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverPointTests.java
index 0563c8f281cb8..f4ee7f264d4f7 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverPointTests.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverPointTests.java
@@ -6,141 +6,23 @@
  */
 package org.elasticsearch.xpack.spatial.search;
 
-import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.geo.ShapeRelation;
-import org.elasticsearch.geometry.GeometryCollection;
-import org.elasticsearch.geometry.Line;
-import org.elasticsearch.geometry.LinearRing;
-import org.elasticsearch.geometry.MultiLine;
-import org.elasticsearch.geometry.MultiPoint;
-import org.elasticsearch.geometry.Point;
-import org.elasticsearch.geometry.Rectangle;
-import org.elasticsearch.geometry.ShapeType;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
-import org.elasticsearch.xpack.spatial.index.query.ShapeQueryBuilder;
-import org.hamcrest.CoreMatchers;
-
-import java.util.List;
 
 public class ShapeQueryOverPointTests extends ShapeQueryTestCase {
     @Override
     protected XContentBuilder createDefaultMapping() throws Exception {
-        XContentBuilder xcb = XContentFactory.jsonBuilder()
+        final boolean isIndexed = randomBoolean();
+        final boolean hasDocValues = isIndexed == false || randomBoolean();
+        return XContentFactory.jsonBuilder()
             .startObject()
             .startObject("properties")
             .startObject(defaultFieldName)
             .field("type", "point")
+            .field("index", isIndexed)
+            .field("doc_values", hasDocValues)
             .endObject()
             .endObject()
             .endObject();
-
-        return xcb;
-    }
-
-    public void testProcessRelationSupport() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate("test").setMapping(mapping).get();
-        ensureGreen();
-
-        Rectangle rectangle = new Rectangle(-35, -25, -25, -35);
-
-        for (ShapeRelation shapeRelation : ShapeRelation.values()) {
-            if (shapeRelation.equals(ShapeRelation.INTERSECTS) == false) {
-                SearchPhaseExecutionException e = expectThrows(
-                    SearchPhaseExecutionException.class,
-                    () -> client().prepareSearch("test")
-                        .setQuery(new ShapeQueryBuilder(defaultFieldName, rectangle).relation(shapeRelation))
-                        .get()
-                );
-                assertThat(
-                    e.getCause().getMessage(),
-                    CoreMatchers.containsString(shapeRelation + " query relation not supported for Field [" + defaultFieldName + "]")
-                );
-            }
-        }
-    }
-
-    public void testQueryLine() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate("test").setMapping(mapping).get();
-        ensureGreen();
-
-        Line line = new Line(new double[] { -25, -25 }, new double[] { -35, -35 });
-
-        try {
-            client().prepareSearch("test").setQuery(new ShapeQueryBuilder(defaultFieldName, line)).get();
-        } catch (SearchPhaseExecutionException e) {
-            assertThat(e.getCause().getMessage(), CoreMatchers.containsString("does not support " + ShapeType.LINESTRING + " queries"));
-        }
-    }
-
-    public void testQueryLinearRing() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate("test").setMapping(mapping).get();
-        ensureGreen();
-
-        LinearRing linearRing = new LinearRing(new double[] { -25, -35, -25 }, new double[] { -25, -35, -25 });
-
-        IllegalArgumentException ex = expectThrows(
-            IllegalArgumentException.class,
-            () -> new ShapeQueryBuilder(defaultFieldName, linearRing)
-        );
-        assertThat(ex.getMessage(), CoreMatchers.containsString("[LINEARRING] geometries are not supported"));
-
-        ex = expectThrows(
-            IllegalArgumentException.class,
-            () -> new ShapeQueryBuilder(defaultFieldName, new GeometryCollection<>(List.of(linearRing)))
-        );
-        assertThat(ex.getMessage(), CoreMatchers.containsString("[LINEARRING] geometries are not supported"));
-    }
-
-    public void testQueryMultiLine() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate("test").setMapping(mapping).get();
-        ensureGreen();
-
-        Line lsb1 = new Line(new double[] { -35, -25 }, new double[] { -35, -25 });
-        Line lsb2 = new Line(new double[] { -15, -5 }, new double[] { -15, -5 });
-
-        MultiLine multiline = new MultiLine(List.of(lsb1, lsb2));
-        try {
-            client().prepareSearch("test").setQuery(new ShapeQueryBuilder(defaultFieldName, multiline)).get();
-        } catch (Exception e) {
-            assertThat(
-                e.getCause().getMessage(),
-                CoreMatchers.containsString("does not support " + ShapeType.MULTILINESTRING + " queries")
-            );
-        }
-    }
-
-    public void testQueryMultiPoint() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate("test").setMapping(mapping).get();
-        ensureGreen();
-
-        MultiPoint multiPoint = new MultiPoint(List.of(new Point(-35, -25), new Point(-15, -5)));
-
-        try {
-            client().prepareSearch("test").setQuery(new ShapeQueryBuilder(defaultFieldName, multiPoint)).get();
-        } catch (Exception e) {
-            assertThat(e.getCause().getMessage(), CoreMatchers.containsString("does not support " + ShapeType.MULTIPOINT + " queries"));
-        }
     }
-
-    public void testQueryPoint() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate("test").setMapping(mapping).get();
-        ensureGreen();
-
-        Point point = new Point(-35, -2);
-
-        try {
-            client().prepareSearch("test").setQuery(new ShapeQueryBuilder(defaultFieldName, point)).get();
-        } catch (Exception e) {
-            assertThat(e.getCause().getMessage(), CoreMatchers.containsString("does not support " + ShapeType.POINT + " queries"));
-        }
-    }
-
 }
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java
index 554c9ff2904dc..1c013aba52261 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java
@@ -51,16 +51,18 @@ public class ShapeQueryOverShapeTests extends ShapeQueryTestCase {
 
     @Override
     protected XContentBuilder createDefaultMapping() throws Exception {
-        XContentBuilder xcb = XContentFactory.jsonBuilder()
+        final boolean isIndexed = randomBoolean();
+        final boolean hasDocValues = isIndexed == false || randomBoolean();
+        return XContentFactory.jsonBuilder()
             .startObject()
             .startObject("properties")
             .startObject(defaultFieldName)
             .field("type", "shape")
+            .field("index", isIndexed)
+            .field("doc_values", hasDocValues)
             .endObject()
             .endObject()
             .endObject();
-
-        return xcb;
     }
 
     @Override
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTestCase.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTestCase.java
index 38d0a30b593b6..1ac6bf3b6fd31 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTestCase.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTestCase.java
@@ -7,16 +7,18 @@
 package org.elasticsearch.xpack.spatial.search;
 
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.geometry.Circle;
+import org.elasticsearch.geometry.GeometryCollection;
+import org.elasticsearch.geometry.Line;
 import org.elasticsearch.geometry.LinearRing;
+import org.elasticsearch.geometry.MultiLine;
+import org.elasticsearch.geometry.MultiPoint;
 import org.elasticsearch.geometry.MultiPolygon;
 import org.elasticsearch.geometry.Point;
 import org.elasticsearch.geometry.Polygon;
 import org.elasticsearch.geometry.Rectangle;
-import org.elasticsearch.geometry.utils.WellKnownText;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -26,6 +28,7 @@
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin;
 import org.elasticsearch.xpack.spatial.index.query.ShapeQueryBuilder;
+import org.hamcrest.CoreMatchers;
 
 import java.util.Collection;
 import java.util.List;
@@ -35,6 +38,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCountAndNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
@@ -46,29 +50,18 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(LocalStateSpatialPlugin.class, LocalStateCompositeXPackPlugin.class);
     }
 
-    protected abstract XContentBuilder createDefaultMapping() throws Exception;
-
-    static String defaultFieldName = "xy";
-    static String defaultIndexName = "test-points";
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
 
-    public void testNullShape() throws Exception {
         String mapping = Strings.toString(createDefaultMapping());
         indicesAdmin().prepareCreate(defaultIndexName).setMapping(mapping).get();
         ensureGreen();
 
         prepareIndex(defaultIndexName).setId("aNullshape")
-            .setSource("{\"geo\": null}", XContentType.JSON)
+            .setSource("{\"" + defaultFieldName + "\": null}", XContentType.JSON)
             .setRefreshPolicy(IMMEDIATE)
             .get();
-        GetResponse result = client().prepareGet(defaultIndexName, "aNullshape").get();
-        assertThat(result.getField("location"), nullValue());
-    };
-
-    public void testIndexPointsFilterRectangle() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate(defaultIndexName).setMapping(mapping).get();
-        ensureGreen();
-
         prepareIndex(defaultIndexName).setId("1")
             .setSource(jsonBuilder().startObject().field("name", "Document 1").field(defaultFieldName, "POINT(-30 -30)").endObject())
             .setRefreshPolicy(IMMEDIATE)
@@ -78,74 +71,82 @@ public void testIndexPointsFilterRectangle() throws Exception {
             .setSource(jsonBuilder().startObject().field("name", "Document 2").field(defaultFieldName, "POINT(-45 -50)").endObject())
             .setRefreshPolicy(IMMEDIATE)
             .get();
+        prepareIndex(defaultIndexName).setId("3")
+            .setSource(jsonBuilder().startObject().field("name", "Document 3").field(defaultFieldName, "POINT(50 50)").endObject())
+            .setRefreshPolicy(IMMEDIATE)
+            .get();
+        prepareIndex(defaultIndexName).setId("4")
+            .setSource(
+                jsonBuilder().startObject()
+                    .field("name", "Document 4")
+                    .field(defaultFieldName, new String[] { "POINT(-30 -30)", "POINT(50 50)" })
+                    .endObject()
+            )
+            .setRefreshPolicy(IMMEDIATE)
+            .get();
+        prepareIndex(defaultIndexName).setId("5")
+            .setSource(
+                jsonBuilder().startObject()
+                    .field("name", "Document 5")
+                    .field(defaultFieldName, new String[] { "POINT(60 60)", "POINT(50 50)" })
+                    .endObject()
+            )
+            .setRefreshPolicy(IMMEDIATE)
+            .get();
+    }
+
+    protected abstract XContentBuilder createDefaultMapping() throws Exception;
+
+    static String defaultFieldName = "xy";
+    static String defaultIndexName = "test-points";
 
+    public void testNullShape() {
+        GetResponse result = client().prepareGet(defaultIndexName, "aNullshape").get();
+        assertThat(result.getField(defaultFieldName), nullValue());
+    };
+
+    public void testIndexPointsFilterRectangle() {
         Rectangle rectangle = new Rectangle(-45, 45, 45, -45);
 
         assertNoFailuresAndResponse(
             client().prepareSearch(defaultIndexName)
                 .setQuery(new ShapeQueryBuilder(defaultFieldName, rectangle).relation(ShapeRelation.INTERSECTS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
-                assertThat(response.getHits().getHits().length, equalTo(1));
-                assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getHits().length, equalTo(2));
+                assertThat(response.getHits().getAt(0).getId(), anyOf(equalTo("1"), equalTo("4")));
+                assertThat(response.getHits().getAt(1).getId(), anyOf(equalTo("1"), equalTo("4")));
             }
         );
 
         // default query, without specifying relation (expect intersects)
-
         assertNoFailuresAndResponse(
             client().prepareSearch(defaultIndexName).setQuery(new ShapeQueryBuilder(defaultFieldName, rectangle)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
-                assertThat(response.getHits().getHits().length, equalTo(1));
-                assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getHits().length, equalTo(2));
+                assertThat(response.getHits().getAt(0).getId(), anyOf(equalTo("1"), equalTo("4")));
+                assertThat(response.getHits().getAt(1).getId(), anyOf(equalTo("1"), equalTo("4")));
             }
         );
     }
 
-    public void testIndexPointsCircle() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate(defaultIndexName).setMapping(mapping).get();
-        ensureGreen();
-
-        prepareIndex(defaultIndexName).setId("1")
-            .setSource(jsonBuilder().startObject().field("name", "Document 1").field(defaultFieldName, "POINT(-30 -30)").endObject())
-            .setRefreshPolicy(IMMEDIATE)
-            .get();
-
-        prepareIndex(defaultIndexName).setId("2")
-            .setSource(jsonBuilder().startObject().field("name", "Document 2").field(defaultFieldName, "POINT(-45 -50)").endObject())
-            .setRefreshPolicy(IMMEDIATE)
-            .get();
-
+    public void testIndexPointsCircle() {
         Circle circle = new Circle(-30, -30, 1);
 
         assertNoFailuresAndResponse(
             client().prepareSearch(defaultIndexName)
                 .setQuery(new ShapeQueryBuilder(defaultFieldName, circle).relation(ShapeRelation.INTERSECTS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(1L));
-                assertThat(response.getHits().getHits().length, equalTo(1));
-                assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+                assertThat(response.getHits().getHits().length, equalTo(2));
+                assertThat(response.getHits().getAt(0).getId(), anyOf(equalTo("1"), equalTo("4")));
+                assertThat(response.getHits().getAt(1).getId(), anyOf(equalTo("1"), equalTo("4")));
             }
         );
     }
 
-    public void testIndexPointsPolygon() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate(defaultIndexName).setMapping(mapping).get();
-        ensureGreen();
-
-        prepareIndex(defaultIndexName).setId("1")
-            .setSource(jsonBuilder().startObject().field(defaultFieldName, "POINT(-30 -30)").endObject())
-            .setRefreshPolicy(IMMEDIATE)
-            .get();
-
-        prepareIndex(defaultIndexName).setId("2")
-            .setSource(jsonBuilder().startObject().field(defaultFieldName, "POINT(-45 -50)").endObject())
-            .setRefreshPolicy(IMMEDIATE)
-            .get();
-
+    public void testIndexPointsPolygon() {
         Polygon polygon = new Polygon(new LinearRing(new double[] { -35, -35, -25, -25, -35 }, new double[] { -35, -25, -25, -35, -35 }));
 
         assertNoFailuresAndResponse(
@@ -153,32 +154,14 @@ public void testIndexPointsPolygon() throws Exception {
                 .setQuery(new ShapeQueryBuilder(defaultFieldName, polygon).relation(ShapeRelation.INTERSECTS)),
             response -> {
                 SearchHits searchHits = response.getHits();
-                assertThat(searchHits.getTotalHits().value, equalTo(1L));
-                assertThat(searchHits.getAt(0).getId(), equalTo("1"));
+                assertThat(searchHits.getTotalHits().value, equalTo(2L));
+                assertThat(searchHits.getAt(0).getId(), anyOf(equalTo("1"), equalTo("4")));
+                assertThat(searchHits.getAt(1).getId(), anyOf(equalTo("1"), equalTo("4")));
             }
         );
     }
 
-    public void testIndexPointsMultiPolygon() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate(defaultIndexName).setMapping(mapping).get();
-        ensureGreen();
-
-        prepareIndex(defaultIndexName).setId("1")
-            .setSource(jsonBuilder().startObject().field("name", "Document 1").field(defaultFieldName, "POINT(-30 -30)").endObject())
-            .setRefreshPolicy(IMMEDIATE)
-            .get();
-
-        prepareIndex(defaultIndexName).setId("2")
-            .setSource(jsonBuilder().startObject().field("name", "Document 2").field(defaultFieldName, "POINT(-40 -40)").endObject())
-            .setRefreshPolicy(IMMEDIATE)
-            .get();
-
-        prepareIndex(defaultIndexName).setId("3")
-            .setSource(jsonBuilder().startObject().field("name", "Document 3").field(defaultFieldName, "POINT(-50 -50)").endObject())
-            .setRefreshPolicy(IMMEDIATE)
-            .get();
-
+    public void testIndexPointsMultiPolygon() {
         Polygon encloseDocument1Shape = new Polygon(
             new LinearRing(new double[] { -35, -35, -25, -25, -35 }, new double[] { -35, -25, -25, -35, -35 })
         );
@@ -192,29 +175,16 @@ public void testIndexPointsMultiPolygon() throws Exception {
             client().prepareSearch(defaultIndexName)
                 .setQuery(new ShapeQueryBuilder(defaultFieldName, mp).relation(ShapeRelation.INTERSECTS)),
             response -> {
-                assertThat(response.getHits().getTotalHits().value, equalTo(2L));
-                assertThat(response.getHits().getHits().length, equalTo(2));
-                assertThat(response.getHits().getAt(0).getId(), not(equalTo("2")));
-                assertThat(response.getHits().getAt(1).getId(), not(equalTo("2")));
+                assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+                assertThat(response.getHits().getHits().length, equalTo(3));
+                assertThat(response.getHits().getAt(0).getId(), not(equalTo("3")));
+                assertThat(response.getHits().getAt(1).getId(), not(equalTo("3")));
+                assertThat(response.getHits().getAt(2).getId(), not(equalTo("3")));
             }
         );
     }
 
-    public void testIndexPointsRectangle() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate(defaultIndexName).setMapping(mapping).get();
-        ensureGreen();
-
-        prepareIndex(defaultIndexName).setId("1")
-            .setSource(jsonBuilder().startObject().field("name", "Document 1").field(defaultFieldName, "POINT(-30 -30)").endObject())
-            .setRefreshPolicy(IMMEDIATE)
-            .get();
-
-        prepareIndex(defaultIndexName).setId("2")
-            .setSource(jsonBuilder().startObject().field("name", "Document 2").field(defaultFieldName, "POINT(-45 -50)").endObject())
-            .setRefreshPolicy(IMMEDIATE)
-            .get();
-
+    public void testIndexPointsRectangle() {
         Rectangle rectangle = new Rectangle(-50, -40, -45, -55);
 
         assertNoFailuresAndResponse(
@@ -229,20 +199,6 @@ public void testIndexPointsRectangle() throws Exception {
     }
 
     public void testIndexPointsIndexedRectangle() throws Exception {
-        String mapping = Strings.toString(createDefaultMapping());
-        indicesAdmin().prepareCreate(defaultIndexName).setMapping(mapping).get();
-        ensureGreen();
-
-        prepareIndex(defaultIndexName).setId("point1")
-            .setSource(jsonBuilder().startObject().field(defaultFieldName, "POINT(-30 -30)").endObject())
-            .setRefreshPolicy(IMMEDIATE)
-            .get();
-
-        prepareIndex(defaultIndexName).setId("point2")
-            .setSource(jsonBuilder().startObject().field(defaultFieldName, "POINT(-45 -50)").endObject())
-            .setRefreshPolicy(IMMEDIATE)
-            .get();
-
         String indexedShapeIndex = "indexed_query_shapes";
         String indexedShapePath = "shape";
         String queryShapesMapping = Strings.toString(
@@ -278,7 +234,7 @@ public void testIndexPointsIndexedRectangle() throws Exception {
             response -> {
                 assertThat(response.getHits().getTotalHits().value, equalTo(1L));
                 assertThat(response.getHits().getHits().length, equalTo(1));
-                assertThat(response.getHits().getAt(0).getId(), equalTo("point2"));
+                assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
             }
         );
 
@@ -291,53 +247,122 @@ public void testIndexPointsIndexedRectangle() throws Exception {
                 ),
             0L
         );
+
     }
 
-    public void testDistanceQuery() throws Exception {
-        indicesAdmin().prepareCreate("test_distance").setMapping("location", "type=shape").get();
-        ensureGreen();
+    public void testDistanceQuery() {
+        Circle circle = new Circle(-25, -25, 10);
 
-        Circle circle = new Circle(1, 0, 10);
-
-        client().index(
-            new IndexRequest("test_distance").source(
-                jsonBuilder().startObject().field("location", WellKnownText.toWKT(new Point(2, 2))).endObject()
-            ).setRefreshPolicy(IMMEDIATE)
-        ).actionGet();
-        client().index(
-            new IndexRequest("test_distance").source(
-                jsonBuilder().startObject().field("location", WellKnownText.toWKT(new Point(3, 1))).endObject()
-            ).setRefreshPolicy(IMMEDIATE)
-        ).actionGet();
-        client().index(
-            new IndexRequest("test_distance").source(
-                jsonBuilder().startObject().field("location", WellKnownText.toWKT(new Point(-20, -30))).endObject()
-            ).setRefreshPolicy(IMMEDIATE)
-        ).actionGet();
-        client().index(
-            new IndexRequest("test_distance").source(
-                jsonBuilder().startObject().field("location", WellKnownText.toWKT(new Point(20, 30))).endObject()
-            ).setRefreshPolicy(IMMEDIATE)
-        ).actionGet();
+        assertHitCount(
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, circle).relation(ShapeRelation.INTERSECTS)),
+            2L
+        );
+
+        assertHitCount(
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, circle).relation(ShapeRelation.WITHIN)),
+            1L
+        );
+
+        assertHitCount(
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, circle).relation(ShapeRelation.DISJOINT)),
+            3L
+        );
 
         assertHitCount(
-            client().prepareSearch("test_distance").setQuery(new ShapeQueryBuilder("location", circle).relation(ShapeRelation.WITHIN)),
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, circle).relation(ShapeRelation.CONTAINS)),
+            0L
+        );
+    }
+
+    public void testIndexPointsQueryLinearRing() {
+        LinearRing linearRing = new LinearRing(new double[] { -50, -50 }, new double[] { 50, 50 });
+
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> new ShapeQueryBuilder(defaultFieldName, linearRing)
+        );
+        assertThat(ex.getMessage(), CoreMatchers.containsString("[LINEARRING] geometries are not supported"));
+
+        ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> new ShapeQueryBuilder(defaultFieldName, new GeometryCollection<>(List.of(linearRing)))
+        );
+        assertThat(ex.getMessage(), CoreMatchers.containsString("[LINEARRING] geometries are not supported"));
+    }
+
+    public void testIndexPointsQueryLine() {
+        Line line = new Line(new double[] { 100, -30 }, new double[] { -100, -30 });
+        assertHitCount(
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, line).relation(ShapeRelation.INTERSECTS)),
             2L
         );
+    }
+
+    public void testIndexPointsQueryMultiLine() {
+        MultiLine multiLine = new MultiLine(
+            List.of(
+                new Line(new double[] { 100, -30 }, new double[] { -100, -30 }),
+                new Line(new double[] { 100, -20 }, new double[] { -100, -20 })
+            )
+        );
 
         assertHitCount(
-            client().prepareSearch("test_distance").setQuery(new ShapeQueryBuilder("location", circle).relation(ShapeRelation.INTERSECTS)),
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, multiLine).relation(ShapeRelation.INTERSECTS)),
             2L
         );
+    }
 
+    public void testIndexPointsQueryPoint() {
+        Point point = new Point(-30, -30);
         assertHitCount(
-            client().prepareSearch("test_distance").setQuery(new ShapeQueryBuilder("location", circle).relation(ShapeRelation.DISJOINT)),
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, point).relation(ShapeRelation.INTERSECTS)),
             2L
         );
+        assertHitCount(
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, point).relation(ShapeRelation.WITHIN)),
+            1L
+        );
+        assertHitCount(
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, point).relation(ShapeRelation.CONTAINS)),
+            2L
+        );
+        assertHitCount(
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, point).relation(ShapeRelation.DISJOINT)),
+            3L
+        );
+    }
 
+    public void testIndexPointsQueryMultiPoint() {
+        MultiPoint multiPoint = new MultiPoint(List.of(new Point(-30, -30), new Point(50, 50)));
         assertHitCount(
-            client().prepareSearch("test_distance").setQuery(new ShapeQueryBuilder("location", circle).relation(ShapeRelation.CONTAINS)),
-            0L
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, multiPoint).relation(ShapeRelation.INTERSECTS)),
+            4L
+        );
+        assertHitCount(
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, multiPoint).relation(ShapeRelation.WITHIN)),
+            3L
+        );
+        assertHitCount(
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, multiPoint).relation(ShapeRelation.CONTAINS)),
+            1L
+        );
+        assertHitCount(
+            client().prepareSearch(defaultIndexName)
+                .setQuery(new ShapeQueryBuilder(defaultFieldName, multiPoint).relation(ShapeRelation.DISJOINT)),
+            1L
         );
     }
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportAction.java
index 0f55905bd0a22..63fd79d736b25 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportAction.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.spatial.action;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportAction.java
index 708594a9a287f..f36ee616996ea 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportAction.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialUsageTransportAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialUsageTransportAction.java
index 40ee3a8dad620..9242961453517 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialUsageTransportAction.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/action/SpatialUsageTransportAction.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java
index d98fe7fdfc6ec..2901e374003dd 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java
@@ -25,11 +25,11 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.lucene.spatial.XYQueriesUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.spatial.common.CartesianPoint;
 import org.elasticsearch.xpack.spatial.index.fielddata.plain.CartesianPointIndexFieldData;
-import org.elasticsearch.xpack.spatial.index.query.ShapeQueryPointProcessor;
 import org.elasticsearch.xpack.spatial.script.field.CartesianPointDocValuesField;
 import org.elasticsearch.xpack.spatial.search.aggregations.support.CartesianPointValuesSourceType;
 
@@ -178,8 +178,6 @@ public FieldMapper.Builder getMergeBuilder() {
 
     public static class PointFieldType extends AbstractPointFieldType<CartesianPoint> implements ShapeQueryable {
 
-        private final ShapeQueryPointProcessor queryProcessor;
-
         private PointFieldType(
             String name,
             boolean indexed,
@@ -190,7 +188,6 @@ private PointFieldType(
             Map<String, String> meta
         ) {
             super(name, indexed, stored, hasDocValues, parser, nullValue, meta);
-            this.queryProcessor = new ShapeQueryPointProcessor();
         }
 
         // only used in test
@@ -215,7 +212,8 @@ public String typeName() {
 
         @Override
         public Query shapeQuery(Geometry shape, String fieldName, ShapeRelation relation, SearchExecutionContext context) {
-            return queryProcessor.shapeQuery(shape, fieldName, relation, context);
+            failIfNotIndexedNorDocValuesFallback(context);
+            return XYQueriesUtils.toXYPointQuery(shape, fieldName, relation, isIndexed(), hasDocValues());
         }
 
         @Override
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java
index 91a118f964064..0ea8c3e22e288 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java
@@ -26,10 +26,12 @@
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
+import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.lucene.spatial.BinaryShapeDocValuesField;
 import org.elasticsearch.lucene.spatial.CartesianShapeIndexer;
 import org.elasticsearch.lucene.spatial.CoordinateEncoder;
+import org.elasticsearch.lucene.spatial.XYQueriesUtils;
 import org.elasticsearch.script.field.AbstractScriptFieldFactory;
 import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
 import org.elasticsearch.script.field.Field;
@@ -38,7 +40,6 @@
 import org.elasticsearch.xpack.spatial.index.fielddata.CartesianShapeValues;
 import org.elasticsearch.xpack.spatial.index.fielddata.plain.AbstractAtomicCartesianShapeFieldData;
 import org.elasticsearch.xpack.spatial.index.fielddata.plain.CartesianShapeIndexFieldData;
-import org.elasticsearch.xpack.spatial.index.query.ShapeQueryProcessor;
 import org.elasticsearch.xpack.spatial.search.aggregations.support.CartesianShapeValuesSourceType;
 
 import java.io.IOException;
@@ -136,8 +137,6 @@ public ShapeFieldMapper build(MapperBuilderContext context) {
 
     public static final class ShapeFieldType extends AbstractShapeGeometryFieldType<Geometry> implements ShapeQueryable {
 
-        private final ShapeQueryProcessor queryProcessor;
-
         public ShapeFieldType(
             String name,
             boolean indexed,
@@ -147,7 +146,6 @@ public ShapeFieldType(
             Map<String, String> meta
         ) {
             super(name, indexed, false, hasDocValues, parser, orientation, meta);
-            this.queryProcessor = new ShapeQueryProcessor();
         }
 
         @Override
@@ -162,7 +160,19 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
 
         @Override
         public Query shapeQuery(Geometry shape, String fieldName, ShapeRelation relation, SearchExecutionContext context) {
-            return queryProcessor.shapeQuery(shape, fieldName, relation, context, hasDocValues());
+            failIfNotIndexedNorDocValuesFallback(context);
+            // CONTAINS queries are not supported by VECTOR strategy for indices created before version 7.5.0 (Lucene 8.3.0);
+            if (relation == ShapeRelation.CONTAINS && context.indexVersionCreated().before(IndexVersions.V_7_5_0)) {
+                throw new QueryShardException(
+                    context,
+                    ShapeRelation.CONTAINS + " query relation not supported for Field [" + fieldName + "]."
+                );
+            }
+            try {
+                return XYQueriesUtils.toXYShapeQuery(shape, fieldName, relation, isIndexed(), hasDocValues());
+            } catch (IllegalArgumentException e) {
+                throw new QueryShardException(context, "Exception creating query on Field [" + fieldName + "] " + e.getMessage(), e);
+            }
         }
 
         @Override
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryPointProcessor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryPointProcessor.java
deleted file mode 100644
index a8c084e7e0f01..0000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryPointProcessor.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.spatial.index.query;
-
-import org.apache.lucene.document.XYDocValuesField;
-import org.apache.lucene.document.XYPointField;
-import org.apache.lucene.geo.XYGeometry;
-import org.apache.lucene.search.IndexOrDocValuesQuery;
-import org.apache.lucene.search.Query;
-import org.elasticsearch.common.geo.LuceneGeometriesUtils;
-import org.elasticsearch.common.geo.ShapeRelation;
-import org.elasticsearch.geometry.Geometry;
-import org.elasticsearch.geometry.ShapeType;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.index.query.SearchExecutionContext;
-import org.elasticsearch.xpack.spatial.index.mapper.PointFieldMapper;
-
-import java.util.function.Consumer;
-
-public class ShapeQueryPointProcessor {
-
-    public Query shapeQuery(Geometry geometry, String fieldName, ShapeRelation relation, SearchExecutionContext context) {
-        final boolean hasDocValues = validateIsPointFieldType(fieldName, context);
-        // only the intersects relation is supported for indexed cartesian point types
-        if (relation != ShapeRelation.INTERSECTS) {
-            throw new QueryShardException(context, relation + " query relation not supported for Field [" + fieldName + "].");
-        }
-        final Consumer<ShapeType> checker = t -> {
-            if (t == ShapeType.POINT || t == ShapeType.MULTIPOINT || t == ShapeType.LINESTRING || t == ShapeType.MULTILINESTRING) {
-                throw new QueryShardException(context, "Field [" + fieldName + "] does not support " + t + " queries");
-            }
-        };
-        final XYGeometry[] luceneGeometries = LuceneGeometriesUtils.toXYGeometry(geometry, checker);
-        Query query = XYPointField.newGeometryQuery(fieldName, luceneGeometries);
-        if (hasDocValues) {
-            final Query queryDocValues = XYDocValuesField.newSlowGeometryQuery(fieldName, luceneGeometries);
-            query = new IndexOrDocValuesQuery(query, queryDocValues);
-        }
-        return query;
-    }
-
-    private boolean validateIsPointFieldType(String fieldName, SearchExecutionContext context) {
-        MappedFieldType fieldType = context.getFieldType(fieldName);
-        if (fieldType instanceof PointFieldMapper.PointFieldType == false) {
-            throw new QueryShardException(
-                context,
-                "Expected " + PointFieldMapper.CONTENT_TYPE + " field type for Field [" + fieldName + "] but found " + fieldType.typeName()
-            );
-        }
-        return fieldType.hasDocValues();
-    }
-}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryProcessor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryProcessor.java
deleted file mode 100644
index cd09b74e99591..0000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryProcessor.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.spatial.index.query;
-
-import org.apache.lucene.document.XYShape;
-import org.apache.lucene.geo.XYGeometry;
-import org.apache.lucene.search.IndexOrDocValuesQuery;
-import org.apache.lucene.search.MatchNoDocsQuery;
-import org.apache.lucene.search.Query;
-import org.elasticsearch.common.geo.LuceneGeometriesUtils;
-import org.elasticsearch.common.geo.ShapeRelation;
-import org.elasticsearch.geometry.Geometry;
-import org.elasticsearch.index.IndexVersions;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.index.query.SearchExecutionContext;
-import org.elasticsearch.lucene.spatial.CartesianShapeDocValuesQuery;
-import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper;
-
-public class ShapeQueryProcessor {
-
-    public Query shapeQuery(
-        Geometry geometry,
-        String fieldName,
-        ShapeRelation relation,
-        SearchExecutionContext context,
-        boolean hasDocValues
-    ) {
-        validateIsShapeFieldType(fieldName, context);
-        // CONTAINS queries are not supported by VECTOR strategy for indices created before version 7.5.0 (Lucene 8.3.0);
-        if (relation == ShapeRelation.CONTAINS && context.indexVersionCreated().before(IndexVersions.V_7_5_0)) {
-            throw new QueryShardException(context, ShapeRelation.CONTAINS + " query relation not supported for Field [" + fieldName + "].");
-        }
-        if (geometry == null || geometry.isEmpty()) {
-            return new MatchNoDocsQuery();
-        }
-        final XYGeometry[] luceneGeometries;
-        try {
-            luceneGeometries = LuceneGeometriesUtils.toXYGeometry(geometry, t -> {});
-        } catch (IllegalArgumentException e) {
-            throw new QueryShardException(context, "Exception creating query on Field [" + fieldName + "] " + e.getMessage(), e);
-        }
-        Query query = XYShape.newGeometryQuery(fieldName, relation.getLuceneRelation(), luceneGeometries);
-        if (hasDocValues) {
-            final Query queryDocValues = new CartesianShapeDocValuesQuery(fieldName, relation.getLuceneRelation(), luceneGeometries);
-            query = new IndexOrDocValuesQuery(query, queryDocValues);
-        }
-        return query;
-    }
-
-    private void validateIsShapeFieldType(String fieldName, SearchExecutionContext context) {
-        MappedFieldType fieldType = context.getFieldType(fieldName);
-        if (fieldType instanceof ShapeFieldMapper.ShapeFieldType == false) {
-            throw new QueryShardException(
-                context,
-                "Expected " + ShapeFieldMapper.CONTENT_TYPE + " field type for Field [" + fieldName + "] but found " + fieldType.typeName()
-            );
-        }
-    }
-}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeScriptFieldTypeTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeScriptFieldTypeTests.java
index 592cb65800b71..5d6f68f8e06a9 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeScriptFieldTypeTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeScriptFieldTypeTests.java
@@ -174,6 +174,11 @@ public boolean needs_score() {
                         return false;
                     }
 
+                    @Override
+                    public boolean needs_termStats() {
+                        return false;
+                    }
+
                     @Override
                     public ScoreScript newInstance(DocReader docReader) {
                         return new ScoreScript(Map.of(), searchContext.lookup(), docReader) {
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderOverPointTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderOverPointTests.java
index db67b1f1e998b..05756168991c9 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderOverPointTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderOverPointTests.java
@@ -30,22 +30,11 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws
 
     @Override
     protected ShapeRelation getShapeRelation(ShapeType type) {
-        return ShapeRelation.INTERSECTS;
+        return randomFrom(ShapeRelation.INTERSECTS, ShapeRelation.CONTAINS, ShapeRelation.DISJOINT, ShapeRelation.WITHIN);
     }
 
     @Override
     protected Geometry getGeometry() {
-        if (randomBoolean()) {
-            if (randomBoolean()) {
-                return ShapeTestUtils.randomMultiPolygon(false);
-            } else {
-                return ShapeTestUtils.randomPolygon(false);
-            }
-        } else if (randomBoolean()) {
-            // it should be a circle
-            return ShapeTestUtils.randomPolygon(false);
-        } else {
-            return ShapeTestUtils.randomRectangle();
-        }
+        return ShapeTestUtils.randomGeometry(false);
     }
 }
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java
index 20b1a906b1dab..66f5597be543e 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java
@@ -27,20 +27,18 @@
 import org.elasticsearch.geometry.utils.WellKnownText;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.GeoShapeIndexer;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.TestIngestDocument;
 import org.elasticsearch.lucene.spatial.CartesianShapeIndexer;
+import org.elasticsearch.lucene.spatial.XYQueriesUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType;
-import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper.ShapeFieldType;
-import org.elasticsearch.xpack.spatial.index.query.ShapeQueryProcessor;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -244,17 +242,12 @@ public void testShapeQuery() throws IOException {
         int numSides = randomIntBetween(4, 1000);
         Geometry geometry = CircleUtils.createRegularShapePolygon(circle, numSides);
 
-        MappedFieldType shapeType = new ShapeFieldType(fieldName, true, true, Orientation.RIGHT, null, Collections.emptyMap());
-
-        ShapeQueryProcessor processor = new ShapeQueryProcessor();
-        SearchExecutionContext mockedContext = mock(SearchExecutionContext.class);
-        when(mockedContext.getFieldType(any())).thenReturn(shapeType);
-        Query sameShapeQuery = processor.shapeQuery(geometry, fieldName, ShapeRelation.INTERSECTS, mockedContext, true);
-        Query centerPointQuery = processor.shapeQuery(
+        Query sameShapeQuery = XYQueriesUtils.toXYShapeQuery(geometry, fieldName, ShapeRelation.INTERSECTS, true, true);
+        Query centerPointQuery = XYQueriesUtils.toXYShapeQuery(
             new Point(circle.getLon(), circle.getLat()),
             fieldName,
             ShapeRelation.INTERSECTS,
-            mockedContext,
+            true,
             true
         );
 
diff --git a/x-pack/plugin/sql/qa/jdbc/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/jdbc/single_node/JdbcShardFailureIT.java b/x-pack/plugin/sql/qa/jdbc/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/jdbc/single_node/JdbcShardFailureIT.java
index f83047411f0b0..0e0f7dc9722d9 100644
--- a/x-pack/plugin/sql/qa/jdbc/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/jdbc/single_node/JdbcShardFailureIT.java
+++ b/x-pack/plugin/sql/qa/jdbc/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/jdbc/single_node/JdbcShardFailureIT.java
@@ -76,7 +76,7 @@ public void createTestIndex() throws IOException {
     public void testPartialResponseHandling() throws SQLException {
         try (Connection c = esJdbc(); Statement s = c.createStatement()) {
             SQLException exception = expectThrows(SQLException.class, () -> s.executeQuery("SELECT * FROM test ORDER BY test_field ASC"));
-            assertThat(exception.getMessage(), containsString("Search rejected due to missing shards"));
+            assertThat(exception.getMessage(), containsString("[open_point_in_time] action requires all shards to be available"));
         }
     }
 }
diff --git a/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlSearchIT.java b/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlSearchIT.java
index 41ebd6adffd41..f05eccb737ca2 100644
--- a/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlSearchIT.java
+++ b/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlSearchIT.java
@@ -13,8 +13,6 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.NotEqualMessageBuilder;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -63,14 +61,7 @@ public void createIndex() throws IOException {
         bwcNodes = new ArrayList<>(nodes.getBWCNodes());
 
         String mappings = readResource(SqlSearchIT.class.getResourceAsStream("/all_field_types.json"));
-        createIndex(
-            index,
-            Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), numShards)
-                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)
-                .build(),
-            mappings
-        );
+        createIndex(index, indexSettings(numShards, numReplicas).build(), mappings);
     }
 
     @After
diff --git a/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShardFailureIT.java b/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShardFailureIT.java
index dc9989b26c3b2..91f3ab029f55c 100644
--- a/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShardFailureIT.java
+++ b/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShardFailureIT.java
@@ -89,7 +89,7 @@ public void testPartialResponseHandling() throws Exception {
         createTestIndex();
         try (Connection c = esJdbc(); Statement s = c.createStatement()) {
             SQLException exception = expectThrows(SQLException.class, () -> s.executeQuery("SELECT * FROM test ORDER BY test_field ASC"));
-            assertThat(exception.getMessage(), containsString("Search rejected due to missing shards"));
+            assertThat(exception.getMessage(), containsString("[open_point_in_time] action requires all shards to be available"));
         }
     }
 
diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec
index 90280676ff6d4..a67ab25bcf66a 100644
--- a/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec
+++ b/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec
@@ -1922,3 +1922,21 @@ SELECT hire_date FROM test_emp WHERE '2005-12-14T00:00:00.000Z'::datetime BETWEE
 ------------------------
 
 ;
+
+// checking regressions after https://github.com/elastic/elasticsearch/pull/110222
+selectDateFunctionsCldr
+schema::month:s|day_of_week:s|day:s|week:s|ad:s|day_of_week2:s|month2:s|ad2:s
+SELECT DATE_FORMAT('2020-04-05T11:22:33.123Z'::date, '%M') AS month,
+DATE_FORMAT('2020-04-05T11:22:33.123Z'::date, '%W') AS day_of_week,
+DATE_FORMAT('2020-04-05T11:22:33.123Z'::date, '%w') AS day,
+DATE_FORMAT('2020-04-05T11:22:33.123Z'::date, '%v') AS week,
+DATETIME_FORMAT('2020-04-05T11:22:33.123Z'::date, 'G') AS ad,
+TO_CHAR('2020-04-05T11:22:33.123Z'::date, 'Day') AS day_of_week2,
+TO_CHAR('2020-04-05T11:22:33.123Z'::date, 'Month') AS month2,
+TO_CHAR('2020-04-05T11:22:33.123Z'::date, 'BC') AS ad2;
+
+    month     |    day_of_week  |  day    |  week   |  ad    |  day_of_week2  |  month2      |  ad2
+--------------+-----------------+---------+---------+--------+----------------+--------------+------------
+  April       |    Sunday       |  0      | 14      | AD     | "Sunday   "    |  "April    " |  AD
+;
+
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlInfoTransportAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlInfoTransportAction.java
index f647c0016cb75..704acd77b5e09 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlInfoTransportAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlInfoTransportAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.sql;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlUsageTransportAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlUsageTransportAction.java
index 5fccc4a3b1214..1cb0e14113394 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlUsageTransportAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlUsageTransportAction.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateFormatter.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateFormatter.java
index 7016d6cb49e46..68f9acd165888 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateFormatter.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateFormatter.java
@@ -42,7 +42,7 @@ class DateFormatter {
             new Builder().pattern("%e").javaPattern("d").build(),
             new Builder().pattern("%f")
                 .javaPattern("n")
-                .additionalMapper(s -> String.format(Locale.ROOT, "%06d", Math.round(Integer.parseInt(s) / 1000.0)))
+                .additionalMapper(s -> String.format(Locale.ENGLISH, "%06d", Math.round(Integer.parseInt(s) / 1000.0)))
                 .build(),
             new Builder().pattern("%H").javaPattern("HH").build(),
             new Builder().pattern("%h").javaPattern("hh").build(),
@@ -59,19 +59,28 @@ class DateFormatter {
             new Builder().pattern("%s").javaPattern("ss").build(),
             new Builder().pattern("%T").javaPattern("HH:mm:ss").build(),
             new Builder().pattern("%U")
-                .javaFormat(t -> String.format(Locale.ROOT, "%02d", t.get(WeekFields.of(DayOfWeek.SUNDAY, 7).weekOfYear())))
+                .javaFormat(t -> String.format(Locale.ENGLISH, "%02d", t.get(WeekFields.of(DayOfWeek.SUNDAY, 7).weekOfYear())))
                 .build(),
-            new Builder().pattern("%u").javaFormat(t -> String.format(Locale.ROOT, "%02d", t.get(WeekFields.ISO.weekOfYear()))).build(),
+            new Builder().pattern("%u")
+                .javaFormat(t -> String.format(Locale.ENGLISH, "%02d", t.get(WeekFields.of(DayOfWeek.MONDAY, 4).weekOfYear())))
+                .build(),
+
             new Builder().pattern("%V")
-                .javaFormat(t -> String.format(Locale.ROOT, "%02d", t.get(WeekFields.of(DayOfWeek.SUNDAY, 7).weekOfWeekBasedYear())))
+                .javaFormat(t -> String.format(Locale.ENGLISH, "%02d", t.get(WeekFields.of(DayOfWeek.SUNDAY, 7).weekOfWeekBasedYear())))
+                .build(),
+            new Builder().pattern("%v")
+                .javaFormat(t -> String.format(Locale.ENGLISH, "%02d", t.get(WeekFields.of(DayOfWeek.MONDAY, 4).weekOfWeekBasedYear())))
                 .build(),
-            new Builder().pattern("%v").javaPattern("ww").build(),
             new Builder().pattern("%W").javaPattern("EEEE").build(),
-            new Builder().pattern("%w").javaPattern("e").additionalMapper(s -> Integer.parseInt(s) == 7 ? String.valueOf(0) : s).build(),
+            new Builder().pattern("%w")
+                .javaFormat(t -> String.format(Locale.ENGLISH, "%01d", t.get(WeekFields.of(DayOfWeek.SUNDAY, 7).dayOfWeek()) - 1))
+                .build(),
             new Builder().pattern("%X")
-                .javaFormat(t -> String.format(Locale.ROOT, "%04d", t.get(WeekFields.of(DayOfWeek.SUNDAY, 7).weekBasedYear())))
+                .javaFormat(t -> String.format(Locale.ENGLISH, "%04d", t.get(WeekFields.of(DayOfWeek.SUNDAY, 7).weekBasedYear())))
+                .build(),
+            new Builder().pattern("%x")
+                .javaFormat(t -> String.format(Locale.ENGLISH, "%04d", t.get(WeekFields.of(DayOfWeek.MONDAY, 7).weekBasedYear())))
                 .build(),
-            new Builder().pattern("%x").javaPattern("Y").build(),
             new Builder().pattern("%Y").javaPattern("yyyy").build(),
             new Builder().pattern("%y").javaPattern("yy").build()
         );
@@ -162,7 +171,7 @@ private Builder pattern(String pattern) {
         }
 
         private Builder javaPattern(String javaPattern) {
-            this.javaFormat = temporalAccessor -> DateTimeFormatter.ofPattern(javaPattern, Locale.ROOT).format(temporalAccessor);
+            this.javaFormat = temporalAccessor -> DateTimeFormatter.ofPattern(javaPattern, Locale.ENGLISH).format(temporalAccessor);
             return this;
         }
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFormatProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFormatProcessor.java
index 14ae0d3d1ed16..422e23f064cbf 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFormatProcessor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFormatProcessor.java
@@ -85,7 +85,7 @@ protected Function<TemporalAccessor, String> formatterFor(String pattern) {
                     return null;
                 }
                 final String javaPattern = msToJavaPattern(pattern);
-                return DateTimeFormatter.ofPattern(javaPattern, Locale.ROOT)::format;
+                return DateTimeFormatter.ofPattern(javaPattern, Locale.ENGLISH)::format;
             }
         },
         DATE_FORMAT {
@@ -97,7 +97,7 @@ protected Function<TemporalAccessor, String> formatterFor(String pattern) {
         DATE_TIME_FORMAT {
             @Override
             protected Function<TemporalAccessor, String> formatterFor(String pattern) {
-                return DateTimeFormatter.ofPattern(pattern, Locale.ROOT)::format;
+                return DateTimeFormatter.ofPattern(pattern, Locale.ENGLISH)::format;
             }
         },
         TO_CHAR {
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessor.java
index 1d2e2735d86e3..c62f4303d93d6 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessor.java
@@ -40,8 +40,8 @@ public final String extract(ZonedDateTime millis, String tzId) {
     }
 
     public static final String NAME = "ndt";
-    private static final DateTimeFormatter DAY_NAME_FORMATTER = DateTimeFormatter.ofPattern("EEEE", Locale.ROOT);
-    private static final DateTimeFormatter MONTH_NAME_FORMATTER = DateTimeFormatter.ofPattern("MMMM", Locale.ROOT);
+    private static final DateTimeFormatter DAY_NAME_FORMATTER = DateTimeFormatter.ofPattern("EEEE", Locale.ENGLISH);
+    private static final DateTimeFormatter MONTH_NAME_FORMATTER = DateTimeFormatter.ofPattern("MMMM", Locale.ENGLISH);
 
     private final NameExtractor extractor;
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/ToCharFormatter.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/ToCharFormatter.java
index 6e7b0fcb47a06..4c9e851f54e36 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/ToCharFormatter.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/ToCharFormatter.java
@@ -41,7 +41,7 @@ class ToCharFormatter {
             of("HH12").formatFn("hh").numeric(),
             of("HH24").formatFn("HH").numeric(),
             of("MI").formatFn("mm").numeric(),
-            of("SS").formatFn("s", x -> String.format(Locale.ROOT, "%02d", parseInt(x))).numeric(),
+            of("SS").formatFn("s", x -> String.format(Locale.ENGLISH, "%02d", parseInt(x))).numeric(),
             of("MS").formatFn("n", nano -> firstDigitsOfNanos(nano, 3)).numericWithLeadingZeros(),
             of("US").formatFn("n", nano -> firstDigitsOfNanos(nano, 6)).numericWithLeadingZeros(),
             of("FF1").formatFn("n", nano -> firstDigitsOfNanos(nano, 1)).numericWithLeadingZeros(),
@@ -52,14 +52,14 @@ class ToCharFormatter {
             of("FF6").formatFn("n", nano -> firstDigitsOfNanos(nano, 6)).numericWithLeadingZeros(),
             of("SSSSS").formatFn("A", milliSecondOfDay -> String.valueOf(parseInt(milliSecondOfDay) / 1000)).numeric(),
             of("SSSS").formatFn("A", milliSecondOfDay -> String.valueOf(parseInt(milliSecondOfDay) / 1000)).numeric(),
-            of("AM").formatFn("a", x -> x.toUpperCase(Locale.ROOT)).text(),
-            of("am").formatFn("a", x -> x.toLowerCase(Locale.ROOT)).text(),
-            of("PM").formatFn("a", x -> x.toUpperCase(Locale.ROOT)).text(),
-            of("pm").formatFn("a", x -> x.toLowerCase(Locale.ROOT)).text(),
+            of("AM").formatFn("a", x -> x.toUpperCase(Locale.ENGLISH)).text(),
+            of("am").formatFn("a", x -> x.toLowerCase(Locale.ENGLISH)).text(),
+            of("PM").formatFn("a", x -> x.toUpperCase(Locale.ENGLISH)).text(),
+            of("pm").formatFn("a", x -> x.toLowerCase(Locale.ENGLISH)).text(),
             of("A.M.").formatFn("a", x -> x.charAt(0) + "." + x.charAt(1) + ".").text(),
-            of("a.m.").formatFn("a", x -> (x.charAt(0) + "." + x.charAt(1) + ".").toLowerCase(Locale.ROOT)).text(),
+            of("a.m.").formatFn("a", x -> (x.charAt(0) + "." + x.charAt(1) + ".").toLowerCase(Locale.ENGLISH)).text(),
             of("P.M.").formatFn("a", x -> x.charAt(0) + "." + x.charAt(1) + ".").text(),
-            of("p.m.").formatFn("a", x -> (x.charAt(0) + "." + x.charAt(1) + ".").toLowerCase(Locale.ROOT)).text(),
+            of("p.m.").formatFn("a", x -> (x.charAt(0) + "." + x.charAt(1) + ".").toLowerCase(Locale.ENGLISH)).text(),
             of("Y,YYY").formatFn("yyyy", year -> year.charAt(0) + "," + year.substring(1)).numericWithLeadingZeros(),
             of("YYYY").formatFn("yyyy").numeric(),
             of("YYY").formatFn("yyyy", year -> year.substring(1)).numeric(),
@@ -70,51 +70,53 @@ class ToCharFormatter {
             of("IY").formatFn(t -> lastNCharacter(absoluteWeekBasedYear(t), 2)).numeric(),
             of("I").formatFn(t -> lastNCharacter(absoluteWeekBasedYear(t), 1)).numeric(),
             of("BC").formatFn("G").text(),
-            of("bc").formatFn("G", x -> x.toLowerCase(Locale.ROOT)).text(),
+            of("bc").formatFn("G", x -> x.toLowerCase(Locale.ENGLISH)).text(),
             of("AD").formatFn("G").text(),
-            of("ad").formatFn("G", x -> x.toLowerCase(Locale.ROOT)).text(),
+            of("ad").formatFn("G", x -> x.toLowerCase(Locale.ENGLISH)).text(),
             of("B.C.").formatFn("G", x -> x.charAt(0) + "." + x.charAt(1) + ".").text(),
-            of("b.c.").formatFn("G", x -> (x.charAt(0) + "." + x.charAt(1) + ".").toLowerCase(Locale.ROOT)).text(),
+            of("b.c.").formatFn("G", x -> (x.charAt(0) + "." + x.charAt(1) + ".").toLowerCase(Locale.ENGLISH)).text(),
             of("A.D.").formatFn("G", x -> x.charAt(0) + "." + x.charAt(1) + ".").text(),
-            of("a.d.").formatFn("G", x -> (x.charAt(0) + "." + x.charAt(1) + ".").toLowerCase(Locale.ROOT)).text(),
-            of("MONTH").formatFn("MMMM", x -> String.format(Locale.ROOT, "%-9s", x.toUpperCase(Locale.ROOT))).text(),
-            of("Month").formatFn("MMMM", x -> String.format(Locale.ROOT, "%-9s", x)).text(),
-            of("month").formatFn("MMMM", x -> String.format(Locale.ROOT, "%-9s", x.toLowerCase(Locale.ROOT))).text(),
-            of("MON").formatFn("MMM", x -> x.toUpperCase(Locale.ROOT)).text(),
+            of("a.d.").formatFn("G", x -> (x.charAt(0) + "." + x.charAt(1) + ".").toLowerCase(Locale.ENGLISH)).text(),
+            of("MONTH").formatFn("MMMM", x -> String.format(Locale.ENGLISH, "%-9s", x.toUpperCase(Locale.ENGLISH))).text(),
+            of("Month").formatFn("MMMM", x -> String.format(Locale.ENGLISH, "%-9s", x)).text(),
+            of("month").formatFn("MMMM", x -> String.format(Locale.ENGLISH, "%-9s", x.toLowerCase(Locale.ENGLISH))).text(),
+            of("MON").formatFn("MMM", x -> x.toUpperCase(Locale.ENGLISH)).text(),
             of("Mon").formatFn("MMM").text(),
-            of("mon").formatFn("MMM", x -> x.toLowerCase(Locale.ROOT)).text(),
+            of("mon").formatFn("MMM", x -> x.toLowerCase(Locale.ENGLISH)).text(),
             of("MM").formatFn("MM").numeric(),
-            of("DAY").formatFn("EEEE", x -> String.format(Locale.ROOT, "%-9s", x.toUpperCase(Locale.ROOT))).text(),
-            of("Day").formatFn("EEEE", x -> String.format(Locale.ROOT, "%-9s", x)).text(),
-            of("day").formatFn("EEEE", x -> String.format(Locale.ROOT, "%-9s", x.toLowerCase(Locale.ROOT))).text(),
-            of("DY").formatFn("E", x -> x.toUpperCase(Locale.ROOT)).text(),
+            of("DAY").formatFn("EEEE", x -> String.format(Locale.ENGLISH, "%-9s", x.toUpperCase(Locale.ENGLISH))).text(),
+            of("Day").formatFn("EEEE", x -> String.format(Locale.ENGLISH, "%-9s", x)).text(),
+            of("day").formatFn("EEEE", x -> String.format(Locale.ENGLISH, "%-9s", x.toLowerCase(Locale.ENGLISH))).text(),
+            of("DY").formatFn("E", x -> x.toUpperCase(Locale.ENGLISH)).text(),
             of("Dy").formatFn("E").text(),
-            of("dy").formatFn("E", x -> x.toLowerCase(Locale.ROOT)).text(),
+            of("dy").formatFn("E", x -> x.toLowerCase(Locale.ENGLISH)).text(),
             of("DDD").formatFn("DDD").numeric(),
             of("IDDD").formatFn(
                 t -> String.format(
-                    Locale.ROOT,
+                    Locale.ENGLISH,
                     "%03d",
                     (t.get(WeekFields.ISO.weekOfWeekBasedYear()) - 1) * 7 + t.get(ChronoField.DAY_OF_WEEK)
                 )
             ).numeric(),
-            of("DD").formatFn("d", x -> String.format(Locale.ROOT, "%02d", parseInt(x))).numeric(),
+            of("DD").formatFn("d", x -> String.format(Locale.ENGLISH, "%02d", parseInt(x))).numeric(),
             of("ID").formatFn(t -> String.valueOf(t.get(ChronoField.DAY_OF_WEEK))).numeric(),
             of("D").formatFn(t -> String.valueOf(t.get(WeekFields.SUNDAY_START.dayOfWeek()))).numeric(),
             of("W").formatFn(t -> String.valueOf(t.get(ChronoField.ALIGNED_WEEK_OF_MONTH))).numeric(),
-            of("WW").formatFn(t -> String.format(Locale.ROOT, "%02d", t.get(ChronoField.ALIGNED_WEEK_OF_YEAR))).numeric(),
-            of("IW").formatFn(t -> String.format(Locale.ROOT, "%02d", t.get(WeekFields.ISO.weekOfWeekBasedYear()))).numeric(),
+            of("WW").formatFn(t -> String.format(Locale.ENGLISH, "%02d", t.get(ChronoField.ALIGNED_WEEK_OF_YEAR))).numeric(),
+            of("IW").formatFn(t -> String.format(Locale.ENGLISH, "%02d", t.get(WeekFields.ISO.weekOfWeekBasedYear()))).numeric(),
             of("CC").formatFn(t -> {
                 int century = yearToCentury(t.get(ChronoField.YEAR));
-                return String.format(Locale.ROOT, century < 0 ? "%03d" : "%02d", century);
+                return String.format(Locale.ENGLISH, century < 0 ? "%03d" : "%02d", century);
             }).numeric(),
             of("J").formatFn(t -> String.valueOf(t.getLong(JulianFields.JULIAN_DAY))).numeric(),
             of("Q").formatFn("Q").numeric(),
-            of("RM").formatFn("MM", month -> String.format(Locale.ROOT, "%-4s", monthToRoman(parseInt(month)))).text(),
-            of("rm").formatFn("MM", month -> String.format(Locale.ROOT, "%-4s", monthToRoman(parseInt(month)).toLowerCase(Locale.ROOT)))
-                .text(),
+            of("RM").formatFn("MM", month -> String.format(Locale.ENGLISH, "%-4s", monthToRoman(parseInt(month)))).text(),
+            of("rm").formatFn(
+                "MM",
+                month -> String.format(Locale.ENGLISH, "%-4s", monthToRoman(parseInt(month)).toLowerCase(Locale.ENGLISH))
+            ).text(),
             of("TZ").formatFn(ToCharFormatter::zoneAbbreviationOf).text(),
-            of("tz").formatFn(t -> zoneAbbreviationOf(t).toLowerCase(Locale.ROOT)).text(),
+            of("tz").formatFn(t -> zoneAbbreviationOf(t).toLowerCase(Locale.ENGLISH)).text(),
             of("TZH").acceptsLowercase(false).formatFn("ZZ", s -> s.substring(0, 3)).text(),
             of("TZM").acceptsLowercase(false).formatFn("ZZ", s -> lastNCharacter(s, 2)).text(),
             of("OF").acceptsLowercase(false).formatFn("ZZZZZ", ToCharFormatter::formatOffset).offset()
@@ -127,7 +129,7 @@ class ToCharFormatter {
         // also index the lower case version of the patterns if accepted
         for (ToCharFormatter formatter : formatters) {
             if (formatter.acceptsLowercase) {
-                formatterMap.putIfAbsent(formatter.pattern.toLowerCase(Locale.ROOT), formatter);
+                formatterMap.putIfAbsent(formatter.pattern.toLowerCase(Locale.ENGLISH), formatter);
             }
         }
         FORMATTER_MAP = formatterMap;
@@ -274,8 +276,8 @@ private static String appendOrdinalSuffix(String defaultSuffix, String s) {
             // the Y,YYY pattern might can cause problems with the parsing, but thankfully the last 3
             // characters is enough to calculate the suffix
             int i = parseInt(lastNCharacter(s, 3));
-            final boolean upperCase = defaultSuffix.equals(defaultSuffix.toUpperCase(Locale.ROOT));
-            return s + (upperCase ? ordinalSuffix(i).toUpperCase(Locale.ROOT) : ordinalSuffix(i));
+            final boolean upperCase = defaultSuffix.equals(defaultSuffix.toUpperCase(Locale.ENGLISH));
+            return s + (upperCase ? ordinalSuffix(i).toUpperCase(Locale.ENGLISH) : ordinalSuffix(i));
         } catch (NumberFormatException ex) {
             return s + defaultSuffix;
         }
@@ -312,11 +314,11 @@ private static String removeLeadingZerosFromOffset(String offset) {
     private static String absoluteWeekBasedYear(TemporalAccessor t) {
         int year = t.get(IsoFields.WEEK_BASED_YEAR);
         year = year > 0 ? year : -(year - 1);
-        return String.format(Locale.ROOT, "%04d", year);
+        return String.format(Locale.ENGLISH, "%04d", year);
     }
 
     private static String firstDigitsOfNanos(String nano, int digits) {
-        return String.format(Locale.ROOT, "%09d", parseInt(nano)).substring(0, digits);
+        return String.format(Locale.ENGLISH, "%09d", parseInt(nano)).substring(0, digits);
     }
 
     private static String lastNCharacter(String s, int n) {
@@ -324,7 +326,7 @@ private static String lastNCharacter(String s, int n) {
     }
 
     private static String zoneAbbreviationOf(TemporalAccessor temporalAccessor) {
-        String zone = ZoneId.from(temporalAccessor).getDisplayName(TextStyle.SHORT, Locale.ROOT);
+        String zone = ZoneId.from(temporalAccessor).getDisplayName(TextStyle.SHORT, Locale.ENGLISH);
         return "Z".equals(zone) ? "UTC" : zone;
     }
 
@@ -344,7 +346,7 @@ public Builder formatFn(final String javaPattern) {
 
         public Builder formatFn(final String javaPattern, final Function<String, String> additionalMapper) {
             this.formatFn = temporalAccessor -> {
-                String formatted = DateTimeFormatter.ofPattern(javaPattern != null ? javaPattern : "'" + pattern + "'", Locale.ROOT)
+                String formatted = DateTimeFormatter.ofPattern(javaPattern != null ? javaPattern : "'" + pattern + "'", Locale.ENGLISH)
                     .format(temporalAccessor);
                 return additionalMapper == null ? formatted : additionalMapper.apply(formatted);
             };
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlAsyncGetResultsAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlAsyncGetResultsAction.java
index 367f5e93fd6f8..b76ee154aa1a4 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlAsyncGetResultsAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlAsyncGetResultsAction.java
@@ -9,10 +9,10 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.ql.plugin.AbstractTransportQlAsyncGetResultsAction;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlAsyncGetStatusAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlAsyncGetStatusAction.java
index 906fe6ec1d140..04cf892d7cfc7 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlAsyncGetStatusAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlAsyncGetStatusAction.java
@@ -9,10 +9,10 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.ql.plugin.AbstractTransportQlAsyncGetStatusAction;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java
index 27400c7393d83..8347642667273 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlClearCursorAction.java
@@ -9,8 +9,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.sql.action.SqlClearCursorRequest;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java
index e39420d1fbefb..7a76ffe8eb109 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java
@@ -14,12 +14,12 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlStatsAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlStatsAction.java
index 4004724019af8..c334c5779050a 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlStatsAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlStatsAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlTranslateAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlTranslateAction.java
index 75663b38c9c2f..c79162d3e1f26 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlTranslateAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlTranslateAction.java
@@ -10,9 +10,9 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java
index 10d6b04d7505c..3e1f910c9f72e 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/CancellationTests.java
@@ -190,7 +190,7 @@ public void testCancellationDuringSearch(String query) throws InterruptedExcepti
         doAnswer(invocation -> {
             @SuppressWarnings("unchecked")
             ActionListener<OpenPointInTimeResponse> listener = (ActionListener<OpenPointInTimeResponse>) invocation.getArguments()[2];
-            listener.onResponse(new OpenPointInTimeResponse(pitId));
+            listener.onResponse(new OpenPointInTimeResponse(pitId, 1, 1, 0, 0));
             return null;
         }).when(client).execute(eq(TransportOpenPointInTimeAction.TYPE), any(), any());
 
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/40_view_role_descriptors.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/40_view_role_descriptors.yml
index a7fb00a052528..e9f8359e41ce3 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/40_view_role_descriptors.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/40_view_role_descriptors.yml
@@ -21,7 +21,7 @@ setup:
               ],
               "applications": [
                 {
-                  "application": "myapp",
+                  "application": "apm",
                   "privileges": ["*"],
                   "resources": ["*"]
                 }
@@ -497,7 +497,7 @@ teardown:
       ],
       "applications" : [
         {
-          "application" : "myapp",
+          "application" : "apm",
           "privileges" : [
             "*"
           ],
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/60_admin_user.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/60_admin_user.yml
index 1a85dda7379be..643287bad4833 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/60_admin_user.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/60_admin_user.yml
@@ -15,14 +15,14 @@ setup:
       security.put_privileges:
         body: >
           {
-            "myapp": {
+            "apm": {
               "read": {
-                "application": "myapp",
+                "application": "apm",
                 "name": "read",
                 "actions": [ "data:read/*" ]
               },
               "write": {
-                "application": "myapp",
+                "application": "apm",
                 "name": "write",
                 "actions": [ "data:write/*" ]
               }
@@ -33,7 +33,7 @@ setup:
 teardown:
   - do:
       security.delete_privileges:
-        application: myapp
+        application: apm
         name: "read,write"
         ignore: 404
 
@@ -254,7 +254,7 @@ teardown:
                   ],
                   "applications": [
                     {
-                      "application": "myapp",
+                      "application": "apm",
                       "privileges": ["read"],
                       "resources": ["*"]
                     }
@@ -299,7 +299,7 @@ teardown:
             ],
             "application": [
               {
-                "application" : "myapp",
+                "application" : "apm",
                 "resources" : [ "*", "some-other-res" ],
                 "privileges" : [ "data:read/me", "data:write/me" ]
               }
@@ -324,7 +324,7 @@ teardown:
       }
     } }
   - match: { "application" : {
-      "myapp" : {
+      "apm" : {
         "*" : {
           "data:read/me" : true,
           "data:write/me" : false
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml
index 1c95e961d0535..e168fc589d11f 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml
@@ -109,6 +109,30 @@ setup:
           - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275225, "color": "green" }
           - { "index": { } }
           - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275226, "color": "red" }
+  - do:
+      indices.create:
+        index:  test_range
+        body:
+          settings:
+            number_of_shards: 5
+          mappings:
+            properties:
+              date_1:
+                type: date
+                format: "dd-MM-yyyy-MM-dd HH:mm Z"
+              date_2:
+                type: date
+  - do:
+      bulk:
+        index: "test_range"
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "date_1": "12-01-1981-01-12 15:00 +0000", "date_2": "2000" }
+          - { "index": { } }
+          - { "date_1": "31-12-1999-12-31 23:59 +0300" }
+          - { "index": { } }
+          - { "date_1": "31-12-1999-12-31 23:59 -0300" }
 
 ---
 "Test From":
@@ -380,3 +404,21 @@ version is not allowed:
         body:
           query: 'from test'
           version: cat
+
+---
+"Test Internal Range":
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'from test_range | where date_1 > "1990" and date_1 < "2000" or date_2 >= "2000" | sort date_1'
+
+  - length: {columns: 2}
+  - match: {columns.0.name: "date_1"}
+  - match: {columns.0.type: "date"}
+  - match: {columns.1.name: "date_2"}
+  - match: {columns.1.type: "date"}
+  - length: {values: 2}
+  - match: {values.0: ["1981-01-12T15:00:00.000Z","2000-01-01T00:00:00.000Z"]}
+  - match: {values.1: ["1999-12-31T20:59:00.000Z", null]}
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml
index 053d33ee9bf43..ad3adbf93adaf 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml
@@ -115,6 +115,14 @@ geo_point unsortable:
         body:
           query: 'from geo_points | sort location'
 
+---
+geo_point unsortable status:
+  - do:
+      catch: bad_request
+      esql.query:
+        body:
+          query: 'from geo_points | sort location'
+
 ---
 geo_point unsortable with limit:
   - do:
@@ -123,6 +131,14 @@ geo_point unsortable with limit:
         body:
           query: 'from geo_points | LIMIT 10 | sort location'
 
+---
+geo_point unsortable with limit status:
+  - do:
+      catch: bad_request
+      esql.query:
+        body:
+          query: 'from geo_points | LIMIT 10 | sort location'
+
 ---
 geo_point unsortable with limit from row:
   - do:
@@ -134,7 +150,15 @@ geo_point unsortable with limit from row:
 ---
 values unsupported for geo_point:
   - do:
-      catch: '/.+argument of \[VALUES\(location\)\] must be \[boolean, date, double, integer, ip, keyword, long, null, text, unsigned_long or version\].+/'
+      catch: '/.+argument of \[VALUES\(location\)\] must be .+/'
+      esql.query:
+        body:
+          query: 'FROM geo_points | STATS VALUES(location)'
+
+---
+values unsupported for geo_point status code:
+  - do:
+      catch: bad_request
       esql.query:
         body:
           query: 'FROM geo_points | STATS VALUES(location)'
@@ -165,6 +189,14 @@ cartesian_point unsortable:
         body:
           query: 'from cartesian_points | sort location'
 
+---
+cartesian_point unsortable status code:
+  - do:
+      catch: bad_request
+      esql.query:
+        body:
+          query: 'from cartesian_points | sort location'
+
 ---
 cartesian_point unsortable with limit:
   - do:
@@ -173,6 +205,14 @@ cartesian_point unsortable with limit:
         body:
           query: 'from cartesian_points | LIMIT 10 | sort location'
 
+---
+cartesian_point unsortable with limit status code:
+  - do:
+      catch: bad_request
+      esql.query:
+        body:
+          query: 'from cartesian_points | LIMIT 10 | sort location'
+
 ---
 cartesian_point unsortable with limit from row:
   - do:
@@ -207,6 +247,14 @@ geo_shape unsortable:
         body:
           query: 'from geo_shapes | sort shape'
 
+---
+geo_shape unsortable status code:
+  - do:
+      catch: bad_request
+      esql.query:
+        body:
+          query: 'from geo_shapes | sort shape'
+
 ---
 geo_shape unsortable with limit:
   - do:
@@ -215,6 +263,14 @@ geo_shape unsortable with limit:
         body:
           query: 'from geo_shapes | LIMIT 10 | sort shape'
 
+---
+geo_shape unsortable with limit status code:
+  - do:
+      catch: bad_request
+      esql.query:
+        body:
+          query: 'from geo_shapes | LIMIT 10 | sort shape'
+
 ---
 geo_shape unsortable with limit from row:
   - do:
@@ -249,6 +305,14 @@ cartesian_shape unsortable:
         body:
           query: 'from cartesian_shapes | sort shape'
 
+---
+cartesian_shape unsortable status code:
+  - do:
+      catch: bad_request
+      esql.query:
+        body:
+          query: 'from cartesian_shapes | sort shape'
+
 ---
 cartesian_shape unsortable with limit:
   - do:
@@ -257,6 +321,14 @@ cartesian_shape unsortable with limit:
         body:
           query: 'from cartesian_shapes | LIMIT 10 | sort shape'
 
+---
+cartesian_shape unsortable with limit status code:
+  - do:
+      catch: bad_request
+      esql.query:
+        body:
+          query: 'from cartesian_shapes | LIMIT 10 | sort shape'
+
 ---
 cartesian_shape unsortable with limit from row:
   - do:
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml
index 003b1d0651d11..92b3f4d1b084d 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml
@@ -296,12 +296,44 @@ load two indices, showing unsupported type and null value for event_duration:
 
 ---
 load two indices with no conversion function, but needs TO_LONG conversion:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [union_types_fix_rename_resolution]
+      reason: "Union type resolution fix for rename also allows direct usage of unsupported fields in KEEP"
+
   - do:
-      catch: '/Cannot use field \[event_duration\] due to ambiguities being mapped as \[2\] incompatible types: \[keyword\] in \[events_ip_keyword\], \[long\] in \[events_ip_long\]/'
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
       esql.query:
         body:
           query: 'FROM events_ip_* METADATA _index | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC'
 
+  - match: { columns.0.name: "_index" }
+  - match: { columns.0.type: "keyword" }
+  - match: { columns.1.name: "@timestamp" }
+  - match: { columns.1.type: "date" }
+  - match: { columns.2.name: "client_ip" }
+  - match: { columns.2.type: "ip" }
+  - match: { columns.3.name: "event_duration" }
+  - match: { columns.3.type: "unsupported" }
+  - match: { columns.4.name: "message" }
+  - match: { columns.4.type: "keyword" }
+  - length: { values: 14 }
+  - match: { values.0.0: "events_ip_keyword" }
+  - match: { values.0.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.0.2: "172.21.3.15" }
+  - match: { values.0.3: null }
+  - match: { values.0.4: "Connected to 10.1.0.1" }
+  - match: { values.7.0: "events_ip_long" }
+  - match: { values.7.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.7.2: "172.21.3.15" }
+  - match: { values.7.3: null }
+  - match: { values.7.4: "Connected to 10.1.0.1" }
+
 ---
 load two indices with incorrect conversion function, TO_IP instead of TO_LONG:
   - do:
@@ -450,12 +482,44 @@ load two indices, showing unsupported type and null value for client_ip:
 
 ---
 load two indices with no conversion function, but needs TO_IP conversion:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [union_types_fix_rename_resolution]
+      reason: "Union type resolution fix for rename also allows direct usage of unsupported fields in KEEP"
+
   - do:
-      catch: '/Cannot use field \[client_ip\] due to ambiguities being mapped as \[2\] incompatible types: \[ip\] in \[events_ip_long\], \[keyword\] in \[events_keyword_long\]/'
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
       esql.query:
         body:
           query: 'FROM events_*_long METADATA _index | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC'
 
+  - match: { columns.0.name: "_index" }
+  - match: { columns.0.type: "keyword" }
+  - match: { columns.1.name: "@timestamp" }
+  - match: { columns.1.type: "date" }
+  - match: { columns.2.name: "client_ip" }
+  - match: { columns.2.type: "unsupported" }
+  - match: { columns.3.name: "event_duration" }
+  - match: { columns.3.type: "long" }
+  - match: { columns.4.name: "message" }
+  - match: { columns.4.type: "keyword" }
+  - length: { values: 14 }
+  - match: { values.0.0: "events_ip_long" }
+  - match: { values.0.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.0.2: null }
+  - match: { values.0.3: 1756467 }
+  - match: { values.0.4: "Connected to 10.1.0.1" }
+  - match: { values.7.0: "events_keyword_long" }
+  - match: { values.7.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.7.2: null }
+  - match: { values.7.3: 1756467 }
+  - match: { values.7.4: "Connected to 10.1.0.1" }
+
 ---
 load two indices with incorrect conversion function, TO_LONG instead of TO_IP:
   - do:
@@ -629,20 +693,103 @@ load two indexes, convert client_ip and group by something invalid:
 
 ---
 load four indices with single conversion function TO_LONG:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [union_types_fix_rename_resolution]
+      reason: "Union type resolution fix for rename also allows direct usage of unsupported fields in KEEP"
+
   - do:
-      catch: '/Cannot use field \[client_ip\] due to ambiguities being mapped as \[2\] incompatible types: \[ip\] in \[events_ip_keyword, events_ip_long\], \[keyword\] in \[events_keyword_keyword, events_keyword_long\]/'
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
       esql.query:
         body:
           query: 'FROM events_* METADATA _index | EVAL event_duration = TO_LONG(event_duration) | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC'
 
+  - match: { columns.0.name: "_index" }
+  - match: { columns.0.type: "keyword" }
+  - match: { columns.1.name: "@timestamp" }
+  - match: { columns.1.type: "date" }
+  - match: { columns.2.name: "client_ip" }
+  - match: { columns.2.type: "unsupported" }
+  - match: { columns.3.name: "event_duration" }
+  - match: { columns.3.type: "long" }
+  - match: { columns.4.name: "message" }
+  - match: { columns.4.type: "keyword" }
+  - length: { values: 28 }
+  - match: { values.0.0: "events_ip_keyword" }
+  - match: { values.0.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.0.2: null }
+  - match: { values.0.3: 1756467 }
+  - match: { values.0.4: "Connected to 10.1.0.1" }
+  - match: { values.7.0: "events_ip_long" }
+  - match: { values.7.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.7.2: null }
+  - match: { values.7.3: 1756467 }
+  - match: { values.7.4: "Connected to 10.1.0.1" }
+  - match: { values.14.0: "events_keyword_keyword" }
+  - match: { values.14.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.14.2: null }
+  - match: { values.14.3: 1756467 }
+  - match: { values.14.4: "Connected to 10.1.0.1" }
+  - match: { values.21.0: "events_keyword_long" }
+  - match: { values.21.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.21.2: null }
+  - match: { values.21.3: 1756467 }
+  - match: { values.21.4: "Connected to 10.1.0.1" }
+
 ---
 load four indices with single conversion function TO_IP:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [union_types_fix_rename_resolution]
+      reason: "Union type resolution fix for rename also allows direct usage of unsupported fields in KEEP"
+
   - do:
-      catch: '/Cannot use field \[event_duration\] due to ambiguities being mapped as \[2\] incompatible types: \[keyword\] in \[events_ip_keyword, events_keyword_keyword\], \[long\] in \[events_ip_long, events_keyword_long\]/'
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
       esql.query:
         body:
           query: 'FROM events_* METADATA _index | EVAL client_ip = TO_IP(client_ip) | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC'
 
+  - match: { columns.0.name: "_index" }
+  - match: { columns.0.type: "keyword" }
+  - match: { columns.1.name: "@timestamp" }
+  - match: { columns.1.type: "date" }
+  - match: { columns.2.name: "client_ip" }
+  - match: { columns.2.type: "ip" }
+  - match: { columns.3.name: "event_duration" }
+  - match: { columns.3.type: "unsupported" }
+  - match: { columns.4.name: "message" }
+  - match: { columns.4.type: "keyword" }
+  - length: { values: 28 }
+  - match: { values.0.0: "events_ip_keyword" }
+  - match: { values.0.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.0.2: "172.21.3.15" }
+  - match: { values.0.3: null }
+  - match: { values.0.4: "Connected to 10.1.0.1" }
+  - match: { values.7.0: "events_ip_long" }
+  - match: { values.7.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.7.2: "172.21.3.15" }
+  - match: { values.7.3: null }
+  - match: { values.7.4: "Connected to 10.1.0.1" }
+  - match: { values.14.0: "events_keyword_keyword" }
+  - match: { values.14.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.14.2: "172.21.3.15" }
+  - match: { values.14.3: null }
+  - match: { values.14.4: "Connected to 10.1.0.1" }
+  - match: { values.21.0: "events_keyword_long" }
+  - match: { values.21.1: "2023-10-23T13:55:01.543Z" }
+  - match: { values.21.2: "172.21.3.15" }
+  - match: { values.21.3: null }
+  - match: { values.21.4: "Connected to 10.1.0.1" }
 ---
 load four indices with multiple conversion functions TO_LONG and TO_IP:
   - do:
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml
new file mode 100644
index 0000000000000..959581b18c11a
--- /dev/null
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml
@@ -0,0 +1,197 @@
+---
+setup:
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [ method, path, parameters, capabilities ]
+          capabilities: [ match_operator ]
+      cluster_features: [ "gte_v8.16.0" ]
+      reason: "Match operator added in 8.16.0"
+      test_runner_features: [capabilities, allowed_warnings_regex]
+  - do:
+      indices.create:
+        index:  test
+        body:
+          mappings:
+            properties:
+              content:
+                type: text
+              id:
+                type: integer
+  - do:
+      bulk:
+        index: "test"
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "content": "This is a brown fox", "id": 1 }
+          - { "index": { } }
+          - { "content": "This is a brown dog", "id": 2 }
+          - { "index": { } }
+          - { "content": "This dog is really brown", "id": 3 }
+          - { "index": { } }
+          - { "content": "The dog is brown but this document is very very long", "id": 4 }
+          - { "index": { } }
+          - { "content": "There is also a white cat", "id": 5 }
+          - { "index": { } }
+          - { "content": "The quick brown fox jumps over the lazy dog", "id": 6 }
+
+---
+"simple where match":
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | WHERE content MATCH "fox" | KEEP id | SORT id'
+
+  - match: { columns.0.name: "id" }
+  - match: { columns.0.type: "integer" }
+  - length: { values: 2 }
+  - match: { values.0.0: 1 }
+  - match: { values.1.0: 6 }
+
+---
+"combined where match":
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | WHERE content MATCH "fox" AND id > 5 | KEEP id | SORT id'
+
+  - match: { columns.0.name: "id" }
+  - match: { columns.0.type: "integer" }
+  - length: { values: 1 }
+  - match: { values.0.0: 6 }
+
+---
+"multiple match":
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | WHERE content MATCH "fox" OR content MATCH "brown" | KEEP id | SORT id'
+
+  - match: { columns.0.name: "id" }
+  - match: { columns.0.type: "integer" }
+  - length: { values: 5 }
+  - match: { values.0.0: 1 }
+  - match: { values.1.0: 2 }
+  - match: { values.2.0: 3 }
+  - match: { values.3.0: 4 }
+  - match: { values.4.0: 6 }
+
+---
+"not where match":
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | WHERE NOT content MATCH "brown fox" | KEEP id | SORT id'
+
+  - match: { columns.0.name: "id" }
+  - match: { columns.0.type: "integer" }
+  - length: { values: 1 }
+  - match: { values.0.0: 5 }
+
+---
+"match on non existing column":
+  - do:
+      catch: bad_request
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | WHERE something match "fox"'
+
+  - match: { status: 400 }
+  - match: { error.type: verification_exception }
+  - match: { error.reason: "Found 1 problem\nline 1:19: Unknown column [something]" }
+
+---
+"match on eval column":
+  - do:
+      catch: bad_request
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | EVAL upper_content = to_upper(content) | WHERE upper_content MATCH "FOX" | KEEP id'
+
+  - match: { status: 400 }
+  - match: { error.type: verification_exception }
+  - match: { error.reason: "Found 1 problem\nline 1:60: MATCH requires a mapped index field, found [upper_content]" }
+
+---
+"match on overwritten column":
+  - do:
+      catch: bad_request
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | DROP content | EVAL content = CONCAT("ID: ", to_str(id)) | WHERE content match "fox"'
+
+  - match: { status: 400 }
+  - match: { error.type: verification_exception }
+  - match: { error.reason: "Found 1 problem\nline 1:78: MATCH requires a mapped index field, found [content]" }
+
+---
+"match after stats":
+  - do:
+      catch: bad_request
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | STATS count(*) | WHERE content match "fox"'
+
+  - match: { status: 400 }
+  - match: { error.type: verification_exception }
+  - match: { error.reason: "Found 1 problem\nline 1:36: Unknown column [content], did you mean [count(*)]?" }
+
+---
+"match with functions":
+  - do:
+      catch: bad_request
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | WHERE content MATCH "fox" OR to_upper(content) == "FOX"'
+
+  - match: { status: 400 }
+  - match: { error.type: verification_exception }
+  - match: { error.reason: "Found 1 problem\nline 1:19: Invalid condition using MATCH" }
+
+---
+"match within eval":
+  - do:
+      catch: bad_request
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | EVAL matches_query = content MATCH "fox"'
+
+  - match: { status: 400 }
+  - match: { error.type: verification_exception }
+  - match: { error.reason: "Found 1 problem\nline 1:34: EVAL does not support MATCH expressions" }
+
+---
+"match with non text field":
+  - do:
+      catch: bad_request
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | WHERE id MATCH "fox"'
+
+  - match: { status: 400 }
+  - match: { error.type: verification_exception }
+  - match: { error.reason: "Found 1 problem\nline 1:19: MATCH requires a text or keyword field, but [id] has type [integer]" }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/26_aggs_bucket.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/26_aggs_bucket.yml
new file mode 100644
index 0000000000000..7d0989a6e1886
--- /dev/null
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/26_aggs_bucket.yml
@@ -0,0 +1,188 @@
+---
+"friendlier BUCKET interval hourly: #110916":
+  - requires:
+      test_runner_features: [allowed_warnings_regex, capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [bucket_inclusive_upper_bound]
+      reason: "BUCKET auto mode now generates different bucket sizes"
+  - do:
+      indices.create:
+        index: test_bucket
+        body:
+          mappings:
+            properties:
+              ts :
+                type : date
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - { "index": { "_index": "test_bucket" } }
+          - { "ts": "2024-07-16T08:10:00Z" }
+          - { "index": { "_index": "test_bucket" } }
+          - { "ts": "2024-07-16T09:20:00Z" }
+          - { "index": { "_index": "test_bucket" } }
+          - { "ts": "2024-07-16T10:30:00Z" }
+          - { "index": { "_index": "test_bucket" } }
+          - { "ts": "2024-07-16T11:40:00Z" }
+
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test_bucket | STATS c = COUNT(*) BY b = BUCKET(ts, 4, "2024-07-16T08:00:00Z", "2024-07-16T12:00:00Z") | SORT b'
+  - match: { columns.0.name: c }
+  - match: { columns.0.type: long }
+  - match: { columns.1.name: b }
+  - match: { columns.1.type: date }
+  - length: { values: 4 }
+  - match: { values.0.0: 1 }
+  - match: { values.0.1: "2024-07-16T08:00:00.000Z" }
+  - match: { values.1.0: 1 }
+  - match: { values.1.1: "2024-07-16T09:00:00.000Z" }
+  - match: { values.2.0: 1 }
+  - match: { values.2.1: "2024-07-16T10:00:00.000Z" }
+  - match: { values.3.0: 1 }
+  - match: { values.3.1: "2024-07-16T11:00:00.000Z" }
+
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test_bucket | STATS c = COUNT(*) BY b = BUCKET(ts, 4, "2024-07-16T08:00:00Z", "2024-07-16T12:00:00.001Z") | SORT b'
+  - match: { columns.0.name: c }
+  - match: { columns.0.type: long }
+  - match: { columns.1.name: b }
+  - match: { columns.1.type: date }
+  - length: { values: 2 }
+  - match: { values.0.0: 1 }
+  - match: { values.0.1: "2024-07-16T06:00:00.000Z" }
+  - match: { values.1.0: 3 }
+  - match: { values.1.1: "2024-07-16T09:00:00.000Z" }
+
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test_bucket | STATS c = COUNT(*) BY b = BUCKET(ts, 4, "2024-07-16T08:09:00Z", "2024-07-16T12:00:00Z") | SORT b'
+  - match: { columns.0.name: c }
+  - match: { columns.0.type: long }
+  - match: { columns.1.name: b }
+  - match: { columns.1.type: date }
+  - length: { values: 4 }
+  - match: { values.0.0: 1 }
+  - match: { values.0.1: "2024-07-16T08:00:00.000Z" }
+  - match: { values.1.0: 1 }
+  - match: { values.1.1: "2024-07-16T09:00:00.000Z" }
+  - match: { values.2.0: 1 }
+  - match: { values.2.1: "2024-07-16T10:00:00.000Z" }
+  - match: { values.3.0: 1 }
+  - match: { values.3.1: "2024-07-16T11:00:00.000Z" }
+
+---
+"friendlier BUCKET interval: monthly #110916":
+  - requires:
+      cluster_features: ["gte_v8.14.0"]
+      test_runner_features: [allowed_warnings_regex, capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [bucket_inclusive_upper_bound]
+      reason: "BUCKET auto mode now generates different bucket sizes"
+  - do:
+      indices.create:
+        index: test_bucket
+        body:
+          mappings:
+            properties:
+              ts :
+                type : date
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - { "index": { "_index": "test_bucket" } }
+          - { "ts": "2024-06-16" }
+          - { "index": { "_index": "test_bucket" } }
+          - { "ts": "2024-07-16" }
+          - { "index": { "_index": "test_bucket" } }
+          - { "ts": "2024-08-16" }
+          - { "index": { "_index": "test_bucket" } }
+          - { "ts": "2024-09-16" }
+
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test_bucket | STATS c = COUNT(*) BY b = BUCKET(ts, 11, "2024-01-01", "2025-01-01") | SORT b'
+  - match: { columns.0.name: c }
+  - match: { columns.0.type: long }
+  - match: { columns.1.name: b }
+  - match: { columns.1.type: date }
+  - length: { values: 1 }
+  - match: { values.0.0: 4 }
+  - match: { values.0.1: "2024-01-01T00:00:00.000Z" }
+
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test_bucket | STATS c = COUNT(*) BY b = BUCKET(ts, 12, "2024-01-01", "2025-01-01") | SORT b'
+  - match: { columns.0.name: c }
+  - match: { columns.0.type: long }
+  - match: { columns.1.name: b }
+  - match: { columns.1.type: date }
+  - length: { values: 4 }
+  - match: { values.0.0: 1 }
+  - match: { values.0.1: "2024-06-01T00:00:00.000Z" }
+  - match: { values.1.0: 1 }
+  - match: { values.1.1: "2024-07-01T00:00:00.000Z" }
+  - match: { values.2.0: 1 }
+  - match: { values.2.1: "2024-08-01T00:00:00.000Z" }
+  - match: { values.3.0: 1 }
+  - match: { values.3.1: "2024-09-01T00:00:00.000Z" }
+
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test_bucket | STATS c = COUNT(*) BY b = BUCKET(ts, 12, "2024-01-01", "2025-01-01T00:00:00.001") | SORT b'
+  - match: { columns.0.name: c }
+  - match: { columns.0.type: long }
+  - match: { columns.1.name: b }
+  - match: { columns.1.type: date }
+  - length: { values: 1 }
+  - match: { values.0.0: 4 }
+  - match: { values.0.1: "2024-01-01T00:00:00.000Z" }
+
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test_bucket | STATS c = COUNT(*) BY b = BUCKET(ts, 13, "2024-01-01T12:13:14Z", "2025-01-01") | SORT b'
+  - match: { columns.0.name: c }
+  - match: { columns.0.type: long }
+  - match: { columns.1.name: b }
+  - match: { columns.1.type: date }
+  - length: { values: 4 }
+  - match: { values.0.0: 1 }
+  - match: { values.0.1: "2024-06-01T00:00:00.000Z" }
+  - match: { values.1.0: 1 }
+  - match: { values.1.1: "2024-07-01T00:00:00.000Z" }
+  - match: { values.2.0: 1 }
+  - match: { values.2.1: "2024-08-01T00:00:00.000Z" }
+  - match: { values.3.0: 1 }
+  - match: { values.3.1: "2024-09-01T00:00:00.000Z" }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml
index 8f291600acbf6..642407ac6d45b 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml
@@ -253,12 +253,24 @@ from index pattern unsupported counter:
 
 ---
 from index pattern explicit counter use:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [union_types_fix_rename_resolution]
+      reason: "Union type resolution fix for rename also allows direct usage of unsupported fields in KEEP"
+
   - do:
-      catch: '/Cannot use field \[k8s.pod.network.tx\] due to ambiguities being mapped as different metric types in indices: \[test, test2\]/'
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
       esql.query:
         body:
           query: 'FROM test* | keep *.tx'
-
+  - match: {columns.0.name: "k8s.pod.network.tx"}
+  - match: {columns.0.type: "unsupported"}
+  - length: {values: 10}
 
 ---
 _source:
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml
index c34666bb12b02..e100f30717aef 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml
@@ -113,6 +113,15 @@ unsupported:
       cluster_features: ["gte_v8.13.0"]
       reason: "Latest types supported in ESQL as of 8.13.0"
 
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [ ]
+          capabilities: [ date_nanos_type ]
+      reason: "support for date nanos type"
+
   - do:
       allowed_warnings_regex:
         - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null"
@@ -128,7 +137,7 @@ unsupported:
   - match: { columns.2.name: completion }
   - match: { columns.2.type: unsupported }
   - match: { columns.3.name: date_nanos }
-  - match: { columns.3.type: unsupported }
+  - match: { columns.3.type: date_nanos }
   - match: { columns.4.name: date_range }
   - match: { columns.4.type: unsupported }
   - match: { columns.5.name: dense_vector }
@@ -184,7 +193,7 @@ unsupported:
   - match: { values.0.0: null }
   - match: { values.0.1: null }
   - match: { values.0.2: null }
-  - match: { values.0.3: null }
+  - match: { values.0.3: "2015-01-01T12:10:30.123456789Z" }
   - match: { values.0.4: null }
   - match: { values.0.5: null }
   - match: { values.0.6: null }
@@ -224,7 +233,7 @@ unsupported:
   - match: { columns.2.name: completion }
   - match: { columns.2.type: unsupported }
   - match: { columns.3.name: date_nanos }
-  - match: { columns.3.type: unsupported }
+  - match: { columns.3.type: date_nanos }
   - match: { columns.4.name: date_range }
   - match: { columns.4.type: unsupported }
   - match: { columns.5.name: dense_vector }
@@ -293,6 +302,15 @@ unsupported with sort:
       cluster_features: ["gte_v8.13.0"]
       reason: "Latest types supported in ESQL as of 8.13.0"
 
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [ ]
+          capabilities: [ date_nanos_type ]
+      reason: "support for date nanos type"
+
   - do:
       allowed_warnings_regex:
         - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null"
@@ -308,7 +326,7 @@ unsupported with sort:
   - match: { columns.2.name: completion }
   - match: { columns.2.type: unsupported }
   - match: { columns.3.name: date_nanos }
-  - match: { columns.3.type: unsupported }
+  - match: { columns.3.type: date_nanos }
   - match: { columns.4.name: date_range }
   - match: { columns.4.type: unsupported }
   - match: { columns.5.name: dense_vector }
@@ -364,7 +382,7 @@ unsupported with sort:
   - match: { values.0.0: null }
   - match: { values.0.1: null }
   - match: { values.0.2: null }
-  - match: { values.0.3: null }
+  - match: { values.0.3: "2015-01-01T12:10:30.123456789Z" }
   - match: { values.0.4: null }
   - match: { values.0.5: null }
   - match: { values.0.6: null }
@@ -390,3 +408,99 @@ unsupported with sort:
   - match: { values.0.26: xy }
   - match: { values.0.27: "foo bar" }
   - match: { values.0.28: 3 }
+
+---
+nested declared inline:
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - { "index": { } }
+          - {
+            "find_me": 1,
+            "nested": {
+              "foo": 1,
+              "bar": "bar",
+              "baz": 1.9
+            }
+          }
+
+  - do:
+      allowed_warnings_regex:
+        - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null"
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test | WHERE find_me == 1 | KEEP n*'
+
+  # The `nested` field is not visible, nor are any of it's subfields.
+  - match: { columns: [{name: name, type: keyword}] }
+  - match: { values: [[null]] }
+
+---
+nested declared in mapping:
+  - do:
+      indices.create:
+        index: test_nested
+        body:
+          settings:
+            number_of_shards: 5
+          mappings:
+            properties:
+              name:
+                type: keyword
+              nested:
+                type: nested
+                properties:
+                  foo:
+                    type: keyword
+                  bar:
+                    type: keyword
+
+  - do:
+      allowed_warnings_regex:
+        - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null"
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test_nested'
+
+  # The `nested` field is not visible, nor are any of it's subfields.
+  - match: { columns: [{name: name, type: keyword}] }
+
+---
+double nested declared in mapping:
+  - do:
+      indices.create:
+        index: test_nested
+        body:
+          settings:
+            number_of_shards: 5
+          mappings:
+            properties:
+              name:
+                type: keyword
+              nested:
+                type: nested
+                properties:
+                  bort:
+                    type: keyword
+                  nested:
+                    type: nested
+                    properties:
+                      foo:
+                        type: keyword
+                      bar:
+                        type: keyword
+
+  - do:
+      allowed_warnings_regex:
+        - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null"
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test_nested'
+
+  # The `nested` field is not visible, nor are any of it's subfields.
+  - match: { columns: [{name: name, type: keyword}] }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/51_many_indexes.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/51_many_indexes.yml
new file mode 100644
index 0000000000000..eb589cb810cc3
--- /dev/null
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/51_many_indexes.yml
@@ -0,0 +1,126 @@
+setup:
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [method, path, parameters, capabilities]
+          capabilities: [short_error_messages_for_unsupported_fields]
+      reason: "We changed error messages for unsupported fields in v 8.16"
+      test_runner_features: [capabilities, allowed_warnings_regex]
+
+  - do:
+      indices.create:
+        index: ambiguous_1
+        body:
+          mappings:
+            properties:
+              "name":
+                type: keyword
+
+  - do:
+      indices.create:
+        index: ambiguous_2
+        body:
+          mappings:
+            properties:
+              "name":
+                type: keyword
+
+  - do:
+      indices.create:
+        index: ambiguous_3
+        body:
+          mappings:
+            properties:
+              "name":
+                type: keyword
+
+  - do:
+      indices.create:
+        index: ambiguous_4
+        body:
+          mappings:
+            properties:
+              "name":
+                type: integer
+
+  - do:
+      indices.create:
+        index: ambiguous_5
+        body:
+          mappings:
+            properties:
+              "name":
+                type: integer
+
+  - do:
+      indices.create:
+        index: ambiguous_6
+        body:
+          mappings:
+            properties:
+              "name":
+                type: integer
+
+  - do:
+      indices.create:
+        index: ambiguous_7
+        body:
+          mappings:
+            properties:
+              "name":
+                type: integer
+
+  - do:
+      indices.create:
+        index: ambiguous_8
+        body:
+          mappings:
+            properties:
+              "name":
+                type: ip
+
+  - do:
+      indices.create:
+        index: ambiguous_9
+        body:
+          mappings:
+            properties:
+              "name":
+                type: ip
+
+  - do:
+      indices.create:
+        index: ambiguous_10
+        body:
+          mappings:
+            properties:
+              "name":
+                type: ip
+
+  - do:
+      indices.create:
+        index: ambiguous_11
+        body:
+          mappings:
+            properties:
+              "name":
+                type: ip
+
+  - do:
+      indices.create:
+        index: ambiguous_12
+        body:
+          mappings:
+            properties:
+              "name":
+                type: ip
+
+---
+load many indices with ambiguities:
+  - do:
+      catch: '/Cannot use field \[name\] due to ambiguities being mapped as \[3\] incompatible types: \[integer\] in \[ambiguous_4, ambiguous_5, ambiguous_6\] and \[1\] other index, \[ip\] in \[ambiguous_10, ambiguous_11, ambiguous_12\] and \[2\] other indices, \[keyword\] in \[ambiguous_1, ambiguous_2, ambiguous_3\]/'
+      esql.query:
+        body:
+          query: 'FROM ambiguous* | SORT name'
+
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/security_tests.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/security_tests.yml
deleted file mode 100644
index 5bae9469a3c47..0000000000000
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/security_tests.yml
+++ /dev/null
@@ -1,191 +0,0 @@
-setup:
-  - skip:
-      features: headers
-
-  - do:
-      cluster.health:
-          wait_for_status: yellow
-
-  - do:
-      indices.create:
-        index: dummy-rollup-index
-        body:
-          settings:
-            number_of_shards: 1
-            number_of_replicas: 0
-          mappings:
-            _meta:
-              _rollup:
-                my-id: { }
-
----
-teardown:
-  - do:
-      security.delete_user:
-        username: "test_user"
-        ignore: 404
-
-  - do:
-      security.delete_role:
-        name: "foo_only_access"
-        ignore: 404
-
----
-"Index-based access":
-
-  - do:
-      security.put_role:
-        name: "foo_only_access"
-        body:  >
-            {
-              "cluster": [ "all" ],
-              "indices": [
-                { "names": ["foo"], "privileges": ["all"] },
-                { "names": ["rollup"], "privileges": ["all"] }
-              ]
-            }
-
-  - do:
-      security.put_user:
-        username: "test_user"
-        body:  >
-          {
-            "password" : "x-pack-test-password",
-            "roles" : [ "foo_only_access" ],
-            "full_name" : "foo only"
-          }
-
-  - do:
-      indices.create:
-        index: foo
-        body:
-          mappings:
-            properties:
-              timestamp:
-                type: date
-              value_field:
-                type: integer
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      index:
-        index: foo
-        body:
-          timestamp: 123
-          value_field: 1232
-
-  - do:
-      indices.create:
-        index: foobar
-        body:
-          mappings:
-            properties:
-              timestamp:
-                type: date
-              value_field:
-                type: integer
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      index:
-        index: foobar
-        body:
-          timestamp: 123
-          value_field: 456
-
-  - do:
-      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
-      indices.refresh:
-        index: foo
-
-  # This index pattern will match both indices, but we only have permission to read one
-  - do:
-      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
-      rollup.put_job:
-        id: foo
-        body:  >
-          {
-            "index_pattern": "foo*",
-            "rollup_index": "rollup",
-            "cron": "*/1 * * * * ?",
-            "page_size" :10,
-            "groups" : {
-              "date_histogram": {
-                "field": "timestamp",
-                "interval": "1s"
-              }
-            },
-            "metrics": [
-              {
-                "field": "value_field",
-                "metrics": ["min", "max", "sum"]
-              }
-            ]
-          }
-
-  - is_true: acknowledged
-
-  - do:
-      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
-      rollup.start_job:
-        id: foo
-  - is_true: started
-
-  - do:
-      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
-      indices.refresh:
-        index: rollup
-
-  # this is a hacky way to sleep for 5s, since we will never have 10 nodes
-  - do:
-      catch: request_timeout
-      cluster.health:
-        wait_for_nodes: 10
-        timeout: "5s"
-  - match:
-     timed_out: true
-
-  - do:
-      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
-      rollup.get_jobs:
-        id: foo
-  - match:
-        jobs.0.stats.documents_processed: 1
-
-  - do:
-      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
-      search:
-        rest_total_hits_as_int: true
-        index: foo
-        body:
-          query:
-            match_all: {}
-
-  - match:
-      hits.total: 1
-
-  - do:
-      headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user
-      search:
-        rest_total_hits_as_int: true
-        index: rollup
-        body:
-          query:
-            match_all: {}
-
-  - match:
-      hits.total: 1
-  - match:
-      hits.hits.0._id: "foo$VxMkzTqILshClbtbFi4-rQ"
-  - match:
-      hits.hits.0._source:
-        timestamp.date_histogram.time_zone: "UTC"
-        timestamp.date_histogram.timestamp: 0
-        value_field.max.value: 1232.0
-        _rollup.version: 2
-        timestamp.date_histogram.interval: "1s"
-        value_field.sum.value: 1232.0
-        value_field.min.value: 1232.0
-        timestamp.date_histogram._count: 1
-        _rollup.id: "foo"
-
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/service_accounts/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/service_accounts/10_basic.yml
index 47d6cdec2858b..a72e2d15c8e85 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/service_accounts/10_basic.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/service_accounts/10_basic.yml
@@ -31,7 +31,8 @@ teardown:
 "Test get service accounts":
   - do:
       security.get_service_accounts: {}
-  - length: { '': 4 }
+  - length: { '': 5 }
+  - is_true: "elastic/auto-ops"
   - is_true: "elastic/enterprise-search-server"
   - is_true: "elastic/fleet-server"
   - is_true: "elastic/fleet-server-remote"
@@ -40,7 +41,8 @@ teardown:
   - do:
       security.get_service_accounts:
         namespace: elastic
-  - length: { '': 4 }
+  - length: { '': 5 }
+  - is_true: "elastic/auto-ops"
   - is_true: "elastic/enterprise-search-server"
   - is_true: "elastic/fleet-server"
   - is_true: "elastic/fleet-server-remote"
diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java
index 8bdf7b30a9997..423865eaab617 100644
--- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java
+++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java
@@ -63,7 +63,7 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase {
 
     private static Map<String, Object> ecsDynamicTemplates;
     private static Map<String, Map<String, Object>> ecsFlatFieldDefinitions;
-    private static Map<String, String> ecsFlatMultiFieldDefinitions;
+    private static Map<String, Map<String, Object>> ecsFlatMultiFieldDefinitions;
 
     @BeforeClass
     public static void setupSuiteScopeCluster() throws Exception {
@@ -142,12 +142,11 @@ private static void prepareEcsDefinitions() throws IOException {
                 iterator.remove();
             }
 
-            List<Map<String, String>> multiFields = (List<Map<String, String>>) definitions.get("multi_fields");
+            List<Map<String, Object>> multiFields = (List<Map<String, Object>>) definitions.get("multi_fields");
             if (multiFields != null) {
                 multiFields.forEach(multiFieldsDefinitions -> {
-                    String subfieldFlatName = Objects.requireNonNull(multiFieldsDefinitions.get("flat_name"));
-                    String subfieldType = Objects.requireNonNull(multiFieldsDefinitions.get("type"));
-                    ecsFlatMultiFieldDefinitions.put(subfieldFlatName, subfieldType);
+                    String subfieldFlatName = (String) Objects.requireNonNull(multiFieldsDefinitions.get("flat_name"));
+                    ecsFlatMultiFieldDefinitions.put(subfieldFlatName, multiFieldsDefinitions);
                 });
             }
         }
@@ -166,6 +165,22 @@ public void testFlattenedFields() throws IOException {
         verifyEcsMappings(indexName);
     }
 
+    public void testFlattenedFieldsWithinAttributes() throws IOException {
+        String indexName = "test-flattened-attributes";
+        createTestIndex(indexName);
+        Map<String, Object> flattenedFieldsMap = createTestDocument(true);
+        indexDocument(indexName, Map.of("attributes", flattenedFieldsMap));
+        verifyEcsMappings(indexName, "attributes.");
+    }
+
+    public void testFlattenedFieldsWithinResourceAttributes() throws IOException {
+        String indexName = "test-flattened-attributes";
+        createTestIndex(indexName);
+        Map<String, Object> flattenedFieldsMap = createTestDocument(true);
+        indexDocument(indexName, Map.of("resource.attributes", flattenedFieldsMap));
+        verifyEcsMappings(indexName, "resource.attributes.");
+    }
+
     public void testFlattenedFieldsWithoutSubobjects() throws IOException {
         String indexName = "test_flattened_fields_subobjects_false";
         createTestIndex(indexName, Map.of("subobjects", false));
@@ -191,7 +206,7 @@ public void testNumericMessage() throws IOException {
         verifyEcsMappings(indexName);
     }
 
-    private void assertType(String expectedType, Map<String, Object> actualMappings) throws IOException {
+    private void assertType(String expectedType, Map<String, Object> actualMappings) {
         assertNotNull("expected to get non-null mappings for field", actualMappings);
         assertEquals(expectedType, actualMappings.get("type"));
     }
@@ -395,12 +410,19 @@ private void processRawMappingsSubtree(
     }
 
     private void verifyEcsMappings(String indexName) throws IOException {
+        verifyEcsMappings(indexName, "");
+    }
+
+    private void verifyEcsMappings(String indexName, String fieldPrefix) throws IOException {
         final Map<String, Object> rawMappings = getMappings(indexName);
         final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
         final Map<String, Map<String, Object>> flatMultiFieldsMappings = new HashMap<>();
         processRawMappingsSubtree(rawMappings, flatFieldMappings, flatMultiFieldsMappings, "");
 
-        Map<String, Map<String, Object>> shallowFieldMapCopy = new HashMap<>(ecsFlatFieldDefinitions);
+        Map<String, Map<String, Object>> shallowFieldMapCopy = ecsFlatFieldDefinitions.entrySet()
+            .stream()
+            .collect(Collectors.toMap(e -> fieldPrefix + e.getKey(), Map.Entry::getValue));
+
         logger.info("Testing mapping of {} ECS fields", shallowFieldMapCopy.size());
         List<String> nonEcsFields = new ArrayList<>();
         Map<String, String> fieldToWrongMappingType = new HashMap<>();
@@ -411,32 +433,35 @@ private void verifyEcsMappings(String indexName) throws IOException {
             if (expectedMappings == null) {
                 nonEcsFields.add(fieldName);
             } else {
-                String expectedType = (String) expectedMappings.get("type");
-                String actualMappingType = (String) actualMappings.get("type");
-                if (actualMappingType.equals(expectedType) == false) {
-                    fieldToWrongMappingType.put(fieldName, actualMappingType);
-                }
-                if (expectedMappings.get("index") != actualMappings.get("index")) {
-                    wronglyIndexedFields.add(fieldName);
-                }
-                if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) {
-                    wronglyDocValuedFields.add(fieldName);
-                }
+                compareExpectedToActualMappings(
+                    fieldName,
+                    actualMappings,
+                    expectedMappings,
+                    fieldToWrongMappingType,
+                    wronglyIndexedFields,
+                    wronglyDocValuedFields
+                );
             }
         });
 
-        Map<String, String> shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions);
+        Map<String, Map<String, Object>> shallowMultiFieldMapCopy = ecsFlatMultiFieldDefinitions.entrySet()
+            .stream()
+            .collect(Collectors.toMap(e -> fieldPrefix + e.getKey(), Map.Entry::getValue));
         logger.info("Testing mapping of {} ECS multi-fields", shallowMultiFieldMapCopy.size());
         flatMultiFieldsMappings.forEach((fieldName, actualMappings) -> {
-            String expectedType = shallowMultiFieldMapCopy.remove(fieldName);
-            if (expectedType != null) {
+            Map<String, Object> expectedMultiFieldMappings = shallowMultiFieldMapCopy.remove(fieldName);
+            if (expectedMultiFieldMappings != null) {
                 // not finding an entry in the expected multi-field mappings map is acceptable: our dynamic templates are required to
                 // ensure multi-field mapping for all fields with such ECS definitions. However, the patterns in these templates may lead
                 // to multi-field mapping for ECS fields for which such are not defined
-                String actualMappingType = (String) actualMappings.get("type");
-                if (actualMappingType.equals(expectedType) == false) {
-                    fieldToWrongMappingType.put(fieldName, actualMappingType);
-                }
+                compareExpectedToActualMappings(
+                    fieldName,
+                    actualMappings,
+                    expectedMultiFieldMappings,
+                    fieldToWrongMappingType,
+                    wronglyIndexedFields,
+                    wronglyDocValuedFields
+                );
             }
         });
 
@@ -460,7 +485,11 @@ private void verifyEcsMappings(String indexName) throws IOException {
             );
         });
         fieldToWrongMappingType.forEach((fieldName, actualMappingType) -> {
-            String ecsExpectedType = (String) ecsFlatFieldDefinitions.get(fieldName).get("type");
+            Map<String, Object> fieldMappings = ecsFlatFieldDefinitions.get(fieldName);
+            if (fieldMappings == null) {
+                fieldMappings = ecsFlatMultiFieldDefinitions.get(fieldName);
+            }
+            String ecsExpectedType = (String) fieldMappings.get("type");
             logger.error(
                 "ECS field '{}' should be mapped to type '{}' but is mapped to type '{}'. Update {} accordingly.",
                 fieldName,
@@ -493,4 +522,25 @@ private void verifyEcsMappings(String indexName) throws IOException {
             wronglyDocValuedFields.isEmpty()
         );
     }
+
+    private static void compareExpectedToActualMappings(
+        String fieldName,
+        Map<String, Object> actualMappings,
+        Map<String, Object> expectedMappings,
+        Map<String, String> fieldToWrongMappingType,
+        List<String> wronglyIndexedFields,
+        List<String> wronglyDocValuedFields
+    ) {
+        String expectedType = (String) expectedMappings.get("type");
+        String actualMappingType = (String) actualMappings.get("type");
+        if (actualMappingType.equals(expectedType) == false) {
+            fieldToWrongMappingType.put(fieldName, actualMappingType);
+        }
+        if (expectedMappings.get("index") != actualMappings.get("index")) {
+            wronglyIndexedFields.add(fieldName);
+        }
+        if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) {
+            wronglyDocValuedFields.add(fieldName);
+        }
+    }
 }
diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
index 6a9936f4f27d3..9e847455d2c86 100644
--- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
+++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
@@ -48,7 +48,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
 
     // The stack template registry version. This number must be incremented when we make changes
     // to built-in templates.
-    public static final int REGISTRY_VERSION = 12;
+    public static final int REGISTRY_VERSION = 13;
 
     public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version";
     public static final Setting<Boolean> STACK_TEMPLATES_ENABLED = Setting.boolSetting(
@@ -106,6 +106,12 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
     public static final String METRICS_ILM_POLICY_NAME = "metrics@lifecycle";
     public static final String METRICS_INDEX_TEMPLATE_NAME = "metrics";
 
+    //////////////////////////////////////////////////////////
+    // Base traces components
+    //////////////////////////////////////////////////////////
+    public static final String TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME = "traces@mappings";
+    public static final String TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME = "traces@settings";
+
     //////////////////////////////////////////////////////////
     // Synthetics components (for matching synthetics-*-* indices)
     //////////////////////////////////////////////////////////
@@ -192,6 +198,20 @@ private Map<String, ComponentTemplate> loadComponentTemplateConfigs(boolean logs
                 TEMPLATE_VERSION_VARIABLE,
                 ADDITIONAL_TEMPLATE_VARIABLES
             ),
+            new IndexTemplateConfig(
+                TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME,
+                "/traces@settings.json",
+                REGISTRY_VERSION,
+                TEMPLATE_VERSION_VARIABLE,
+                ADDITIONAL_TEMPLATE_VARIABLES
+            ),
+            new IndexTemplateConfig(
+                TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME,
+                "/traces@mappings.json",
+                REGISTRY_VERSION,
+                TEMPLATE_VERSION_VARIABLE,
+                ADDITIONAL_TEMPLATE_VARIABLES
+            ),
             new IndexTemplateConfig(
                 SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
                 "/synthetics@mappings.json",
diff --git a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java
index abb2d5765b128..25ff3b5311fa2 100644
--- a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java
+++ b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java
@@ -430,10 +430,12 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
         versions.put(StackTemplateRegistry.SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
+        versions.put(StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
+        versions.put(StackTemplateRegistry.TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(versions, nodes);
         client.setVerifier((action, request, listener) -> {
-            if (action instanceof PutComponentTemplateAction) {
-                fail("template should not have been re-installed");
+            if (request instanceof PutComponentTemplateAction.Request put) {
+                fail("template should not have been re-installed: " + put.name());
                 return null;
             } else if (action == ILMActions.PUT) {
                 // Ignore this, it's verified in another test
@@ -489,6 +491,14 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
             StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME,
             StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
         );
+        versions.put(
+            StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME,
+            StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
+        );
+        versions.put(
+            StackTemplateRegistry.TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME,
+            StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
+        );
         ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(versions, nodes);
         registry.clusterChanged(higherVersionEvent);
     }
diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java
index 89b21c8dfbcf5..6d8e4cb78ba30 100644
--- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java
+++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.common.CheckedSupplier;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.concurrent.ExecutorService;
diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java
index 88e60dc3ffd9f..827d775c35ea5 100644
--- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java
+++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java
index d915a7babcbfe..5ccbfe3edf22f 100644
--- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java
+++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java
index aa546e1747ba4..81cdbf4bd9b61 100644
--- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java
+++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportTestGrokPatternAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportTestGrokPatternAction.java
index f8ce7a1099952..442881dc5985f 100644
--- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportTestGrokPatternAction.java
+++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportTestGrokPatternAction.java
@@ -10,12 +10,11 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.grok.Grok;
 import org.elasticsearch.grok.GrokBuiltinPatterns;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -32,18 +31,15 @@ public class TransportTestGrokPatternAction extends TransportAction<TestGrokPatt
 
     private static final Logger logger = LogManager.getLogger(TransportTestGrokPatternAction.class);
 
-    private final ThreadPool threadPool;
-
     @Inject
     public TransportTestGrokPatternAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool) {
-        super(TestGrokPatternAction.INSTANCE.name(), actionFilters, transportService.getTaskManager());
-        this.threadPool = threadPool;
+        // As matching a regular expression might take a while, we run in a different thread to avoid blocking the network thread.
+        super(TestGrokPatternAction.INSTANCE.name(), actionFilters, transportService.getTaskManager(), threadPool.generic());
     }
 
     @Override
     protected void doExecute(Task task, TestGrokPatternAction.Request request, ActionListener<TestGrokPatternAction.Response> listener) {
-        // As matching a regular expression might take a while, we run in a different thread to avoid blocking the network thread.
-        threadPool.generic().execute(ActionRunnable.supply(listener, () -> getResponse(request)));
+        listener.onResponse(getResponse(request));
     }
 
     private TestGrokPatternAction.Response getResponse(TestGrokPatternAction.Request request) {
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java
index b0d56f47cbec6..477ce2127ed42 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
@@ -95,9 +94,7 @@ public void testAuditorWritesAudits() throws Exception {
     }
 
     public void testAliasCreatedforBWCIndexes() throws Exception {
-        Settings.Builder settings = Settings.builder()
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0);
+        Settings.Builder settings = indexSettings(1, 0);
 
         // These indices should only exist if created in previous versions, ignore the deprecation warning for this test
         RequestOptions options = expectWarnings(
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java
index ec613f1038fda..fd98c9f8feaa4 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java
@@ -15,9 +15,9 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java
index d4e03475af22e..8ee23e38f9ffe 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformUsageTransportAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java
index 6f26df549efc7..aad3cc4dc6ac0 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java
@@ -21,11 +21,11 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java
index bf18f7257d906..ce590773fe72e 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java
@@ -30,11 +30,11 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointNodeAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointNodeAction.java
index 177f00c704c3c..63b9dc7ae3fbb 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointNodeAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointNodeAction.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexService;
@@ -20,6 +19,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformAction.java
index 880067facab75..cb2985b5b1b3a 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformAction.java
@@ -13,10 +13,10 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.SortOrder;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformNodeStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformNodeStatsAction.java
index bbe8f6ea05b4c..83e7f55df04b0 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformNodeStatsAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformNodeStatsAction.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
index 302db8816f4bf..ae0eb000e70e5 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
@@ -23,12 +23,12 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata.Assignment;
 import org.elasticsearch.tasks.CancellableTask;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java
index e79602887b728..36237d2705205 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -29,6 +28,7 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java
index ef42a2781962a..35c97f50b6824 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPutTransformAction.java
@@ -21,9 +21,9 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java
index 473aafb6efa91..e817463385a13 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java
@@ -23,10 +23,10 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java
index 4c0fb58390a1d..ee82c28913143 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformResetModeAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformResetModeAction.java
index 45450b04070d7..b5b50ec6d3997 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformResetModeAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformResetModeAction.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.action.AbstractTransportSetResetModeAction;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
index 59df3fa67074d..4ad56bf3a661a 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
@@ -23,11 +23,11 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.health.HealthStatus;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.rest.RestStatus;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
index 39874a9b4f9fc..34e89986b5bcd 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
@@ -23,12 +23,12 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask;
 import org.elasticsearch.persistent.PersistentTasksService;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
index f254294cd104c..6251dab36b2ca 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpdateTransformAction.java
@@ -21,10 +21,10 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java
index 2d0ec21eaee60..5e2fa7f7a590e 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java
@@ -20,10 +20,10 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportValidateTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportValidateTransformAction.java
index 7041f18df1e4a..7fc7ec49797f7 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportValidateTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportValidateTransformAction.java
@@ -17,10 +17,10 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ValidationException;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java
index 062c951f67c96..c8677c2816fc9 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerTests.java
@@ -538,7 +538,7 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
             if (request instanceof OpenPointInTimeRequest) {
                 if (pitSupported) {
                     pitContextCounter.incrementAndGet();
-                    OpenPointInTimeResponse response = new OpenPointInTimeResponse(new BytesArray("the_pit_id"));
+                    OpenPointInTimeResponse response = new OpenPointInTimeResponse(new BytesArray("the_pit_id"), 1, 1, 0, 0);
                     listener.onResponse((Response) response);
                 } else {
                     listener.onFailure(new ActionNotFoundTransportException("_pit"));
diff --git a/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodeFeatureSet.java b/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodeFeatureSet.java
index 8dfc68064fcac..54bb265321799 100644
--- a/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodeFeatureSet.java
+++ b/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodeFeatureSet.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
index c8dd4d42ac4a1..f59123fd66593 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.QueryWatchesAction;
@@ -45,6 +44,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.putJsonStoredScript;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
@@ -221,21 +221,17 @@ public void testConditionSearchWithSource() throws Exception {
 
     public void testConditionSearchWithIndexedTemplate() throws Exception {
         SearchSourceBuilder searchSourceBuilder = searchSource().query(matchQuery("level", "a"));
-        assertAcked(
-            clusterAdmin().preparePutStoredScript()
-                .setId("my-template")
-                .setContent(
-                    BytesReference.bytes(
-                        jsonBuilder().startObject()
-                            .startObject("script")
-                            .field("lang", "mustache")
-                            .field("source")
-                            .value(searchSourceBuilder)
-                            .endObject()
-                            .endObject()
-                    ),
-                    XContentType.JSON
-                )
+        putJsonStoredScript(
+            "my-template",
+            BytesReference.bytes(
+                jsonBuilder().startObject()
+                    .startObject("script")
+                    .field("lang", "mustache")
+                    .field("source")
+                    .value(searchSourceBuilder)
+                    .endObject()
+                    .endObject()
+            )
         );
 
         Script template = new Script(ScriptType.STORED, null, "my-template", Collections.emptyMap());
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
index 19cd37400a01c..0070554d99d27 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
@@ -8,6 +8,8 @@
 
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.SortBuilders;
@@ -17,6 +19,7 @@
 import org.elasticsearch.xpack.core.watcher.actions.ActionStatus;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
+import org.elasticsearch.xpack.core.watcher.history.WatchRecord;
 import org.elasticsearch.xpack.core.watcher.input.Input;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
@@ -29,6 +32,7 @@
 import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule;
 
 import java.util.Locale;
+import java.util.Map;
 
 import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -44,11 +48,21 @@
 import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.templateRequest;
 import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
 import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interval;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
 public class HistoryIntegrationTests extends AbstractWatcherIntegrationTestCase {
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put("xpack.watcher.max.history.record.size", "100kb") // used for testThatHistoryIsTruncated()
+            .build();
+    }
+
     // issue: https://github.com/elastic/x-plugins/issues/2338
     public void testThatHistoryIsWrittenWithChainedInput() throws Exception {
         XContentBuilder xContentBuilder = jsonBuilder().startObject()
@@ -225,6 +239,69 @@ public void testThatHistoryContainsStatus() throws Exception {
         });
     }
 
+    public void testThatHistoryIsTruncated() throws Exception {
+        {
+            /*
+             * The input for this watch is 20 KB, smaller than the configured 100 KB of HistoryStore's MAX_HISTORY_SIZE_SETTING. So we do
+             * not expect its history record to be truncated.
+             */
+            new PutWatchRequestBuilder(client()).setId("test_watch_small")
+                .setSource(
+                    watchBuilder().trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.HOURS)))
+                        .input(simpleInput("foo", randomAlphaOfLength((int) ByteSizeValue.ofKb(20).getBytes())))
+                        .addAction("_logger", loggingAction("#### randomLogging"))
+                )
+                .get();
+            new ExecuteWatchRequestBuilder(client()).setId("test_watch_small").setRecordExecution(true).get();
+            assertBusy(() -> {
+                assertResponse(getWatchHistory(), searchResponse -> {
+                    assertHitCount(searchResponse, 1);
+                    SearchHit hit = searchResponse.getHits().getAt(0);
+                    XContentSource source = new XContentSource(hit.getSourceRef(), XContentType.JSON);
+                    Map<String, Object> input = source.getValue("input");
+                    assertThat(input.containsKey(WatchRecord.TRUNCATED_RECORD_KEY), equalTo(false));
+                    assertThat(input.containsKey("simple"), equalTo(true));
+                    Map<String, Object> result = source.getValue("result");
+                    assertThat(result.containsKey(WatchRecord.TRUNCATED_RECORD_KEY), equalTo(false));
+                    assertThat(result.containsKey("input"), equalTo(true));
+                    assertThat(result.containsKey("actions"), equalTo(true));
+                    assertThat(result.containsKey("condition"), equalTo(true));
+                });
+            });
+        }
+        {
+            /*
+             * The input for this watch is 500 KB, much bigger than the configured 100 KB of HistoryStore's MAX_HISTORY_SIZE_SETTING. So we
+             * expect to see its history record truncated before being stored.
+             */
+            new PutWatchRequestBuilder(client()).setId("test_watch_large")
+                .setSource(
+                    watchBuilder().trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.HOURS)))
+                        .input(simpleInput("foo", randomAlphaOfLength((int) ByteSizeValue.ofKb(500).getBytes())))
+                        .addAction("_logger", loggingAction("#### randomLogging"))
+                )
+                .get();
+            new ExecuteWatchRequestBuilder(client()).setId("test_watch_large").setRecordExecution(true).get();
+            assertBusy(() -> {
+                assertResponse(getWatchHistory(), searchResponse -> {
+                    assertHitCount(searchResponse, 2);
+                    SearchHit hit = searchResponse.getHits().getAt(1);
+                    XContentSource source = new XContentSource(hit.getSourceRef(), XContentType.JSON);
+                    Map<String, Object> input = source.getValue("input");
+                    assertThat(input.containsKey(WatchRecord.TRUNCATED_RECORD_KEY), equalTo(true));
+                    assertThat(input.get(WatchRecord.TRUNCATED_RECORD_KEY), equalTo(WatchRecord.TRUNCATED_RECORD_VALUE));
+                    assertThat(input.containsKey("simple"), equalTo(false));
+                    Map<String, Object> result = source.getValue("result");
+                    assertThat(result.containsKey(WatchRecord.TRUNCATED_RECORD_KEY), equalTo(true));
+                    assertThat(result.get(WatchRecord.TRUNCATED_RECORD_KEY), equalTo(WatchRecord.TRUNCATED_RECORD_VALUE));
+                    assertThat(result.containsKey("input"), equalTo(false));
+                    assertThat(result.containsKey("actions"), equalTo(false));
+                    assertThat(result.containsKey("condition"), equalTo(false));
+                });
+            });
+        }
+    }
+
     /*
      * Returns a SearchRequestBuilder containing up to the default number of watch history records (10) if the .watcher-history* is ready.
      * Otherwise it throws an AssertionError.
@@ -232,7 +309,7 @@ public void testThatHistoryContainsStatus() throws Exception {
     private SearchRequestBuilder getWatchHistory() {
         ensureGreen(HistoryStoreField.DATA_STREAM);
         flushAndRefresh(".watcher-history-*");
-        return prepareSearch(".watcher-history-*");
+        return prepareSearch(".watcher-history-*").addSort("@timestamp", SortOrder.ASC);
     }
 
 }
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java
index 1308597b7bcf9..6402b71d3b810 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.condition.CompareCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@@ -75,7 +74,6 @@ public void testAckSingleAction() throws Exception {
             .get();
 
         assertThat(putWatchResponse.isCreated(), is(true));
-        assertThat(new WatcherStatsRequestBuilder(client()).get().getWatchesCount(), is(1L));
 
         timeWarp().trigger("_id", 4, TimeValue.timeValueSeconds(5));
         AckWatchResponse ackResponse = new AckWatchRequestBuilder(client(), "_id").setActionIds("_a1").get();
@@ -148,7 +146,6 @@ public void testAckAllActions() throws Exception {
             .get();
 
         assertThat(putWatchResponse.isCreated(), is(true));
-        assertThat(new WatcherStatsRequestBuilder(client()).get().getWatchesCount(), is(1L));
 
         timeWarp().trigger("_id", 4, TimeValue.timeValueSeconds(5));
 
@@ -226,7 +223,6 @@ public void testAckWithRestart() throws Exception {
             )
             .get();
         assertThat(putWatchResponse.isCreated(), is(true));
-        assertThat(new WatcherStatsRequestBuilder(client()).get().getWatchesCount(), is(1L));
 
         timeWarp().trigger("_name", 4, TimeValue.timeValueSeconds(5));
         restartWatcherRandomly();
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
index bdc040ff2eca7..92ac91a63e097 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.watcher.transform;
 
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Strings;
@@ -15,7 +14,6 @@
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
-import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
@@ -33,9 +31,9 @@
 import java.util.function.Function;
 
 import static java.util.Collections.singletonMap;
+import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.putJsonStoredScript;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse;
 import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.indexAction;
 import static org.elasticsearch.xpack.watcher.client.WatchSourceBuilders.watchBuilder;
@@ -106,13 +104,13 @@ public void testScriptTransform() throws Exception {
             script = mockScript("['key3' : ctx.payload.key1 + ctx.payload.key2]");
         } else {
             logger.info("testing script transform with an indexed script");
-            assertAcked(clusterAdmin().preparePutStoredScript().setId("my-script").setContent(new BytesArray(Strings.format("""
+            putJsonStoredScript("my-script", Strings.format("""
                 {
                   "script": {
                     "lang": "%s",
                     "source": "['key3' : ctx.payload.key1 + ctx.payload.key2]"
                   }
-                }""", MockScriptPlugin.NAME)), XContentType.JSON).get());
+                }""", MockScriptPlugin.NAME));
             script = new Script(ScriptType.STORED, null, "my-script", Collections.emptyMap());
         }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
index 2d71aef08ea13..5cff1db9dd174 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
@@ -490,7 +490,7 @@ public void afterBulk(long executionId, BulkRequest request, Exception failure)
             .setBulkSize(SETTING_BULK_SIZE.get(settings))
             .build();
 
-        HistoryStore historyStore = new HistoryStore(bulkProcessor);
+        HistoryStore historyStore = new HistoryStore(bulkProcessor, settings);
 
         // schedulers
         final Set<Schedule.Parser<?>> scheduleParsers = new HashSet<>();
@@ -613,6 +613,7 @@ public List<Setting<?>> getSettings() {
         settings.add(SETTING_BULK_CONCURRENT_REQUESTS);
         settings.add(SETTING_BULK_FLUSH_INTERVAL);
         settings.add(SETTING_BULK_SIZE);
+        settings.add(HistoryStore.MAX_HISTORY_SIZE_SETTING);
 
         // notification services
         settings.addAll(SlackService.getSettings());
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportAction.java
index e93081359ca92..dff56a9423f5a 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportAction.java
@@ -7,8 +7,8 @@
 package org.elasticsearch.xpack.watcher;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java
index 7a888c2d0c65e..1256b21b4c70d 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/history/HistoryStore.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/history/HistoryStore.java
index b4f6d82eab965..d8ba0c7e7a506 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/history/HistoryStore.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/history/HistoryStore.java
@@ -13,6 +13,9 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
@@ -22,16 +25,24 @@
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.settings.Setting.Property.NodeScope;
 import static org.elasticsearch.xpack.core.watcher.support.Exceptions.ioException;
 
 public class HistoryStore {
 
     private static final Logger logger = LogManager.getLogger(HistoryStore.class);
+    public static final Setting<ByteSizeValue> MAX_HISTORY_SIZE_SETTING = Setting.byteSizeSetting(
+        "xpack.watcher.max.history.record.size",
+        ByteSizeValue.ofMb(10),
+        NodeScope
+    );
 
     private final BulkProcessor2 bulkProcessor;
+    private final ByteSizeValue maxHistoryRecordSize;
 
-    public HistoryStore(BulkProcessor2 bulkProcessor) {
+    public HistoryStore(BulkProcessor2 bulkProcessor, Settings settings) {
         this.bulkProcessor = bulkProcessor;
+        maxHistoryRecordSize = MAX_HISTORY_SIZE_SETTING.get(settings);
     }
 
     /**
@@ -41,9 +52,15 @@ public HistoryStore(BulkProcessor2 bulkProcessor) {
     public void put(WatchRecord watchRecord) throws Exception {
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
             watchRecord.toXContent(builder, WatcherParams.HIDE_SECRETS);
-
             IndexRequest request = new IndexRequest(HistoryStoreField.DATA_STREAM).id(watchRecord.id().value()).source(builder);
             request.opType(IndexRequest.OpType.CREATE);
+            if (request.source().length() > maxHistoryRecordSize.getBytes()) {
+                WatchRecord redactedWatchRecord = watchRecord.dropLargeFields();
+                try (XContentBuilder redactedBuilder = XContentFactory.jsonBuilder()) {
+                    redactedWatchRecord.toXContent(redactedBuilder, WatcherParams.HIDE_SECRETS);
+                    request.source(redactedBuilder);
+                }
+            }
             bulkProcessor.add(request);
         } catch (IOException ioe) {
             throw ioException("failed to persist watch record [{}]", ioe, watchRecord);
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Attachment.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Attachment.java
index 33995abd9ac71..d1d406238e908 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Attachment.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/Attachment.java
@@ -8,8 +8,8 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Provider;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.injection.guice.Provider;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchAction.java
index 9f25c65bc9d3c..02c8275e5b911 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchAction.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.routing.Preference;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportActivateWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportActivateWatchAction.java
index b5a74aecea5b5..cdc279c922a0c 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportActivateWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportActivateWatchAction.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.routing.Preference;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.XContentBuilder;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportDeleteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportDeleteWatchAction.java
index 095bbe8b65321..19c8efded609b 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportDeleteWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportDeleteWatchAction.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportExecuteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportExecuteWatchAction.java
index 2eec48ac07db9..8504ba75e88fb 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportExecuteWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportExecuteWatchAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.cluster.routing.Preference;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatchAction.java
index 15e8887bb65a0..2ed807330cf71 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatchAction.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.routing.Preference;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.XContentBuilder;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java
index f0fe010962697..29349735afcd2 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchAction.java
index f19a1ecd0c0fc..c894117f69523 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchAction.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportQueryWatchesAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportQueryWatchesAction.java
index 09e9299730bd0..97ae29a26e68c 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportQueryWatchesAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportQueryWatchesAction.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.FieldSortBuilder;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java
index 8064a17d451e3..378ee642cf105 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java
@@ -22,10 +22,10 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.tasks.Task;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java
index 4d7612d843379..9212780d11fd3 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java
@@ -22,9 +22,9 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsAction.java
index fc181b38f5944..220415cf9d094 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsAction.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java
index 2f5df9b51ab7e..89968aa2cf19b 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -38,6 +39,7 @@
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
 import org.elasticsearch.xpack.watcher.notification.jira.JiraAccount;
 import org.elasticsearch.xpack.watcher.notification.jira.JiraIssue;
+import org.elasticsearch.xpack.watcher.test.WatcherTestUtils;
 import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTriggerEvent;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
@@ -45,14 +47,18 @@
 import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
@@ -71,9 +77,18 @@ public void init() {
         when(client.threadPool()).thenReturn(threadPool);
         when(client.settings()).thenReturn(settings);
         when(threadPool.getThreadContext()).thenReturn(new ThreadContext(settings));
+        historyStore = createHistoryStore(null);
+    }
+
+    private HistoryStore createHistoryStore(ByteSizeValue maxHistoryRecordSize) {
         BulkProcessor2.Listener listener = mock(BulkProcessor2.Listener.class);
-        BulkProcessor2 bulkProcessor = BulkProcessor2.builder(client::bulk, listener, threadPool).setBulkActions(1).build();
-        historyStore = new HistoryStore(bulkProcessor);
+        BulkProcessor2 bulkProcessor = BulkProcessor2.builder(client::bulk, listener, client.threadPool()).setBulkActions(1).build();
+        Settings.Builder settingsBuilder = Settings.builder();
+        if (maxHistoryRecordSize != null) {
+            settingsBuilder.put(HistoryStore.MAX_HISTORY_SIZE_SETTING.getKey(), maxHistoryRecordSize);
+        }
+        Settings settings = settingsBuilder.build();
+        return new HistoryStore(bulkProcessor, settings);
     }
 
     public void testPut() throws Exception {
@@ -83,11 +98,11 @@ public void testPut() throws Exception {
         WatchRecord watchRecord = new WatchRecord.MessageWatchRecord(wid, event, ExecutionState.EXECUTED, null, randomAlphaOfLength(10));
 
         IndexResponse indexResponse = mock(IndexResponse.class);
-
+        AtomicBoolean historyItemIndexed = new AtomicBoolean(false);
         doAnswer(invocation -> {
-            BulkRequest request = (BulkRequest) invocation.getArguments()[1];
+            BulkRequest request = (BulkRequest) invocation.getArguments()[0];
             @SuppressWarnings("unchecked")
-            ActionListener<BulkResponse> listener = (ActionListener<BulkResponse>) invocation.getArguments()[2];
+            ActionListener<BulkResponse> listener = (ActionListener<BulkResponse>) invocation.getArguments()[1];
 
             IndexRequest indexRequest = (IndexRequest) request.requests().get(0);
             if (indexRequest.id().equals(wid.value())
@@ -96,14 +111,108 @@ public void testPut() throws Exception {
                 listener.onResponse(
                     new BulkResponse(new BulkItemResponse[] { BulkItemResponse.success(1, OpType.CREATE, indexResponse) }, 1)
                 );
+                historyItemIndexed.set(true);
             } else {
                 listener.onFailure(new ElasticsearchException("test issue"));
+                fail("Unexpected indexRequest");
             }
             return null;
         }).when(client).bulk(any(), any());
 
         historyStore.put(watchRecord);
         verify(client).bulk(any(), any());
+        assertThat(historyItemIndexed.get(), equalTo(true));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testPutLargeHistory() throws Exception {
+        IndexResponse indexResponse = mock(IndexResponse.class);
+        AtomicBoolean historyRedacted = new AtomicBoolean(false);
+        doAnswer(invocation -> {
+            BulkRequest request = (BulkRequest) invocation.getArguments()[0];
+            ActionListener<BulkResponse> listener = (ActionListener<BulkResponse>) invocation.getArguments()[1];
+            IndexRequest indexRequest = (IndexRequest) request.requests().get(0);
+            Map<String, Object> sourceMap = indexRequest.sourceAsMap();
+            if (indexRequest.opType() == OpType.CREATE && indexRequest.index().equals(HistoryStoreField.DATA_STREAM)) {
+                if (sourceMap.containsKey("input")
+                    && ((Map<String, Object>) sourceMap.get("input")).containsKey(WatchRecord.TRUNCATED_RECORD_KEY)
+                    && sourceMap.containsKey("result")
+                    && ((Map<String, Object>) sourceMap.get("result")).containsKey(WatchRecord.TRUNCATED_RECORD_KEY)) {
+                    assertThat(
+                        ((Map<String, Object>) sourceMap.get("input")).get(WatchRecord.TRUNCATED_RECORD_KEY),
+                        equalTo(WatchRecord.TRUNCATED_RECORD_VALUE)
+                    );
+                    assertThat(
+                        ((Map<String, Object>) sourceMap.get("result")).get(WatchRecord.TRUNCATED_RECORD_KEY),
+                        equalTo(WatchRecord.TRUNCATED_RECORD_VALUE)
+                    );
+                    historyRedacted.set(true);
+                }
+                listener.onResponse(
+                    new BulkResponse(new BulkItemResponse[] { BulkItemResponse.success(1, OpType.CREATE, indexResponse) }, 1)
+                );
+            } else {
+                listener.onFailure(new ElasticsearchException("test issue"));
+                fail("Should never get here");
+            }
+            return null;
+        }).when(client).bulk(any(), any());
+        HistoryStore historyStoreSmallLimit = createHistoryStore(ByteSizeValue.ofBytes(10));
+        HistoryStore historyStoreLargeLimit = createHistoryStore(ByteSizeValue.ofBytes(10_000_000));
+        {
+            /*
+             * First, create a history record with input and results. We expect this to not be truncated when the store has a high limit,
+             * and we expect it to be truncated when we have the artificially low limit.
+             */
+            WatchExecutionContext context = WatcherTestUtils.createWatchExecutionContext();
+            WatchExecutionResult result = new WatchExecutionResult(context, randomNonNegativeLong());
+            String message = randomAlphaOfLength(100);
+            WatchRecord watchRecord = new WatchRecord.MessageWatchRecord(context, result, message);
+            historyStoreLargeLimit.put(watchRecord);
+            verify(client, atLeastOnce()).bulk(any(), any());
+            assertThat(historyRedacted.get(), equalTo(false));
+            historyStoreSmallLimit.put(watchRecord);
+            verify(client, atLeastOnce()).bulk(any(), any());
+            assertThat(historyRedacted.get(), equalTo(true));
+        }
+        {
+            /*
+             * Now make sure that we don't blow up when the input and result are null
+             */
+            historyRedacted.set(false);
+            ZonedDateTime now = Instant.ofEpochMilli(0).atZone(ZoneOffset.UTC);
+            Wid wid = new Wid("_name", now);
+            ScheduleTriggerEvent event = new ScheduleTriggerEvent(wid.watchId(), now, now);
+            WatchRecord watchRecord = new WatchRecord.MessageWatchRecord(
+                wid,
+                event,
+                ExecutionState.EXECUTED,
+                null,
+                randomAlphaOfLength(10)
+            );
+            historyStoreLargeLimit.put(watchRecord);
+            verify(client, atLeastOnce()).bulk(any(), any());
+            assertThat(historyRedacted.get(), equalTo(false));
+            historyStoreSmallLimit.put(watchRecord);
+            verify(client, atLeastOnce()).bulk(any(), any());
+            assertThat(historyRedacted.get(), equalTo(false));
+        }
+        {
+            /*
+             * Now make sure that we don't blow up when the input and result are null
+             */
+            historyRedacted.set(false);
+            WatchExecutionContext context = WatcherTestUtils.createWatchExecutionContext();
+            WatchExecutionResult result = new WatchExecutionResult(context, randomNonNegativeLong());
+            Exception exception = new RuntimeException(randomAlphaOfLength(100));
+            WatchRecord watchRecord = new WatchRecord.ExceptionWatchRecord(context, result, exception);
+            historyStoreLargeLimit.put(watchRecord);
+            verify(client, atLeastOnce()).bulk(any(), any());
+            assertThat(historyRedacted.get(), equalTo(false));
+            historyStoreSmallLimit.put(watchRecord);
+            verify(client, atLeastOnce()).bulk(any(), any());
+            assertThat(historyRedacted.get(), equalTo(true));
+        }
     }
 
     public void testStoreWithHideSecrets() throws Exception {
@@ -148,12 +257,14 @@ public void testStoreWithHideSecrets() throws Exception {
         watchRecord.result().actionsResults().put(JiraAction.TYPE, result);
 
         ArgumentCaptor<BulkRequest> requestCaptor = ArgumentCaptor.forClass(BulkRequest.class);
+        AtomicBoolean historyItemIndexed = new AtomicBoolean(false);
         doAnswer(invocation -> {
             @SuppressWarnings("unchecked")
-            ActionListener<BulkResponse> listener = (ActionListener<BulkResponse>) invocation.getArguments()[2];
+            ActionListener<BulkResponse> listener = (ActionListener<BulkResponse>) invocation.getArguments()[1];
 
             IndexResponse indexResponse = mock(IndexResponse.class);
             listener.onResponse(new BulkResponse(new BulkItemResponse[] { BulkItemResponse.success(1, OpType.CREATE, indexResponse) }, 1));
+            historyItemIndexed.set(true);
             return null;
         }).when(client).bulk(requestCaptor.capture(), any());
 
@@ -162,7 +273,7 @@ public void testStoreWithHideSecrets() throws Exception {
         } else {
             historyStore.forcePut(watchRecord);
         }
-
+        assertThat(historyItemIndexed.get(), equalTo(true));
         assertThat(requestCaptor.getAllValues(), hasSize(1));
         assertThat(requestCaptor.getValue().requests().get(0), instanceOf(IndexRequest.class));
         IndexRequest capturedIndexRequest = (IndexRequest) requestCaptor.getValue().requests().get(0);
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchStoreUtilsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchStoreUtilsTests.java
index ffceb054ae54d..a04eb3e7d5091 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchStoreUtilsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchStoreUtilsTests.java
@@ -133,10 +133,7 @@ public void testGetConcreteIndexForConcreteIndex() {
 
     private IndexMetadata createIndexMetaData(String indexName, AliasMetadata aliasMetadata) {
         IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder(indexName);
-        Settings settings = Settings.builder()
-            .put(IndexMetadata.SETTING_PRIORITY, 5)
-            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+        Settings settings = indexSettings(1, 1).put(IndexMetadata.SETTING_PRIORITY, 5)
             .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current())
             .build();
         indexMetadataBuilder.settings(settings);
diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
index 52dbc3d9e1ecb..a661322899a8d 100644
--- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
+++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java
@@ -61,6 +61,7 @@
 import org.elasticsearch.index.mapper.BinaryFieldMapper.CustomBinaryDocValuesField;
 import org.elasticsearch.index.mapper.BlockDocValuesReader;
 import org.elasticsearch.index.mapper.BlockLoader;
+import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
@@ -88,8 +89,6 @@
 import java.util.Set;
 import java.util.stream.Stream;
 
-import static java.util.Collections.emptyList;
-
 /**
  * A {@link FieldMapper} for indexing fields with ngrams for efficient wildcard matching
  */
@@ -1003,21 +1002,33 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
                 "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        return new WildcardSyntheticFieldLoader();
+
+        var loader = new WildcardSyntheticFieldLoader();
+        if (ignoreAbove != Defaults.IGNORE_ABOVE) {
+            return new CompositeSyntheticFieldLoader(
+                leafName(),
+                fullPath(),
+                loader,
+                new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) {
+                    @Override
+                    protected void writeValue(Object value, XContentBuilder b) throws IOException {
+                        BytesRef r = (BytesRef) value;
+                        b.utf8Value(r.bytes, r.offset, r.length);
+                    }
+                }
+            );
+        }
+
+        return new CompositeSyntheticFieldLoader(leafName(), fullPath(), loader);
     }
 
-    private class WildcardSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader {
+    private class WildcardSyntheticFieldLoader implements CompositeSyntheticFieldLoader.Layer {
         private final ByteArrayStreamInput docValuesStream = new ByteArrayStreamInput();
         private int docValueCount;
         private BytesRef docValueBytes;
 
-        private List<Object> storedValues = emptyList();
-
         @Override
         public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
-            if (ignoreAbove != Defaults.IGNORE_ABOVE) {
-                return Stream.of(Map.entry(originalName(), storedValues -> this.storedValues = storedValues));
-            }
             return Stream.empty();
         }
 
@@ -1044,33 +1055,21 @@ public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf
 
         @Override
         public boolean hasValue() {
-            return docValueCount > 0 || storedValues.isEmpty() == false;
+            return docValueCount > 0;
+        }
+
+        @Override
+        public long valueCount() {
+            return docValueCount;
         }
 
         @Override
         public void write(XContentBuilder b) throws IOException {
-            switch (docValueCount + storedValues.size()) {
-                case 0:
-                    return;
-                case 1:
-                    b.field(leafName());
-                    break;
-                default:
-                    b.startArray(leafName());
-            }
             for (int i = 0; i < docValueCount; i++) {
                 int length = docValuesStream.readVInt();
                 b.utf8Value(docValueBytes.bytes, docValuesStream.getPosition(), length);
                 docValuesStream.skipBytes(length);
             }
-            for (Object o : storedValues) {
-                BytesRef r = (BytesRef) o;
-                b.utf8Value(r.bytes, r.offset, r.length);
-            }
-            if (docValueCount + storedValues.size() > 1) {
-                b.endArray();
-            }
-            storedValues = emptyList();
         }
 
         @Override
diff --git a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java
index cb93725b320d1..b37b026b853e2 100644
--- a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java
+++ b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java
@@ -161,7 +161,12 @@ private void setUpDataStreamWriteDocsAndRollover(String dataStreamName, Settings
                 )
             )
         );
-        assertAcked(client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet());
+        assertAcked(
+            client().execute(
+                CreateDataStreamAction.INSTANCE,
+                new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName)
+            ).actionGet()
+        );
 
         final int numberOfRollovers = randomIntBetween(5, 10);
         for (int i = 0; i < numberOfRollovers; i++) {
diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
index 80f1c706a34af..d76490e885592 100644
--- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.WarningsHandler;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
@@ -98,6 +99,7 @@ public void testSingleDoc() throws IOException {
             Request createDoc = new Request("PUT", docLocation);
             createDoc.addParameter("refresh", "true");
             createDoc.setJsonEntity(doc);
+            createDoc.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(fieldNamesFieldOk()));
             client().performRequest(createDoc);
         }
 
@@ -447,6 +449,7 @@ public void testRollupAfterRestart() throws Exception {
                         }
                     }
                     """);
+                req.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(fieldNamesFieldOk()));
                 client().performRequest(req);
             }
 
@@ -541,7 +544,7 @@ public void testTransformLegacyTemplateCleanup() throws Exception {
                     }
                   }
                 }""");
-
+            createIndexRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(fieldNamesFieldOk()));
             Map<String, Object> createIndexResponse = entityAsMap(client().performRequest(createIndexRequest));
             assertThat(createIndexResponse.get("acknowledged"), equalTo(Boolean.TRUE));
 
@@ -972,6 +975,7 @@ public void testDataStreams() throws Exception {
                 .field("@timestamp", System.currentTimeMillis())
                 .endObject();
             indexRequest.setJsonEntity(Strings.toString(builder));
+            indexRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(fieldNamesFieldOk()));
             assertOK(client().performRequest(indexRequest));
         }
 
@@ -1027,7 +1031,7 @@ public void testDisableFieldNameField() throws IOException {
                 {
                   "settings": {
                     "index": {
-                      "number_of_replicas": 0
+                      "number_of_replicas": 1
                     }
                   },
                   "mappings": {
@@ -1038,11 +1042,7 @@ public void testDisableFieldNameField() throws IOException {
                     }
                   }
                 }""");
-            createIndex.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> switch (warnings.size()) {
-                case 0 -> false;  // old versions don't return a warning
-                case 1 -> false == warnings.get(0).contains("_field_names");
-                default -> true;
-            }));
+            createIndex.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(fieldNamesFieldOk()));
             client().performRequest(createIndex);
 
             Request createDoc = new Request("PUT", docLocation);
@@ -1061,14 +1061,37 @@ public void testDisableFieldNameField() throws IOException {
                   "query": "FROM nofnf | LIMIT 1"
                 }""");
             // {"columns":[{"name":"dv","type":"keyword"},{"name":"no_dv","type":"keyword"}],"values":[["test",null]]}
-            assertMap(
-                entityAsMap(client().performRequest(esql)),
-                matchesMap().entry("columns", List.of(Map.of("name", "dv", "type", "keyword"), Map.of("name", "no_dv", "type", "keyword")))
-                    .entry("values", List.of(List.of("test", "test")))
-            );
+            try {
+                assertMap(
+                    entityAsMap(client().performRequest(esql)),
+                    matchesMap().entry(
+                        "columns",
+                        List.of(Map.of("name", "dv", "type", "keyword"), Map.of("name", "no_dv", "type", "keyword"))
+                    ).entry("values", List.of(List.of("test", "test")))
+                );
+            } catch (ResponseException e) {
+                logger.error(
+                    "failed to query index without field name field. Existing indices:\n{}",
+                    EntityUtils.toString(client().performRequest(new Request("GET", "_cat/indices")).getEntity())
+                );
+                throw e;
+            }
         }
     }
 
+    /**
+     * Ignore the warning about the {@code _field_names} field. We intentionally
+     * turn that field off sometimes. And other times old versions spuriously
+     * send it back to us.
+     */
+    private WarningsHandler fieldNamesFieldOk() {
+        return warnings -> switch (warnings.size()) {
+            case 0 -> false;  // old versions don't return a warning
+            case 1 -> false == warnings.get(0).contains("_field_names");
+            default -> true;
+        };
+    }
+
     private static void createComposableTemplate(RestClient client, String templateName, String indexPattern) throws IOException {
         StringEntity templateJSON = new StringEntity(Strings.format("""
             {
diff --git a/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java b/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java
index 0625ec166e32c..8e9dca396fb8e 100644
--- a/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java
+++ b/x-pack/qa/password-protected-keystore/src/javaRestTest/java/org/elasticsearch/password_protected_keystore/ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT.java
@@ -7,7 +7,9 @@
 package org.elasticsearch.password_protected_keystore;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -18,6 +20,7 @@
 import org.elasticsearch.xcontent.ObjectPath;
 import org.junit.ClassRule;
 
+import java.io.IOException;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.anyOf;
@@ -39,6 +42,7 @@ public class ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT extends ESR
         .name("javaRestTest")
         .keystore(nodeSpec -> Map.of("xpack.security.transport.ssl.secure_key_passphrase", "transport-password"))
         .setting("xpack.security.enabled", "true")
+        .setting("xpack.ml.enabled", "false")
         .setting("xpack.security.authc.anonymous.roles", "anonymous")
         .setting("xpack.security.transport.ssl.enabled", "true")
         .setting("xpack.security.transport.ssl.certificate", "transport.crt")
@@ -50,6 +54,9 @@ public class ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT extends ESR
         .configFile("ca.crt", Resource.fromClasspath("ssl/ca.crt"))
         .user("admin_user", "admin-password")
         .user("test-user", "test-user-password", "user_role", false)
+        .user("manage-user", "test-user-password", "manage_role", false)
+        .user("manage-security-user", "test-user-password", "manage_security_role", false)
+        .user("monitor-user", "test-user-password", "monitor_role", false)
         .build();
 
     @Override
@@ -74,6 +81,33 @@ public void testReloadSecureSettingsWithCorrectPassword() throws Exception {
         }
     }
 
+    @SuppressWarnings("unchecked")
+    public void testReloadSecureSettingsWithDifferentPrivileges() throws Exception {
+        final Request request = new Request("POST", "/_nodes/reload_secure_settings");
+        request.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}");
+        final Response response = performRequestWithUser("manage-user", request);
+        final Map<String, Object> map = entityAsMap(response);
+        assertThat(ObjectPath.eval("cluster_name", map), equalTo("javaRestTest"));
+        assertThat(map.get("nodes"), instanceOf(Map.class));
+        final Map<String, Object> nodes = (Map<String, Object>) map.get("nodes");
+        assertThat(nodes.size(), equalTo(NUM_NODES));
+        for (Map.Entry<String, Object> entry : nodes.entrySet()) {
+            assertThat(entry.getValue(), instanceOf(Map.class));
+            final Map<String, Object> node = (Map<String, Object>) entry.getValue();
+            assertThat(node.get("reload_exception"), nullValue());
+        }
+        expectThrows403(() -> {
+            final Request innerRequest = new Request("POST", "/_nodes/reload_secure_settings");
+            innerRequest.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}");
+            performRequestWithUser("manage-security-user", innerRequest);
+        });
+        expectThrows403(() -> {
+            final Request innerRequest = new Request("POST", "/_nodes/reload_secure_settings");
+            innerRequest.setJsonEntity("{\"secure_settings_password\":\"" + KEYSTORE_PASSWORD + "\"}");
+            performRequestWithUser("monitor-user", request);
+        });
+    }
+
     @SuppressWarnings("unchecked")
     public void testReloadSecureSettingsWithIncorrectPassword() throws Exception {
         final Request request = new Request("POST", "_nodes/reload_secure_settings");
@@ -136,4 +170,16 @@ protected Settings restAdminSettings() {
         String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
         return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
     }
+
+    private static void expectThrows403(ThrowingRunnable runnable) {
+        assertThat(expectThrows(ResponseException.class, runnable).getResponse().getStatusLine().getStatusCode(), equalTo(403));
+    }
+
+    private Response performRequestWithUser(final String username, final Request request) throws IOException {
+        request.setOptions(
+            RequestOptions.DEFAULT.toBuilder()
+                .addHeader("Authorization", basicAuthHeaderValue(username, new SecureString("test-user-password")))
+        );
+        return client().performRequest(request);
+    }
 }
diff --git a/x-pack/qa/password-protected-keystore/src/javaRestTest/resources/roles.yml b/x-pack/qa/password-protected-keystore/src/javaRestTest/resources/roles.yml
index 60dd459d0f824..c551d8200bfe0 100644
--- a/x-pack/qa/password-protected-keystore/src/javaRestTest/resources/roles.yml
+++ b/x-pack/qa/password-protected-keystore/src/javaRestTest/resources/roles.yml
@@ -2,3 +2,15 @@
 user_role:
   cluster: [ALL]
   indices: []
+
+manage_role:
+  cluster: [MANAGE]
+  indices: []
+
+manage_security_role:
+  cluster: [MANAGE_SECURITY]
+  indices: []
+
+monitor_role:
+  cluster: [MONITOR]
+  indices: []
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java
index 4f4ff1d5743ee..650779cfbc85d 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RolesBackwardsCompatibilityIT.java
@@ -29,6 +29,7 @@
 
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomApplicationPrivileges;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomIndicesPrivileges;
+import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomManageRolesPrivileges;
 import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTestHelper.randomRoleDescriptorMetadata;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
@@ -40,7 +41,7 @@ public class RolesBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
     private RestClient oldVersionClient = null;
     private RestClient newVersionClient = null;
 
-    public void testCreatingAndUpdatingRoles() throws Exception {
+    public void testRolesWithDescription() throws Exception {
         assumeTrue(
             "The role description is supported after transport version: " + TransportVersions.SECURITY_ROLE_DESCRIPTION,
             minimumTransportVersion().before(TransportVersions.SECURITY_ROLE_DESCRIPTION)
@@ -48,14 +49,14 @@ public void testCreatingAndUpdatingRoles() throws Exception {
         switch (CLUSTER_TYPE) {
             case OLD -> {
                 // Creating role in "old" cluster should succeed when description is not provided
-                final String initialRole = randomRoleDescriptorSerialized(false);
+                final String initialRole = randomRoleDescriptorSerialized();
                 createRole(client(), "my-old-role", initialRole);
-                updateRole("my-old-role", randomValueOtherThan(initialRole, () -> randomRoleDescriptorSerialized(false)));
+                updateRole("my-old-role", randomValueOtherThan(initialRole, RolesBackwardsCompatibilityIT::randomRoleDescriptorSerialized));
 
                 // and fail if we include description
                 var createException = expectThrows(
                     Exception.class,
-                    () -> createRole(client(), "my-invalid-old-role", randomRoleDescriptorSerialized(true))
+                    () -> createRole(client(), "my-invalid-old-role", randomRoleDescriptorWithDescriptionSerialized())
                 );
                 assertThat(
                     createException.getMessage(),
@@ -65,7 +66,7 @@ public void testCreatingAndUpdatingRoles() throws Exception {
                 RestClient client = client();
                 var updateException = expectThrows(
                     Exception.class,
-                    () -> updateRole(client, "my-old-role", randomRoleDescriptorSerialized(true))
+                    () -> updateRole(client, "my-old-role", randomRoleDescriptorWithDescriptionSerialized())
                 );
                 assertThat(
                     updateException.getMessage(),
@@ -74,17 +75,20 @@ public void testCreatingAndUpdatingRoles() throws Exception {
             }
             case MIXED -> {
                 try {
-                    this.createClientsByVersion();
+                    this.createClientsByVersion(TransportVersions.SECURITY_ROLE_DESCRIPTION);
                     // succeed when role description is not provided
-                    final String initialRole = randomRoleDescriptorSerialized(false);
+                    final String initialRole = randomRoleDescriptorSerialized();
                     createRole(client(), "my-valid-mixed-role", initialRole);
-                    updateRole("my-valid-mixed-role", randomValueOtherThan(initialRole, () -> randomRoleDescriptorSerialized(false)));
+                    updateRole(
+                        "my-valid-mixed-role",
+                        randomValueOtherThan(initialRole, RolesBackwardsCompatibilityIT::randomRoleDescriptorSerialized)
+                    );
 
                     // against old node, fail when description is provided either in update or create request
                     {
                         Exception e = expectThrows(
                             Exception.class,
-                            () -> updateRole(oldVersionClient, "my-valid-mixed-role", randomRoleDescriptorSerialized(true))
+                            () -> updateRole(oldVersionClient, "my-valid-mixed-role", randomRoleDescriptorWithDescriptionSerialized())
                         );
                         assertThat(
                             e.getMessage(),
@@ -94,7 +98,7 @@ public void testCreatingAndUpdatingRoles() throws Exception {
                     {
                         Exception e = expectThrows(
                             Exception.class,
-                            () -> createRole(oldVersionClient, "my-invalid-mixed-role", randomRoleDescriptorSerialized(true))
+                            () -> createRole(oldVersionClient, "my-invalid-mixed-role", randomRoleDescriptorWithDescriptionSerialized())
                         );
                         assertThat(
                             e.getMessage(),
@@ -106,7 +110,7 @@ public void testCreatingAndUpdatingRoles() throws Exception {
                     {
                         Exception e = expectThrows(
                             Exception.class,
-                            () -> createRole(newVersionClient, "my-invalid-mixed-role", randomRoleDescriptorSerialized(true))
+                            () -> createRole(newVersionClient, "my-invalid-mixed-role", randomRoleDescriptorWithDescriptionSerialized())
                         );
                         assertThat(
                             e.getMessage(),
@@ -120,7 +124,7 @@ public void testCreatingAndUpdatingRoles() throws Exception {
                     {
                         Exception e = expectThrows(
                             Exception.class,
-                            () -> updateRole(newVersionClient, "my-valid-mixed-role", randomRoleDescriptorSerialized(true))
+                            () -> updateRole(newVersionClient, "my-valid-mixed-role", randomRoleDescriptorWithDescriptionSerialized())
                         );
                         assertThat(
                             e.getMessage(),
@@ -138,11 +142,129 @@ public void testCreatingAndUpdatingRoles() throws Exception {
             case UPGRADED -> {
                 // on upgraded cluster which supports new description field
                 // create/update requests should succeed either way (with or without description)
-                final String initialRole = randomRoleDescriptorSerialized(randomBoolean());
+                final String initialRole = randomFrom(randomRoleDescriptorSerialized(), randomRoleDescriptorWithDescriptionSerialized());
                 createRole(client(), "my-valid-upgraded-role", initialRole);
                 updateRole(
                     "my-valid-upgraded-role",
-                    randomValueOtherThan(initialRole, () -> randomRoleDescriptorSerialized(randomBoolean()))
+                    randomValueOtherThan(
+                        initialRole,
+                        () -> randomFrom(randomRoleDescriptorSerialized(), randomRoleDescriptorWithDescriptionSerialized())
+                    )
+                );
+            }
+        }
+    }
+
+    public void testRolesWithManageRoles() throws Exception {
+        assumeTrue(
+            "The manage roles privilege is supported after transport version: " + TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE,
+            minimumTransportVersion().before(TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE)
+        );
+        switch (CLUSTER_TYPE) {
+            case OLD -> {
+                // Creating role in "old" cluster should succeed when manage roles is not provided
+                final String initialRole = randomRoleDescriptorSerialized();
+                createRole(client(), "my-old-role", initialRole);
+                updateRole("my-old-role", randomValueOtherThan(initialRole, RolesBackwardsCompatibilityIT::randomRoleDescriptorSerialized));
+
+                // and fail if we include manage roles
+                var createException = expectThrows(
+                    Exception.class,
+                    () -> createRole(client(), "my-invalid-old-role", randomRoleDescriptorWithManageRolesSerialized())
+                );
+                assertThat(
+                    createException.getMessage(),
+                    allOf(containsString("failed to parse privilege"), containsString("but found [role] instead"))
+                );
+
+                RestClient client = client();
+                var updateException = expectThrows(
+                    Exception.class,
+                    () -> updateRole(client, "my-old-role", randomRoleDescriptorWithManageRolesSerialized())
+                );
+                assertThat(
+                    updateException.getMessage(),
+                    allOf(containsString("failed to parse privilege"), containsString("but found [role] instead"))
+                );
+            }
+            case MIXED -> {
+                try {
+                    this.createClientsByVersion(TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE);
+                    // succeed when role manage roles is not provided
+                    final String initialRole = randomRoleDescriptorSerialized();
+                    createRole(client(), "my-valid-mixed-role", initialRole);
+                    updateRole(
+                        "my-valid-mixed-role",
+                        randomValueOtherThan(initialRole, RolesBackwardsCompatibilityIT::randomRoleDescriptorSerialized)
+                    );
+
+                    // against old node, fail when manage roles is provided either in update or create request
+                    {
+                        Exception e = expectThrows(
+                            Exception.class,
+                            () -> updateRole(oldVersionClient, "my-valid-mixed-role", randomRoleDescriptorWithManageRolesSerialized())
+                        );
+                        assertThat(
+                            e.getMessage(),
+                            allOf(containsString("failed to parse privilege"), containsString("but found [role] instead"))
+                        );
+                    }
+                    {
+                        Exception e = expectThrows(
+                            Exception.class,
+                            () -> createRole(oldVersionClient, "my-invalid-mixed-role", randomRoleDescriptorWithManageRolesSerialized())
+                        );
+                        assertThat(
+                            e.getMessage(),
+                            allOf(containsString("failed to parse privilege"), containsString("but found [role] instead"))
+                        );
+                    }
+
+                    // and against new node in a mixed cluster we should fail
+                    {
+                        Exception e = expectThrows(
+                            Exception.class,
+                            () -> createRole(newVersionClient, "my-invalid-mixed-role", randomRoleDescriptorWithManageRolesSerialized())
+                        );
+
+                        assertThat(
+                            e.getMessage(),
+                            containsString(
+                                "all nodes must have version ["
+                                    + TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE.toReleaseVersion()
+                                    + "] or higher to support the manage roles privilege"
+                            )
+                        );
+                    }
+                    {
+                        Exception e = expectThrows(
+                            Exception.class,
+                            () -> updateRole(newVersionClient, "my-valid-mixed-role", randomRoleDescriptorWithManageRolesSerialized())
+                        );
+                        assertThat(
+                            e.getMessage(),
+                            containsString(
+                                "all nodes must have version ["
+                                    + TransportVersions.ADD_MANAGE_ROLES_PRIVILEGE.toReleaseVersion()
+                                    + "] or higher to support the manage roles privilege"
+                            )
+                        );
+                    }
+                } finally {
+                    this.closeClientsByVersion();
+                }
+            }
+            case UPGRADED -> {
+                // on upgraded cluster which supports new description field
+                // create/update requests should succeed either way (with or without description)
+                final String initialRole = randomFrom(randomRoleDescriptorSerialized(), randomRoleDescriptorWithManageRolesSerialized());
+                createRole(client(), "my-valid-upgraded-role", initialRole);
+                updateRole(
+                    "my-valid-upgraded-role",
+                    randomValueOtherThan(
+                        initialRole,
+                        () -> randomFrom(randomRoleDescriptorSerialized(), randomRoleDescriptorWithManageRolesSerialized())
+                    )
                 );
             }
         }
@@ -166,10 +288,22 @@ private void updateRole(RestClient client, String roleName, String payload) thro
         assertThat(created, equalTo(false));
     }
 
-    private static String randomRoleDescriptorSerialized(boolean includeDescription) {
+    private static String randomRoleDescriptorSerialized() {
+        return randomRoleDescriptorSerialized(false, false);
+    }
+
+    private static String randomRoleDescriptorWithDescriptionSerialized() {
+        return randomRoleDescriptorSerialized(true, false);
+    }
+
+    private static String randomRoleDescriptorWithManageRolesSerialized() {
+        return randomRoleDescriptorSerialized(false, true);
+    }
+
+    private static String randomRoleDescriptorSerialized(boolean includeDescription, boolean includeManageRoles) {
         try {
             return XContentTestUtils.convertToXContent(
-                XContentTestUtils.convertToMap(randomRoleDescriptor(includeDescription)),
+                XContentTestUtils.convertToMap(randomRoleDescriptor(includeDescription, includeManageRoles)),
                 XContentType.JSON
             ).utf8ToString();
         } catch (IOException e) {
@@ -177,26 +311,26 @@ private static String randomRoleDescriptorSerialized(boolean includeDescription)
         }
     }
 
-    private boolean nodeSupportRoleDescription(Map<String, Object> nodeDetails) {
+    private boolean nodeSupportTransportVersion(Map<String, Object> nodeDetails, TransportVersion transportVersion) {
         String nodeVersionString = (String) nodeDetails.get("version");
-        TransportVersion transportVersion = getTransportVersionWithFallback(
+        TransportVersion nodeTransportVersion = getTransportVersionWithFallback(
             nodeVersionString,
             nodeDetails.get("transport_version"),
             () -> TransportVersions.ZERO
         );
 
-        if (transportVersion.equals(TransportVersions.ZERO)) {
+        if (nodeTransportVersion.equals(TransportVersions.ZERO)) {
             // In cases where we were not able to find a TransportVersion, a pre-8.8.0 node answered about a newer (upgraded) node.
             // In that case, the node will be current (upgraded), and remote indices are supported for sure.
             var nodeIsCurrent = nodeVersionString.equals(Build.current().version());
             assertTrue(nodeIsCurrent);
             return true;
         }
-        return transportVersion.onOrAfter(TransportVersions.SECURITY_ROLE_DESCRIPTION);
+        return nodeTransportVersion.onOrAfter(transportVersion);
     }
 
-    private void createClientsByVersion() throws IOException {
-        var clientsByCapability = getRestClientByCapability();
+    private void createClientsByVersion(TransportVersion transportVersion) throws IOException {
+        var clientsByCapability = getRestClientByCapability(transportVersion);
         if (clientsByCapability.size() == 2) {
             for (Map.Entry<Boolean, RestClient> client : clientsByCapability.entrySet()) {
                 if (client.getKey() == false) {
@@ -224,7 +358,7 @@ private void closeClientsByVersion() throws IOException {
     }
 
     @SuppressWarnings("unchecked")
-    private Map<Boolean, RestClient> getRestClientByCapability() throws IOException {
+    private Map<Boolean, RestClient> getRestClientByCapability(TransportVersion transportVersion) throws IOException {
         Response response = client().performRequest(new Request("GET", "_nodes"));
         assertOK(response);
         ObjectPath objectPath = ObjectPath.createFromResponse(response);
@@ -232,7 +366,7 @@ private Map<Boolean, RestClient> getRestClientByCapability() throws IOException
         Map<Boolean, List<HttpHost>> hostsByCapability = new HashMap<>();
         for (Map.Entry<String, Object> entry : nodesAsMap.entrySet()) {
             Map<String, Object> nodeDetails = (Map<String, Object>) entry.getValue();
-            var capabilitySupported = nodeSupportRoleDescription(nodeDetails);
+            var capabilitySupported = nodeSupportTransportVersion(nodeDetails, transportVersion);
             Map<String, Object> httpInfo = (Map<String, Object>) nodeDetails.get("http");
             hostsByCapability.computeIfAbsent(capabilitySupported, k -> new ArrayList<>())
                 .add(HttpHost.create((String) httpInfo.get("publish_address")));
@@ -244,7 +378,7 @@ private Map<Boolean, RestClient> getRestClientByCapability() throws IOException
         return clientsByCapability;
     }
 
-    private static RoleDescriptor randomRoleDescriptor(boolean includeDescription) {
+    private static RoleDescriptor randomRoleDescriptor(boolean includeDescription, boolean includeManageRoles) {
         final Set<String> excludedPrivileges = Set.of(
             "cross_cluster_replication",
             "cross_cluster_replication_internal",
@@ -255,7 +389,7 @@ private static RoleDescriptor randomRoleDescriptor(boolean includeDescription) {
             randomSubsetOf(Set.of("all", "monitor", "none")).toArray(String[]::new),
             randomIndicesPrivileges(0, 3, excludedPrivileges),
             randomApplicationPrivileges(),
-            null,
+            includeManageRoles ? randomManageRolesPrivileges() : null,
             generateRandomStringArray(5, randomIntBetween(2, 8), false, true),
             randomRoleDescriptorMetadata(false),
             Map.of(),
diff --git a/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java
index 3b602a53a76a4..2bea4bb247d8f 100644
--- a/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java
+++ b/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java
@@ -341,7 +341,7 @@ private boolean handleBulk(ApiCallSection bulk) {
                     try (XContentBuilder b = new XContentBuilder(JsonXContent.jsonXContent, bos)) {
                         b.map(body);
                     }
-                    bos.write(JsonXContent.jsonXContent.streamSeparator());
+                    bos.write(JsonXContent.jsonXContent.bulkSeparator());
                 }
                 List<IndexRequest> indexRequests = new ArrayList<>();
                 new BulkRequestParser(false, RestApiVersion.current()).parse(
diff --git a/x-pack/rest-resources-zip/build.gradle b/x-pack/rest-resources-zip/build.gradle
index 3d0533b4ec57e..0133ff80dfadf 100644
--- a/x-pack/rest-resources-zip/build.gradle
+++ b/x-pack/rest-resources-zip/build.gradle
@@ -20,6 +20,8 @@ dependencies {
   apis project(path: ':rest-api-spec', configuration: 'restSpecs')
   freeTests project(path: ':rest-api-spec', configuration: 'restTests')
   freeTests project(path: ':modules:aggregations', configuration: 'restTests')
+  freeTests project(path: ':modules:analysis-common', configuration: 'restTests')
+  freeTests project(path: ':modules:ingest-geoip', configuration: 'restTests')
   compatApis project(path: ':rest-api-spec', configuration: 'restCompatSpecs')
   compatApis project(path: ':x-pack:plugin', configuration: 'restCompatSpecs')
   freeCompatTests project(path: ':rest-api-spec', configuration: 'restCompatTests')